diff --git a/BUILDING.txt b/BUILDING.txt
index 06bef1fc33048..6e38ad374a174 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -4,7 +4,7 @@ Build instructions for Hadoop
 Requirements:
 
 * Unix System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
@@ -204,15 +204,19 @@ Building on Windows
 Requirements:
 
 * Windows System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
 * CMake 2.6 or newer
 * Windows SDK or Visual Studio 2010 Professional
-* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
 * zlib headers (if building native code bindings for zlib)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
+  tools must be present on your PATH.
+
+Unix command-line tools are also included with the Windows Git package which
+can be downloaded from http://git-scm.com/download/win.
 
 If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
 Do not use Visual Studio Express.  It does not support compiling for 64-bit,
@@ -221,6 +225,8 @@ download here:
 
 http://www.microsoft.com/en-us/download/details.aspx?id=8279
 
+Cygwin is neither required nor supported.
+
 ----------------------------------------------------------------------------------
 Building:
 
diff --git a/dev-support/determine-flaky-tests-hadoop.py b/dev-support/determine-flaky-tests-hadoop.py
new file mode 100755
index 0000000000000..ce152bacaefd5
--- /dev/null
+++ b/dev-support/determine-flaky-tests-hadoop.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Given a jenkins test job, this script examines all runs of the job done
+# within specified period of time (number of days prior to the execution
+# time of this script), and reports all failed tests.
+#
+# The output of this script includes a section for each run that has failed
+# tests, with each failed test name listed.
+#
+# More importantly, at the end, it outputs a summary section to list all failed
+# tests within all examined runs, and indicate how many runs a same test
+# failed, and sorted all failed tests by how many runs each test failed.
+#
+# This way, when we see failed tests in PreCommit build, we can quickly tell
+# whether a failed test is a new failure, or it failed before and how often it
+# failed, so to have idea whether it may just be a flaky test.
+#
+# Of course, to be 100% sure about the reason of a test failure, closer look
+# at the failed test for the specific run is necessary.
+#
+import sys
+import platform
+sysversion = sys.hexversion
+onward30 = False
+if sysversion < 0x020600F0:
+  sys.exit("Minimum supported python version is 2.6, the current version is " +
+      "Python" + platform.python_version())
+
+if sysversion == 0x030000F0:
+  sys.exit("There is a known bug with Python" + platform.python_version() +
+      ", please try a different version");
+
+if sysversion < 0x03000000:
+  import urllib2
+else:
+  onward30 = True
+  import urllib.request
+
+import datetime
+import json as simplejson
+import logging
+from optparse import OptionParser
+import time
+
+# Configuration
+DEFAULT_JENKINS_URL = "https://builds.apache.org"
+DEFAULT_JOB_NAME = "Hadoop-Common-trunk"
+DEFAULT_NUM_PREVIOUS_DAYS = 14
+
+SECONDS_PER_DAY = 86400
+
+# total number of runs to examine
+numRunsToExamine = 0
+
+""" Parse arguments """
+def parse_args():
+  parser = OptionParser()
+  parser.add_option("-J", "--jenkins-url", type="string",
+                    dest="jenkins_url", help="Jenkins URL",
+                    default=DEFAULT_JENKINS_URL)
+  parser.add_option("-j", "--job-name", type="string",
+                    dest="job_name", help="Job name to look at",
+                    default=DEFAULT_JOB_NAME)
+  parser.add_option("-n", "--num-days", type="int",
+                    dest="num_prev_days", help="Number of days to examine",
+                    default=DEFAULT_NUM_PREVIOUS_DAYS)
+
+  (options, args) = parser.parse_args()
+  if args:
+    parser.error("unexpected arguments: " + repr(args))
+  return options
+
+""" Load data from specified url """
+def load_url_data(url):
+  if onward30:
+    ourl = urllib.request.urlopen(url)
+    codec = ourl.info().get_param('charset')
+    content = ourl.read().decode(codec)
+    data = simplejson.loads(content, strict=False)
+  else:
+    ourl = urllib2.urlopen(url)
+    data = simplejson.load(ourl, strict=False)
+  return data
+ 
+""" List all builds of the target project. """
+def list_builds(jenkins_url, job_name):
+  url = "%(jenkins)s/job/%(job_name)s/api/json?tree=builds[url,result,timestamp]" % dict(
+      jenkins=jenkins_url,
+      job_name=job_name)
+
+  try:
+    data = load_url_data(url)
+
+  except:
+    logging.error("Could not fetch: %s" % url)
+    raise
+  return data['builds']
+
+""" Find the names of any tests which failed in the given build output URL. """
+def find_failing_tests(testReportApiJson, jobConsoleOutput):
+  ret = set()
+  try:
+    data = load_url_data(testReportApiJson)
+
+  except:
+    logging.error("    Could not open testReport, check " +
+        jobConsoleOutput + " for why it was reported failed")
+    return ret
+
+  for suite in data['suites']:
+    for cs in suite['cases']:
+      status = cs['status']
+      errDetails = cs['errorDetails']
+      if (status == 'REGRESSION' or status == 'FAILED' or (errDetails is not None)):
+        ret.add(cs['className'] + "." + cs['name'])
+
+  if len(ret) == 0:
+    logging.info("    No failed tests in testReport, check " +
+        jobConsoleOutput + " for why it was reported failed.")
+  return ret
+
+""" Iterate runs of specfied job within num_prev_days and collect results """
+def find_flaky_tests(jenkins_url, job_name, num_prev_days):
+  global numRunsToExamine
+  all_failing = dict()
+  # First list all builds
+  builds = list_builds(jenkins_url, job_name)
+
+  # Select only those in the last N days
+  min_time = int(time.time()) - SECONDS_PER_DAY * num_prev_days
+  builds = [b for b in builds if (int(b['timestamp']) / 1000) > min_time]
+
+  # Filter out only those that failed
+  failing_build_urls = [(b['url'] , b['timestamp']) for b in builds
+      if (b['result'] in ('UNSTABLE', 'FAILURE'))]
+
+  tnum = len(builds)
+  num = len(failing_build_urls)
+  numRunsToExamine = tnum
+  logging.info("    THERE ARE " + str(num) + " builds (out of " + str(tnum)
+      + ") that have failed tests in the past " + str(num_prev_days) + " days"
+      + ((".", ", as listed below:\n")[num > 0]))
+
+  for failed_build_with_time in failing_build_urls:
+    failed_build = failed_build_with_time[0];
+    jobConsoleOutput = failed_build + "Console";
+    testReport = failed_build + "testReport";
+    testReportApiJson = testReport + "/api/json";
+
+    ts = float(failed_build_with_time[1]) / 1000.
+    st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
+    logging.info("===>%s" % str(testReport) + " (" + st + ")")
+    failing = find_failing_tests(testReportApiJson, jobConsoleOutput)
+    if failing:
+      for ftest in failing:
+        logging.info("    Failed test: %s" % ftest)
+        all_failing[ftest] = all_failing.get(ftest,0)+1
+
+  return all_failing
+
+def main():
+  global numRunsToExamine
+  logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
+  # set up logger to write to stdout
+  soh = logging.StreamHandler(sys.stdout)
+  soh.setLevel(logging.INFO)
+  logger = logging.getLogger()
+  logger.removeHandler(logger.handlers[0])
+  logger.addHandler(soh)
+
+  opts = parse_args()
+  logging.info("****Recently FAILED builds in url: " + opts.jenkins_url
+      + "/job/" + opts.job_name + "")
+
+  all_failing = find_flaky_tests(opts.jenkins_url, opts.job_name,
+      opts.num_prev_days)
+  if len(all_failing) == 0:
+    raise SystemExit(0)
+  logging.info("\nAmong " + str(numRunsToExamine) + " runs examined, all failed "
+      + "tests <#failedRuns: testName>:")
+
+  # print summary section: all failed tests sorted by how many times they failed
+  for tn in sorted(all_failing, key=all_failing.get, reverse=True):
+    logging.info("    " + str(all_failing[tn])+ ": " + tn)
+
+if __name__ == "__main__":
+  main()
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index e6512abf81809..b0fbb80a8b03c 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -857,74 +857,6 @@ findModules () {
   rm $TMP_MODULES
   echo $CHANGED_MODULES
 }
-###############################################################################
-### Run the test-contrib target
-runContribTests () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Running contrib tests."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then
-    echo "No contrib tests in this project."
-    return 0
-  fi
-
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
-  echo "NOP"
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 contrib tests{color}.  The patch failed contrib unit tests."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 contrib tests{color}.  The patch passed contrib unit tests."
-  return 0
-}
-
-###############################################################################
-### Run the inject-system-faults target
-checkInjectSystemFaults () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Checking the integrity of system test framework code."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-  
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
-  echo "NOP"
-  return 0
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 system test framework{color}.  The patch failed system test framework compile."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 system test framework{color}.  The patch passed system test framework compile."
-  return 0
-}
-
 ###############################################################################
 ### Submit a comment to the defect's Jira
 submitJiraComment () {
@@ -1059,11 +991,7 @@ checkReleaseAuditWarnings
 if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
-  runContribTests
-  (( RESULT = RESULT + $? ))
 fi
-checkInjectSystemFaults
-(( RESULT = RESULT + $? ))
 JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
 $JIRA_COMMENT_FOOTER"
 
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 66b6bdb16bd14..b53baccaad516 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -45,10 +45,10 @@
         <configuration>
           <rules>
             <requireMavenVersion>
-              <version>[3.0.0,)</version>
+              <version>${enforced.maven.version}</version>
             </requireMavenVersion>
             <requireJavaVersion>
-              <version>1.6</version>
+              <version>${enforced.java.version}</version>
             </requireJavaVersion>
           </rules>
         </configuration>
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
index f01953532077d..1a5d7d00cef13 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
@@ -48,6 +48,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>${basedir}/src/main/shellprofile.d</directory>
+      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>${basedir}/src/main/bin</directory>
       <outputDirectory>/sbin</outputDirectory>
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index 749e16af05581..247b09c6fa768 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -41,6 +41,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>shellprofile.d</directory>
+      <outputDirectory>libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>bin</directory>
       <outputDirectory>sbin</outputDirectory>
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index a15e1243b2426..6d386f1d1cea2 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -46,6 +46,11 @@
       </includes>
       <fileMode>0755</fileMode>
     </fileSet>
+    <fileSet>
+      <directory>hadoop-yarn/shellprofile.d</directory>
+      <outputDirectory>libexec/shellprofile.d</outputDirectory>
+      <fileMode>0755</fileMode>
+    </fileSet>
     <fileSet>
       <directory>hadoop-yarn/bin</directory>
       <outputDirectory>sbin</outputDirectory>
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index 2b66790f62a5b..f8b6d97ca2efe 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -13,7 +13,7 @@
  limitations under the License. See accompanying LICENSE file.
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 <parent>
    <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 84a106e665325..c011b4581ea30 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -39,23 +39,6 @@
   </dependencies>
 
   <profiles>
-    <profile>
-      <id>os.linux</id>
-      <activation>
-        <os>
-          <family>!Mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>jdk.tools</groupId>
-          <artifactId>jdk.tools</artifactId>
-          <version>1.6</version>
-          <scope>system</scope>
-          <systemPath>${java.home}/../lib/tools.jar</systemPath>
-        </dependency>
-      </dependencies>
-    </profile>
     <profile>
       <id>jdk1.7</id>
       <activation>
diff --git a/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/WhoClient.java b/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/WhoClient.java
index 2299ae1fd8089..f5cff2b529a5f 100644
--- a/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/WhoClient.java
+++ b/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/WhoClient.java
@@ -19,6 +19,7 @@
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.nio.charset.Charset;
 
 /**
  * Example that uses <code>AuthenticatedURL</code>.
@@ -39,7 +40,9 @@ public static void main(String[] args) {
       System.out.println("Status code: " + conn.getResponseCode() + " " + conn.getResponseMessage());
       System.out.println();
       if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
-        BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+        BufferedReader reader = new BufferedReader(
+            new InputStreamReader(
+                conn.getInputStream(), Charset.forName("UTF-8")));
         String line = reader.readLine();
         while (line != null) {
           System.out.println(line);
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index 61c3c6d5f53c3..c50a5164a5780 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -24,19 +24,18 @@
 /**
  * The {@link AuthenticatedURL} class enables the use of the JDK {@link URL} class
  * against HTTP endpoints protected with the {@link AuthenticationFilter}.
- * <p/>
+ * <p>
  * The authentication mechanisms supported by default are Hadoop Simple  authentication
  * (also known as pseudo authentication) and Kerberos SPNEGO authentication.
- * <p/>
+ * <p>
  * Additional authentication mechanisms can be supported via {@link Authenticator} implementations.
- * <p/>
+ * <p>
  * The default {@link Authenticator} is the {@link KerberosAuthenticator} class which supports
  * automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication.
- * <p/>
+ * <p>
  * <code>AuthenticatedURL</code> instances are not thread-safe.
- * <p/>
+ * <p>
  * The usage pattern of the {@link AuthenticatedURL} is:
- * <p/>
  * <pre>
  *
  * // establishing an initial connection
@@ -240,7 +239,7 @@ public static void injectToken(HttpURLConnection conn, Token token) {
 
   /**
    * Helper method that extracts an authentication token received from a connection.
-   * <p/>
+   * <p>
    * This method is used by {@link Authenticator} implementations.
    *
    * @param conn connection to extract the authentication token from.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java
index e7bae4a891593..6828970fdbb59 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java
@@ -19,7 +19,7 @@
 
 /**
  * Interface for client authentication mechanisms.
- * <p/>
+ * <p>
  * Implementations are use-once instances, they don't need to be thread safe.
  */
 public interface Authenticator {
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
index 928866c532cce..323b019eb827b 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
@@ -43,9 +43,9 @@
 
 /**
  * The {@link KerberosAuthenticator} implements the Kerberos SPNEGO authentication sequence.
- * <p/>
+ * <p>
  * It uses the default principal for the Kerberos cache (normally set via kinit).
- * <p/>
+ * <p>
  * It falls back to the {@link PseudoAuthenticator} if the HTTP endpoint does not trigger an SPNEGO authentication
  * sequence.
  */
@@ -162,9 +162,9 @@ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
 
   /**
    * Performs SPNEGO authentication against the specified URL.
-   * <p/>
+   * <p>
    * If a token is given it does a NOP and returns the given token.
-   * <p/>
+   * <p>
    * If no token is given, it will perform the SPNEGO authentication sequence using an
    * HTTP <code>OPTIONS</code> request.
    *
@@ -211,7 +211,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
 
   /**
    * If the specified URL does not support SPNEGO authentication, a fallback {@link Authenticator} will be used.
-   * <p/>
+   * <p>
    * This implementation returns a {@link PseudoAuthenticator}.
    *
    * @return the fallback {@link Authenticator}.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
index f534be9b20bc0..46d94b88dec67 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
@@ -20,7 +20,7 @@
 /**
  * The {@link PseudoAuthenticator} implementation provides an authentication equivalent to Hadoop's
  * Simple authentication, it trusts the value of the 'user.name' Java System property.
- * <p/>
+ * <p>
  * The 'user.name' value is propagated using an additional query string parameter {@link #USER_NAME} ('user.name').
  */
 public class PseudoAuthenticator implements Authenticator {
@@ -47,13 +47,13 @@ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
 
   /**
    * Performs simple authentication against the specified URL.
-   * <p/>
+   * <p>
    * If a token is given it does a NOP and returns the given token.
-   * <p/>
+   * <p>
    * If no token is given, it will perform an HTTP <code>OPTIONS</code> request injecting an additional
    * parameter {@link #USER_NAME} in the query string with the value returned by the {@link #getUserName()}
    * method.
-   * <p/>
+   * <p>
    * If the response is successful it will update the authentication token.
    *
    * @param url the URl to authenticate against.
@@ -79,7 +79,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token) throws IOExcepti
 
   /**
    * Returns the current user name.
-   * <p/>
+   * <p>
    * This implementation returns the value of the Java system property 'user.name'
    *
    * @return the current user name.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AltKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AltKerberosAuthenticationHandler.java
index e786e37df8ed7..987330fa0e5be 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AltKerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AltKerberosAuthenticationHandler.java
@@ -28,7 +28,6 @@
  * to allow a developer to implement their own custom authentication for browser
  * access.  The alternateAuthenticate method will be called whenever a request
  * comes from a browser.
- * <p/>
  */
 public abstract class AltKerberosAuthenticationHandler
                         extends KerberosAuthenticationHandler {
@@ -52,7 +51,6 @@ public abstract class AltKerberosAuthenticationHandler
   /**
    * Returns the authentication type of the authentication handler,
    * 'alt-kerberos'.
-   * <p/>
    *
    * @return the authentication type of the authentication handler,
    * 'alt-kerberos'.
@@ -80,7 +78,6 @@ public void init(Properties config) throws ServletException {
    * completed successfully (in the case of Java access) and only after the
    * custom authentication implemented by the subclass in alternateAuthenticate
    * has completed successfully (in the case of browser access).
-   * <p/>
    *
    * @param request the HTTP client request.
    * @param response the HTTP client response.
@@ -109,7 +106,7 @@ public AuthenticationToken authenticate(HttpServletRequest request,
    * refers to a browser.  If its not a browser, then Kerberos authentication
    * will be used; if it is a browser, alternateAuthenticate from the subclass
    * will be used.
-   * <p/>
+   * <p>
    * A User-Agent String is considered to be a browser if it does not contain
    * any of the values from alt-kerberos.non-browser.user-agents; the default
    * behavior is to consider everything a browser unless it contains one of:
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 0ac352ba2d116..e891ed2623dd5 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -44,18 +44,20 @@
 import java.util.*;
 
 /**
- * The {@link AuthenticationFilter} enables protecting web application resources with different (pluggable)
+ * <p>The {@link AuthenticationFilter} enables protecting web application
+ * resources with different (pluggable)
  * authentication mechanisms and signer secret providers.
- * <p/>
+ * </p>
+ * <p>
  * Out of the box it provides 2 authentication mechanisms: Pseudo and Kerberos SPNEGO.
- * <p/>
+ * </p>
  * Additional authentication mechanisms are supported via the {@link AuthenticationHandler} interface.
- * <p/>
+ * <p>
  * This filter delegates to the configured authentication handler for authentication and once it obtains an
  * {@link AuthenticationToken} from it, sets a signed HTTP cookie with the token. For client requests
  * that provide the signed HTTP cookie, it verifies the validity of the cookie, extracts the user information
  * and lets the request proceed to the target resource.
- * <p/>
+ * </p>
  * The supported configuration properties are:
  * <ul>
  * <li>config.prefix: indicates the prefix to be used by all other configuration properties, the default value
@@ -73,18 +75,19 @@
  * <li>[#PREFIX#.]cookie.domain: domain to use for the HTTP cookie that stores the authentication token.</li>
  * <li>[#PREFIX#.]cookie.path: path to use for the HTTP cookie that stores the authentication token.</li>
  * </ul>
- * <p/>
+ * <p>
  * The rest of the configuration properties are specific to the {@link AuthenticationHandler} implementation and the
  * {@link AuthenticationFilter} will take all the properties that start with the prefix #PREFIX#, it will remove
  * the prefix from it and it will pass them to the the authentication handler for initialization. Properties that do
  * not start with the prefix will not be passed to the authentication handler initialization.
- * <p/>
+ * </p>
+ * <p>
  * Out of the box it provides 3 signer secret provider implementations:
  * "string", "random", and "zookeeper"
- * <p/>
+ * </p>
  * Additional signer secret providers are supported via the
  * {@link SignerSecretProvider} class.
- * <p/>
+ * <p>
  * For the HTTP cookies mentioned above, the SignerSecretProvider is used to
  * determine the secret to use for signing the cookies. Different
  * implementations can have different behaviors.  The "string" implementation
@@ -94,7 +97,7 @@
  * [#PREFIX#.]token.validity mentioned above.  The "zookeeper" implementation
  * is like the "random" one, except that it synchronizes the random secret
  * and rollovers between multiple servers; it's meant for HA services.
- * <p/>
+ * </p>
  * The relevant configuration properties are:
  * <ul>
  * <li>signer.secret.provider: indicates the name of the SignerSecretProvider
@@ -108,10 +111,10 @@
  * implementations are specified, this value is used as the rollover
  * interval.</li>
  * </ul>
- * <p/>
+ * <p>
  * The "zookeeper" implementation has additional configuration properties that
  * must be specified; see {@link ZKSignerSecretProvider} for details.
- * <p/>
+ * </p>
  * For subclasses of AuthenticationFilter that want additional control over the
  * SignerSecretProvider, they can use the following attribute set in the
  * ServletContext:
@@ -190,10 +193,9 @@ public class AuthenticationFilter implements Filter {
   private String cookiePath;
 
   /**
-   * Initializes the authentication filter and signer secret provider.
-   * <p/>
-   * It instantiates and initializes the specified {@link AuthenticationHandler}.
-   * <p/>
+   * <p>Initializes the authentication filter and signer secret provider.</p>
+   * It instantiates and initializes the specified {@link
+   * AuthenticationHandler}.
    *
    * @param filterConfig filter configuration.
    *
@@ -375,7 +377,7 @@ protected String getCookiePath() {
 
   /**
    * Destroys the filter.
-   * <p/>
+   * <p>
    * It invokes the {@link AuthenticationHandler#destroy()} method to release any resources it may hold.
    */
   @Override
@@ -393,7 +395,7 @@ public void destroy() {
    * Returns the filtered configuration (only properties starting with the specified prefix). The property keys
    * are also trimmed from the prefix. The returned {@link Properties} object is used to initialized the
    * {@link AuthenticationHandler}.
-   * <p/>
+   * <p>
    * This method can be overriden by subclasses to obtain the configuration from other configuration source than
    * the web.xml file.
    *
@@ -419,7 +421,7 @@ protected Properties getConfiguration(String configPrefix, FilterConfig filterCo
 
   /**
    * Returns the full URL of the request including the query string.
-   * <p/>
+   * <p>
    * Used as a convenience method for logging purposes.
    *
    * @param request the request object.
@@ -436,11 +438,11 @@ protected String getRequestURL(HttpServletRequest request) {
 
   /**
    * Returns the {@link AuthenticationToken} for the request.
-   * <p/>
+   * <p>
    * It looks at the received HTTP cookies and extracts the value of the {@link AuthenticatedURL#AUTH_COOKIE}
    * if present. It verifies the signature and if correct it creates the {@link AuthenticationToken} and returns
    * it.
-   * <p/>
+   * <p>
    * If this method returns <code>null</code> the filter will invoke the configured {@link AuthenticationHandler}
    * to perform user authentication.
    *
@@ -597,7 +599,7 @@ protected void doFilter(FilterChain filterChain, HttpServletRequest request,
    *
    * @param token authentication token for the cookie.
    * @param expires UNIX timestamp that indicates the expire date of the
-   *                cookie. It has no effect if its value < 0.
+   *                cookie. It has no effect if its value &lt; 0.
    *
    * XXX the following code duplicate some logic in Jetty / Servlet API,
    * because of the fact that Hadoop is stuck at servlet 2.5 and jetty 6
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
index 04984be5a7d58..797e95a689dd5 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationHandler.java
@@ -24,9 +24,7 @@
 
 /**
  * Interface for server authentication mechanisms.
- * <p/>
  * The {@link AuthenticationFilter} manages the lifecycle of the authentication handler.
- * <p/>
  * Implementations must be thread-safe as one instance is initialized and used for all requests.
  */
 public interface AuthenticationHandler {
@@ -35,7 +33,6 @@ public interface AuthenticationHandler {
 
   /**
    * Returns the authentication type of the authentication handler.
-   * <p/>
    * This should be a name that uniquely identifies the authentication type.
    * For example 'simple' or 'kerberos'.
    *
@@ -45,7 +42,7 @@ public interface AuthenticationHandler {
 
   /**
    * Initializes the authentication handler instance.
-   * <p/>
+   * <p>
    * This method is invoked by the {@link AuthenticationFilter#init} method.
    *
    * @param config configuration properties to initialize the handler.
@@ -56,21 +53,21 @@ public interface AuthenticationHandler {
 
   /**
    * Destroys the authentication handler instance.
-   * <p/>
+   * <p>
    * This method is invoked by the {@link AuthenticationFilter#destroy} method.
    */
   public void destroy();
 
   /**
    * Performs an authentication management operation.
-   * <p/>
+   * <p>
    * This is useful for handling operations like get/renew/cancel
    * delegation tokens which are being handled as operations of the
    * service end-point.
-   * <p/>
+   * <p>
    * If the method returns <code>TRUE</code> the request will continue normal
    * processing, this means the method has not produced any HTTP response.
-   * <p/>
+   * <p>
    * If the method returns <code>FALSE</code> the request will end, this means 
    * the method has produced the corresponding HTTP response.
    *
@@ -91,17 +88,17 @@ public boolean managementOperation(AuthenticationToken token,
 
   /**
    * Performs an authentication step for the given HTTP client request.
-   * <p/>
+   * <p>
    * This method is invoked by the {@link AuthenticationFilter} only if the HTTP client request is
    * not yet authenticated.
-   * <p/>
+   * <p>
    * Depending upon the authentication mechanism being implemented, a particular HTTP client may
    * end up making a sequence of invocations before authentication is successfully established (this is
    * the case of Kerberos SPNEGO).
-   * <p/>
+   * <p>
    * This method must return an {@link AuthenticationToken} only if the the HTTP client request has
    * been successfully and fully authenticated.
-   * <p/>
+   * <p>
    * If the HTTP client request has not been completely authenticated, this method must take over
    * the corresponding HTTP response and it must return <code>null</code>.
    *
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java
index ff68847c8a0bc..bb3e71da61c32 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationToken.java
@@ -29,7 +29,7 @@
  * The {@link AuthenticationToken} contains information about an authenticated
  * HTTP client and doubles as the {@link Principal} to be returned by
  * authenticated {@link HttpServletRequest}s
- * <p/>
+ * <p>
  * The token can be serialized/deserialized to and from a string as it is sent
  * and received in HTTP client responses and requests as a HTTP cookie (this is
  * done by the {@link AuthenticationFilter}).
@@ -170,7 +170,7 @@ public boolean isExpired() {
 
   /**
    * Returns the string representation of the token.
-   * <p/>
+   * <p>
    * This string representation is parseable by the {@link #parse} method.
    *
    * @return the string representation of the token.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
index 92bc57c413b45..846541b162b5f 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
@@ -51,7 +51,7 @@
 
 /**
  * The {@link KerberosAuthenticationHandler} implements the Kerberos SPNEGO authentication mechanism for HTTP.
- * <p/>
+ * <p>
  * The supported configuration properties are:
  * <ul>
  * <li>kerberos.principal: the Kerberos principal to used by the server. As stated by the Kerberos SPNEGO
@@ -168,9 +168,9 @@ public KerberosAuthenticationHandler(String type) {
 
   /**
    * Initializes the authentication handler instance.
-   * <p/>
+   * <p>
    * It creates a Kerberos context using the principal and keytab specified in the configuration.
-   * <p/>
+   * <p>
    * This method is invoked by the {@link AuthenticationFilter#init} method.
    *
    * @param config configuration properties to initialize the handler.
@@ -243,7 +243,7 @@ public GSSManager run() throws Exception {
 
   /**
    * Releases any resources initialized by the authentication handler.
-   * <p/>
+   * <p>
    * It destroys the Kerberos context.
    */
   @Override
@@ -262,7 +262,7 @@ public void destroy() {
 
   /**
    * Returns the authentication type of the authentication handler, 'kerberos'.
-   * <p/>
+   * <p>
    *
    * @return the authentication type of the authentication handler, 'kerberos'.
    */
@@ -313,7 +313,6 @@ public boolean managementOperation(AuthenticationToken token,
   /**
    * It enforces the the Kerberos SPNEGO authentication sequence returning an {@link AuthenticationToken} only
    * after the Kerberos SPNEGO sequence has completed successfully.
-   * <p/>
    *
    * @param request the HTTP client request.
    * @param response the HTTP client response.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
index 2c7db8898fa56..50f0cf11fe2f5 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/PseudoAuthenticationHandler.java
@@ -30,12 +30,12 @@
 /**
  * The <code>PseudoAuthenticationHandler</code> provides a pseudo authentication mechanism that accepts
  * the user name specified as a query string parameter.
- * <p/>
+ * <p>
  * This mimics the model of Hadoop Simple authentication which trust the 'user.name' property provided in
  * the configuration object.
- * <p/>
+ * <p>
  * This handler can be configured to support anonymous users.
- * <p/>
+ * <p>
  * The only supported configuration property is:
  * <ul>
  * <li>simple.anonymous.allowed: <code>true|false</code>, default value is <code>false</code></li>
@@ -80,7 +80,7 @@ public PseudoAuthenticationHandler(String type) {
 
   /**
    * Initializes the authentication handler instance.
-   * <p/>
+   * <p>
    * This method is invoked by the {@link AuthenticationFilter#init} method.
    *
    * @param config configuration properties to initialize the handler.
@@ -103,7 +103,7 @@ protected boolean getAcceptAnonymous() {
 
   /**
    * Releases any resources initialized by the authentication handler.
-   * <p/>
+   * <p>
    * This implementation does a NOP.
    */
   @Override
@@ -112,7 +112,6 @@ public void destroy() {
 
   /**
    * Returns the authentication type of the authentication handler, 'simple'.
-   * <p/>
    *
    * @return the authentication type of the authentication handler, 'simple'.
    */
@@ -156,14 +155,14 @@ private String getUserName(HttpServletRequest request) {
 
   /**
    * Authenticates an HTTP client request.
-   * <p/>
+   * <p>
    * It extracts the {@link PseudoAuthenticator#USER_NAME} parameter from the query string and creates
    * an {@link AuthenticationToken} with it.
-   * <p/>
+   * <p>
    * If the HTTP client request does not contain the {@link PseudoAuthenticator#USER_NAME} parameter and
    * the handler is configured to allow anonymous users it returns the {@link AuthenticationToken#ANONYMOUS}
    * token.
-   * <p/>
+   * <p>
    * If the HTTP client request does not contain the {@link PseudoAuthenticator#USER_NAME} parameter and
    * the handler is configured to disallow anonymous users it throws an {@link AuthenticationException}.
    *
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
index 62bb00acab424..7ae8ab2672e9d 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
@@ -92,7 +92,7 @@ public class KerberosName {
 
   /**
    * Create a name from the full Kerberos principal name.
-   * @param name
+   * @param name full Kerberos principal name.
    */
   public KerberosName(String name) {
     Matcher match = nameParser.matcher(name);
@@ -367,7 +367,7 @@ public static class NoMatchingRule extends IOException {
    * Get the translation of the principal name into an operating system
    * user name.
    * @return the short name
-   * @throws IOException
+   * @throws IOException throws if something is wrong with the rules
    */
   public String getShortName() throws IOException {
     String[] params;
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
index ca0fce2251ea0..0e8d8db8ea520 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
@@ -135,12 +135,10 @@ static final String[] getPrincipalNames(String keytabFileName) throws IOExceptio
   /**
    * Get all the unique principals from keytabfile which matches a pattern.
    * 
-   * @param keytab 
-   *          Name of the keytab file to be read.
-   * @param pattern 
-   *         pattern to be matched.
+   * @param keytab Name of the keytab file to be read.
+   * @param pattern pattern to be matched.
    * @return list of unique principals which matches the pattern.
-   * @throws IOException 
+   * @throws IOException if cannot get the principal name
    */
   public static final String[] getPrincipalNames(String keytab,
       Pattern pattern) throws IOException {
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java
index 29e5661cb0bd5..41059a7e00900 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java
@@ -14,6 +14,8 @@
 package org.apache.hadoop.security.authentication.util;
 
 import com.google.common.annotations.VisibleForTesting;
+
+import java.nio.charset.Charset;
 import java.util.Random;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -46,6 +48,6 @@ public RandomSignerSecretProvider(long seed) {
 
   @Override
   protected byte[] generateNewSecret() {
-    return Long.toString(rand.nextLong()).getBytes();
+    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
   }
 }
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java
index e29301bc4ba05..aa63e403c6364 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/Signer.java
@@ -15,6 +15,7 @@
 
 import org.apache.commons.codec.binary.Base64;
 
+import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
@@ -41,8 +42,6 @@ public Signer(SignerSecretProvider secretProvider) {
 
   /**
    * Returns a signed string.
-   * <p/>
-   * The signature '&s=SIGNATURE' is appended at the end of the string.
    *
    * @param str string to sign.
    *
@@ -88,7 +87,7 @@ public String verifyAndExtract(String signedStr) throws SignerException {
   protected String computeSignature(byte[] secret, String str) {
     try {
       MessageDigest md = MessageDigest.getInstance("SHA");
-      md.update(str.getBytes());
+      md.update(str.getBytes(Charset.forName("UTF-8")));
       md.update(secret);
       byte[] digest = md.digest();
       return new Base64(0).encodeToString(digest);
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
index 7aaccd2914c22..57ddd372fe4b1 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
@@ -13,6 +13,7 @@
  */
 package org.apache.hadoop.security.authentication.util;
 
+import java.nio.charset.Charset;
 import java.util.Properties;
 import javax.servlet.ServletContext;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -36,7 +37,7 @@ public void init(Properties config, ServletContext servletContext,
           long tokenValidity) throws Exception {
     String signatureSecret = config.getProperty(
             AuthenticationFilter.SIGNATURE_SECRET, null);
-    secret = signatureSecret.getBytes();
+    secret = signatureSecret.getBytes(Charset.forName("UTF-8"));
     secrets = new byte[][]{secret};
   }
 
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 6c0fbbb0a26ee..11bfccd05c6e9 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -15,6 +15,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -44,7 +45,7 @@
 /**
  * A SignerSecretProvider that synchronizes a rolling random secret between
  * multiple servers using ZooKeeper.
- * <p/>
+ * <p>
  * It works by storing the secrets and next rollover time in a ZooKeeper znode.
  * All ZKSignerSecretProviders looking at that znode will use those
  * secrets and next rollover time to ensure they are synchronized.  There is no
@@ -55,7 +56,7 @@
  * your own Curator client, you can pass it to ZKSignerSecretProvider; see
  * {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter}
  * for more details.
- * <p/>
+ * <p>
  * The supported configuration properties are:
  * <ul>
  * <li>signer.secret.provider.zookeeper.connection.string: indicates the
@@ -77,11 +78,13 @@
  * </ul>
  *
  * The following attribute in the ServletContext can also be set if desired:
+ * <ul>
  * <li>signer.secret.provider.zookeeper.curator.client: A CuratorFramework
  * client object can be passed here. If given, the "zookeeper" implementation
  * will use this Curator client instead of creating its own, which is useful if
  * you already have a Curator client or want more control over its
  * configuration.</li>
+ * </ul>
  */
 @InterfaceStability.Unstable
 @InterfaceAudience.Private
@@ -367,14 +370,14 @@ private synchronized void pullFromZK(boolean isInit) {
   }
 
   private byte[] generateRandomSecret() {
-    return Long.toString(rand.nextLong()).getBytes();
+    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
   }
 
   /**
    * This method creates the Curator client and connects to ZooKeeper.
    * @param config configuration properties
    * @return A Curator client
-   * @throws java.lang.Exception
+   * @throws Exception
    */
   protected CuratorFramework createCuratorClient(Properties config)
           throws Exception {
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
new file mode 100644
index 0000000000000..9728f977bba70
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -0,0 +1,10 @@
+  BREAKDOWN OF HADOOP-11264 SUBTASKS AND RELATED JIRAS (Common part of HDFS-7285)
+
+    HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding
+    (Kai Zheng via umamahesh)
+
+    HADOOP-11534. Minor improvements for raw erasure coders
+    ( Kai Zheng via vinayakumarb )
+
+    HADOOP-11541. Raw XOR coder
+    ( Kai Zheng )
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a6263887f0487..b02e6954a9430 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -11,18 +11,19 @@ Trunk (Unreleased)
 
     HADOOP-9902. Shell script rewrite (aw)
 
+    HADOOP-10950. rework heap management vars (John Smith via aw)
+
   NEW FEATURES
 
-    HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop.
-    (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
-    Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
-    Alexander Stojanovic, Brian Swan, and Min Wei via cnauroth)
+    HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via aw)
+
+    HADOOP-11353. Add support for .hadooprc (aw)
+
+    HADOOP-9044. add FindClass main class to provide classpath checking
+    of installations (Steve Loughran via aw)
+
+    HADOOP-11485. Pluggable shell integration (aw)
 
-    HADOOP-10728. Metrics system for Windows Azure Storage Filesystem.
-    (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
-    Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
-    Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth)
-    
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
@@ -127,12 +128,6 @@ Trunk (Unreleased)
     HADOOP-11092. hadoop shell commands should print usage if not given a
     a class (aw)
 
-    HADOOP-10809. hadoop-azure: page blob support. (Dexter Bradshaw,
-    Mostafa Elhemali, Eric Hanson, and Mike Liddell via cnauroth)
-
-    HADOOP-11188. hadoop-azure: automatically expand page blobs when they become
-    full. (Eric Hanson via cnauroth)
-
     HADOOP-11231. Remove dead code in ServletUtil. (Li Lu via wheat9)
 
     HADOOP-11025. hadoop-daemons.sh should just call hdfs directly (Masatake
@@ -148,8 +143,33 @@ Trunk (Unreleased)
     HADOOP-11081. Document hadoop properties expected to be set by the shell 
     code in *-env.sh (aw)
 
+    HADOOP-11352 Clean up test-patch.sh to disable "+1 contrib tests"
+    (Akira AJISAKA via stevel)
+
+    HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)
+
+    HADOOP-11058. Missing HADOOP_CONF_DIR generates strange results
+    (Masatake Iwasaki via aw)
+
+    HADOOP-11460. Deprecate shell vars (John Smith via aw)
+
+    HADOOP-11346. Rewrite sls/rumen to use new shell framework (John Smith
+    via aw)
+
+    HADOOP-10976. moving the source code of hadoop-tools docs to the
+    directory under hadoop-tools (Masatake Iwasaki via aw)
+
+    HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
+    via aw)
+
+    HADOOP-6964. Allow compact property description in xml (Kengo Seki
+    via aw)
+
   BUG FIXES
 
+    HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
+    (Jason Lowe via raviprak)
+
     HADOOP-9451. Fault single-layer config if node group topology is enabled.
     (Junping Du via llu)
 
@@ -319,20 +339,10 @@ Trunk (Unreleased)
     HADOOP-10625. Trim configuration names when putting/getting them
     to properties. (Wangda Tan via xgong)
 
-    HADOOP-10689. InputStream is not closed in
-    AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth)
-
-    HADOOP-10690. Lack of synchronization on access to InputStream in
-    NativeAzureFileSystem#NativeAzureFsInputStream#close().
-    (Chen He via cnauroth)
-
     HADOOP-10831. UserProvider is not thread safe. (Benoy Antony via umamahesh)
 
     HADOOP-10834. Typo in CredentialShell usage. (Benoy Antony via umamahesh)
 
-    HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File
-    System. (Shanyu Zhao via cnauroth)
-
     HADOOP-11002. shell escapes are incompatible with previous releases (aw)
 
     HADOOP-10996. Stop violence in the *_HOME (aw)
@@ -347,9 +357,6 @@ Trunk (Unreleased)
     HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., 
     sbin) (aw)
 
-    HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
-    collection. (Shanyu Zhao via cnauroth)
-
     HADOOP-11284. Fix variable name mismatches in hadoop-functions.sh (Masatake
     Iwasaki via aw)
 
@@ -358,6 +365,12 @@ Trunk (Unreleased)
     HADOOP-11296. hadoop-daemons.sh throws 'host1: bash: host3: 
     command not found...' (vinayakumarb)
 
+    HADOOP-11380. Restore Rack Awareness documenation (aw)
+
+    HADOOP-11397. Can't override HADOOP_IDENT_STRING (Kengo Seki via aw)
+
+    HADOOP-10908. Common needs updates for shell rewrite (aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -368,6 +381,8 @@ Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
+    HADOOP-10530 Make hadoop build on Java7+ only (stevel)
+
   NEW FEATURES
 
     HADOOP-10987. Provide an iterator-based listing API for FileSystem (kihwal)
@@ -375,10 +390,28 @@ Release 2.7.0 - UNRELEASED
     HADOOP-7984. Add hadoop --loglevel option to change log level.
     (Akira AJISAKA via cnauroth)
 
+    HADOOP-9629. Support Windows Azure Storage - Blob as a file system in Hadoop.
+    (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+    Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+    Alexander Stojanovic, Brian Swan, and Min Wei via cnauroth)
+
+    HADOOP-10728. Metrics system for Windows Azure Storage Filesystem.
+    (Dexter Bradshaw, Mostafa Elhemali, Xi Fang, Johannes Klein, David Lao,
+    Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys,
+    Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth)
+
     HADOOP-8989. hadoop fs -find feature (Jonathan Allen via aw)
 
+    HADOOP-11490. Expose truncate API via FileSystem and shell command.
+    (Milan Desai via shv)
+
+    HADOOP-11045. Introducing a tool to detect flaky tests of hadoop jenkins testing
+    job. (Yongjun Zhang and Todd Lipcon via ozawa)
+
   IMPROVEMENTS
 
+    HADOOP-11483. HardLink.java should use the jdk7 createLink method (aajisaka)
+
     HADOOP-11156. DelegateToFileSystem should implement
     getFsStatus(final Path f). (Zhihai Xu via wang)
 
@@ -416,13 +449,151 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11313. Adding a document about NativeLibraryChecker.
     (Tsuyoshi OZAWA via cnauroth)
 
+    HADOOP-11287. Simplify UGI#reloginFromKeytab for Java 7+.
+    (Li Lu via wheat9)
+
+    HADOOP-10476) Bumping the findbugs version to 3.0.0. (wheat9)
+
+    HADOOP-11410. Make the rpath of libhadoop.so configurable (cmccabe)
+
+    HADOOP-11416. Move ChunkedArrayList into hadoop-common (cmccabe)
+
+    HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File
+    System. (Shanyu Zhao via cnauroth)
+
+    HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
+    collection. (Shanyu Zhao via cnauroth)
+
+    HADOOP-11421. Add IOUtils#listDirectory (cmccabe)
+
+    HADOOP-11427. ChunkedArrayList: fix removal via iterator and implement get
+    (cmccabe)
+
+    HADOOP-11430. Add GenericTestUtils#disableLog, GenericTestUtils#setLogLevel
+    (cmccabe)
+
+    HADOOP-11422. Check CryptoCodec is AES-CTR for Crypto input/output stream
+    (Yi Liu via Colin P. McCabe)
+
+    HADOOP-11213. Typos in html pages: SecureMode and EncryptedShuffle. 
+    (Wei Yan via kasha)
+
+    HADOOP-11395. Add site documentation for Azure Storage FileSystem
+    integration. (Chris Nauroth via Arpit Agarwal)
+
+    HDFS-7555. Remove the support of unmanaged connectors in HttpServer2.
+    (wheat9)
+
+    HADOOP-11399. Java Configuration file and .xml files should be
+    automatically cross-compared (rchiang via rkanter)
+
+    HADOOP-11455. KMS and Credential CLI should request confirmation for
+    deletion by default. (Charles Lamb via yliu)
+
+    HADOOP-11390 Metrics 2 ganglia provider to include hostname in
+    unresolved address problems. (Varun Saxena via stevel)
+
+    HADOOP-11032. Replace use of Guava's Stopwatch with Hadoop's StopWatch
+    (ozawa)
+
+    HADOOP-11464. Reinstate support for launching Hadoop processes on Windows
+    using Cygwin. (cnauroth)
+
+    HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job
+    (Akshay Radia via brandonli)
+
+    HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via
+    Arpit Agarwal)
+
+    HADOOP-11481. ClassCastException while using a key created by keytool to
+    create encryption zone. (Charles Lamb via Colin P. Mccabe)
+
+    HADOOP-8757. Metrics should disallow names with invalid characters
+    (rchiang via rkanter)
+
+    HADOOP-11261 Set custom endpoint for S3A. (Thomas Demoor via stevel)
+
+    HADOOP-11171 Enable using a proxy server to connect to S3a.
+    (Thomas Demoor via stevel)
+
+    HADOOP-11489 Dropping dependency on io.netty from hadoop-nfs' pom.xml
+    (Ted Yu via ozawa)
+
+    HADOOP-11419 Improve hadoop-maven-plugins. (Herve Boutemy via stevel)
+
+    HADOOP-11450. Cleanup DistCpV1 not to use deprecated methods and fix
+    javadocs. (Varun Saxena via ozawa)
+
+    HADOOP-4297. Enable Java assertions when running tests.
+    (Tsz Wo Nicholas Sze via wheat9)
+
+    HADOOP-10626. Limit Returning Attributes for LDAP search. (Jason Hubbard
+    via atm)
+
+    HADOOP-11317. Increment SLF4J version to 1.7.10. (Tim Robertson via ozawa)
+
+    HADOOP-10525. Remove DRFA.MaxBackupIndex config from log4j.properties
+    (aajisaka)
+
+    HADOOP-10574. Bump the maven plugin versions too -moving the numbers into
+    properties. (aajisaka)
+
+    HADOOP-11441. Hadoop-azure: Change few methods scope to public.
+    (Shashank Khandelwal via cnauroth)
+
+    HADOOP-9137. Support connection limiting in IPC server (kihwal)
+
+    HADOOP-11498. Bump the version of HTrace to 3.1.0-incubating (Masatake
+    Iwasaki via Colin P. McCabe)
+
+    HADOOP-11442. hadoop-azure: Create test jar.
+    (Shashank Khandelwal via cnauroth)
+
+    HADOOP-11544. Remove unused configuration keys for tracing. (Masatake
+    Iwasaki via aajisaka)
+
+    HADOOP-11492. Bump up curator version to 2.7.1. (Arun Suresh and
+    Karthik Kambatla via kasha)
+
+    HADOOP-11463 Replace method-local TransferManager object with
+    S3AFileSystem#transfers. (Ted Yu via stevel)
+
+    HADOOP-11543. Improve help message for hadoop/yarn command.
+    (Brahma Reddy Battula via ozawa)
+
+    HADOOP-11520. Clean incomplete multi-part uploads in S3A tests.
+    (Thomas Demoor via stevel)
+
   OPTIMIZATIONS
 
     HADOOP-11323. WritableComparator#compare keeps reference to byte array.
     (Wilfred Spiegelenburg via wang)
 
+    HADOOP-11238. Update the NameNode's Group Cache in the background when
+    possible (Chris Li via Colin P. McCabe)
+
+    HADOOP-10809. hadoop-azure: page blob support. (Dexter Bradshaw,
+    Mostafa Elhemali, Eric Hanson, and Mike Liddell via cnauroth)
+
+    HADOOP-11188. hadoop-azure: automatically expand page blobs when they become
+    full. (Eric Hanson via cnauroth)
+
+    HADOOP-11506. Configuration variable expansion regex expensive for long
+    values. (Gera Shegalov via gera)
+
   BUG FIXES
 
+    HADOOP-11488. Difference in default connection timeout for S3A FS
+    (Daisuke Kobayashi via harsh)
+
+    HADOOP-11256. Some site docs have inconsistent appearance (Masatake 
+    Iwasaki via aw)
+
+    HADOOP-11318. Update the document for hadoop fs -stat (aajisaka)
+
+    HADOOP 11400. GraphiteSink does not reconnect to Graphite after 'broken pipe' 
+    (Kamil Gorlo via raviprak)
+
     HADOOP-11236. NFS: Fix javadoc warning in RpcProgram.java (Abhiraj Butala via harsh)
 
     HADOOP-11166. Remove ulimit from test-patch.sh. (wang)
@@ -514,6 +685,218 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11343. Overflow is not properly handled in caclulating final iv for
     AES CTR. (Jerry Chen via wang)
 
+    HADOOP-11354. ThrottledInputStream doesn't perform effective throttling.
+    (Ted Yu via jing9)
+
+    HADOOP-11329. Add JAVA_LIBRARY_PATH to KMS startup options. (Arun Suresh via wang)
+
+    HADOOP-11363 Hadoop maven surefire-plugin uses must set heap size. (stevel)
+
+    HADOOP-10134 [JDK8] Fix Javadoc errors caused by incorrect or illegal tags in doc
+    comments. (apurtell via stevel)
+
+    HADOOP-11367. Fix warnings from findbugs 3.0 in hadoop-streaming. (Li Lu via wheat9)
+
+    HADOOP-11369. Fix new findbugs warnings in hadoop-mapreduce-client,
+    non-core directories. (Li Lu via wheat9)
+
+    HADOOP-11368. Fix SSLFactory truststore reloader thread leak in
+    KMSClientProvider. (Arun Suresh via wang)
+
+    HADOOP-11372. Fix new findbugs warnings in mapreduce-examples.
+    (Li Lu via wheat9)
+
+    HADOOP-11273. TestMiniKdc failure: login options not compatible with IBM
+    JDK. (Gao Zhong Liang via wheat9)
+
+    HADOOP-11379. Fix new findbugs warnings in hadoop-auth*. (Li Lu via wheat9)
+
+    HADOOP-11378. Fix new findbugs warnings in hadoop-kms. (Li Lu via wheat9)
+
+    HADOOP-11349. RawLocalFileSystem leaks file descriptor while creating a
+    file if creat succeeds but chmod fails. (Varun Saxena via Colin P. McCabe)
+
+    HADOOP-11381. Fix findbugs warnings in hadoop-distcp, hadoop-aws,
+    hadoop-azure, and hadoop-openstack. (Li Lu via wheat9)
+
+    HADOOP-10482. Fix various findbugs warnings in hadoop-common. (wheat9)
+
+    HADOOP-11388. Remove deprecated o.a.h.metrics.file.FileContext.
+    (Li Lu via wheat9)
+
+    HADOOP-11386. Replace \n by %n in format hadoop-common format strings.
+    (Li Lu via wheat9)
+
+    HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be
+    be order-independent. (Yitong Zhou via gera)
+
+    HADOOP-11389. Clean up byte to string encoding issues in hadoop-common.
+    (wheat9)
+
+    HADOOP-11394. hadoop-aws documentation missing. (cnauroth)
+
+    HADOOP-11396. Provide navigation in the site documentation linking to the
+    Hadoop Compatible File Systems. (cnauroth)
+
+    HADOOP-11412 POMs mention "The Apache Software License" rather than
+    "Apache License". (Herve Boutemy via stevel)
+
+    HADOOP-11321. copyToLocal cannot save a file to an SMB share unless the user
+    has Full Control permissions. (cnauroth)
+
+    HADOOP-11420. Use latest maven-site-plugin and replace link to svn with
+    link to git. (Herve Boutemy via wheat9)
+
+    HADOOP-10689. InputStream is not closed in
+    AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth)
+
+    HADOOP-10690. Lack of synchronization on access to InputStream in
+    NativeAzureFileSystem#NativeAzureFsInputStream#close().
+    (Chen He via cnauroth)
+
+    HADOOP-11358. Tests for encryption/decryption with IV calculation
+    overflow. (yliu)
+
+    HADOOP-11125. Remove redundant tests in TestOsSecureRandom.
+    (Masanori Oyama via wheat9)
+
+    HADOOP-11385. Prevent cross site scripting attack on JMXJSONServlet.
+    (wheat9)
+
+    HADOOP-11409. FileContext.getFileContext can stack overflow if default fs
+    misconfigured (Gera Shegalov via jlowe)
+
+    HADOOP-11428. Remove obsolete reference to Cygwin in BUILDING.txt.
+    (Arpit Agarwal via wheat9)
+
+    HADOOP-11431. clean up redundant maven-site-plugin configuration.
+    (Herve Boutemy via wheat9)
+
+    HADOOP-11429. Findbugs warnings in hadoop extras.
+    (Varun Saxena via wheat9)
+
+    HADOOP-11414. FileBasedIPList#readLines() can leak file descriptors.
+    (ozawa)
+
+    HADOOP-11283. SequenceFile.Writer can leak file descriptors in
+    DistCpV1#setup(). (Varun Saxena via ozawa)
+
+    HADOOP-11448. Fix findbugs warnings in FileBasedIPList. (ozawa)
+
+    HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
+    implementations. (Yi Liu via Colin P. McCabe)
+
+    HADOOP-11446. S3AOutputStream should use shared thread pool to
+    avoid OutOfMemoryError. (Ted Yu via stevel)	
+
+    HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils
+    and ShellBasedIdMapping (vinayakumarb)
+
+    HADOOP-11445. Bzip2Codec: Data block is skipped when position of newly
+    created stream is equal to start of split (Ankit Kamboj via jlowe)
+
+    HADOOP-11462. TestSocketIOWithTimeout needs change for PowerPC platform.
+    (Ayappan via cnauroth)
+
+    HADOOP-11350. The size of header buffer of HttpServer is too small when
+    HTTPS is enabled. (Benoy Antony via wheat9)
+
+    HADOOP-10542 Potential null pointer dereference in Jets3tFileSystemStore
+    retrieveBlock(). (Ted Yu via stevel)	
+
+    HADOOP-10668. TestZKFailoverControllerStress#testExpireBackAndForth
+    occasionally fails. (Ming Ma via cnauroth)
+
+    HADOOP-11327. BloomFilter#not() omits the last bit, resulting in an
+    incorrect filter (Eric Payne via jlowe)
+
+    HADOOP-11209. Configuration#updatingResource/finalParameters are not
+    thread-safe. (Varun Saxena via ozawa)
+
+    HADOOP-11500. InputStream is left unclosed in ApplicationClassLoader.
+    (Ted Yu via ozawa)
+
+    HADOOP-11008. Remove duplicated description about proxy-user in site 
+    documents (Masatake Iwasaki via aw)
+
+    HADOOP-11493. Fix some typos in kms-acls.xml description.
+    (Charles Lamb via aajisaka)
+
+    HADOOP-11507 Hadoop RPC Authentication problem with different user locale.
+    (Talat UYARER via stevel)
+
+    HADOOP-11482. Use correct UGI when KMSClientProvider is called by a proxy
+    user. Contributed by Arun Suresh.
+
+    HADOOP-11499. Check of executorThreadsStarted in
+    ValueQueue#submitRefillTask() evades lock acquisition (Ted Yu via jlowe)
+
+    HADOOP-6221 RPC Client operations cannot be interrupted. (stevel)
+
+    HADOOP-11509. Change parsing sequence in GenericOptionsParser to parse -D
+    parameters before -files. (xgong)
+
+    HADOOP-11469. KMS should skip default.key.acl and whitelist.key.acl when
+    loading key acl. (Dian Fu via yliu)
+
+    HADOOP-11316. "mvn package -Pdist,docs -DskipTests -Dtar" fails because
+    of non-ascii characters. (ozawa)
+
+    HADOOP-9907. Webapp http://hostname:port/metrics link is not working.
+    (aajisaka)
+
+    HADOOP-11403. Avoid using sys_errlist on Solaris, which lacks support for it
+    (Malcolm Kavalsky via Colin P. McCabe)
+
+    HADOOP-11523. StorageException complaining " no lease ID" when updating
+    FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+
+    HADOOP-11432. Fix SymlinkBaseTest#testCreateLinkUsingPartQualPath2.
+    (Liang Xie via gera)
+
+    HADOOP-10181. GangliaContext does not work with multicast ganglia setup.
+    (Andrew Johnson via cnauroth)
+
+    HADOOP-11529. Fix findbugs warnings in hadoop-archives.
+    (Masatake Iwasaki via wheat9)
+
+    HADOOP-11546. Checkstyle failing: Unable to instantiate 
+    DoubleCheckedLockingCheck. (ozawa)
+
+    HADOOP-11548. checknative should display a nicer error message when openssl
+    support is not compiled in. (Anu Engineer via cnauroth)
+
+    HADOOP-11547. hadoop-common native compilation fails on Windows due to
+    missing support for __attribute__ declaration. (cnauroth)
+
+    HADOOP-11549. flaky test detection tool failed to handle special control
+    characters in test result. (Yongjun Zhang via aajisaka)
+
+    HADOOP-10062. race condition in MetricsSystemImpl#publishMetricsNow that 
+    causes incorrect results. (Sangjin Lee via junping_du)
+
+    HADOOP-11526. Memory leak in Bzip2Compressor and Bzip2Decompressor.
+    (Anu Engineer via cnauroth)
+
+    HADOOP-11535 TableMapping related tests failed due to 'successful'
+    resolving of invalid test hostname. (Kai Zheng via stevel)
+
+Release 2.6.1 - UNRELEASED
+
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+  IMPROVEMENTS
+
+  OPTIMIZATIONS
+
+  BUG FIXES
+
+    HADOOP-11466. FastByteComparisons: do not use UNSAFE_COMPARER on the SPARC
+    architecture because it is slower there (Suman Somasundar via Colin P.
+    McCabe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-common-project/hadoop-common/dev-support/checkstyle.xml b/hadoop-common-project/hadoop-common/dev-support/checkstyle.xml
index ad70bee1764a6..4caa03f05dfd2 100644
--- a/hadoop-common-project/hadoop-common/dev-support/checkstyle.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/checkstyle.xml
@@ -150,7 +150,6 @@
         <!-- Checks for common coding problems               -->
         <!-- See http://checkstyle.sf.net/config_coding.html -->
         <!-- module name="AvoidInlineConditionals"/-->
-        <module name="DoubleCheckedLocking"/>
         <module name="EmptyStatement"/>
         <module name="EqualsHashCode"/>
         <module name="HiddenField">
diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index 8de3c378bd6f7..ab8673b860e96 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -241,6 +241,16 @@
        <Method name="writeVLong" />
        <Bug pattern="SF_SWITCH_FALLTHROUGH" />
      </Match>
+     <Match>
+       <Class name="org.apache.hadoop.io.Text" />
+       <Method name="bytesToCodePoint" />
+       <Bug pattern="SF_SWITCH_NO_DEFAULT" />
+     </Match>
+     <Match>
+       <Class name="org.apache.hadoop.util.PureJavaCrc32C" />
+       <Method name="update" />
+       <Bug pattern="SF_SWITCH_NO_DEFAULT" />
+     </Match>
     <!--
 	  The switch condition fall through is intentional and for performance
 	  purposes.
@@ -376,4 +386,10 @@
     <Bug pattern="REC_CATCH_EXCEPTION"/>
   </Match>
 
+  <Match>
+    <Class name="org.apache.hadoop.conf.Configuration"/>
+    <Method name="loadProperty"/>
+    <Bug pattern="NP_NULL_PARAM_DEREF"/>
+  </Match>
+
 </FindBugsFilter>
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 6a63ccdb03ebb..3ae78f4300042 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -240,7 +240,7 @@
     </dependency>
 
     <dependency>
-      <groupId>org.htrace</groupId>
+      <groupId>org.apache.htrace</groupId>
       <artifactId>htrace-core</artifactId>
     </dependency>
     <dependency>
@@ -530,6 +530,7 @@
         <require.openssl>false</require.openssl>
         <runningWithNative>true</runningWithNative>
         <bundle.openssl.in.bin>false</bundle.openssl.in.bin>
+        <extra.libhadoop.rpath></extra.libhadoop.rpath>
       </properties>
       <build>
         <plugins>
@@ -601,7 +602,7 @@
                 <configuration>
                   <target>
                     <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include} -DEXTRA_LIBHADOOP_RPATH=${extra.libhadoop.rpath}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                       <arg line="VERBOSE=1"/>
diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index 29fa2b8f50a61..942b19c752672 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
@@ -185,7 +185,6 @@ find_path(OPENSSL_INCLUDE_DIR NAMES openssl/evp.h)
 SET(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
 SET(USABLE_OPENSSL 0)
 if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
-    GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
     INCLUDE(CheckCSourceCompiles)
     SET(OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
     SET(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
@@ -198,6 +197,7 @@ if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
     endif(NOT HAS_NEW_ENOUGH_OPENSSL)
 endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
 if (USABLE_OPENSSL)
+    GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
     SET(OPENSSL_SOURCE_FILES
         "${D}/crypto/OpensslCipher.c"
         "${D}/crypto/random/OpensslSecureRandom.c")
@@ -267,8 +267,12 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     # dlopen will look in the directory containing libhadoop.so.
     # However, $ORIGIN is not supported by all operating systems.
     #
+    set(RPATH "\$ORIGIN/")
+    if (EXTRA_LIBHADOOP_RPATH)
+        set(RPATH "${RPATH}:${EXTRA_LIBHADOOP_RPATH}/")
+    endif(EXTRA_LIBHADOOP_RPATH)
     SET_TARGET_PROPERTIES(hadoop 
-        PROPERTIES INSTALL_RPATH "\$ORIGIN/")
+        PROPERTIES INSTALL_RPATH "${RPATH}")
 ENDIF()
 
 target_link_dual_libraries(hadoop
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 9df2c7dfa8942..d689bdad14b4f 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -17,8 +17,10 @@
 
 function hadoop_usage()
 {
-  echo "Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND"
-  echo "       where COMMAND is one of:"
+  echo "Usage: hadoop [--config confdir] [COMMAND | CLASSNAME]"
+  echo "  CLASSNAME            run the class named CLASSNAME"
+  echo " or"
+  echo "  where COMMAND is one of:"
   echo "  archive -archiveName NAME -p <parent path> <src>* <dest>"
   echo "                       create a Hadoop archive"
   echo "  checknative [-a|-h]  check native Hadoop and compression "
@@ -39,8 +41,6 @@ function hadoop_usage()
   echo "  key                  manage keys via the KeyProvider"
   echo "  trace                view and modify Hadoop tracing settings"
   echo "  version              print the version"
-  echo " or"
-  echo "  CLASSNAME            run the class named CLASSNAME"
   echo ""
   echo "Most commands print help when invoked w/o parameters."
 }
@@ -124,13 +124,7 @@ case ${COMMAND} in
     CLASS=org.apache.hadoop.util.NativeLibraryChecker
   ;;
   classpath)
-    if [[ "$#" -eq 1 ]]; then
-      CLASS=org.apache.hadoop.util.Classpath
-    else   
-      hadoop_finalize
-      echo "${CLASSPATH}"
-      exit 0
-    fi
+    hadoop_do_classpath_subcommand "$@"
   ;;
   credential)
     CLASS=org.apache.hadoop.security.alias.CredentialShell
@@ -179,12 +173,12 @@ case ${COMMAND} in
   ;;
 esac
 
+hadoop_verify_user "${COMMAND}"
+
 # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
 hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
-
 hadoop_finalize
 hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
 
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index 89b0c932b32a6..f0cabbe3aeab2 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -1,4 +1,4 @@
-#
+
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -76,7 +76,7 @@ fi
 #
 
 # Let's go!  Base definitions so we can move forward
-hadoop_bootstrap_init
+hadoop_bootstrap
 
 # let's find our conf.
 #
@@ -107,8 +107,6 @@ while [[ -z "${_hadoop_common_done}" ]]; do
       confdir=$1
       shift
       if [[ -d "${confdir}" ]]; then
-        # shellcheck disable=SC2034
-        YARN_CONF_DIR="${confdir}"
         # shellcheck disable=SC2034
         HADOOP_CONF_DIR="${confdir}"
       elif [[ -z "${confdir}" ]]; then
@@ -160,14 +158,21 @@ while [[ -z "${_hadoop_common_done}" ]]; do
   esac
 done
 
+#
+# Setup the base-line environment
+#
 hadoop_find_confdir
 hadoop_exec_hadoopenv
+hadoop_import_shellprofiles
 hadoop_exec_userfuncs
 
 #
 # IMPORTANT! User provided code is now available!
 #
 
+hadoop_exec_hadooprc
+hadoop_verify_confdir
+
 # do all the OS-specific startup bits here
 # this allows us to get a decent JAVA_HOME,
 # call crle for LD_LIBRARY_PATH, etc.
@@ -182,22 +187,20 @@ if declare -F hadoop_subproject_init >/dev/null ; then
   hadoop_subproject_init
 fi
 
+hadoop_shellprofiles_init
+
 # get the native libs in there pretty quick
 hadoop_add_javalibpath "${HADOOP_PREFIX}/build/native"
 hadoop_add_javalibpath "${HADOOP_PREFIX}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
 
+hadoop_shellprofiles_nativelib
+
 # get the basic java class path for these subprojects
 # in as quickly as possible since other stuff
 # will definitely depend upon it.
-#
-# at some point, this will get replaced with something pluggable
-# so that these functions can sit in their projects rather than
-# common
-#
-for i in common hdfs yarn mapred
-do
-  hadoop_add_to_classpath_$i
-done
+
+hadoop_add_common_to_classpath
+hadoop_shellprofiles_classpath
 
 #
 # backwards compatibility. new stuff should
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 2b56634a0fbd9..142ac22409e87 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -28,13 +28,35 @@ function hadoop_debug
   fi
 }
 
-function hadoop_bootstrap_init
+function hadoop_deprecate_envvar
+{
+  #
+  # Deprecate $1 with $2
+  local oldvar=$1
+  local newvar=$2
+  local oldval=${!oldvar}
+  local newval=${!newvar}
+
+  if [[ -n "${oldval}" ]]; then
+    hadoop_error "WARNING: ${oldvar} has been replaced by ${newvar}. Using value of ${oldvar}."
+    # shellcheck disable=SC2086
+    eval ${newvar}=\"${oldval}\"
+
+    # shellcheck disable=SC2086
+    newval=${oldval}
+
+    # shellcheck disable=SC2086
+    eval ${newvar}=\"${newval}\"
+  fi
+}
+
+function hadoop_bootstrap
 {
   # NOTE: This function is not user replaceable.
 
   # the root of the Hadoop installation
   # See HADOOP-6255 for the expected directory structure layout
-  
+
   # By now, HADOOP_LIBEXEC_DIR should have been defined upstream
   # We can piggyback off of that to figure out where the default
   # HADOOP_FREFIX should be.  This allows us to run without
@@ -48,7 +70,7 @@ function hadoop_bootstrap_init
   HADOOP_DEFAULT_PREFIX=$(cd -P -- "${HADOOP_LIBEXEC_DIR}/.." >/dev/null && pwd -P)
   HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}
   export HADOOP_PREFIX
-  
+
   #
   # short-cuts. vendors may redefine these as well, preferably
   # in hadoop-layouts.sh
@@ -66,7 +88,7 @@ function hadoop_bootstrap_init
   TOOL_PATH=${TOOL_PATH:-${HADOOP_PREFIX}/share/hadoop/tools/lib/*}
 
   export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
- 
+
   # defaults
   export HADOOP_OPTS=${HADOOP_OPTS:-"-Djava.net.preferIPv4Stack=true"}
   hadoop_debug "Initial HADOOP_OPTS=${HADOOP_OPTS}"
@@ -92,6 +114,15 @@ function hadoop_find_confdir
   hadoop_debug "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}"
 }
 
+function hadoop_verify_confdir
+{
+  # Check only log4j.properties by default.
+  # --loglevel does not work without logger settings in log4j.log4j.properties.
+  if [[ ! -f "${HADOOP_CONF_DIR}/log4j.properties" ]]; then
+    hadoop_error "WARNING: log4j.properties is not found. HADOOP_CONF_DIR may be incomplete."
+  fi
+}
+
 function hadoop_exec_hadoopenv
 {
   # NOTE: This function is not user replaceable.
@@ -113,6 +144,98 @@ function hadoop_exec_userfuncs
   fi
 }
 
+function hadoop_exec_hadooprc
+{
+  # Read the user's settings.  This provides for users to override
+  # and/or append hadoop-env.sh. It is not meant as a complete system override.
+
+  if [[ -f "${HOME}/.hadooprc" ]]; then
+    hadoop_debug "Applying the user's .hadooprc"
+    . "${HOME}/.hadooprc"
+  fi
+}
+
+function hadoop_import_shellprofiles
+{
+  local i
+  local files1
+  local files2
+
+  if [[ -d "${HADOOP_LIBEXEC_DIR}/shellprofile.d" ]]; then
+    files1=(${HADOOP_LIBEXEC_DIR}/shellprofile.d/*)
+  else
+    hadoop_error "WARNING: ${HADOOP_LIBEXEC_DIR}/shellprofile.d doesn't exist. Functionality may not work."
+  fi
+
+  if [[ -d "${HADOOP_CONF_DIR}/shellprofile.d" ]]; then
+    files2=(${HADOOP_CONF_DIR}/shellprofile.d/*)
+  fi
+
+  for i in "${files1[@]}" "${files2[@]}"
+  do
+    if [[ -n "${i}" ]]; then
+      hadoop_debug "Profiles: importing ${i}"
+      . "${i}"
+    fi
+  done
+}
+
+function hadoop_shellprofiles_init
+{
+  local i
+
+  for i in ${HADOOP_SHELL_PROFILES}
+  do
+    if declare -F _${i}_hadoop_init >/dev/null ; then
+       hadoop_debug "Profiles: ${i} init"
+       # shellcheck disable=SC2086
+       _${i}_hadoop_init
+    fi
+  done
+}
+
+function hadoop_shellprofiles_classpath
+{
+  local i
+
+  for i in ${HADOOP_SHELL_PROFILES}
+  do
+    if declare -F _${i}_hadoop_classpath >/dev/null ; then
+       hadoop_debug "Profiles: ${i} classpath"
+       # shellcheck disable=SC2086
+       _${i}_hadoop_classpath
+    fi
+  done
+}
+
+function hadoop_shellprofiles_nativelib
+{
+  local i
+
+  for i in ${HADOOP_SHELL_PROFILES}
+  do
+    if declare -F _${i}_hadoop_nativelib >/dev/null ; then
+       hadoop_debug "Profiles: ${i} nativelib"
+       # shellcheck disable=SC2086
+       _${i}_hadoop_nativelib
+    fi
+  done
+}
+
+function hadoop_shellprofiles_finalize
+{
+  local i
+
+  for i in ${HADOOP_SHELL_PROFILES}
+  do
+    if declare -F _${i}_hadoop_finalize >/dev/null ; then
+       hadoop_debug "Profiles: ${i} finalize"
+       # shellcheck disable=SC2086
+       _${i}_hadoop_finalize
+    fi
+  done
+}
+
 function hadoop_basic_init
 {
   # Some of these are also set in hadoop-env.sh.
@@ -121,38 +244,37 @@ function hadoop_basic_init
   #
   # but it is important to note that if you update these
   # you also need to update hadoop-env.sh as well!!!
-  
-  # CLASSPATH initially contains $HADOOP_CONF_DIR
-  CLASSPATH="${HADOOP_CONF_DIR}"
-  hadoop_debug "Initial CLASSPATH=${HADOOP_CONF_DIR}"
-  
+
+  CLASSPATH=""
+  hadoop_debug "Initialize CLASSPATH"
+
   if [[ -z "${HADOOP_COMMON_HOME}" ]] &&
   [[ -d "${HADOOP_PREFIX}/${HADOOP_COMMON_DIR}" ]]; then
     export HADOOP_COMMON_HOME="${HADOOP_PREFIX}"
   fi
-  
+
   # default policy file for service-level authorization
   HADOOP_POLICYFILE=${HADOOP_POLICYFILE:-"hadoop-policy.xml"}
-  
+
   # define HADOOP_HDFS_HOME
   if [[ -z "${HADOOP_HDFS_HOME}" ]] &&
      [[ -d "${HADOOP_PREFIX}/${HDFS_DIR}" ]]; then
     export HADOOP_HDFS_HOME="${HADOOP_PREFIX}"
   fi
-  
+
   # define HADOOP_YARN_HOME
   if [[ -z "${HADOOP_YARN_HOME}" ]] &&
      [[ -d "${HADOOP_PREFIX}/${YARN_DIR}" ]]; then
     export HADOOP_YARN_HOME="${HADOOP_PREFIX}"
   fi
-  
+
   # define HADOOP_MAPRED_HOME
   if [[ -z "${HADOOP_MAPRED_HOME}" ]] &&
      [[ -d "${HADOOP_PREFIX}/${MAPRED_DIR}" ]]; then
     export HADOOP_MAPRED_HOME="${HADOOP_PREFIX}"
   fi
-  
-  HADOOP_IDENT_STRING=${HADOP_IDENT_STRING:-$USER}
+
+  HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER}
   HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_PREFIX}/logs"}
   HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log}
   HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
@@ -162,7 +284,6 @@ function hadoop_basic_init
   HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}
   HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},RFA}
   HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}
-  HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024}
   HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS:-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"}
   HADOOP_SECURE_LOG_DIR=${HADOOP_SECURE_LOG_DIR:-${HADOOP_LOG_DIR}}
   HADOOP_SECURE_PID_DIR=${HADOOP_SECURE_PID_DIR:-${HADOOP_PID_DIR}}
@@ -181,8 +302,6 @@ function hadoop_populate_slaves_file()
   elif [[ -f "${HADOOP_CONF_DIR}/${slavesfile}" ]]; then
     # shellcheck disable=2034
     HADOOP_SLAVES="${HADOOP_CONF_DIR}/${slavesfile}"
-    # shellcheck disable=2034
-    YARN_SLAVES="${HADOOP_CONF_DIR}/${slavesfile}"
   else
     hadoop_error "ERROR: Cannot find hosts file \"${slavesfile}\""
     hadoop_exit_with_usage 1
@@ -201,7 +320,7 @@ function hadoop_rotate_log
   #
   local log=$1;
   local num=${2:-5};
-  
+
   if [[ -f "${log}" ]]; then # rotate logs
     while [[ ${num} -gt 1 ]]; do
       #shellcheck disable=SC2086
@@ -221,7 +340,7 @@ function hadoop_actual_ssh
   # should get hostname followed by rest of command line
   local slave=$1
   shift
-  
+
   # shellcheck disable=SC2086
   ssh ${HADOOP_SSH_OPTS} ${slave} $"${@// /\\ }" 2>&1 | sed "s/^/$slave: /"
 }
@@ -230,7 +349,7 @@ function hadoop_connect_to_hosts
 {
   # shellcheck disable=SC2124
   local params="$@"
-  
+
   #
   # ssh (or whatever) to a host
   #
@@ -239,13 +358,13 @@ function hadoop_connect_to_hosts
     hadoop_error "ERROR: Both HADOOP_SLAVES and HADOOP_SLAVE_NAME were defined. Aborting."
     exit 1
   fi
-  
+
   if [[ -n "${HADOOP_SLAVE_NAMES}" ]] ; then
     SLAVE_NAMES=${HADOOP_SLAVE_NAMES}
   else
     SLAVE_FILE=${HADOOP_SLAVES:-${HADOOP_CONF_DIR}/slaves}
   fi
-  
+
   # if pdsh is available, let's use it.  otherwise default
   # to a loop around ssh.  (ugh)
   if [[ -e '/usr/bin/pdsh' ]]; then
@@ -265,7 +384,7 @@ function hadoop_connect_to_hosts
     if [[ -z "${SLAVE_NAMES}" ]]; then
       SLAVE_NAMES=$(sed 's/#.*$//;/^$/d' "${SLAVE_FILE}")
     fi
-    
+
     # quoting here gets tricky. it's easier to push it into a function
     # so that we don't have to deal with it. However...
     # xargs can't use a function so instead we'll export it out
@@ -275,8 +394,8 @@ function hadoop_connect_to_hosts
     export HADOOP_SSH_OPTS
 
     # xargs is used with option -I to replace the placeholder in arguments
-    # list with each hostname read from stdin/pipe. But it consider one 
-    # line as one argument while reading from stdin/pipe. So place each 
+    # list with each hostname read from stdin/pipe. But it consider one
+    # line as one argument while reading from stdin/pipe. So place each
     # hostname in different lines while passing via pipe.
     SLAVE_NAMES=$(echo "$SLAVE_NAMES" | tr ' ' '\n' )
     echo "${SLAVE_NAMES}" | \
@@ -321,6 +440,12 @@ function hadoop_add_param
   fi
 }
 
+function hadoop_add_profile
+{
+  # shellcheck disable=SC2086
+  hadoop_add_param HADOOP_SHELL_PROFILES $1 $1
+}
+
 function hadoop_add_classpath
 {
   # two params:
@@ -329,7 +454,7 @@ function hadoop_add_classpath
   #      classpath this object should go. default is after
   # return 0 = success (added or duplicate)
   # return 1 = failure (doesn't exist, whatever)
-  
+
   # However, with classpath (& JLP), we can do dedupe
   # along with some sanity checking (e.g., missing directories)
   # since we have a better idea of what is legal
@@ -342,7 +467,7 @@ function hadoop_add_classpath
       hadoop_debug "Rejected CLASSPATH: $1 (not a dir)"
       return 1
     fi
-    
+
     # no wildcard in the middle, so check existence
     # (doesn't matter *what* it is)
   elif [[ ! $1 =~ ^.*\*.*$ ]] && [[ ! -e "$1" ]]; then
@@ -374,7 +499,7 @@ function hadoop_add_colonpath
   #      classpath this object should go
   # return 0 = success
   # return 1 = failure (duplicate)
-  
+
   # this is CLASSPATH, JLP, etc but with dedupe but no
   # other checking
   if [[ -d "${2}" ]] && [[ ":${!1}:" != *":$2:"* ]]; then
@@ -407,101 +532,30 @@ function hadoop_add_ldlibpath
 {
   # specialized function for a common use case
   hadoop_add_colonpath LD_LIBRARY_PATH "$1" "$2"
-  
+
   # note that we export this
   export LD_LIBRARY_PATH
 }
 
-function hadoop_add_to_classpath_common
+function hadoop_add_common_to_classpath
 {
-  
   #
   # get all of the common jars+config in the path
   #
-  
+
   # developers
   if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
     hadoop_add_classpath "${HADOOP_COMMON_HOME}/hadoop-common/target/classes"
   fi
-  
+
   if [[ -d "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}/webapps" ]]; then
     hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"
   fi
-  
+
   hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"'/*'
   hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*'
 }
 
-function hadoop_add_to_classpath_hdfs
-{
-  #
-  # get all of the hdfs jars+config in the path
-  #
-  # developers
-  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
-    hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
-  fi
-  
-  # put hdfs in classpath if present
-  if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
-    hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
-  fi
-  
-  hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
-  hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
-}
-
-function hadoop_add_to_classpath_yarn
-{
-  local i
-  #
-  # get all of the yarn jars+config in the path
-  #
-  # developers
-  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
-    for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
-    yarn-server/yarn-server-nodemanager \
-    yarn-server/yarn-server-common \
-    yarn-server/yarn-server-resourcemanager; do
-      hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
-    done
-    
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
-  fi
-  
-  if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
-    hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
-  fi
-  
-  hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
-  hadoop_add_classpath  "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
-}
-
-function hadoop_add_to_classpath_mapred
-{
-  #
-  # get all of the mapreduce jars+config in the path
-  #
-  # developers
-  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
-  fi
-  
-  if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
-    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
-  fi
-  
-  hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
-  hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
-}
-
 function hadoop_add_to_classpath_userpath
 {
   # Add the user-specified HADOOP_CLASSPATH to the
@@ -524,7 +578,7 @@ function hadoop_add_to_classpath_userpath
       let c+=1
     done
     let j=c-1
-    
+
     if [[ -z "${HADOOP_USE_CLIENT_CLASSLOADER}" ]]; then
       if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
         for ((i=j; i>=0; i--)); do
@@ -543,8 +597,10 @@ function hadoop_os_tricks
 {
   local bindv6only
 
-  # some OSes have special needs. here's some out of the box
-  # examples for OS X and Linux. Vendors, replace this with your special sauce.
+  # Some OSes have special needs.  Here's some out of the box examples for OS X,
+  # Linux and Windows on Cygwin.
+  # Vendors, replace this with your special sauce.
+  HADOOP_IS_CYGWIN=false
   case ${HADOOP_OS_TYPE} in
     Darwin)
       if [[ -z "${JAVA_HOME}" ]]; then
@@ -561,8 +617,8 @@ function hadoop_os_tricks
       # NOTE! HADOOP_ALLOW_IPV6 is a developer hook.  We leave it
       # undocumented in hadoop-env.sh because we don't want users to
       # shoot themselves in the foot while devs make IPv6 work.
-      if [[ -n "${bindv6only}" ]] && 
-         [[ "${bindv6only}" -eq "1" ]] && 
+      if [[ -n "${bindv6only}" ]] &&
+         [[ "${bindv6only}" -eq "1" ]] &&
          [[ "${HADOOP_ALLOW_IPV6}" != "yes" ]]; then
         hadoop_error "ERROR: \"net.ipv6.bindv6only\" is set to 1 "
         hadoop_error "ERROR: Hadoop networking could be broken. Aborting."
@@ -575,6 +631,10 @@ function hadoop_os_tricks
       # down to prevent vmem explosion.
       export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
     ;;
+    CYGWIN*)
+      # Flag that we're running on Cygwin to trigger path translation later.
+      HADOOP_IS_CYGWIN=true
+    ;;
   esac
 }
 
@@ -585,46 +645,83 @@ function hadoop_java_setup
     hadoop_error "ERROR: JAVA_HOME is not set and could not be found."
     exit 1
   fi
-  
+
   if [[ ! -d "${JAVA_HOME}" ]]; then
     hadoop_error "ERROR: JAVA_HOME ${JAVA_HOME} does not exist."
     exit 1
   fi
-  
+
   JAVA="${JAVA_HOME}/bin/java"
-  
+
   if [[ ! -x "$JAVA" ]]; then
     hadoop_error "ERROR: $JAVA is not executable."
     exit 1
   fi
-  # shellcheck disable=SC2034
-  JAVA_HEAP_MAX=-Xmx1g
-  HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024}
-  
-  # check envvars which might override default args
-  if [[ -n "$HADOOP_HEAPSIZE" ]]; then
-    # shellcheck disable=SC2034
-    JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m"
-  fi
 }
 
 function hadoop_finalize_libpaths
 {
   if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
+    hadoop_translate_cygwin_path JAVA_LIBRARY_PATH
     hadoop_add_param HADOOP_OPTS java.library.path \
     "-Djava.library.path=${JAVA_LIBRARY_PATH}"
     export LD_LIBRARY_PATH
   fi
 }
 
+function hadoop_finalize_hadoop_heap
+{
+  if [[ -n "${HADOOP_HEAPSIZE_MAX}" ]]; then
+    if [[ "${HADOOP_HEAPSIZE_MAX}" =~ ^[0-9]+$ ]]; then
+      HADOOP_HEAPSIZE_MAX="${HADOOP_HEAPSIZE_MAX}m"
+    fi
+    hadoop_add_param HADOOP_OPTS Xmx "-Xmx${HADOOP_HEAPSIZE_MAX}"
+  fi
+
+  # backwards compatibility
+  if [[ -n "${HADOOP_HEAPSIZE}" ]]; then
+    if [[ "${HADOOP_HEAPSIZE}" =~ ^[0-9]+$ ]]; then
+      HADOOP_HEAPSIZE="${HADOOP_HEAPSIZE}m"
+    fi
+    hadoop_add_param HADOOP_OPTS Xmx "-Xmx${HADOOP_HEAPSIZE}"
+  fi
+
+  if [[ -n "${HADOOP_HEAPSIZE_MIN}" ]]; then
+    if [[ "${HADOOP_HEAPSIZE_MIN}" =~ ^[0-9]+$ ]]; then
+      HADOOP_HEAPSIZE_MIN="${HADOOP_HEAPSIZE_MIN}m"
+    fi
+    hadoop_add_param HADOOP_OPTS Xms "-Xms${HADOOP_HEAPSIZE_MIN}"
+  fi
+}
+
+# Accepts a variable name.  If running on Cygwin, sets the variable value to the
+# equivalent translated Windows path by running the cygpath utility.  If the
+# second argument is true, then the variable is treated as a path list.
+function hadoop_translate_cygwin_path
+{
+  if [[ "${HADOOP_IS_CYGWIN}" = "true" ]]; then
+    if [[ "$2" = "true" ]]; then
+      #shellcheck disable=SC2016
+      eval "$1"='$(cygpath -p -w "${!1}" 2>/dev/null)'
+    else
+      #shellcheck disable=SC2016
+      eval "$1"='$(cygpath -w "${!1}" 2>/dev/null)'
+    fi
+  fi
+}
+
 #
 # fill in any last minute options that might not have been defined yet
 #
 function hadoop_finalize_hadoop_opts
 {
+  hadoop_translate_cygwin_path HADOOP_LOG_DIR
   hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR}"
   hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE}"
-  hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}"
+  HADOOP_HOME=${HADOOP_PREFIX}
+  hadoop_translate_cygwin_path HADOOP_HOME
+  export HADOOP_HOME
+  hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}"
   hadoop_add_param HADOOP_OPTS hadoop.id.str "-Dhadoop.id.str=${HADOOP_IDENT_STRING}"
   hadoop_add_param HADOOP_OPTS hadoop.root.logger "-Dhadoop.root.logger=${HADOOP_ROOT_LOGGER}"
   hadoop_add_param HADOOP_OPTS hadoop.policy.file "-Dhadoop.policy.file=${HADOOP_POLICYFILE}"
@@ -634,19 +731,48 @@ function hadoop_finalize_hadoop_opts
 function hadoop_finalize_classpath
 {
   hadoop_add_classpath "${HADOOP_CONF_DIR}" before
-  
+
   # user classpath gets added at the last minute. this allows
   # override of CONF dirs and more
   hadoop_add_to_classpath_userpath
+  hadoop_translate_cygwin_path CLASSPATH true
+}
+
+function hadoop_finalize_catalina_opts
+{
+
+  local prefix=${HADOOP_CATALINA_PREFIX}
+
+  hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}"
+  if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
+    hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}"
+  fi
+  hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_PREFIX}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.admin.port" "-D${prefix}.admin.port=${HADOOP_CATALINA_ADMIN_PORT}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.http.port" "-D${prefix}.http.port=${HADOOP_CATALINA_HTTP_PORT}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.max.threads" "-D${prefix}.max.threads=${HADOOP_CATALINA_MAX_THREADS}"
+  hadoop_add_param CATALINA_OPTS "${prefix}.ssl.keystore.file" "-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}"
 }
 
 function hadoop_finalize
 {
-  # user classpath gets added at the last minute. this allows
-  # override of CONF dirs and more
+
+  hadoop_shellprofiles_finalize
+
   hadoop_finalize_classpath
   hadoop_finalize_libpaths
+  hadoop_finalize_hadoop_heap
   hadoop_finalize_hadoop_opts
+
+  hadoop_translate_cygwin_path HADOOP_PREFIX
+  hadoop_translate_cygwin_path HADOOP_CONF_DIR
+  hadoop_translate_cygwin_path HADOOP_COMMON_HOME
+  hadoop_translate_cygwin_path HADOOP_HDFS_HOME
+  hadoop_translate_cygwin_path HADOOP_YARN_HOME
+  hadoop_translate_cygwin_path HADOOP_MAPRED_HOME
 }
 
 function hadoop_exit_with_usage
@@ -672,7 +798,7 @@ function hadoop_verify_secure_prereq
   # if you are on an OS like Illumos that has functional roles
   # and you are using pfexec, you'll probably want to change
   # this.
-  
+
   # ${EUID} comes from the shell itself!
   if [[ "${EUID}" -ne 0 ]] && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
     hadoop_error "ERROR: You must be a privileged user in order to run a secure service."
@@ -685,7 +811,7 @@ function hadoop_verify_secure_prereq
 function hadoop_setup_secure_service
 {
   # need a more complicated setup? replace me!
-  
+
   HADOOP_PID_DIR=${HADOOP_SECURE_PID_DIR}
   HADOOP_LOG_DIR=${HADOOP_SECURE_LOG_DIR}
 }
@@ -734,7 +860,7 @@ function hadoop_verify_logdir
   rm "${HADOOP_LOG_DIR}/$$" >/dev/null 2>&1
 }
 
-function hadoop_status_daemon() 
+function hadoop_status_daemon()
 {
   #
   # LSB 4.1.0 compatible status command (1)
@@ -751,12 +877,12 @@ function hadoop_status_daemon()
   #     them the benefit of a doubt and returning 1 even if
   #     our pid is not in in /var/run .
   #
-  
+
   local pidfile=$1
   shift
-  
+
   local pid
-  
+
   if [[ -f "${pidfile}" ]]; then
     pid=$(cat "${pidfile}")
     if ps -p "${pid}" > /dev/null 2>&1; then
@@ -775,7 +901,7 @@ function hadoop_java_exec
   local command=$1
   local class=$2
   shift 2
-  
+
   hadoop_debug "Final CLASSPATH: ${CLASSPATH}"
   hadoop_debug "Final HADOOP_OPTS: ${HADOOP_OPTS}"
 
@@ -822,14 +948,14 @@ function hadoop_start_daemon_wrapper
   local counter
 
   hadoop_rotate_log "${outfile}"
-  
+
   hadoop_start_daemon "${daemonname}" \
     "$class" \
     "${pidfile}" \
     "$@" >> "${outfile}" 2>&1 < /dev/null &
 
   # we need to avoid a race condition here
-  # so let's wait for the fork to finish 
+  # so let's wait for the fork to finish
   # before overriding with the daemonized pid
   (( counter=0 ))
   while [[ ! -f ${pidfile} && ${counter} -le 5 ]]; do
@@ -843,23 +969,23 @@ function hadoop_start_daemon_wrapper
   if [[ $? -gt 0 ]]; then
     hadoop_error "ERROR:  Cannot write ${daemonname} pid ${pidfile}."
   fi
-  
+
   # shellcheck disable=SC2086
   renice "${HADOOP_NICENESS}" $! >/dev/null 2>&1
   if [[ $? -gt 0 ]]; then
     hadoop_error "ERROR: Cannot set priority of ${daemonname} process $!"
   fi
-  
+
   # shellcheck disable=SC2086
   disown %+ >/dev/null 2>&1
   if [[ $? -gt 0 ]]; then
     hadoop_error "ERROR: Cannot disconnect ${daemonname} process $!"
   fi
   sleep 1
-  
+
   # capture the ulimit output
   ulimit -a >> "${outfile}" 2>&1
-  
+
   # shellcheck disable=SC2086
   if ! ps -p $! >/dev/null 2>&1; then
     return 1
@@ -873,22 +999,22 @@ function hadoop_start_secure_daemon
   #
   local daemonname=$1
   local class=$2
-  
+
   # pid file to create for our deamon
   local daemonpidfile=$3
-  
+
   # where to send stdout. jsvc has bad habits so this *may* be &1
   # which means you send it to stdout!
   local daemonoutfile=$4
-  
+
   # where to send stderr.  same thing, except &2 = stderr
   local daemonerrfile=$5
   local privpidfile=$6
   shift 6
- 
+
   hadoop_rotate_log "${daemonoutfile}"
   hadoop_rotate_log "${daemonerrfile}"
-  
+
   jsvc="${JSVC_HOME}/jsvc"
   if [[ ! -f "${jsvc}" ]]; then
     hadoop_error "JSVC_HOME is not set or set incorrectly. jsvc is required to run secure"
@@ -897,7 +1023,7 @@ function hadoop_start_secure_daemon
     hadoop_error "and set JSVC_HOME to the directory containing the jsvc binary."
     exit 1
   fi
-  
+
   # note that shellcheck will throw a
   # bogus for-our-use-case 2086 here.
   # it doesn't properly support multi-line situations
@@ -910,7 +1036,7 @@ function hadoop_start_secure_daemon
   if [[ $? -gt 0 ]]; then
     hadoop_error "ERROR:  Cannot write ${daemonname} pid ${privpidfile}."
   fi
-  
+
   exec "${jsvc}" \
     "-Dproc_${daemonname}" \
     -outfile "${daemonoutfile}" \
@@ -929,29 +1055,29 @@ function hadoop_start_secure_daemon_wrapper
   # of the dirty work to launch a daemon in the background!
   local daemonname=$1
   local class=$2
-  
+
   # same rules as hadoop_start_secure_daemon except we
   # have some additional parameters
-  
+
   local daemonpidfile=$3
-  
+
   local daemonoutfile=$4
-  
+
   # the pid file of the subprocess that spawned our
   # secure launcher
   local jsvcpidfile=$5
-  
+
   # the output of the subprocess that spawned our secure
   # launcher
   local jsvcoutfile=$6
-  
+
   local daemonerrfile=$7
   shift 7
 
   local counter
-  
+
   hadoop_rotate_log "${jsvcoutfile}"
-  
+
   hadoop_start_secure_daemon \
     "${daemonname}" \
     "${class}" \
@@ -961,7 +1087,7 @@ function hadoop_start_secure_daemon_wrapper
     "${jsvcpidfile}"  "$@" >> "${jsvcoutfile}" 2>&1 < /dev/null &
 
   # we need to avoid a race condition here
-  # so let's wait for the fork to finish 
+  # so let's wait for the fork to finish
   # before overriding with the daemonized pid
   (( counter=0 ))
   while [[ ! -f ${daemonpidfile} && ${counter} -le 5 ]]; do
@@ -975,7 +1101,7 @@ function hadoop_start_secure_daemon_wrapper
   if [[ $? -gt 0 ]]; then
     hadoop_error "ERROR:  Cannot write ${daemonname} pid ${daemonpidfile}."
   fi
-  
+
   sleep 1
   #shellcheck disable=SC2086
   renice "${HADOOP_NICENESS}" $! >/dev/null 2>&1
@@ -1008,12 +1134,12 @@ function hadoop_stop_daemon
   local cmd=$1
   local pidfile=$2
   shift 2
-  
+
   local pid
-  
+
   if [[ -f "${pidfile}" ]]; then
     pid=$(cat "$pidfile")
-    
+
     kill "${pid}" >/dev/null 2>&1
     sleep "${HADOOP_STOP_TIMEOUT}"
     if kill -0 "${pid}" > /dev/null 2>&1; then
@@ -1035,7 +1161,7 @@ function hadoop_stop_secure_daemon
   local privpidfile=$3
   shift 3
   local ret
-  
+
   hadoop_stop_daemon "${command}" "${daemonpidfile}"
   ret=$?
   rm -f "${daemonpidfile}" "${privpidfile}" 2>/dev/null
@@ -1050,18 +1176,18 @@ function hadoop_daemon_handler
   local daemon_pidfile=$4
   local daemon_outfile=$5
   shift 5
-  
+
   case ${daemonmode} in
     status)
       hadoop_status_daemon "${daemon_pidfile}"
       exit $?
     ;;
-    
+
     stop)
       hadoop_stop_daemon "${daemonname}" "${daemon_pidfile}"
       exit $?
     ;;
-    
+
     ##COMPAT  -- older hadoops would also start daemons by default
     start|default)
       hadoop_verify_piddir
@@ -1097,19 +1223,19 @@ function hadoop_secure_daemon_handler
   local priv_outfile=$7
   local priv_errfile=$8
   shift 8
-  
+
   case ${daemonmode} in
     status)
       hadoop_status_daemon "${daemon_pidfile}"
       exit $?
     ;;
-    
+
     stop)
       hadoop_stop_secure_daemon "${daemonname}" \
       "${daemon_pidfile}" "${priv_pidfile}"
       exit $?
     ;;
-    
+
     ##COMPAT  -- older hadoops would also start daemons by default
     start|default)
       hadoop_verify_piddir
@@ -1122,7 +1248,7 @@ function hadoop_secure_daemon_handler
         # stale pid file, so just remove it and continue on
         rm -f "${daemon_pidfile}" >/dev/null 2>&1
       fi
-      
+
       ##COMPAT  - differenticate between --daemon start and nothing
       # "nothing" shouldn't detach
       if [[ "${daemonmode}" = "default" ]]; then
@@ -1138,3 +1264,26 @@ function hadoop_secure_daemon_handler
   esac
 }
 
+function hadoop_verify_user
+{
+  local command=$1
+  local uservar="HADOOP_${command}_USER"
+
+  if [[ -n ${!uservar} ]]; then
+    if [[ ${!uservar} !=  ${USER} ]]; then
+      hadoop_error "ERROR: ${command} can only be executed by ${!uservar}."
+      exit 1
+    fi
+  fi
+}
+
+function hadoop_do_classpath_subcommand
+{
+  if [[ "$#" -gt 0 ]]; then
+    CLASS=org.apache.hadoop.util.Classpath
+  else
+    hadoop_finalize
+    echo "${CLASSPATH}"
+    exit 0
+  fi
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/rcc b/hadoop-common-project/hadoop-common/src/main/bin/rcc
index 74253539fbeb6..512fc2cc52f89 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/rcc
+++ b/hadoop-common-project/hadoop-common/src/main/bin/rcc
@@ -37,7 +37,5 @@ CLASS='org.apache.hadoop.record.compiler.generated.Rcc'
 hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
-
 hadoop_finalize
 hadoop_java_exec rcc "${CLASS}" "$@"
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index ed9382b345364..ae18542d40b1d 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -64,8 +64,19 @@
 # path.
 # export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
 
-# The maximum amount of heap to use, in MB. Default is 1024.
-# export HADOOP_HEAPSIZE=1024
+# The maximum amount of heap to use (Java -Xmx).  If no unit 
+# is provided, it will be converted to MB.  Daemons will 
+# prefer any Xmx setting in their respective _OPT variable.
+# There is no default; the JVM will autoscale based upon machine
+# memory size.
+# export HADOOP_HEAPSIZE_MAX=
+
+# The minimum amount of heap to use (Java -Xms).  If no unit 
+# is provided, it will be converted to MB.  Daemons will 
+# prefer any Xms setting in their respective _OPT variable.
+# There is no default; the JVM will autoscale based upon machine
+# memory size.
+# export HADOOP_HEAPSIZE_MIN=
 
 # Extra Java runtime options for all Hadoop commands. We don't support
 # IPv6 yet/still, so by default the preference is set to IPv4.
@@ -387,3 +398,9 @@ esac
 # via this special env var:
 # export HADOOP_ENABLE_BUILD_PATHS="true"
 
+#
+# To prevent accidents, shell commands be (superficially) locked
+# to only allow certain users to execute certain subcommands.
+#
+# For example, to limit who can execute the namenode command,
+# export HADOOP_namenode_USER=hdfs
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
index 7699c406ba5db..b2f78f8a41f47 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
@@ -84,11 +84,4 @@
 #    echo "ERROR: ${JAVA} is not executable." 1>&2
 #    exit 1
 #  fi
-#  JAVA_HEAP_MAX=-Xmx1g
-#  HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128}
-#
-#  # check envvars which might override default args
-#  if [[ -n "$HADOOP_HEAPSIZE" ]]; then
-#    JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m"
-#  fi
 #}
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 5fa21fad9e9fa..316c48e7575cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -54,11 +54,9 @@ log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
 
-# Rollver at midnight
+# Rollover at midnight
 log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
 
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
 log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
 
 # Pattern format: Date LogLevel LoggerName LogMessage
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/shellprofile.d/example b/hadoop-common-project/hadoop-common/src/main/conf/shellprofile.d/example
new file mode 100644
index 0000000000000..dc50821ef16d8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/conf/shellprofile.d/example
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This is an example shell profile.  It does not do anything other than
+# show an example of what the general structure and API of the pluggable
+# shell profile code looks like.
+#
+#
+
+#
+#  First, register the profile:
+#
+# hadoop_add_profile example
+#
+#
+# This profile name determines what the name of the functions will
+# be. The general pattern is _(profilename)_hadoop_(capability).  There
+# are currently four capabilities:
+#  * init
+#  * classpath
+#  * nativelib
+#  * finalize
+#
+# None of these functions are required.  Examples of all four follow...
+
+#
+# The _hadoop_init function is called near the very beginning of the
+# execution cycle. System and site-level shell env vars have been set,
+# command line processing finished, etc.  Note that the user's .hadooprc
+# has not yet been processed.  This is to allow them to override anything
+# that may be set here or potentially a dependency!
+#
+# function _example_hadoop_init
+# {
+#   # This example expects a home.  So set a default if not set.
+#   EXAMPLE_HOME="${EXAMPLE_HOME:-/usr/example}"
+# }
+#
+
+#
+# The _hadoop_classpath function is called when the shell code is
+# establishing the classpath.  This function should use the
+# shell hadoop_add_classpath function rather than directly
+# manipulating the CLASSPATH variable.  This ensures that the
+# CLASSPATH does not have duplicates and provides basic
+# sanity checks
+#
+# function _example_hadoop_classpath
+# {
+#   # jars that should be near the front
+#   hadoop_add_classpath "${EXAMPLE_HOME}/share/pre-jars/*" before
+#
+#   # jars that should be near the back
+#   hadoop_add_classpath "${EXAMPLE_HOME}/share/post-jars/*" after
+# }
+
+#
+# The _hadoop_nativelib function is called when the shell code is
+# buliding the locations for linkable shared libraries. Depending
+# upon needs, there are shell function calls that are useful
+# to use here:
+#
+# hadoop_add_javalibpath will push the path onto the command line
+# and into the java.library.path system property.  In the majority
+# of cases, this should be sufficient, especially if the shared
+# library has been linked correctly with $ORIGIN.
+#
+# hadoop_add_ldlibpath will push the path into the LD_LIBRARY_PATH
+# env var.  This should be unnecessary for most code.
+#
+# function _example_hadoop_nativelib
+# {
+#   # our library is standalone, so just need the basic path
+#   # added. Using after so we are later in the link list
+#   hadoop_add_javalibpath "${EXAMPLE_HOME}/lib" after
+# }
+
+#
+# The _hadoop_finalize function is called to finish up whatever
+# extra work needs to be done prior to exec'ing java or some other
+# binary. This is where command line properties should get added
+# and any last minute work.  This is called prior to Hadoop common
+# which means that one can override any parameters that Hadoop
+# would normally put here... so be careful!
+#
+# Useful functions here include hadoop_add_param and for
+# Windows compabitility, hadoop_translate_cygwin_path.
+#
+# function _example_hadoop_finalize
+# {
+#   # we need a property for our feature
+#   hadoop_add_param HADOOP_OPTS Dexample.feature "-Dexample.feature=awesome"
+# }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index a3fae19b5b124..ea0d3a65880de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.conf;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import java.io.BufferedInputStream;
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -50,6 +52,7 @@
 import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -67,6 +70,7 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
+import com.google.common.base.Charsets;
 import org.apache.commons.collections.map.UnmodifiableMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -86,6 +90,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
+import org.w3c.dom.Attr;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -177,7 +182,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     LogFactory.getLog("org.apache.hadoop.conf.Configuration.deprecation");
 
   private boolean quietmode = true;
-  
+
+  private static final String DEFAULT_STRING_CHECK =
+    "testingforemptydefaultvalue";
+
+  private boolean allowNullValueProperties = false;
+
   private static class Resource {
     private final Object resource;
     private final String name;
@@ -220,7 +230,8 @@ public String toString() {
   /**
    * List of configuration parameters marked <b>final</b>. 
    */
-  private Set<String> finalParameters = new HashSet<String>();
+  private Set<String> finalParameters = Collections.newSetFromMap(
+      new ConcurrentHashMap<String, Boolean>());
   
   private boolean loadDefaults = true;
   
@@ -250,7 +261,7 @@ public String toString() {
    * Stores the mapping of key to the resource which modifies or loads 
    * the key most recently
    */
-  private HashMap<String, String[]> updatingResource;
+  private Map<String, String[]> updatingResource;
  
   /**
    * Class to keep the information about the keys which replace the deprecated
@@ -677,7 +688,7 @@ public Configuration() {
    */
   public Configuration(boolean loadDefaults) {
     this.loadDefaults = loadDefaults;
-    updatingResource = new HashMap<String, String[]>();
+    updatingResource = new ConcurrentHashMap<String, String[]>();
     synchronized(Configuration.class) {
       REGISTRY.put(this, null);
     }
@@ -700,8 +711,11 @@ public Configuration(Configuration other) {
        this.overlay = (Properties)other.overlay.clone();
      }
 
-     this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
-     this.finalParameters = new HashSet<String>(other.finalParameters);
+     this.updatingResource = new ConcurrentHashMap<String, String[]>(
+         other.updatingResource);
+     this.finalParameters = Collections.newSetFromMap(
+         new ConcurrentHashMap<String, Boolean>());
+     this.finalParameters.addAll(other.finalParameters);
    }
    
     synchronized(Configuration.class) {
@@ -831,31 +845,106 @@ private synchronized void addResourceObject(Resource resource) {
     resources.add(resource);                      // add to resources
     reloadConfiguration();
   }
-  
-  private static final Pattern VAR_PATTERN =
-      Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
 
   private static final int MAX_SUBST = 20;
 
+  private static final int SUB_START_IDX = 0;
+  private static final int SUB_END_IDX = SUB_START_IDX + 1;
+
+  /**
+   * This is a manual implementation of the following regex
+   * "\\$\\{[^\\}\\$\u0020]+\\}". It can be 15x more efficient than
+   * a regex matcher as demonstrated by HADOOP-11506. This is noticeable with
+   * Hadoop apps building on the assumption Configuration#get is an O(1)
+   * hash table lookup, especially when the eval is a long string.
+   *
+   * @param eval a string that may contain variables requiring expansion.
+   * @return a 2-element int array res such that
+   * eval.substring(res[0], res[1]) is "var" for the left-most occurrence of
+   * ${var} in eval. If no variable is found -1, -1 is returned.
+   */
+  private static int[] findSubVariable(String eval) {
+    int[] result = {-1, -1};
+
+    int matchStart;
+    int leftBrace;
+
+    // scanning for a brace first because it's less frequent than $
+    // that can occur in nested class names
+    //
+    match_loop:
+    for (matchStart = 1, leftBrace = eval.indexOf('{', matchStart);
+         // minimum left brace position (follows '$')
+         leftBrace > 0
+         // right brace of a smallest valid expression "${c}"
+         && leftBrace + "{c".length() < eval.length();
+         leftBrace = eval.indexOf('{', matchStart)) {
+      int matchedLen = 0;
+      if (eval.charAt(leftBrace - 1) == '$') {
+        int subStart = leftBrace + 1; // after '{'
+        for (int i = subStart; i < eval.length(); i++) {
+          switch (eval.charAt(i)) {
+            case '}':
+              if (matchedLen > 0) { // match
+                result[SUB_START_IDX] = subStart;
+                result[SUB_END_IDX] = subStart + matchedLen;
+                break match_loop;
+              }
+              // fall through to skip 1 char
+            case ' ':
+            case '$':
+              matchStart = i + 1;
+              continue match_loop;
+            default:
+              matchedLen++;
+          }
+        }
+        // scanned from "${"  to the end of eval, and no reset via ' ', '$':
+        //    no match!
+        break match_loop;
+      } else {
+        // not a start of a variable
+        //
+        matchStart = leftBrace + 1;
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Attempts to repeatedly expand the value {@code expr} by replacing the
+   * left-most substring of the form "${var}" in the following precedence order
+   * <ol>
+   *   <li>by the value of the Java system property "var" if defined</li>
+   *   <li>by the value of the configuration key "var" if defined</li>
+   * </ol>
+   *
+   * If var is unbounded the current state of expansion "prefix${var}suffix" is
+   * returned.
+   *
+   * If a cycle is detected: replacing var1 requires replacing var2 ... requires
+   * replacing var1, i.e., the cycle is shorter than
+   * {@link Configuration#MAX_SUBST} then the original expr is returned.
+   *
+   * @param expr the literal value of a config key
+   * @return null if expr is null, otherwise the value resulting from expanding
+   * expr using the algorithm above.
+   * @throws IllegalArgumentException when more than
+   * {@link Configuration#MAX_SUBST} replacements are required
+   */
   private String substituteVars(String expr) {
     if (expr == null) {
       return null;
     }
-    Matcher match = VAR_PATTERN.matcher("");
     String eval = expr;
-    Set<String> evalSet = new HashSet<String>();
-    for(int s=0; s<MAX_SUBST; s++) {
-      if (evalSet.contains(eval)) {
-        // Cyclic resolution pattern detected. Return current expression.
+    Set<String> evalSet = null;
+    for(int s = 0; s < MAX_SUBST; s++) {
+      final int[] varBounds = findSubVariable(eval);
+      if (varBounds[SUB_START_IDX] == -1) {
         return eval;
       }
-      evalSet.add(eval);
-      match.reset(eval);
-      if (!match.find()) {
-        return eval;
-      }
-      String var = match.group();
-      var = var.substring(2, var.length()-1); // remove ${ .. }
+      final String var = eval.substring(varBounds[SUB_START_IDX],
+          varBounds[SUB_END_IDX]);
       String val = null;
       try {
         val = System.getProperty(var);
@@ -868,8 +957,23 @@ private String substituteVars(String expr) {
       if (val == null) {
         return eval; // return literal ${var}: var is unbound
       }
+
+      // prevent recursive resolution
+      //
+      final int dollar = varBounds[SUB_START_IDX] - "${".length();
+      final int afterRightBrace = varBounds[SUB_END_IDX] + "}".length();
+      final String refVar = eval.substring(dollar, afterRightBrace);
+      if (evalSet == null) {
+        evalSet = new HashSet<String>();
+      }
+      if (!evalSet.add(refVar)) {
+        return expr; // return original expression if there is a loop
+      }
+
       // substitute
-      eval = eval.substring(0, match.start())+val+eval.substring(match.end());
+      eval = eval.substring(0, dollar)
+             + val
+             + eval.substring(afterRightBrace);
     }
     throw new IllegalStateException("Variable substitution depth too large: " 
                                     + MAX_SUBST + " " + expr);
@@ -895,7 +999,38 @@ public String get(String name) {
     }
     return result;
   }
-  
+
+  /**
+   * Set Configuration to allow keys without values during setup.  Intended
+   * for use during testing.
+   *
+   * @param val If true, will allow Configuration to store keys without values
+   */
+  @VisibleForTesting
+  public void setAllowNullValueProperties( boolean val ) {
+    this.allowNullValueProperties = val;
+  }
+
+  /**
+   * Return existence of the <code>name</code> property, but only for
+   * names which have no valid value, usually non-existent or commented
+   * out in XML.
+   *
+   * @param name the property name
+   * @return true if the property <code>name</code> exists without value
+   */
+  @VisibleForTesting
+  public boolean onlyKeyExists(String name) {
+    String[] names = handleDeprecation(deprecationContext.get(), name);
+    for(String n : names) {
+      if ( getProps().getProperty(n,DEFAULT_STRING_CHECK)
+               .equals(DEFAULT_STRING_CHECK) ) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
    * <code>null</code> if no such property exists. 
@@ -1487,11 +1622,8 @@ public Pattern getPattern(String name, Pattern defaultValue) {
    * @param pattern new value
    */
   public void setPattern(String name, Pattern pattern) {
-    if (null == pattern) {
-      set(name, null);
-    } else {
-      set(name, pattern.pattern());
-    }
+    assert pattern != null : "Pattern cannot be null";
+    set(name, pattern.pattern());
   }
 
   /**
@@ -2266,7 +2398,7 @@ public Reader getConfResourceAsReader(String name) {
         LOG.info("found resource " + name + " at " + url);
       }
 
-      return new InputStreamReader(url.openStream());
+      return new InputStreamReader(url.openStream(), Charsets.UTF_8);
     } catch (Exception e) {
       return null;
     }
@@ -2278,20 +2410,27 @@ public Reader getConfResourceAsReader(String name) {
    * @return final parameter set.
    */
   public Set<String> getFinalParameters() {
-    return new HashSet<String>(finalParameters);
+    Set<String> setFinalParams = Collections.newSetFromMap(
+        new ConcurrentHashMap<String, Boolean>());
+    setFinalParams.addAll(finalParameters);
+    return setFinalParams;
   }
 
   protected synchronized Properties getProps() {
     if (properties == null) {
       properties = new Properties();
-      HashMap<String, String[]> backup = 
-        new HashMap<String, String[]>(updatingResource);
+      Map<String, String[]> backup =
+          new ConcurrentHashMap<String, String[]>(updatingResource);
       loadResources(properties, resources, quietmode);
-      if (overlay!= null) {
+
+      if (overlay != null) {
         properties.putAll(overlay);
         for (Map.Entry<Object,Object> item: overlay.entrySet()) {
           String key = (String)item.getKey();
-          updatingResource.put(key, backup.get(key));
+          String[] source = backup.get(key);
+          if(source != null) {
+            updatingResource.put(key, source);
+          }
         }
       }
     }
@@ -2329,9 +2468,9 @@ public Iterator<Map.Entry<String, String>> iterator() {
     // code.
     Map<String,String> result = new HashMap<String,String>();
     for(Map.Entry<Object,Object> item: getProps().entrySet()) {
-      if (item.getKey() instanceof String && 
+      if (item.getKey() instanceof String &&
           item.getValue() instanceof String) {
-        result.put((String) item.getKey(), (String) item.getValue());
+          result.put((String) item.getKey(), (String) item.getValue());
       }
     }
     return result.entrySet().iterator();
@@ -2466,11 +2605,26 @@ private Resource loadResource(Properties properties, Resource wrapper, boolean q
         }
         if (!"property".equals(prop.getTagName()))
           LOG.warn("bad conf file: element not <property>");
-        NodeList fields = prop.getChildNodes();
+
         String attr = null;
         String value = null;
         boolean finalParameter = false;
         LinkedList<String> source = new LinkedList<String>();
+
+        Attr propAttr = prop.getAttributeNode("name");
+        if (propAttr != null)
+          attr = StringInterner.weakIntern(propAttr.getValue());
+        propAttr = prop.getAttributeNode("value");
+        if (propAttr != null)
+          value = StringInterner.weakIntern(propAttr.getValue());
+        propAttr = prop.getAttributeNode("final");
+        if (propAttr != null)
+          finalParameter = "true".equals(propAttr.getValue());
+        propAttr = prop.getAttributeNode("source");
+        if (propAttr != null)
+          source.add(StringInterner.weakIntern(propAttr.getValue()));
+
+        NodeList fields = prop.getChildNodes();
         for (int j = 0; j < fields.getLength(); j++) {
           Node fieldNode = fields.item(j);
           if (!(fieldNode instanceof Element))
@@ -2537,16 +2691,21 @@ private void overlay(Properties to, Properties from) {
   
   private void loadProperty(Properties properties, String name, String attr,
       String value, boolean finalParameter, String[] source) {
-    if (value != null) {
+    if (value != null || allowNullValueProperties) {
       if (!finalParameters.contains(attr)) {
+        if (value==null && allowNullValueProperties) {
+          value = DEFAULT_STRING_CHECK;
+        }
         properties.setProperty(attr, value);
-        updatingResource.put(attr, source);
+        if(source != null) {
+          updatingResource.put(attr, source);
+        }
       } else if (!value.equals(properties.getProperty(attr))) {
         LOG.warn(name+":an attempt to override final parameter: "+attr
             +";  Ignoring.");
       }
     }
-    if (finalParameter) {
+    if (finalParameter && attr != null) {
       finalParameters.add(attr);
     }
   }
@@ -2749,7 +2908,9 @@ public void readFields(DataInput in) throws IOException {
       String value = org.apache.hadoop.io.Text.readString(in);
       set(key, value); 
       String sources[] = WritableUtils.readCompressedStringArray(in);
-      updatingResource.put(key, sources);
+      if(sources != null) {
+        updatingResource.put(key, sources);
+      }
     }
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
index eb1fb6b7d584f..bb221ee361fc7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
@@ -71,10 +71,10 @@ private Reconfigurable getReconfigurable(HttpServletRequest req) {
 
   private void printHeader(PrintWriter out, String nodeName) {
     out.print("<html><head>");
-    out.printf("<title>%s Reconfiguration Utility</title>\n", 
+    out.printf("<title>%s Reconfiguration Utility</title>%n",
                StringEscapeUtils.escapeHtml(nodeName));
     out.print("</head><body>\n");
-    out.printf("<h1>%s Reconfiguration Utility</h1>\n",
+    out.printf("<h1>%s Reconfiguration Utility</h1>%n",
                StringEscapeUtils.escapeHtml(nodeName));
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index 4b5356384d576..f3e5b90afa99d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -113,6 +113,7 @@ public CryptoInputStream(InputStream in, CryptoCodec codec,
   public CryptoInputStream(InputStream in, CryptoCodec codec,
       int bufferSize, byte[] key, byte[] iv, long streamOffset) throws IOException {
     super(in);
+    CryptoStreamUtils.checkCodec(codec);
     this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize);
     this.codec = codec;
     this.key = key.clone();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
index 4f9f7f5c6c4f9..876ffd6376615 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
@@ -83,6 +83,7 @@ public CryptoOutputStream(OutputStream out, CryptoCodec codec,
       int bufferSize, byte[] key, byte[] iv, long streamOffset) 
       throws IOException {
     super(out);
+    CryptoStreamUtils.checkCodec(codec);
     this.bufferSize = CryptoStreamUtils.checkBufferSize(codec, bufferSize);
     this.codec = codec;
     this.key = key.clone();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
index 820d77580cd1c..be85497209be0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
@@ -49,6 +49,13 @@ public static int getBufferSize(Configuration conf) {
         HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT);
   }
   
+  /** AES/CTR/NoPadding is required */
+  public static void checkCodec(CryptoCodec codec) {
+    if (codec.getCipherSuite() != CipherSuite.AES_CTR_NOPADDING) {
+      throw new UnsupportedCodecException("AES/CTR/NoPadding is required");
+    }
+  }
+
   /** Check and floor buffer size */
   public static int checkBufferSize(CryptoCodec codec, int bufferSize) {
     Preconditions.checkArgument(bufferSize >= MIN_BUFFER_SIZE, 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/UnsupportedCodecException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/UnsupportedCodecException.java
new file mode 100644
index 0000000000000..ad6b525ead4c0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/UnsupportedCodecException.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+/**
+ * Thrown to indicate that the specific codec is not supported.
+ */
+public class UnsupportedCodecException extends RuntimeException {
+
+  /** Default constructor */
+  public UnsupportedCodecException() {
+  }
+
+  /**
+   * Constructs an UnsupportedCodecException with the specified
+   * detail message.
+   * 
+   * @param message the detail message
+   */
+  public UnsupportedCodecException(String message) {
+    super(message);
+  }
+
+  /**
+   * Constructs a new exception with the specified detail message and
+   * cause.
+   * 
+   * @param message the detail message
+   * @param cause the cause
+   */
+  public UnsupportedCodecException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  /**
+   * Constructs a new exception with the specified cause.
+   * 
+   * @param cause the cause
+   */
+  public UnsupportedCodecException(Throwable cause) {
+    super(cause);
+  }
+
+  private static final long serialVersionUID = 6713920435487942224L;
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 75981c473ba82..bfec1ef1a90da 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -21,7 +21,6 @@
 import com.google.common.base.Preconditions;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -144,13 +143,8 @@ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
           // Provided Password file does not exist
           throw new IOException("Password file does not exists");
         }
-        if (pwdFile != null) {
-          InputStream is = pwdFile.openStream();
-          try {
-            password = IOUtils.toString(is).trim().toCharArray();
-          } finally {
-            is.close();
-          }
+        try (InputStream is = pwdFile.openStream()) {
+          password = IOUtils.toString(is).trim().toCharArray();
         }
       }
     }
@@ -406,6 +400,10 @@ public Metadata getMetadata(String name) throws IOException {
         Metadata meta = ((KeyMetadata) keyStore.getKey(name, password)).metadata;
         cache.put(name, meta);
         return meta;
+      } catch (ClassCastException e) {
+        throw new IOException("Can't cast key for " + name + " in keystore " +
+            path + " to a KeyMetadata. Key may have been added using " +
+            " keytool or some other non-Hadoop method.", e);
       } catch (KeyStoreException e) {
         throw new IOException("Can't get metadata for " + name +
             " from keystore " + path, e);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index dd2d5b99fb252..a0675c2f16b9b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -32,6 +32,7 @@
 
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -207,7 +208,8 @@ protected int addVersion() {
      */
     protected byte[] serialize() throws IOException {
       ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-      JsonWriter writer = new JsonWriter(new OutputStreamWriter(buffer));
+      JsonWriter writer = new JsonWriter(
+          new OutputStreamWriter(buffer, Charsets.UTF_8));
       try {
         writer.beginObject();
         if (cipher != null) {
@@ -251,7 +253,7 @@ protected Metadata(byte[] bytes) throws IOException {
       String description = null;
       Map<String, String> attributes = null;
       JsonReader reader = new JsonReader(new InputStreamReader
-        (new ByteArrayInputStream(bytes)));
+        (new ByteArrayInputStream(bytes), Charsets.UTF_8));
       try {
         reader.beginObject();
         while (reader.hasNext()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index e0ca6240e3f18..4c72d06d068c5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -47,7 +47,7 @@ public class KeyShell extends Configured implements Tool {
       "   [" + ListCommand.USAGE + "]\n";
   private static final String LIST_METADATA = "keyShell.list.metadata";
 
-  private boolean interactive = false;
+  private boolean interactive = true;
   private Command command = null;
 
   /** allows stdout to be captured if necessary */
@@ -169,8 +169,8 @@ private int init(String[] args) throws IOException {
         getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
       } else if ("-metadata".equals(args[i])) {
         getConf().setBoolean(LIST_METADATA, true);
-      } else if ("-i".equals(args[i]) || ("-interactive".equals(args[i]))) {
-        interactive = true;
+      } else if ("-f".equals(args[i]) || ("-force".equals(args[i]))) {
+        interactive = false;
       } else if ("-help".equals(args[i])) {
         printKeyShellUsage();
         return 1;
@@ -367,11 +367,13 @@ public String getUsage() {
   }
 
   private class DeleteCommand extends Command {
-    public static final String USAGE = "delete <keyname> [-provider <provider>] [-help]";
+    public static final String USAGE =
+        "delete <keyname> [-provider <provider>] [-f] [-help]";
     public static final String DESC =
         "The delete subcommand deletes all versions of the key\n" +
         "specified by the <keyname> argument from within the\n" +
-        "provider specified -provider.";
+        "provider specified -provider. The command asks for\n" +
+        "user confirmation unless -f is specified.";
 
     String keyName = null;
     boolean cont = true;
@@ -397,10 +399,10 @@ public boolean validate() {
         try {
           cont = ToolRunner
               .confirmPrompt("You are about to DELETE all versions of "
-                  + " key: " + keyName + " from KeyProvider "
-                  + provider + ". Continue?:");
+                  + " key " + keyName + " from KeyProvider "
+                  + provider + ". Continue? ");
           if (!cont) {
-            out.println("Nothing has been be deleted.");
+            out.println(keyName + " has not been deleted.");
           }
           return cont;
         } catch (IOException e) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index cb03683f5300c..97ab2535d9ac2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.crypto.key.kms;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
@@ -209,7 +210,7 @@ private static Metadata parseJSONMetadata(Map valueMap) {
   }
 
   private static void writeJson(Map map, OutputStream os) throws IOException {
-    Writer writer = new OutputStreamWriter(os);
+    Writer writer = new OutputStreamWriter(os, Charsets.UTF_8);
     ObjectMapper jsonMapper = new ObjectMapper();
     jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, map);
   }
@@ -786,25 +787,44 @@ public int getEncKeyQueueSize(String keyName) throws IOException {
   }
 
   @Override
-  public Token<?>[] addDelegationTokens(String renewer,
+  public Token<?>[] addDelegationTokens(final String renewer,
       Credentials credentials) throws IOException {
     Token<?>[] tokens = null;
     Text dtService = getDelegationTokenService();
     Token<?> token = credentials.getToken(dtService);
     if (token == null) {
-      URL url = createURL(null, null, null, null);
-      DelegationTokenAuthenticatedURL authUrl =
+      final URL url = createURL(null, null, null, null);
+      final DelegationTokenAuthenticatedURL authUrl =
           new DelegationTokenAuthenticatedURL(configurator);
       try {
-        token = authUrl.getDelegationToken(url, authToken, renewer);
+        // 'actualUGI' is the UGI of the user creating the client 
+        // It is possible that the creator of the KMSClientProvier
+        // calls this method on behalf of a proxyUser (the doAsUser).
+        // In which case this call has to be made as the proxy user.
+        UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
+        final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+            UserGroupInformation.AuthenticationMethod.PROXY)
+                                ? currentUgi.getShortUserName() : null;
+
+        token = actualUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
+          @Override
+          public Token<?> run() throws Exception {
+            // Not using the cached token here.. Creating a new token here
+            // everytime.
+            return authUrl.getDelegationToken(url,
+                new DelegationTokenAuthenticatedURL.Token(), renewer, doAsUser);
+          }
+        });
         if (token != null) {
           credentials.addToken(token.getService(), token);
           tokens = new Token<?>[] { token };
         } else {
           throw new IOException("Got NULL as delegation token");
         }
-      } catch (AuthenticationException ex) {
-        throw new IOException(ex);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+      } catch (Exception e) {
+        throw new IOException(e);
       }
     }
     return tokens;
@@ -827,6 +847,10 @@ public void close() throws IOException {
       encKeyVersionQueue.shutdown();
     } catch (Exception e) {
       throw new IOException(e);
+    } finally {
+      if (sslFactory != null) {
+        sslFactory.destroy();
+      }
     }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index 8e67eccf84a11..32451d8360d34 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -308,11 +308,13 @@ private void submitRefillTask(final String keyName,
       final Queue<E> keyQueue) throws InterruptedException {
     if (!executorThreadsStarted) {
       synchronized (this) {
-        // To ensure all requests are first queued, make coreThreads =
-        // maxThreads
-        // and pre-start all the Core Threads.
-        executor.prestartAllCoreThreads();
-        executorThreadsStarted = true;
+        if (!executorThreadsStarted) {
+          // To ensure all requests are first queued, make coreThreads =
+          // maxThreads
+          // and pre-start all the Core Threads.
+          executor.prestartAllCoreThreads();
+          executorThreadsStarted = true;
+        }
       }
     }
     // The submit/execute method of the ThreadPoolExecutor is bypassed and
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index a9a19cdc29b5d..92d4ecae166b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -45,12 +45,13 @@
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Progressable;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * This class provides an interface for implementors of a Hadoop file system
  * (analogous to the VFS of Unix). Applications do not access this class;
@@ -79,6 +80,9 @@ public abstract class AbstractFileSystem {
   
   /** The statistics for this file system. */
   protected Statistics statistics;
+
+  @VisibleForTesting
+  static final String NO_ABSTRACT_FS_ERROR = "No AbstractFileSystem configured for scheme";
   
   private final URI myUri;
   
@@ -148,11 +152,14 @@ static <T> T newInstance(Class<T> theClass,
    */
   public static AbstractFileSystem createFileSystem(URI uri, Configuration conf)
       throws UnsupportedFileSystemException {
-    Class<?> clazz = conf.getClass("fs.AbstractFileSystem." + 
-                                uri.getScheme() + ".impl", null);
+    final String fsImplConf = String.format("fs.AbstractFileSystem.%s.impl",
+        uri.getScheme());
+
+    Class<?> clazz = conf.getClass(fsImplConf, null);
     if (clazz == null) {
-      throw new UnsupportedFileSystemException(
-          "No AbstractFileSystem for scheme: " + uri.getScheme());
+      throw new UnsupportedFileSystemException(String.format(
+          "%s=null: %s: %s",
+          fsImplConf, NO_ABSTRACT_FS_ERROR, uri.getScheme()));
     }
     return (AbstractFileSystem) newInstance(clazz, uri, conf);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
index e18a115149912..aa6e85ea88937 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
@@ -29,9 +29,8 @@ public interface ByteBufferReadable {
    * Reads up to buf.remaining() bytes into buf. Callers should use
    * buf.limit(..) to control the size of the desired read.
    * <p/>
-   * After a successful call, buf.position() and buf.limit() should be
-   * unchanged, and therefore any data can be immediately read from buf.
-   * buf.mark() may be cleared or updated.
+   * After a successful call, buf.position() will be advanced by the number 
+   * of bytes read and buf.limit() should be unchanged.
    * <p/>
    * In the case of an exception, the values of buf.position() and buf.limit()
    * are undefined, and callers should be prepared to recover from this
@@ -45,9 +44,9 @@ public interface ByteBufferReadable {
    * signal an error upon their receipt.
    *
    * @param buf
-   *          the ByteBuffer to receive the results of the read operation. Up to
-   *          buf.limit() - buf.position() bytes may be read.
-   * @return the number of bytes available to read from buf
+   *          the ByteBuffer to receive the results of the read operation.
+   * @return the number of bytes read, possibly zero, or -1 if 
+   *         reach end-of-stream
    * @throws IOException
    *           if there is some error performing the read
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index dfc895101cb5f..dddf0ce815fb3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -352,6 +352,11 @@ public FSDataOutputStream append(Path f, int bufferSize,
     throw new IOException("Not supported");
   }
 
+  @Override
+  public boolean truncate(Path f, long newLength) throws IOException {
+    throw new IOException("Not supported");
+  }
+
   /**
    * Calculated the length of the checksum file in bytes.
    * @param size the length of the data file in bytes
@@ -379,17 +384,19 @@ public ChecksumFSOutputSummer(ChecksumFileSystem fs,
                           int bufferSize,
                           short replication,
                           long blockSize,
-                          Progressable progress)
+                          Progressable progress,
+                          FsPermission permission)
       throws IOException {
       super(DataChecksum.newDataChecksum(DataChecksum.Type.CRC32,
           fs.getBytesPerSum()));
       int bytesPerSum = fs.getBytesPerSum();
-      this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, 
-                                         replication, blockSize, progress);
+      this.datas = fs.getRawFileSystem().create(file, permission, overwrite,
+                                         bufferSize, replication, blockSize,
+                                         progress);
       int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
-      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, 
-                                               sumBufferSize, replication,
-                                               blockSize);
+      this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file),
+                                               permission, true, sumBufferSize,
+                                               replication, blockSize, null);
       sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
       sums.writeInt(bytesPerSum);
     }
@@ -448,7 +455,7 @@ private FSDataOutputStream create(Path f, FsPermission permission,
     if (writeChecksum) {
       out = new FSDataOutputStream(
           new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication,
-              blockSize, progress), null);
+              blockSize, progress, permission), null);
     } else {
       out = fs.create(f, permission, overwrite, bufferSize, replication,
           blockSize, progress);
@@ -458,9 +465,6 @@ private FSDataOutputStream create(Path f, FsPermission permission,
         fs.delete(checkFile, true);
       }
     }
-    if (permission != null) {
-      setPermission(f, permission);
-    }
     return out;
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 748c552805857..442dc7de9a306 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -286,7 +286,4 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
   public static final String NFS_EXPORTS_ALLOWED_HOSTS_SEPARATOR = ";";
   public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY = "nfs.exports.allowed.hosts";
   public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY_DEFAULT = "* rw";
-
-  public static final String  HADOOP_TRACE_SAMPLER = "hadoop.htrace.sampler";
-  public static final String  HADOOP_TRACE_SAMPLER_DEFAULT = "NeverSampler";
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e4ee78a09cdb7..00c8d78bf77d9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -227,6 +227,11 @@ public class CommonConfigurationKeysPublic {
     "ipc.server.tcpnodelay";
   /** Default value for IPC_SERVER_TCPNODELAY_KEY */
   public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = true;
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String  IPC_SERVER_MAX_CONNECTIONS_KEY =
+    "ipc.server.max.connections";
+  /** Default value for IPC_SERVER_MAX_CONNECTIONS_KEY */
+  public static final int     IPC_SERVER_MAX_CONNECTIONS_DEFAULT = 0;
 
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String  HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY =
@@ -307,7 +312,7 @@ public class CommonConfigurationKeysPublic {
   public static final String  HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS =
     "hadoop.security.impersonation.provider.class";
 
-  //  <!--- KMSClientProvider configurations —>
+  //  <!-- KMSClientProvider configurations -->
   /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
   public static final String KMS_CLIENT_ENC_KEY_CACHE_SIZE =
       "hadoop.security.kms.client.encrypted.key.cache.size";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 5d01637a0fb30..6276dda2addbd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -97,28 +97,35 @@ public void readFields(DataInput in) throws IOException {
     this.spaceConsumed = in.readLong();
     this.spaceQuota = in.readLong();
   }
-  
-  /** 
+
+  /**
    * Output format:
    * <----12----> <----12----> <-------18------->
-   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE FILE_NAME    
+   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE
    */
-  private static final String STRING_FORMAT = "%12s %12s %18s ";
-  /** 
+  private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
+  /**
    * Output format:
-   * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
-   *    QUOTA   REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT   FILE_COUNT   CONTENT_SIZE     FILE_NAME    
+   * <----12----> <------15-----> <------15-----> <------15----->
+   *        QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA
+   * <----12----> <----12----> <-------18------->
+   *    DIR_COUNT   FILE_COUNT       CONTENT_SIZE
    */
-  private static final String QUOTA_STRING_FORMAT = "%12s %15s ";
-  private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";
-  
+  private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
+  private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";
+
+  private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
+      "FILE_COUNT", "CONTENT_SIZE"};
+  private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
+      "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };
+
   /** The header string */
   private static final String HEADER = String.format(
-      STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes");
+      SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);
 
   private static final String QUOTA_HEADER = String.format(
-      QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, 
-      "name quota", "rem name quota", "space quota", "rem space quota") +
+      QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
+      (Object[]) QUOTA_HEADER_FIELDS) +
       HEADER;
   
   /** Return the header of the output.
@@ -131,7 +138,25 @@ public void readFields(DataInput in) throws IOException {
   public static String getHeader(boolean qOption) {
     return qOption ? QUOTA_HEADER : HEADER;
   }
-  
+
+  /**
+   * Returns the names of the fields from the summary header.
+   * 
+   * @return names of fields as displayed in the header
+   */
+  public static String[] getHeaderFields() {
+    return HEADER_FIELDS;
+  }
+
+  /**
+   * Returns the names of the fields used in the quota summary.
+   * 
+   * @return names of quota fields as displayed in the header
+   */
+  public static String[] getQuotaHeaderFields() {
+    return QUOTA_HEADER_FIELDS;
+  }
+
   @Override
   public String toString() {
     return toString(true);
@@ -175,11 +200,11 @@ public String toString(boolean qOption, boolean hOption) {
         spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
       }
       
-      prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, 
+      prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
                              quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
     }
     
-    return prefix + String.format(STRING_FORMAT,
+    return prefix + String.format(SUMMARY_FORMAT,
      formatSize(directoryCount, hOption),
      formatSize(fileCount, hOption),
      formatSize(length, hOption));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index c5d23b48e82e8..e008ecc59fd3f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -47,6 +47,10 @@
  * <li> SYNC_BLOCK - to force closed blocks to the disk device.
  * In addition {@link Syncable#hsync()} should be called after each write,
  * if true synchronous behavior is required.</li>
+ * <li> LAZY_PERSIST - Create the block on transient storage (RAM) if
+ * available.</li>
+ * <li> APPEND_NEWBLOCK - Append data to a new block instead of end of the last
+ * partial block.</li>
  * </ol>
  * 
  * Following combination is not valid and will result in 
@@ -93,7 +97,13 @@ public enum CreateFlag {
    * This flag must only be used for intermediate data whose loss can be
    * tolerated by the application.
    */
-  LAZY_PERSIST((short) 0x10);
+  LAZY_PERSIST((short) 0x10),
+
+  /**
+   * Append data to a new block instead of the end of the last partial block.
+   * This is only useful for APPEND.
+   */
+  NEW_BLOCK((short) 0x20);
 
   private final short mode;
 
@@ -149,4 +159,16 @@ public static void validate(Object path, boolean pathExists,
           + ". Create option is not specified in " + flag);
     }
   }
+
+  /**
+   * Validate the CreateFlag for the append operation. The flag must contain
+   * APPEND, and cannot contain OVERWRITE.
+   */
+  public static void validateForAppend(EnumSet<CreateFlag> flag) {
+    validate(flag);
+    if (!flag.contains(APPEND)) {
+      throw new HadoopIllegalArgumentException(flag
+          + " does not contain APPEND");
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
index c8609d450f27c..6d39d1e0b89f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
@@ -58,7 +58,7 @@ public FSDataInputStream(InputStream in) {
    * @param desired offset to seek to
    */
   @Override
-  public synchronized void seek(long desired) throws IOException {
+  public void seek(long desired) throws IOException {
     ((Seekable)in).seek(desired);
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 934421a1884dc..13a5e26423b90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -165,7 +165,7 @@ protected synchronized int flushBuffer(boolean keep,
         count = partialLen;
         System.arraycopy(buf, bufLen - count, buf, 0, count);
       } else {
-      count = 0;
+        count = 0;
       }
     }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 85f8136c0ac4f..e710ec0261216 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -457,9 +457,15 @@ public static FileContext getFileContext(final URI defaultFsUri,
    */
   public static FileContext getFileContext(final Configuration aConf)
       throws UnsupportedFileSystemException {
-    return getFileContext(
-      URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)), 
-      aConf);
+    final URI defaultFsUri = URI.create(aConf.get(FS_DEFAULT_NAME_KEY,
+        FS_DEFAULT_NAME_DEFAULT));
+    if (   defaultFsUri.getScheme() != null
+        && !defaultFsUri.getScheme().trim().isEmpty()) {
+      return getFileContext(defaultFsUri, aConf);
+    }
+    throw new UnsupportedFileSystemException(String.format(
+        "%s: URI configured via %s carries no scheme",
+        defaultFsUri, FS_DEFAULT_NAME_KEY));
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9edc54bac5922..cfa519861aea9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1317,6 +1317,29 @@ protected void rename(final Path src, final Path dst,
       throw new IOException("rename from " + src + " to " + dst + " failed.");
     }
   }
+
+  /**
+   * Truncate the file in the indicated path to the indicated size.
+   * <ul>
+   * <li>Fails if path is a directory.
+   * <li>Fails if path does not exist.
+   * <li>Fails if path is not closed.
+   * <li>Fails if new size is greater than current size.
+   * </ul>
+   * @param f The path to the file to be truncated
+   * @param newLength The size the file is to be truncated to
+   *
+   * @return <code>true</code> if the file has been truncated to the desired
+   * <code>newLength</code> and is immediately available to be reused for
+   * write operations such as <code>append</code>, or
+   * <code>false</code> if a background process of adjusting the length of
+   * the last block has been started, and clients should wait for it to
+   * complete before proceeding with further file updates.
+   */
+  public boolean truncate(Path f, long newLength) throws IOException {
+    throw new UnsupportedOperationException("Not implemented by the " +
+        getClass().getSimpleName() + " FileSystem implementation");
+  }
   
   /**
    * Delete a file 
@@ -2618,9 +2641,6 @@ public static Class<? extends FileSystem> getFileSystemClass(String scheme,
   private static FileSystem createFileSystem(URI uri, Configuration conf
       ) throws IOException {
     Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
-    if (clazz == null) {
-      throw new IOException("No FileSystem for scheme: " + uri.getScheme());
-    }
     FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
     fs.initialize(uri, conf);
     return fs;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 3d5a753b0ece1..d4080adc74dcb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -225,6 +225,11 @@ public boolean setReplication(Path src, short replication) throws IOException {
   public boolean rename(Path src, Path dst) throws IOException {
     return fs.rename(src, dst);
   }
+
+  @Override
+  public boolean truncate(Path f, final long newLength) throws IOException {
+    return fs.truncate(f, newLength);
+  }
   
   /** Delete a file */
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index 3ba6de1f9a335..e89bc4949b8b0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -220,12 +220,7 @@ private URI decodeHarURI(URI rawURI, Configuration conf) throws IOException {
       return FileSystem.getDefaultUri(conf);
     }
     String authority = rawURI.getAuthority();
-    if (authority == null) {
-      throw new IOException("URI: " + rawURI
-          + " is an invalid Har URI since authority==null."
-          + "  Expecting har://<scheme>-<host>/<path>.");
-    }
- 
+
     int i = authority.indexOf('-');
     if (i < 0) {
       throw new IOException("URI: " + rawURI
@@ -489,19 +484,12 @@ public static int getHarHash(Path p) {
   }
   
   static class Store {
-    public Store() {
-      begin = end = startHash = endHash = 0;
-    }
-    public Store(long begin, long end, int startHash, int endHash) {
+    public Store(long begin, long end) {
       this.begin = begin;
       this.end = end;
-      this.startHash = startHash;
-      this.endHash = endHash;
     }
     public long begin;
     public long end;
-    public int startHash;
-    public int endHash;
   }
   
   /**
@@ -594,7 +582,7 @@ private class HarStatus {
     public HarStatus(String harString) throws UnsupportedEncodingException {
       String[] splits = harString.split(" ");
       this.name = decodeFileName(splits[0]);
-      this.isDir = "dir".equals(splits[1]) ? true: false;
+      this.isDir = "dir".equals(splits[1]);
       // this is equal to "none" if its a directory
       this.partName = splits[2];
       this.startIndex = Long.parseLong(splits[3]);
@@ -769,6 +757,14 @@ public FSDataOutputStream append(Path f) throws IOException {
     throw new IOException("Har: append not allowed");
   }
 
+  /**
+   * Not implemented.
+   */
+  @Override
+  public boolean truncate(Path f, long newLength) throws IOException {
+    throw new IOException("Har: truncate not allowed");
+  }
+
   /**
    * Not implemented.
    */
@@ -1167,11 +1163,8 @@ private void parseMetaData() throws IOException {
           int b = lin.readLine(line);
           read += b;
           readStr = line.toString().split(" ");
-          int startHash = Integer.parseInt(readStr[0]);
-          int endHash  = Integer.parseInt(readStr[1]);
           stores.add(new Store(Long.parseLong(readStr[2]), 
-              Long.parseLong(readStr[3]), startHash,
-              endHash));
+              Long.parseLong(readStr[3])));
           line.clear();
         }
       } catch (IOException ioe) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
index e48354dba790d..209ba6997ae87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
@@ -23,13 +23,16 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.StringReader;
-import java.util.Arrays;
 
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 
+import com.google.common.annotations.VisibleForTesting;
+
+import static java.nio.file.Files.createLink;
+
 /**
  * Class for creating hardlinks.
  * Supports Unix/Linux, Windows via winutils , and Mac OS X.
@@ -75,114 +78,30 @@ public HardLink() {
   
   /**
    * This abstract class bridges the OS-dependent implementations of the 
-   * needed functionality for creating hardlinks and querying link counts.
+   * needed functionality for querying link counts.
    * The particular implementation class is chosen during 
    * static initialization phase of the HardLink class.
-   * The "getter" methods construct shell command strings for various purposes.
+   * The "getter" methods construct shell command strings.
    */
   private static abstract class HardLinkCommandGetter {
-
-    /**
-     * Get the command string needed to hardlink a bunch of files from
-     * a single source directory into a target directory.  The source directory
-     * is not specified here, but the command will be executed using the source
-     * directory as the "current working directory" of the shell invocation.
-     * 
-     * @param fileBaseNames - array of path-less file names, relative
-     *            to the source directory
-     * @param linkDir - target directory where the hardlinks will be put
-     * @return - an array of Strings suitable for use as a single shell command
-     * @throws IOException - if any of the file or path names misbehave
-     */
-    abstract String[] linkMult(String[] fileBaseNames, File linkDir) 
-                          throws IOException;
-    
-    /**
-     * Get the command string needed to hardlink a single file
-     */
-    abstract String[] linkOne(File file, File linkName) throws IOException;
-    
     /**
      * Get the command string to query the hardlink count of a file
      */
     abstract String[] linkCount(File file) throws IOException;
-    
-    /**
-     * Calculate the total string length of the shell command
-     * resulting from execution of linkMult, plus the length of the
-     * source directory name (which will also be provided to the shell)
-     * 
-     * @param fileDir - source directory, parent of fileBaseNames
-     * @param fileBaseNames - array of path-less file names, relative
-     *            to the source directory
-     * @param linkDir - target directory where the hardlinks will be put
-     * @return - total data length (must not exceed maxAllowedCmdArgLength)
-     * @throws IOException
-     */
-    abstract int getLinkMultArgLength(
-                     File fileDir, String[] fileBaseNames, File linkDir) 
-                     throws IOException;
-    
-    /**
-     * Get the maximum allowed string length of a shell command on this OS,
-     * which is just the documented minimum guaranteed supported command
-     * length - aprx. 32KB for Unix, and 8KB for Windows.
-     */
-    abstract int getMaxAllowedCmdArgLength(); 
   }
   
   /**
    * Implementation of HardLinkCommandGetter class for Unix
    */
-  static class HardLinkCGUnix extends HardLinkCommandGetter {
-    private static String[] hardLinkCommand = {"ln", null, null};
-    private static String[] hardLinkMultPrefix = {"ln"};
-    private static String[] hardLinkMultSuffix = {null};
+  private static class HardLinkCGUnix extends HardLinkCommandGetter {
     private static String[] getLinkCountCommand = {"stat","-c%h", null};
-    //Unix guarantees at least 32K bytes cmd length.
-    //Subtract another 64b to allow for Java 'exec' overhead
-    private static final int maxAllowedCmdArgLength = 32*1024 - 65;
-    
     private static synchronized 
     void setLinkCountCmdTemplate(String[] template) {
       //May update this for specific unix variants, 
       //after static initialization phase
       getLinkCountCommand = template;
     }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
-     */
-    @Override
-    String[] linkOne(File file, File linkName) 
-    throws IOException {
-      String[] buf = new String[hardLinkCommand.length];
-      System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
-      //unix wants argument order: "ln <existing> <new>"
-      buf[1] = FileUtil.makeShellPath(file, true); 
-      buf[2] = FileUtil.makeShellPath(linkName, true);
-      return buf;
-    }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
-     */
-    @Override
-    String[] linkMult(String[] fileBaseNames, File linkDir) 
-    throws IOException {
-      String[] buf = new String[fileBaseNames.length 
-                                + hardLinkMultPrefix.length 
-                                + hardLinkMultSuffix.length];
-      int mark=0;
-      System.arraycopy(hardLinkMultPrefix, 0, buf, mark, 
-                       hardLinkMultPrefix.length);
-      mark += hardLinkMultPrefix.length;
-      System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
-      mark += fileBaseNames.length;
-      buf[mark] = FileUtil.makeShellPath(linkDir, true);
-      return buf;
-    }
-    
+
     /*
      * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
      */
@@ -195,169 +114,30 @@ String[] linkCount(File file)
       buf[getLinkCountCommand.length - 1] = FileUtil.makeShellPath(file, true);
       return buf;
     }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
-     */
-    @Override
-    int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir) 
-    throws IOException{
-      int sum = 0;
-      for (String x : fileBaseNames) {
-        // add 1 to account for terminal null or delimiter space
-        sum += 1 + ((x == null) ? 0 : x.length());
-      }
-      sum += 2 + FileUtil.makeShellPath(fileDir, true).length()
-             + FileUtil.makeShellPath(linkDir, true).length();
-      //add the fixed overhead of the hardLinkMult prefix and suffix
-      sum += 3; //length("ln") + 1
-      return sum;
-    }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
-     */
-    @Override
-    int getMaxAllowedCmdArgLength() {
-      return maxAllowedCmdArgLength;
-    }
   }
   
-  
   /**
    * Implementation of HardLinkCommandGetter class for Windows
    */
+  @VisibleForTesting
   static class HardLinkCGWin extends HardLinkCommandGetter {
-    //The Windows command getter impl class and its member fields are
-    //package-private ("default") access instead of "private" to assist 
-    //unit testing (sort of) on non-Win servers
 
-    static String CMD_EXE = "cmd.exe";
-    static String[] hardLinkCommand = {
-                        Shell.WINUTILS,"hardlink","create", null, null};
-    static String[] hardLinkMultPrefix = {
-        CMD_EXE, "/q", "/c", "for", "%f", "in", "("};
-    static String   hardLinkMultDir = "\\%f";
-    static String[] hardLinkMultSuffix = {
-        ")", "do", Shell.WINUTILS, "hardlink", "create", null,
-        "%f"};
     static String[] getLinkCountCommand = {
         Shell.WINUTILS, "hardlink", "stat", null};
-    //Windows guarantees only 8K - 1 bytes cmd length.
-    //Subtract another 64b to allow for Java 'exec' overhead
-    static final int maxAllowedCmdArgLength = 8*1024 - 65;
 
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkOne(java.io.File, java.io.File)
-     */
-    @Override
-    String[] linkOne(File file, File linkName) 
-    throws IOException {
-      String[] buf = new String[hardLinkCommand.length];
-      System.arraycopy(hardLinkCommand, 0, buf, 0, hardLinkCommand.length);
-      //windows wants argument order: "create <new> <existing>"
-      buf[4] = file.getCanonicalPath(); 
-      buf[3] = linkName.getCanonicalPath();
-      return buf;
-    }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkMult(java.lang.String[], java.io.File)
-     */
-    @Override
-    String[] linkMult(String[] fileBaseNames, File linkDir) 
-    throws IOException {
-      String[] buf = new String[fileBaseNames.length 
-                                + hardLinkMultPrefix.length 
-                                + hardLinkMultSuffix.length];
-      String td = linkDir.getCanonicalPath() + hardLinkMultDir;
-      int mark=0;
-      System.arraycopy(hardLinkMultPrefix, 0, buf, mark, 
-                       hardLinkMultPrefix.length);
-      mark += hardLinkMultPrefix.length;
-      System.arraycopy(fileBaseNames, 0, buf, mark, fileBaseNames.length);
-      mark += fileBaseNames.length;
-      System.arraycopy(hardLinkMultSuffix, 0, buf, mark, 
-                       hardLinkMultSuffix.length);
-      mark += hardLinkMultSuffix.length;
-      buf[mark - 2] = td;
-      return buf;
-    }
-    
     /*
      * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#linkCount(java.io.File)
      */
     @Override
-    String[] linkCount(File file) 
-    throws IOException {
+    String[] linkCount(File file) throws IOException {
       String[] buf = new String[getLinkCountCommand.length];
       System.arraycopy(getLinkCountCommand, 0, buf, 0, 
                        getLinkCountCommand.length);
       buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
       return buf;
     }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getLinkMultArgLength(java.io.File, java.lang.String[], java.io.File)
-     */
-    @Override
-    int getLinkMultArgLength(File fileDir, String[] fileBaseNames, File linkDir) 
-    throws IOException {
-      int sum = 0;
-      for (String x : fileBaseNames) {
-        // add 1 to account for terminal null or delimiter space
-        sum += 1 + ((x == null) ? 0 : x.length());
-      }
-      sum += 2 + fileDir.getCanonicalPath().length() +
-               linkDir.getCanonicalPath().length();
-      //add the fixed overhead of the hardLinkMult command 
-      //(prefix, suffix, and Dir suffix)
-      sum += (CMD_EXE + " /q /c for %f in ( ) do "
-              + Shell.WINUTILS + " hardlink create \\%f %f").length();
-      return sum;
-    }
-    
-    /*
-     * @see org.apache.hadoop.fs.HardLink.HardLinkCommandGetter#getMaxAllowedCmdArgLength()
-     */
-    @Override
-    int getMaxAllowedCmdArgLength() {
-      return maxAllowedCmdArgLength;
-    }
   }
-  
-  
-  /**
-   * Calculate the nominal length of all contributors to the total 
-   * commandstring length, including fixed overhead of the OS-dependent 
-   * command.  It's protected rather than private, to assist unit testing,
-   * but real clients are not expected to need it -- see the way 
-   * createHardLinkMult() uses it internally so the user doesn't need to worry
-   * about it.
-   * 
-   * @param fileDir - source directory, parent of fileBaseNames
-   * @param fileBaseNames - array of path-less file names, relative
-   *            to the source directory
-   * @param linkDir - target directory where the hardlinks will be put
-   * @return - total data length (must not exceed maxAllowedCmdArgLength)
-   * @throws IOException
-   */
-  protected static int getLinkMultArgLength(
-          File fileDir, String[] fileBaseNames, File linkDir) 
-  throws IOException {
-    return getHardLinkCommand.getLinkMultArgLength(fileDir, 
-          fileBaseNames, linkDir);
-  }
-  
-  /**
-   * Return this private value for use by unit tests.
-   * Shell commands are not allowed to have a total string length
-   * exceeding this size.
-   */
-  protected static int getMaxAllowedCmdArgLength() {
-    return getHardLinkCommand.getMaxAllowedCmdArgLength();
-  }
-  
+
   /*
    * ****************************************************
    * Complexity is above.  User-visible functionality is below
@@ -370,7 +150,7 @@ protected static int getMaxAllowedCmdArgLength() {
    * @param linkName - desired target link file
    */
   public static void createHardLink(File file, File linkName) 
-  throws IOException {
+      throws IOException {
     if (file == null) {
       throw new IOException(
           "invalid arguments to createHardLink: source file is null");
@@ -379,17 +159,7 @@ public static void createHardLink(File file, File linkName)
       throw new IOException(
           "invalid arguments to createHardLink: link name is null");
     }
-	  // construct and execute shell command
-    String[] hardLinkCommand = getHardLinkCommand.linkOne(file, linkName);
-    ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand);
-    try {
-      shexec.execute();
-    } catch (ExitCodeException e) {
-      throw new IOException("Failed to execute command " +
-          Arrays.toString(hardLinkCommand) +
-          "; command output: \"" + shexec.getOutput() + "\"" +
-          "; WrappedException: \"" + e.getMessage() + "\"");
-    }
+    createLink(linkName.toPath(), file.toPath());
   }
 
   /**
@@ -398,30 +168,10 @@ public static void createHardLink(File file, File linkName)
    * @param parentDir - directory containing source files
    * @param fileBaseNames - list of path-less file names, as returned by 
    *                        parentDir.list()
-   * @param linkDir - where the hardlinks should be put.  It must already exist.
-   * 
-   * If the list of files is too long (overflows maxAllowedCmdArgLength),
-   * we will automatically split it into multiple invocations of the
-   * underlying method.
+   * @param linkDir - where the hardlinks should be put. It must already exist.
    */
-  public static void createHardLinkMult(File parentDir, String[] fileBaseNames, 
+  public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
       File linkDir) throws IOException {
-    //This is the public method all non-test clients are expected to use.
-    //Normal case - allow up to maxAllowedCmdArgLength characters in the cmd
-    createHardLinkMult(parentDir, fileBaseNames, linkDir, 
-                       getHardLinkCommand.getMaxAllowedCmdArgLength());
-  }
-
-  /*
-   * Implements {@link createHardLinkMult} with added variable  "maxLength",
-   * to ease unit testing of the auto-splitting feature for long lists.
-   * Likewise why it returns "callCount", the number of sub-arrays that
-   * the file list had to be split into.
-   * Non-test clients are expected to call the public method instead.
-   */
-  protected static int createHardLinkMult(File parentDir, 
-      String[] fileBaseNames, File linkDir, int maxLength) 
-  throws IOException {
     if (parentDir == null) {
       throw new IOException(
           "invalid arguments to createHardLinkMult: parent directory is null");
@@ -435,40 +185,13 @@ protected static int createHardLinkMult(File parentDir,
           "invalid arguments to createHardLinkMult: "
           + "filename list can be empty but not null");
     }
-    if (fileBaseNames.length == 0) {
-      //the OS cmds can't handle empty list of filenames, 
-      //but it's legal, so just return.
-      return 0; 
-    }
     if (!linkDir.exists()) {
       throw new FileNotFoundException(linkDir + " not found.");
     }
-
-    //if the list is too long, split into multiple invocations
-    int callCount = 0;
-    if (getLinkMultArgLength(parentDir, fileBaseNames, linkDir) > maxLength
-          && fileBaseNames.length > 1) {
-      String[] list1 = Arrays.copyOf(fileBaseNames, fileBaseNames.length/2);
-      callCount += createHardLinkMult(parentDir, list1, linkDir, maxLength);
-      String[] list2 = Arrays.copyOfRange(fileBaseNames, fileBaseNames.length/2,
-          fileBaseNames.length);
-      callCount += createHardLinkMult(parentDir, list2, linkDir, maxLength);  
-      return callCount;
-    } else {
-      callCount = 1;
-    }
-    
-    // construct and execute shell command
-    String[] hardLinkCommand = getHardLinkCommand.linkMult(fileBaseNames, 
-        linkDir);
-    ShellCommandExecutor shexec = new ShellCommandExecutor(hardLinkCommand,
-      parentDir, null, 0L);
-    try {
-      shexec.execute();
-    } catch (ExitCodeException e) {
-      throw new IOException(shexec.getOutput() + e.getMessage());
+    for (String name : fileBaseNames) {
+      createLink(linkDir.toPath().resolve(name),
+                 parentDir.toPath().resolve(name));
     }
-    return callCount;
   }
 
    /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index 88b4d4e42c45c..8f011ce409f4f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -372,7 +372,7 @@ public synchronized Path getLocalPathForWrite(String pathStr, long size,
         // Keep rolling the wheel till we get a valid path
         Random r = new java.util.Random();
         while (numDirsSearched < numDirs && returnPath == null) {
-          long randomPosition = Math.abs(r.nextLong()) % totalAvailable;
+          long randomPosition = (r.nextLong() >>> 1) % totalAvailable;
           int dir = 0;
           while (randomPosition > availableOnDisk[dir]) {
             randomPosition -= availableOnDisk[dir];
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
index 591899567288b..21f56ed1d948c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
@@ -143,13 +143,13 @@ public static MD5MD5CRC32FileChecksum valueOf(Attributes attrs
       switch (finalCrcType) {
         case CRC32:
           return new MD5MD5CRC32GzipFileChecksum(
-              Integer.valueOf(bytesPerCRC),
-              Integer.valueOf(crcPerBlock),
+              Integer.parseInt(bytesPerCRC),
+              Integer.parseInt(crcPerBlock),
               new MD5Hash(md5));
         case CRC32C:
           return new MD5MD5CRC32CastagnoliFileChecksum(
-              Integer.valueOf(bytesPerCRC),
-              Integer.valueOf(crcPerBlock),
+              Integer.parseInt(bytesPerCRC),
+              Integer.parseInt(crcPerBlock),
               new MD5Hash(md5));
         default:
           // we should never get here since finalCrcType will
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
index 54ddedaff1dc4..caeb7a1b799fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
@@ -60,7 +60,6 @@ public class Path implements Comparable {
 
   /**
    * Pathnames with scheme and relative path are illegal.
-   * @param path to be checked
    */
   void checkNotSchemeWithRelative() {
     if (toUri().isAbsolute() && !isUriPathAbsolute()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index b6b6f5905491f..d7866b84aafd9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -41,7 +41,9 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.NativeIOException;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
@@ -207,8 +209,28 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
   class LocalFSFileOutputStream extends OutputStream {
     private FileOutputStream fos;
     
-    private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
-      this.fos = new FileOutputStream(pathToFile(f), append);
+    private LocalFSFileOutputStream(Path f, boolean append,
+        FsPermission permission) throws IOException {
+      File file = pathToFile(f);
+      if (permission == null) {
+        this.fos = new FileOutputStream(file, append);
+      } else {
+        if (Shell.WINDOWS && NativeIO.isAvailable()) {
+          this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
+              append, permission.toShort());
+        } else {
+          this.fos = new FileOutputStream(file, append);
+          boolean success = false;
+          try {
+            setPermission(f, permission);
+            success = true;
+          } finally {
+            if (!success) {
+              IOUtils.cleanup(LOG, this.fos);
+            }
+          }
+        }
+      }
     }
     
     /*
@@ -247,19 +269,20 @@ public FSDataOutputStream append(Path f, int bufferSize,
       throw new IOException("Cannot append to a diretory (=" + f + " )");
     }
     return new FSDataOutputStream(new BufferedOutputStream(
-        new LocalFSFileOutputStream(f, true), bufferSize), statistics);
+        createOutputStreamWithMode(f, true, null), bufferSize), statistics);
   }
 
   @Override
   public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
     short replication, long blockSize, Progressable progress)
     throws IOException {
-    return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
+    return create(f, overwrite, true, bufferSize, replication, blockSize,
+        progress, null);
   }
 
   private FSDataOutputStream create(Path f, boolean overwrite,
       boolean createParent, int bufferSize, short replication, long blockSize,
-      Progressable progress) throws IOException {
+      Progressable progress, FsPermission permission) throws IOException {
     if (exists(f) && !overwrite) {
       throw new FileAlreadyExistsException("File already exists: " + f);
     }
@@ -268,12 +291,18 @@ private FSDataOutputStream create(Path f, boolean overwrite,
       throw new IOException("Mkdirs failed to create " + parent.toString());
     }
     return new FSDataOutputStream(new BufferedOutputStream(
-        createOutputStream(f, false), bufferSize), statistics);
+        createOutputStreamWithMode(f, false, permission), bufferSize),
+        statistics);
   }
   
   protected OutputStream createOutputStream(Path f, boolean append) 
       throws IOException {
-    return new LocalFSFileOutputStream(f, append); 
+    return createOutputStreamWithMode(f, append, null);
+  }
+
+  protected OutputStream createOutputStreamWithMode(Path f, boolean append,
+      FsPermission permission) throws IOException {
+    return new LocalFSFileOutputStream(f, append, permission);
   }
   
   @Override
@@ -285,7 +314,8 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
       throw new FileAlreadyExistsException("File already exists: " + f);
     }
     return new FSDataOutputStream(new BufferedOutputStream(
-        new LocalFSFileOutputStream(f, false), bufferSize), statistics);
+        createOutputStreamWithMode(f, false, permission), bufferSize),
+            statistics);
   }
 
   @Override
@@ -293,9 +323,8 @@ public FSDataOutputStream create(Path f, FsPermission permission,
     boolean overwrite, int bufferSize, short replication, long blockSize,
     Progressable progress) throws IOException {
 
-    FSDataOutputStream out = create(f,
-        overwrite, bufferSize, replication, blockSize, progress);
-    setPermission(f, permission);
+    FSDataOutputStream out = create(f, overwrite, true, bufferSize, replication,
+        blockSize, progress, permission);
     return out;
   }
 
@@ -304,9 +333,8 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
       boolean overwrite,
       int bufferSize, short replication, long blockSize,
       Progressable progress) throws IOException {
-    FSDataOutputStream out = create(f,
-        overwrite, false, bufferSize, replication, blockSize, progress);
-    setPermission(f, permission);
+    FSDataOutputStream out = create(f, overwrite, false, bufferSize, replication,
+        blockSize, progress, permission);
     return out;
   }
 
@@ -343,6 +371,31 @@ public boolean rename(Path src, Path dst) throws IOException {
     }
     return FileUtil.copy(this, src, this, dst, true, getConf());
   }
+
+  @Override
+  public boolean truncate(Path f, final long newLength) throws IOException {
+    FileStatus status = getFileStatus(f);
+    if(status == null) {
+      throw new FileNotFoundException("File " + f + " not found");
+    }
+    if(status.isDirectory()) {
+      throw new IOException("Cannot truncate a directory (=" + f + ")");
+    }
+    long oldLength = status.getLen();
+    if(newLength > oldLength) {
+      throw new IllegalArgumentException(
+          "Cannot truncate to a larger file size. Current size: " + oldLength +
+          ", truncate size: " + newLength + ".");
+    }
+    try (FileOutputStream out = new FileOutputStream(pathToFile(f), true)) {
+      try {
+        out.getChannel().truncate(newLength);
+      } catch(IOException e) {
+        throw new FSError(e);
+      }
+    }
+    return true;
+  }
   
   /**
    * Delete the given path to a file or directory.
@@ -413,7 +466,34 @@ public FileStatus[] listStatus(Path f) throws IOException {
   }
   
   protected boolean mkOneDir(File p2f) throws IOException {
-    return p2f.mkdir();
+    return mkOneDirWithMode(new Path(p2f.getAbsolutePath()), p2f, null);
+  }
+
+  protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
+      throws IOException {
+    if (permission == null) {
+      return p2f.mkdir();
+    } else {
+      if (Shell.WINDOWS && NativeIO.isAvailable()) {
+        try {
+          NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
+          return true;
+        } catch (IOException e) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug(String.format(
+                "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
+                p2f, permission.toShort()), e);
+          }
+          return false;
+        }
+      } else {
+        boolean b = p2f.mkdir();
+        if (b) {
+          setPermission(p, permission);
+        }
+        return b;
+      }
+    }
   }
 
   /**
@@ -422,6 +502,16 @@ protected boolean mkOneDir(File p2f) throws IOException {
    */
   @Override
   public boolean mkdirs(Path f) throws IOException {
+    return mkdirsWithOptionalPermission(f, null);
+  }
+
+  @Override
+  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+    return mkdirsWithOptionalPermission(f, permission);
+  }
+
+  private boolean mkdirsWithOptionalPermission(Path f, FsPermission permission)
+      throws IOException {
     if(f == null) {
       throw new IllegalArgumentException("mkdirs path arg is null");
     }
@@ -440,25 +530,7 @@ public boolean mkdirs(Path f) throws IOException {
               " and is not a directory: " + p2f.getCanonicalPath());
     }
     return (parent == null || parent2f.exists() || mkdirs(parent)) &&
-      (mkOneDir(p2f) || p2f.isDirectory());
-  }
-
-  @Override
-  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
-    boolean b = mkdirs(f);
-    if(b) {
-      setPermission(f, permission);
-    }
-    return b;
-  }
-  
-
-  @Override
-  protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
-    throws IOException {
-    boolean b = mkdirs(f);
-    setPermission(f, absolutePermission);
-    return b;
+      (mkOneDirWithMode(f, p2f, permission) || p2f.isDirectory());
   }
   
   
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 9d36bcff8b03c..47fb25c2570ff 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -23,6 +23,7 @@
 import java.net.ConnectException;
 import java.net.URI;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.net.ftp.FTP;
@@ -101,17 +102,12 @@ public void initialize(URI uri, Configuration conf) throws IOException { // get
     if (userAndPassword == null) {
       userAndPassword = (conf.get("fs.ftp.user." + host, null) + ":" + conf
           .get("fs.ftp.password." + host, null));
-      if (userAndPassword == null) {
-        throw new IOException("Invalid user/passsword specified");
-      }
     }
     String[] userPasswdInfo = userAndPassword.split(":");
+    Preconditions.checkState(userPasswdInfo.length > 1,
+                             "Invalid username / password");
     conf.set(FS_FTP_USER_PREFIX + host, userPasswdInfo[0]);
-    if (userPasswdInfo.length > 1) {
-      conf.set(FS_FTP_PASSWORD_PREFIX + host, userPasswdInfo[1]);
-    } else {
-      conf.set(FS_FTP_PASSWORD_PREFIX + host, null);
-    }
+    conf.set(FS_FTP_PASSWORD_PREFIX + host, userPasswdInfo[1]);
     setConf(conf);
     this.uri = uri;
   }
@@ -293,7 +289,8 @@ public FSDataOutputStream append(Path f, int bufferSize,
    */
   private boolean exists(FTPClient client, Path file) throws IOException {
     try {
-      return getFileStatus(client, file) != null;
+      getFileStatus(client, file);
+      return true;
     } catch (FileNotFoundException fnfe) {
       return false;
     }
@@ -333,10 +330,8 @@ private boolean delete(FTPClient client, Path file, boolean recursive)
     if (dirEntries != null && dirEntries.length > 0 && !(recursive)) {
       throw new IOException("Directory: " + file + " is not empty.");
     }
-    if (dirEntries != null) {
-      for (int i = 0; i < dirEntries.length; i++) {
-        delete(client, new Path(absolute, dirEntries[i].getPath()), recursive);
-      }
+    for (FileStatus dirEntry : dirEntries) {
+      delete(client, new Path(absolute, dirEntry.getPath()), recursive);
     }
     return client.removeDirectory(pathName);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java
index b65b7a0b438b2..b9def6447a870 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclEntry.java
@@ -146,7 +146,9 @@ public Builder setType(AclEntryType type) {
      * @return Builder this builder, for call chaining
      */
     public Builder setName(String name) {
-      this.name = name;
+      if (name != null && !name.isEmpty()) {
+        this.name = name;
+      }
       return this;
     }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
index 4a7258f0a2738..9d7500a697b1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
 /**
@@ -36,6 +37,7 @@ public class AclStatus {
   private final String group;
   private final boolean stickyBit;
   private final List<AclEntry> entries;
+  private final FsPermission permission;
 
   /**
    * Returns the file owner.
@@ -73,6 +75,14 @@ public List<AclEntry> getEntries() {
     return entries;
   }
 
+  /**
+   * Returns the permission set for the path
+   * @return {@link FsPermission} for the path
+   */
+  public FsPermission getPermission() {
+    return permission;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (o == null) {
@@ -113,6 +123,7 @@ public static class Builder {
     private String group;
     private boolean stickyBit;
     private List<AclEntry> entries = Lists.newArrayList();
+    private FsPermission permission = null;
 
     /**
      * Sets the file owner.
@@ -172,13 +183,22 @@ public Builder stickyBit(boolean stickyBit) {
       return this;
     }
 
+    /**
+     * Sets the permission for the file.
+     * @param permission
+     */
+    public Builder setPermission(FsPermission permission) {
+      this.permission = permission;
+      return this;
+    }
+
     /**
      * Builds a new AclStatus populated with the set properties.
      *
      * @return AclStatus new AclStatus
      */
     public AclStatus build() {
-      return new AclStatus(owner, group, stickyBit, entries);
+      return new AclStatus(owner, group, stickyBit, entries, permission);
     }
   }
 
@@ -190,12 +210,67 @@ public AclStatus build() {
    * @param group String file group
    * @param stickyBit the sticky bit
    * @param entries the ACL entries
+   * @param permission permission of the path
    */
   private AclStatus(String owner, String group, boolean stickyBit,
-      Iterable<AclEntry> entries) {
+      Iterable<AclEntry> entries, FsPermission permission) {
     this.owner = owner;
     this.group = group;
     this.stickyBit = stickyBit;
     this.entries = Lists.newArrayList(entries);
+    this.permission = permission;
+  }
+
+  /**
+   * Get the effective permission for the AclEntry
+   * @param entry AclEntry to get the effective action
+   */
+  public FsAction getEffectivePermission(AclEntry entry) {
+    return getEffectivePermission(entry, permission);
+  }
+
+  /**
+   * Get the effective permission for the AclEntry. <br>
+   * Recommended to use this API ONLY if client communicates with the old
+   * NameNode, needs to pass the Permission for the path to get effective
+   * permission, else use {@link AclStatus#getEffectivePermission(AclEntry)}.
+   * @param entry AclEntry to get the effective action
+   * @param permArg Permission for the path. However if the client is NOT
+   *          communicating with old namenode, then this argument will not have
+   *          any preference.
+   * @return Returns the effective permission for the entry.
+   * @throws IllegalArgumentException If the client communicating with old
+   *           namenode and permission is not passed as an argument.
+   */
+  public FsAction getEffectivePermission(AclEntry entry, FsPermission permArg)
+      throws IllegalArgumentException {
+    // At least one permission bits should be available.
+    Preconditions.checkArgument(this.permission != null || permArg != null,
+        "Permission bits are not available to calculate effective permission");
+    if (this.permission != null) {
+      // permission bits from server response will have the priority for
+      // accuracy.
+      permArg = this.permission;
+    }
+    if ((entry.getName() != null || entry.getType() == AclEntryType.GROUP)) {
+      if (entry.getScope() == AclEntryScope.ACCESS) {
+        FsAction entryPerm = entry.getPermission();
+        return entryPerm.and(permArg.getGroupAction());
+      } else {
+        Preconditions.checkArgument(this.entries.contains(entry)
+            && this.entries.size() >= 3,
+            "Passed default ACL entry not found in the list of ACLs");
+        // default mask entry for effective permission calculation will be the
+        // penultimate entry. This can be mask entry in case of extended ACLs.
+        // In case of minimal ACL, this is the owner group entry, and we end up
+        // intersecting group FsAction with itself, which is a no-op.
+        FsAction defaultMask = this.entries.get(this.entries.size() - 2)
+            .getPermission();
+        FsAction entryPerm = entry.getPermission();
+        return entryPerm.and(defaultMask);
+      }
+    } else {
+      return entry.getPermission();
+    }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
index 206576cfa547c..d139ebadce3bc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
@@ -86,22 +86,26 @@ protected void processPath(PathData item) throws IOException {
           (perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T"));
       }
 
-      List<AclEntry> entries = perm.getAclBit() ?
-        item.fs.getAclStatus(item.path).getEntries() :
-        Collections.<AclEntry>emptyList();
+      AclStatus aclStatus = item.fs.getAclStatus(item.path);
+      List<AclEntry> entries = perm.getAclBit() ? aclStatus.getEntries()
+          : Collections.<AclEntry> emptyList();
       ScopedAclEntries scopedEntries = new ScopedAclEntries(
         AclUtil.getAclFromPermAndEntries(perm, entries));
-      printAclEntriesForSingleScope(scopedEntries.getAccessEntries());
-      printAclEntriesForSingleScope(scopedEntries.getDefaultEntries());
+      printAclEntriesForSingleScope(aclStatus, perm,
+          scopedEntries.getAccessEntries());
+      printAclEntriesForSingleScope(aclStatus, perm,
+          scopedEntries.getDefaultEntries());
       out.println();
     }
 
     /**
      * Prints all the ACL entries in a single scope.
-     *
+     * @param aclStatus AclStatus for the path
+     * @param fsPerm FsPermission for the path
      * @param entries List<AclEntry> containing ACL entries of file
      */
-    private void printAclEntriesForSingleScope(List<AclEntry> entries) {
+    private void printAclEntriesForSingleScope(AclStatus aclStatus,
+        FsPermission fsPerm, List<AclEntry> entries) {
       if (entries.isEmpty()) {
         return;
       }
@@ -110,10 +114,8 @@ private void printAclEntriesForSingleScope(List<AclEntry> entries) {
           out.println(entry);
         }
       } else {
-        // ACL sort order guarantees mask is the second-to-last entry.
-        FsAction maskPerm = entries.get(entries.size() - 2).getPermission();
         for (AclEntry entry: entries) {
-          printExtendedAclEntry(entry, maskPerm);
+          printExtendedAclEntry(aclStatus, fsPerm, entry);
         }
       }
     }
@@ -123,14 +125,16 @@ private void printAclEntriesForSingleScope(List<AclEntry> entries) {
      * permissions of the entry, then also prints the restricted version as the
      * effective permissions.  The mask applies to all named entries and also
      * the unnamed group entry.
-     *
+     * @param aclStatus AclStatus for the path
+     * @param fsPerm FsPermission for the path
      * @param entry AclEntry extended ACL entry to print
-     * @param maskPerm FsAction permissions in the ACL's mask entry
      */
-    private void printExtendedAclEntry(AclEntry entry, FsAction maskPerm) {
+    private void printExtendedAclEntry(AclStatus aclStatus,
+        FsPermission fsPerm, AclEntry entry) {
       if (entry.getName() != null || entry.getType() == AclEntryType.GROUP) {
         FsAction entryPerm = entry.getPermission();
-        FsAction effectivePerm = entryPerm.and(maskPerm);
+        FsAction effectivePerm = aclStatus
+            .getEffectivePermission(entry, fsPerm);
         if (entryPerm != effectivePerm) {
           out.println(String.format("%s\t#effective:%s", entry,
             effectivePerm.SYMBOL));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index ff7a10f2975f5..dd7d1686ca70f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.LinkedList;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -44,18 +45,26 @@ public static void registerCommands(CommandFactory factory) {
 
   private static final String OPTION_QUOTA = "q";
   private static final String OPTION_HUMAN = "h";
+  private static final String OPTION_HEADER = "v";
 
   public static final String NAME = "count";
   public static final String USAGE =
-      "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
-  public static final String DESCRIPTION = 
+      "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+          + "] <path> ...";
+  public static final String DESCRIPTION =
       "Count the number of directories, files and bytes under the paths\n" +
-      "that match the specified file pattern.  The output columns are:\n" +
-      "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
-      "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
-      "      DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
-      "The -h option shows file sizes in human readable format.";
-  
+          "that match the specified file pattern.  The output columns are:\n" +
+          StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+          " PATHNAME\n" +
+          "or, with the -" + OPTION_QUOTA + " option:\n" +
+          StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
+          "      " +
+          StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
+          " PATHNAME\n" +
+          "The -" + OPTION_HUMAN +
+          " option shows file sizes in human readable format.\n" +
+          "The -" + OPTION_HEADER + " option displays a header line.";
+
   private boolean showQuotas;
   private boolean humanReadable;
 
@@ -65,7 +74,7 @@ public Count() {}
   /** Constructor
    * @deprecated invoke via {@link FsShell}
    * @param cmd the count command
-   * @param pos the starting index of the arguments 
+   * @param pos the starting index of the arguments
    * @param conf configuration
    */
   @Deprecated
@@ -77,13 +86,16 @@ public Count(String[] cmd, int pos, Configuration conf) {
   @Override
   protected void processOptions(LinkedList<String> args) {
     CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
-      OPTION_QUOTA, OPTION_HUMAN);
+        OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
     cf.parse(args);
     if (args.isEmpty()) { // default path is the current working directory
       args.add(".");
     }
     showQuotas = cf.getOpt(OPTION_QUOTA);
     humanReadable = cf.getOpt(OPTION_HUMAN);
+    if (cf.getOpt(OPTION_HEADER)) {
+      out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
+    }
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index d437a663f5557..f0d7b8de4453c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -32,6 +32,7 @@
 import org.apache.avro.io.DatumWriter;
 import org.apache.avro.io.EncoderFactory;
 import org.apache.avro.io.JsonEncoder;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -195,11 +196,11 @@ protected void processPath(PathData item) throws IOException {
 
       FileChecksum checksum = item.fs.getFileChecksum(item.path);
       if (checksum == null) {
-        out.printf("%s\tNONE\t\n", item.toString());
+        out.printf("%s\tNONE\t%n", item.toString());
       } else {
         String checksumString = StringUtils.byteToHexString(
             checksum.getBytes(), 0, checksum.getLength());
-        out.printf("%s\t%s\t%s\n",
+        out.printf("%s\t%s\t%s%n",
             item.toString(), checksum.getAlgorithmName(),
             checksumString);
       }
@@ -234,10 +235,10 @@ public int read() throws IOException {
         if (!r.next(key, val)) {
           return -1;
         }
-        byte[] tmp = key.toString().getBytes();
+        byte[] tmp = key.toString().getBytes(Charsets.UTF_8);
         outbuf.write(tmp, 0, tmp.length);
         outbuf.write('\t');
-        tmp = val.toString().getBytes();
+        tmp = val.toString().getBytes(Charsets.UTF_8);
         outbuf.write(tmp, 0, tmp.length);
         outbuf.write('\n');
         inbuf.reset(outbuf.getData(), outbuf.getLength());
@@ -299,7 +300,8 @@ public int read() throws IOException {
       encoder.flush();
       if (!fileReader.hasNext()) {
         // Write a new line after the last Avro record.
-        output.write(System.getProperty("line.separator").getBytes());
+        output.write(System.getProperty("line.separator")
+                         .getBytes(Charsets.UTF_8));
         output.flush();
       }
       pos = 0;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index cc8fbb4f2f1c3..9515fde4c4447 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -60,6 +60,7 @@ public static void registerCommands(CommandFactory factory) {
     factory.registerCommands(Tail.class);
     factory.registerCommands(Test.class);
     factory.registerCommands(Touch.class);
+    factory.registerCommands(Truncate.class);
     factory.registerCommands(SnapshotCommands.class);
     factory.registerCommands(XAttrCommands.class);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 6024d885afb05..c7e80b6b4af1f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -57,7 +57,7 @@ public static void registerCommands(CommandFactory factory) {
 		  
   
 
-  protected static final SimpleDateFormat dateFormat = 
+  protected final SimpleDateFormat dateFormat =
     new SimpleDateFormat("yyyy-MM-dd HH:mm");
 
   protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index ee56fe6492c93..458d3ee7852b5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -30,15 +30,17 @@
 
 /**
  * Print statistics about path in specified format.
- * Format sequences:
- *   %b: Size of file in blocks
- *   %g: Group name of owner
- *   %n: Filename
- *   %o: Block size
- *   %r: replication
- *   %u: User name of owner
- *   %y: UTC date as &quot;yyyy-MM-dd HH:mm:ss&quot;
- *   %Y: Milliseconds since January 1, 1970 UTC
+ * Format sequences:<br>
+ *   %b: Size of file in blocks<br>
+ *   %F: Type<br>
+ *   %g: Group name of owner<br>
+ *   %n: Filename<br>
+ *   %o: Block size<br>
+ *   %r: replication<br>
+ *   %u: User name of owner<br>
+ *   %y: UTC date as &quot;yyyy-MM-dd HH:mm:ss&quot;<br>
+ *   %Y: Milliseconds since January 1, 1970 UTC<br>
+ * If the format is not specified, %y is used by default.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -48,15 +50,22 @@ public static void registerCommands(CommandFactory factory) {
     factory.addClass(Stat.class, "-stat");
   }
 
+  private static final String NEWLINE = System.getProperty("line.separator");
+
   public static final String NAME = "stat";
   public static final String USAGE = "[format] <path> ...";
   public static final String DESCRIPTION =
-    "Print statistics about the file/directory at <path> " +
-    "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
-    "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
+    "Print statistics about the file/directory at <path>" + NEWLINE +
+    "in the specified format. Format accepts filesize in" + NEWLINE +
+    "blocks (%b), type (%F), group name of owner (%g)," + NEWLINE +
+    "name (%n), block size (%o), replication (%r), user name" + NEWLINE +
+    "of owner (%u), modification date (%y, %Y)." + NEWLINE +
+    "%y shows UTC date as \"yyyy-MM-dd HH:mm:ss\" and" + NEWLINE +
+    "%Y shows milliseconds since January 1, 1970 UTC." + NEWLINE +
+    "If the format is not specified, %y is used by default." + NEWLINE;
 
-  protected static final SimpleDateFormat timeFmt;
-  static {
+  protected final SimpleDateFormat timeFmt;
+  {
     timeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     timeFmt.setTimeZone(TimeZone.getTimeZone("UTC"));
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
index 4cfdb084d7eb7..9984cf26fa4d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Test.java
@@ -83,6 +83,8 @@ protected void processPath(PathData item) throws IOException {
       case 'z':
         test = (item.stat.getLen() == 0);
         break;
+      default:
+        break;
     }
     if (!test) exitCode = 1;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java
new file mode 100644
index 0000000000000..99128634900a1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Truncate.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Truncates a file to a new size
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Truncate extends FsCommand {
+  public static void registerCommands(CommandFactory factory) {
+    factory.addClass(Truncate.class, "-truncate");
+  }
+
+  public static final String NAME = "truncate";
+  public static final String USAGE = "[-w] <length> <path> ...";
+  public static final String DESCRIPTION =
+      "Truncate all files that match the specified file pattern to the " +
+      "specified length.\n" +
+      "-w: Requests that the command wait for block recovery to complete, " +
+      "if necessary.";
+
+  protected long newLength = -1;
+  protected List<PathData> waitList = new LinkedList<>();
+  protected boolean waitOpt = false;
+
+  @Override
+  protected void processOptions(LinkedList<String> args) throws IOException {
+    CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "w");
+    cf.parse(args);
+    waitOpt = cf.getOpt("w");
+
+    try {
+      newLength = Long.parseLong(args.removeFirst());
+    } catch(NumberFormatException nfe) {
+      displayWarning("Illegal length, a non-negative integer expected");
+      throw nfe;
+    }
+    if(newLength < 0) {
+      throw new IllegalArgumentException("length must be >= 0");
+    }
+  }
+
+  @Override
+  protected void processArguments(LinkedList<PathData> args)
+      throws IOException {
+    super.processArguments(args);
+    if (waitOpt) waitForRecovery();
+  }
+
+  @Override
+  protected void processPath(PathData item) throws IOException {
+    if(item.stat.isDirectory()) {
+      throw new PathIsDirectoryException(item.toString());
+    }
+    long oldLength = item.stat.getLen();
+    if(newLength > oldLength) {
+      throw new IllegalArgumentException(
+          "Cannot truncate to a larger file size. Current size: " + oldLength +
+          ", truncate size: " + newLength + ".");
+    }
+    if(item.fs.truncate(item.path, newLength)) {
+      out.println("Truncated " + item + " to length: " + newLength);
+    }
+    else if(waitOpt) {
+      waitList.add(item);
+    }
+    else {
+      out.println("Truncating " + item + " to length: " + newLength + ". " +
+          "Wait for block recovery to complete before further updating this " +
+          "file.");
+    }
+  }
+
+  /**
+   * Wait for all files in waitList to have length equal to newLength.
+   */
+  private void waitForRecovery() throws IOException {
+    for(PathData item : waitList) {
+      out.println("Waiting for " + item + " ...");
+      out.flush();
+
+      for(;;) {
+        item.refreshStatus();
+        if(item.stat.getLen() == newLength) break;
+        try {Thread.sleep(1000);} catch(InterruptedException ignored) {}
+      }
+
+      out.println("Truncated " + item + " to length: " + newLength);
+      out.flush();
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 963289f43739b..0f77f470d9723 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -446,6 +446,14 @@ public boolean rename(final Path src, final Path dst) throws IOException {
     return resSrc.targetFileSystem.rename(resSrc.remainingPath,
         resDst.remainingPath);
   }
+
+  @Override
+  public boolean truncate(final Path f, final long newLength)
+      throws IOException {
+    InodeTree.ResolveResult<FileSystem> res =
+        fsState.resolve(getUriPath(f), true);
+    return res.targetFileSystem.truncate(f, newLength);
+  }
   
   @Override
   public void setOwner(final Path f, final String username,
@@ -833,6 +841,11 @@ public boolean rename(Path src, Path dst) throws AccessControlException,
       throw readOnlyMountTable("rename", src);     
     }
 
+    @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+      throw readOnlyMountTable("truncate", f);
+    }
+
     @Override
     public void setOwner(Path f, String username, String groupname)
         throws AccessControlException, IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119f709c5..947baa93e1e8e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1064,7 +1064,9 @@ private void setZooKeeperRef(ZooKeeper zk) {
     public void process(WatchedEvent event) {
       hasReceivedEvent.countDown();
       try {
-        hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
+        if (!hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS)) {
+          LOG.debug("Event received with stale zk");
+        }
         ActiveStandbyElector.this.processWatchEvent(
             zk, event);
       } catch (Throwable t) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index bd6366c76cc44..f72df7739e32f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -69,15 +69,15 @@ public abstract class HAAdmin extends Configured implements Tool {
   protected final static Map<String, UsageInfo> USAGE =
     ImmutableMap.<String, UsageInfo>builder()
     .put("-transitionToActive",
-        new UsageInfo("<serviceId> [--"+FORCEACTIVE+"]", "Transitions the service into Active state"))
+        new UsageInfo("[--"+FORCEACTIVE+"] <serviceId>", "Transitions the service into Active state"))
     .put("-transitionToStandby",
         new UsageInfo("<serviceId>", "Transitions the service into Standby state"))
     .put("-failover",
         new UsageInfo("[--"+FORCEFENCE+"] [--"+FORCEACTIVE+"] <serviceId> <serviceId>",
             "Failover from the first service to the second.\n" +
-            "Unconditionally fence services if the "+FORCEFENCE+" option is used.\n" +
+            "Unconditionally fence services if the --"+FORCEFENCE+" option is used.\n" +
             "Try to failover to the target service even if it is not ready if the " + 
-            FORCEACTIVE + " option is used."))
+            "--" + FORCEACTIVE + " option is used."))
     .put("-getServiceState",
         new UsageInfo("<serviceId>", "Returns the state of the service"))
     .put("-checkHealth",
@@ -168,12 +168,6 @@ private int transitionToActive(final CommandLine cmd)
   private boolean isOtherTargetNodeActive(String targetNodeToActivate, boolean forceActive)
       throws IOException  {
     Collection<String> targetIds = getTargetIds(targetNodeToActivate);
-    if(targetIds == null) {
-      errOut.println("transitionToActive: No target node in the "
-          + "current configuration");
-      printUsage(errOut, "-transitionToActive");
-      return true;
-    }
     targetIds.remove(targetNodeToActivate);
     for(String targetId : targetIds) {
       HAServiceTarget target = resolveTarget(targetId);
@@ -234,7 +228,7 @@ private boolean checkManualStateManagementOK(HAServiceTarget target) {
             "Refusing to manually manage HA state, since it may cause\n" +
             "a split-brain scenario or other incorrect state.\n" +
             "If you are very sure you know what you are doing, please \n" +
-            "specify the " + FORCEMANUAL + " flag.");
+            "specify the --" + FORCEMANUAL + " flag.");
         return false;
       } else {
         LOG.warn("Proceeding with manual HA state management even though\n" +
@@ -468,7 +462,7 @@ protected int runCmd(String[] argv) throws Exception {
   
   private boolean confirmForceManual() throws IOException {
      return ToolRunner.confirmPrompt(
-        "You have specified the " + FORCEMANUAL + " flag. This flag is " +
+        "You have specified the --" + FORCEMANUAL + " flag. This flag is " +
         "dangerous, as it can induce a split-brain scenario that WILL " +
         "CORRUPT your HDFS namespace, possibly irrecoverably.\n" +
         "\n" +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
index 0f5465194b928..90eb915e69ffb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
@@ -310,6 +310,8 @@ public void log(int level, String message) {
       case com.jcraft.jsch.Logger.FATAL:
         LOG.fatal(message);
         break;
+      default:
+        break;
       }
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/StreamPumper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/StreamPumper.java
index 8bc16af2afa99..00c6401d88d9d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/StreamPumper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/StreamPumper.java
@@ -22,6 +22,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 
 /**
@@ -76,7 +77,8 @@ void start() {
   }
 
   protected void pump() throws IOException {
-    InputStreamReader inputStreamReader = new InputStreamReader(stream);
+    InputStreamReader inputStreamReader = new InputStreamReader(
+        stream, Charsets.UTF_8);
     BufferedReader br = new BufferedReader(inputStreamReader);
     String line = null;
     while ((line = br.readLine()) != null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 46c485b1d9a79..f58c3f4de997c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -153,7 +153,9 @@ protected abstract void checkRpcAdminAccess()
   public HAServiceTarget getLocalTarget() {
     return localTarget;
   }
-  
+
+  HAServiceState getServiceState() { return serviceState; }
+
   public int run(final String[] args) throws Exception {
     if (!localTarget.isAutoFailoverEnabled()) {
       LOG.fatal("Automatic failover is not enabled for " + localTarget + "." +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
index 99befeea6eb43..57acebd85f4a7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.http;
 
+import org.apache.commons.io.Charsets;
+
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -25,11 +27,11 @@
  * This class is responsible for quoting HTML characters.
  */
 public class HtmlQuoting {
-  private static final byte[] ampBytes = "&amp;".getBytes();
-  private static final byte[] aposBytes = "&apos;".getBytes();
-  private static final byte[] gtBytes = "&gt;".getBytes();
-  private static final byte[] ltBytes = "&lt;".getBytes();
-  private static final byte[] quotBytes = "&quot;".getBytes();
+  private static final byte[] ampBytes = "&amp;".getBytes(Charsets.UTF_8);
+  private static final byte[] aposBytes = "&apos;".getBytes(Charsets.UTF_8);
+  private static final byte[] gtBytes = "&gt;".getBytes(Charsets.UTF_8);
+  private static final byte[] ltBytes = "&lt;".getBytes(Charsets.UTF_8);
+  private static final byte[] quotBytes = "&quot;".getBytes(Charsets.UTF_8);
 
   /**
    * Does the given string need to be quoted?
@@ -63,7 +65,7 @@ public static boolean needsQuoting(String str) {
     if (str == null) {
       return false;
     }
-    byte[] bytes = str.getBytes();
+    byte[] bytes = str.getBytes(Charsets.UTF_8);
     return needsQuoting(bytes, 0 , bytes.length);
   }
 
@@ -98,15 +100,16 @@ public static String quoteHtmlChars(String item) {
     if (item == null) {
       return null;
     }
-    byte[] bytes = item.getBytes();
+    byte[] bytes = item.getBytes(Charsets.UTF_8);
     if (needsQuoting(bytes, 0, bytes.length)) {
       ByteArrayOutputStream buffer = new ByteArrayOutputStream();
       try {
         quoteHtmlChars(buffer, bytes, 0, bytes.length);
+        return buffer.toString("UTF-8");
       } catch (IOException ioe) {
         // Won't happen, since it is a bytearrayoutputstream
+        return null;
       }
-      return buffer.toString();
     } else {
       return item;
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 45b641957077c..80831e976aa57 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -20,7 +20,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.net.URI;
@@ -123,26 +123,13 @@ public final class HttpServer2 implements FilterContainer {
 
   protected final Server webServer;
 
-  private static class ListenerInfo {
-    /**
-     * Boolean flag to determine whether the HTTP server should clean up the
-     * listener in stop().
-     */
-    private final boolean isManaged;
-    private final Connector listener;
-    private ListenerInfo(boolean isManaged, Connector listener) {
-      this.isManaged = isManaged;
-      this.listener = listener;
-    }
-  }
-
-  private final List<ListenerInfo> listeners = Lists.newArrayList();
+  private final List<Connector> listeners = Lists.newArrayList();
 
   protected final WebAppContext webAppContext;
   protected final boolean findPort;
   protected final Map<Context, Boolean> defaultContexts =
-      new HashMap<Context, Boolean>();
-  protected final List<String> filterNames = new ArrayList<String>();
+      new HashMap<>();
+  protected final List<String> filterNames = new ArrayList<>();
   static final String STATE_DESCRIPTION_ALIVE = " - alive";
   static final String STATE_DESCRIPTION_NOT_LIVE = " - not live";
 
@@ -151,7 +138,6 @@ private ListenerInfo(boolean isManaged, Connector listener) {
    */
   public static class Builder {
     private ArrayList<URI> endpoints = Lists.newArrayList();
-    private Connector connector;
     private String name;
     private Configuration conf;
     private String[] pathSpecs;
@@ -243,11 +229,6 @@ public Builder setConf(Configuration conf) {
       return this;
     }
 
-    public Builder setConnector(Connector connector) {
-      this.connector = connector;
-      return this;
-    }
-
     public Builder setPathSpec(String[] pathSpec) {
       this.pathSpecs = pathSpec;
       return this;
@@ -274,17 +255,11 @@ public Builder setKeytabConfKey(String keytabConfKey) {
     }
 
     public HttpServer2 build() throws IOException {
-      if (this.name == null) {
-        throw new HadoopIllegalArgumentException("name is not set");
-      }
-
-      if (endpoints.size() == 0 && connector == null) {
-        throw new HadoopIllegalArgumentException("No endpoints specified");
-      }
+      Preconditions.checkNotNull(name, "name is not set");
+      Preconditions.checkState(!endpoints.isEmpty(), "No endpoints specified");
 
       if (hostName == null) {
-        hostName = endpoints.size() == 0 ? connector.getHost() : endpoints.get(
-            0).getHost();
+        hostName = endpoints.get(0).getHost();
       }
 
       if (this.conf == null) {
@@ -297,17 +272,14 @@ public HttpServer2 build() throws IOException {
         server.initSpnego(conf, hostName, usernameConfKey, keytabConfKey);
       }
 
-      if (connector != null) {
-        server.addUnmanagedListener(connector);
-      }
-
       for (URI ep : endpoints) {
-        Connector listener = null;
+        final Connector listener;
         String scheme = ep.getScheme();
         if ("http".equals(scheme)) {
           listener = HttpServer2.createDefaultChannelConnector();
         } else if ("https".equals(scheme)) {
           SslSocketConnector c = new SslSocketConnectorSecure();
+          c.setHeaderBufferSize(1024*64);
           c.setNeedClientAuth(needsClientAuth);
           c.setKeyPassword(keyPassword);
 
@@ -330,7 +302,7 @@ public HttpServer2 build() throws IOException {
         }
         listener.setHost(ep.getHost());
         listener.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
-        server.addManagedListener(listener);
+        server.addListener(listener);
       }
       server.loadListeners();
       return server;
@@ -348,7 +320,7 @@ private HttpServer2(final Builder b) throws IOException {
 
   private void initializeWebServer(String name, String hostName,
       Configuration conf, String[] pathSpecs)
-      throws FileNotFoundException, IOException {
+      throws IOException {
 
     Preconditions.checkNotNull(webAppContext);
 
@@ -406,12 +378,8 @@ private void initializeWebServer(String name, String hostName,
     }
   }
 
-  private void addUnmanagedListener(Connector connector) {
-    listeners.add(new ListenerInfo(false, connector));
-  }
-
-  private void addManagedListener(Connector connector) {
-    listeners.add(new ListenerInfo(true, connector));
+  private void addListener(Connector connector) {
+    listeners.add(connector);
   }
 
   private static WebAppContext createWebAppContext(String name,
@@ -442,15 +410,6 @@ private static void addNoCacheFilter(WebAppContext ctxt) {
         Collections.<String, String> emptyMap(), new String[] { "/*" });
   }
 
-  /**
-   * Create a required listener for the Jetty instance listening on the port
-   * provided. This wrapper and all subclasses must create at least one
-   * listener.
-   */
-  public Connector createBaseListener(Configuration conf) {
-    return HttpServer2.createDefaultChannelConnector();
-  }
-
   @InterfaceAudience.Private
   public static Connector createDefaultChannelConnector() {
     SelectChannelConnector ret = new SelectChannelConnector();
@@ -546,23 +505,6 @@ public void addContext(Context ctxt, boolean isFiltered) {
     defaultContexts.put(ctxt, isFiltered);
   }
 
-  /**
-   * Add a context
-   * @param pathSpec The path spec for the context
-   * @param dir The directory containing the context
-   * @param isFiltered if true, the servlet is added to the filter path mapping
-   * @throws IOException
-   */
-  protected void addContext(String pathSpec, String dir, boolean isFiltered) throws IOException {
-    if (0 == webServer.getHandlers().length) {
-      throw new RuntimeException("Couldn't find handler");
-    }
-    WebAppContext webAppCtx = new WebAppContext();
-    webAppCtx.setContextPath(pathSpec);
-    webAppCtx.setWar(dir);
-    addContext(webAppCtx, true);
-  }
-
   /**
    * Set a value in the webapp context. These values are available to the jsp
    * pages as "application.getAttribute(name)".
@@ -654,8 +596,8 @@ public void addFilter(String name, String classname,
 
     final String[] USER_FACING_URLS = { "*.html", "*.jsp" };
     defineFilter(webAppContext, name, classname, parameters, USER_FACING_URLS);
-    LOG.info("Added filter " + name + " (class=" + classname
-        + ") to context " + webAppContext.getDisplayName());
+    LOG.info(
+        "Added filter " + name + " (class=" + classname + ") to context " + webAppContext.getDisplayName());
     final String[] ALL_URLS = { "/*" };
     for (Map.Entry<Context, Boolean> e : defaultContexts.entrySet()) {
       if (e.getValue()) {
@@ -782,7 +724,7 @@ public void setThreads(int min, int max) {
 
   private void initSpnego(Configuration conf, String hostName,
       String usernameConfKey, String keytabConfKey) throws IOException {
-    Map<String, String> params = new HashMap<String, String>();
+    Map<String, String> params = new HashMap<>();
     String principalInConf = conf.get(usernameConfKey);
     if (principalInConf != null && !principalInConf.isEmpty()) {
       params.put("kerberos.principal", SecurityUtil.getServerPrincipal(
@@ -815,8 +757,8 @@ public void start() throws IOException {
       }
       // Make sure there is no handler failures.
       Handler[] handlers = webServer.getHandlers();
-      for (int i = 0; i < handlers.length; i++) {
-        if (handlers[i].isFailed()) {
+      for (Handler handler : handlers) {
+        if (handler.isFailed()) {
           throw new IOException(
               "Problem in starting http server. Server handlers failed");
         }
@@ -841,8 +783,8 @@ public void start() throws IOException {
   }
 
   private void loadListeners() {
-    for (ListenerInfo li : listeners) {
-      webServer.addConnector(li.listener);
+    for (Connector c : listeners) {
+      webServer.addConnector(c);
     }
   }
 
@@ -851,9 +793,8 @@ private void loadListeners() {
    * @throws Exception
    */
   void openListeners() throws Exception {
-    for (ListenerInfo li : listeners) {
-      Connector listener = li.listener;
-      if (!li.isManaged || li.listener.getLocalPort() != -1) {
+    for (Connector listener : listeners) {
+      if (listener.getLocalPort() != -1) {
         // This listener is either started externally or has been bound
         continue;
       }
@@ -886,13 +827,9 @@ void openListeners() throws Exception {
    */
   public void stop() throws Exception {
     MultiException exception = null;
-    for (ListenerInfo li : listeners) {
-      if (!li.isManaged) {
-        continue;
-      }
-
+    for (Connector c : listeners) {
       try {
-        li.listener.close();
+        c.close();
       } catch (Exception e) {
         LOG.error(
             "Error while stopping listener for webapp"
@@ -945,23 +882,17 @@ public boolean isAlive() {
     return webServer != null && webServer.isStarted();
   }
 
-  /**
-   * Return the host and port of the HttpServer, if live
-   * @return the classname and any HTTP URL
-   */
   @Override
   public String toString() {
-    if (listeners.size() == 0) {
-      return "Inactive HttpServer";
-    } else {
-      StringBuilder sb = new StringBuilder("HttpServer (")
-        .append(isAlive() ? STATE_DESCRIPTION_ALIVE : STATE_DESCRIPTION_NOT_LIVE).append("), listening at:");
-      for (ListenerInfo li : listeners) {
-        Connector l = li.listener;
-        sb.append(l.getHost()).append(":").append(l.getPort()).append("/,");
-      }
-      return sb.toString();
+    Preconditions.checkState(!listeners.isEmpty());
+    StringBuilder sb = new StringBuilder("HttpServer (")
+        .append(isAlive() ? STATE_DESCRIPTION_ALIVE
+                    : STATE_DESCRIPTION_NOT_LIVE)
+        .append("), listening at:");
+    for (Connector l : listeners) {
+      sb.append(l.getHost()).append(":").append(l.getPort()).append("/,");
     }
+    return sb.toString();
   }
 
   /**
@@ -999,8 +930,6 @@ public static boolean isInstrumentationAccessAllowed(
    * Does the user sending the HttpServletRequest has the administrator ACLs? If
    * it isn't the case, response will be modified to send an error to the user.
    *
-   * @param servletContext
-   * @param request
    * @param response used to send the error response if user does not have admin access.
    * @return true if admin-authorized, false otherwise
    * @throws IOException
@@ -1065,13 +994,14 @@ public static class StackServlet extends HttpServlet {
     public void doGet(HttpServletRequest request, HttpServletResponse response)
       throws ServletException, IOException {
       if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
-                                                     request, response)) {
+                                                      request, response)) {
         return;
       }
       response.setContentType("text/plain; charset=UTF-8");
-      PrintWriter out = response.getWriter();
-      ReflectionUtils.printThreadInfo(out, "");
-      out.close();
+      try (PrintStream out = new PrintStream(
+          response.getOutputStream(), false, "UTF-8")) {
+        ReflectionUtils.printThreadInfo(out, "");
+      }
       ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
     }
   }
@@ -1138,7 +1068,7 @@ public String[] getParameterValues(String name) {
       @SuppressWarnings("unchecked")
       @Override
       public Map<String, String[]> getParameterMap() {
-        Map<String, String[]> result = new HashMap<String,String[]>();
+        Map<String, String[]> result = new HashMap<>();
         Map<String, String[]> raw = rawRequest.getParameterMap();
         for (Map.Entry<String,String[]> item: raw.entrySet()) {
           String[] rawValue = item.getValue();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
index d32d58b6008be..3ba577fc4f4de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -90,7 +91,7 @@ public String toString(T obj) throws IOException {
     serializer.serialize(obj);
     byte[] buf = new byte[outBuf.getLength()];
     System.arraycopy(outBuf.getData(), 0, buf, 0, buf.length);
-    return new String(Base64.encodeBase64(buf));
+    return new String(Base64.encodeBase64(buf), Charsets.UTF_8);
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
index 3f5881b2dd6ee..a3fea3115cbf5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
@@ -24,6 +24,9 @@
 
 import sun.misc.Unsafe;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import com.google.common.primitives.Longs;
 import com.google.common.primitives.UnsignedBytes;
 
@@ -33,6 +36,7 @@
  * class to be able to compare arrays that start at non-zero offsets.
  */
 abstract class FastByteComparisons {
+  static final Log LOG = LogFactory.getLog(FastByteComparisons.class);
 
   /**
    * Lexicographically compare two byte arrays.
@@ -71,6 +75,13 @@ private static class LexicographicalComparerHolder {
      * implementation if unable to do so.
      */
     static Comparer<byte[]> getBestComparer() {
+      if (System.getProperty("os.arch").equals("sparc")) {
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Lexicographical comparer selected for "
+              + "byte aligned system architecture");
+        }
+        return lexicographicalComparerJavaImpl();
+      }
       try {
         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
 
@@ -78,8 +89,16 @@ static Comparer<byte[]> getBestComparer() {
         @SuppressWarnings("unchecked")
         Comparer<byte[]> comparer =
           (Comparer<byte[]>) theClass.getEnumConstants()[0];
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("Unsafe comparer selected for "
+              + "byte unaligned system architecture");
+        }
         return comparer;
       } catch (Throwable t) { // ensure we really catch *everything*
+        if (LOG.isTraceEnabled()) {
+          LOG.trace(t.getMessage());
+          LOG.trace("Lexicographical comparer selected");
+        }
         return lexicographicalComparerJavaImpl();
       }
     }
@@ -234,4 +253,4 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
index 6be7446c99450..e6c00c940bb24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
@@ -23,6 +23,12 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.WritableByteChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -30,6 +36,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ChunkedArrayList;
 
 /**
  * An utility class for I/O related functionality. 
@@ -313,4 +320,33 @@ public static void writeFully(FileChannel fc, ByteBuffer buf,
       offset += fc.write(buf, offset);
     } while (buf.remaining() > 0);
   }
+
+  /**
+   * Return the complete list of files in a directory as strings.<p/>
+   *
+   * This is better than File#listDir because it does not ignore IOExceptions.
+   *
+   * @param dir              The directory to list.
+   * @param filter           If non-null, the filter to use when listing
+   *                         this directory.
+   * @return                 The list of files in the directory.
+   *
+   * @throws IOException     On I/O error
+   */
+  public static List<String> listDirectory(File dir, FilenameFilter filter)
+      throws IOException {
+    ArrayList<String> list = new ArrayList<String> ();
+    try (DirectoryStream<Path> stream =
+             Files.newDirectoryStream(dir.toPath())) {
+      for (Path entry: stream) {
+        String fileName = entry.getFileName().toString();
+        if ((filter == null) || filter.accept(dir, fileName)) {
+          list.add(fileName);
+        }
+      }
+    } catch (DirectoryIteratorException e) {
+      throw e.getCause();
+    }
+    return list;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
index 6dec4aa618a4f..b77ca6781a639 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
@@ -99,11 +99,11 @@ public static class DecreasingComparator extends Comparator {
     
     @Override
     public int compare(WritableComparable a, WritableComparable b) {
-      return -super.compare(a, b);
+      return super.compare(b, a);
     }
     @Override
     public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
-      return -super.compare(b1, s1, l1, b2, s2, l2);
+      return super.compare(b2, s2, l2, b1, s1, l1);
     }
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 4cda107748208..7a59149ff0596 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -22,6 +22,8 @@
 import java.util.*;
 import java.rmi.server.UID;
 import java.security.MessageDigest;
+
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.*;
 import org.apache.hadoop.util.Options;
 import org.apache.hadoop.fs.*;
@@ -849,7 +851,7 @@ public static class Writer implements java.io.Closeable, Syncable {
       try {                                       
         MessageDigest digester = MessageDigest.getInstance("MD5");
         long time = Time.now();
-        digester.update((new UID()+"@"+time).getBytes());
+        digester.update((new UID()+"@"+time).getBytes(Charsets.UTF_8));
         sync = digester.digest();
       } catch (Exception e) {
         throw new RuntimeException(e);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index 3dc507687f516..0bcaee3ea79ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -584,6 +584,8 @@ public static void validateUTF8(byte[] utf8, int start, int len)
           state = TRAIL_BYTE;
         }
         break;
+      default:
+        break;
       } // switch (state)
       count++;
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
index 37b97f2a641a9..2c5a7bec852ba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 
@@ -224,7 +225,7 @@ public SplitCompressionInputStream createInputStream(InputStream seekableIn,
     // ........................................^^[We align at wrong position!]
     // ...........................................................^^[While this pos is correct]
 
-    if (in.getPos() <= start) {
+    if (in.getPos() < start) {
       ((Seekable)seekableIn).seek(start);
       in = new BZip2CompressionInputStream(seekableIn, start, end, readMode);
     }
@@ -281,7 +282,7 @@ private void writeStreamHeader() throws IOException {
         // The compressed bzip2 stream should start with the
         // identifying characters BZ. Caller of CBZip2OutputStream
         // i.e. this class must write these characters.
-        out.write(HEADER.getBytes());
+        out.write(HEADER.getBytes(Charsets.UTF_8));
       }
     }
 
@@ -415,7 +416,7 @@ private BufferedInputStream readStreamHeader() throws IOException {
         byte[] headerBytes = new byte[HEADER_LEN];
         int actualRead = bufferedIn.read(headerBytes, 0, HEADER_LEN);
         if (actualRead != -1) {
-          String header = new String(headerBytes);
+          String header = new String(headerBytes, Charsets.UTF_8);
           if (header.compareTo(HEADER) != 0) {
             bufferedIn.reset();
           } else {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
index 16e0ad763ac57..6bee6b84e5318 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
@@ -40,7 +40,7 @@ public DecompressorStream(InputStream in, Decompressor decompressor,
   throws IOException {
     super(in);
 
-    if (in == null || decompressor == null) {
+    if (decompressor == null) {
       throw new NullPointerException();
     } else if (bufferSize <= 0) {
       throw new IllegalArgumentException("Illegal bufferSize");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
new file mode 100644
index 0000000000000..01e8f3580e505
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A wrapper for ByteBuffer or bytes array for an erasure code chunk.
+ */
+public class ECChunk {
+
+  private ByteBuffer chunkBuffer;
+
+  /**
+   * Wrapping a ByteBuffer
+   * @param buffer
+   */
+  public ECChunk(ByteBuffer buffer) {
+    this.chunkBuffer = buffer;
+  }
+
+  /**
+   * Wrapping a bytes array
+   * @param buffer
+   */
+  public ECChunk(byte[] buffer) {
+    this.chunkBuffer = ByteBuffer.wrap(buffer);
+  }
+
+  /**
+   * Convert to ByteBuffer
+   * @return ByteBuffer
+   */
+  public ByteBuffer getBuffer() {
+    return chunkBuffer;
+  }
+
+  /**
+   * Convert an array of this chunks to an array of ByteBuffers
+   * @param chunks
+   * @return an array of ByteBuffers
+   */
+  public static ByteBuffer[] toBuffers(ECChunk[] chunks) {
+    ByteBuffer[] buffers = new ByteBuffer[chunks.length];
+
+    for (int i = 0; i < chunks.length; i++) {
+      buffers[i] = chunks[i].getBuffer();
+    }
+
+    return buffers;
+  }
+
+  /**
+   * Convert an array of this chunks to an array of byte array.
+   * Note the chunk buffers are not affected.
+   * @param chunks
+   * @return an array of byte array
+   */
+  public static byte[][] toArray(ECChunk[] chunks) {
+    byte[][] bytesArr = new byte[chunks.length][];
+
+    ByteBuffer buffer;
+    for (int i = 0; i < chunks.length; i++) {
+      buffer = chunks[i].getBuffer();
+      if (buffer.hasArray()) {
+        bytesArr[i] = buffer.array();
+      } else {
+        bytesArr[i] = new byte[buffer.remaining()];
+        // Avoid affecting the original one
+        buffer.mark();
+        buffer.get(bytesArr[i]);
+        buffer.reset();
+      }
+    }
+
+    return bytesArr;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
new file mode 100644
index 0000000000000..74d2ab6fc1ecf
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * A common class of basic facilities to be shared by encoder and decoder
+ *
+ * It implements the {@link RawErasureCoder} interface.
+ */
+public abstract class AbstractRawErasureCoder implements RawErasureCoder {
+
+  private int numDataUnits;
+  private int numParityUnits;
+  private int chunkSize;
+
+  @Override
+  public void initialize(int numDataUnits, int numParityUnits,
+                         int chunkSize) {
+    this.numDataUnits = numDataUnits;
+    this.numParityUnits = numParityUnits;
+    this.chunkSize = chunkSize;
+  }
+
+  @Override
+  public int getNumDataUnits() {
+    return numDataUnits;
+  }
+
+  @Override
+  public int getNumParityUnits() {
+    return numParityUnits;
+  }
+
+  @Override
+  public int getChunkSize() {
+    return chunkSize;
+  }
+
+  @Override
+  public boolean preferNativeBuffer() {
+    return false;
+  }
+
+  @Override
+  public void release() {
+    // Nothing to do by default
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
new file mode 100644
index 0000000000000..4613b25eb13f3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * An abstract raw erasure decoder that's to be inherited by new decoders.
+ *
+ * It implements the {@link RawErasureDecoder} interface.
+ */
+public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder
+    implements RawErasureDecoder {
+
+  @Override
+  public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+                     ByteBuffer[] outputs) {
+    if (erasedIndexes.length == 0) {
+      return;
+    }
+
+    doDecode(inputs, erasedIndexes, outputs);
+  }
+
+  /**
+   * Perform the real decoding using ByteBuffer
+   * @param inputs
+   * @param erasedIndexes
+   * @param outputs
+   */
+  protected abstract void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+                                   ByteBuffer[] outputs);
+
+  @Override
+  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+    if (erasedIndexes.length == 0) {
+      return;
+    }
+
+    doDecode(inputs, erasedIndexes, outputs);
+  }
+
+  /**
+   * Perform the real decoding using bytes array
+   * @param inputs
+   * @param erasedIndexes
+   * @param outputs
+   */
+  protected abstract void doDecode(byte[][] inputs, int[] erasedIndexes,
+                                   byte[][] outputs);
+
+  @Override
+  public void decode(ECChunk[] inputs, int[] erasedIndexes,
+                     ECChunk[] outputs) {
+    doDecode(inputs, erasedIndexes, outputs);
+  }
+
+  /**
+   * Perform the real decoding using chunks
+   * @param inputs
+   * @param erasedIndexes
+   * @param outputs
+   */
+  protected void doDecode(ECChunk[] inputs, int[] erasedIndexes,
+                          ECChunk[] outputs) {
+    if (inputs[0].getBuffer().hasArray()) {
+      byte[][] inputBytesArr = ECChunk.toArray(inputs);
+      byte[][] outputBytesArr = ECChunk.toArray(outputs);
+      doDecode(inputBytesArr, erasedIndexes, outputBytesArr);
+    } else {
+      ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputs);
+      ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputs);
+      doDecode(inputBuffers, erasedIndexes, outputBuffers);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
new file mode 100644
index 0000000000000..4feaf39f28302
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * An abstract raw erasure encoder that's to be inherited by new encoders.
+ *
+ * It implements the {@link RawErasureEncoder} interface.
+ */
+public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder
+    implements RawErasureEncoder {
+
+  @Override
+  public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+    assert (inputs.length == getNumDataUnits());
+    assert (outputs.length == getNumParityUnits());
+
+    doEncode(inputs, outputs);
+  }
+
+  /**
+   * Perform the real encoding work using ByteBuffer
+   * @param inputs
+   * @param outputs
+   */
+  protected abstract void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs);
+
+  @Override
+  public void encode(byte[][] inputs, byte[][] outputs) {
+    assert (inputs.length == getNumDataUnits());
+    assert (outputs.length == getNumParityUnits());
+
+    doEncode(inputs, outputs);
+  }
+
+  /**
+   * Perform the real encoding work using bytes array
+   * @param inputs
+   * @param outputs
+   */
+  protected abstract void doEncode(byte[][] inputs, byte[][] outputs);
+
+  @Override
+  public void encode(ECChunk[] inputs, ECChunk[] outputs) {
+    assert (inputs.length == getNumDataUnits());
+    assert (outputs.length == getNumParityUnits());
+
+    doEncode(inputs, outputs);
+  }
+
+  /**
+   * Perform the real encoding work using chunks.
+   * @param inputs
+   * @param outputs
+   */
+  protected void doEncode(ECChunk[] inputs, ECChunk[] outputs) {
+    /**
+     * Note callers may pass byte array, or ByteBuffer via ECChunk according
+     * to how ECChunk is created. Some implementations of coder use byte array
+     * (ex: pure Java), some use native ByteBuffer (ex: ISA-L), all for the
+     * better performance.
+     */
+    if (inputs[0].getBuffer().hasArray()) {
+      byte[][] inputBytesArr = ECChunk.toArray(inputs);
+      byte[][] outputBytesArr = ECChunk.toArray(outputs);
+      doEncode(inputBytesArr, outputBytesArr);
+    } else {
+      ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputs);
+      ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputs);
+      doEncode(inputBuffers, outputBuffers);
+    }
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
new file mode 100644
index 0000000000000..91a9abfe2aad4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * RawErasureCoder is a common interface for {@link RawErasureEncoder} and
+ * {@link RawErasureDecoder} as both encoder and decoder share some properties.
+ *
+ * RawErasureCoder is part of ErasureCodec framework, where ErasureCoder is
+ * used to encode/decode a group of blocks (BlockGroup) according to the codec
+ * specific BlockGroup layout and logic. An ErasureCoder extracts chunks of
+ * data from the blocks and can employ various low level RawErasureCoders to
+ * perform encoding/decoding against the chunks.
+ *
+ * To distinguish from ErasureCoder, here RawErasureCoder is used to mean the
+ * low level constructs, since it only takes care of the math calculation with
+ * a group of byte buffers.
+ */
+public interface RawErasureCoder {
+
+  /**
+   * Initialize with the important parameters for the code.
+   * @param numDataUnits how many data inputs for the coding
+   * @param numParityUnits how many parity outputs the coding generates
+   * @param chunkSize the size of the input/output buffer
+   */
+  public void initialize(int numDataUnits, int numParityUnits, int chunkSize);
+
+  /**
+   * The number of data input units for the coding. A unit can be a byte,
+   * chunk or buffer or even a block.
+   * @return count of data input units
+   */
+  public int getNumDataUnits();
+
+  /**
+   * The number of parity output units for the coding. A unit can be a byte,
+   * chunk, buffer or even a block.
+   * @return count of parity output units
+   */
+  public int getNumParityUnits();
+
+  /**
+   * Chunk buffer size for the input/output
+   * @return chunk buffer size
+   */
+  public int getChunkSize();
+
+  /**
+   * Tell if native or off-heap buffer is preferred or not. It's for callers to
+   * decide how to allocate coding chunk buffers, either on heap or off heap.
+   * It will return false by default.
+   * @return true if native buffer is preferred for performance consideration,
+   * otherwise false.
+   */
+  public boolean preferNativeBuffer();
+
+  /**
+   * Should be called when release this coder. Good chance to release encoding
+   * or decoding buffers
+   */
+  public void release();
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
new file mode 100644
index 0000000000000..1358b7d0bc534
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * RawErasureDecoder performs decoding given chunks of input data and generates
+ * missing data that corresponds to an erasure code scheme, like XOR and
+ * Reed-Solomon.
+ *
+ * It extends the {@link RawErasureCoder} interface.
+ */
+public interface RawErasureDecoder extends RawErasureCoder {
+
+  /**
+   * Decode with inputs and erasedIndexes, generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+                     ByteBuffer[] outputs);
+
+  /**
+   * Decode with inputs and erasedIndexes, generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs);
+
+  /**
+   * Decode with inputs and erasedIndexes, generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs);
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
new file mode 100644
index 0000000000000..974f86ca6ca62
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * RawErasureEncoder performs encoding given chunks of input data and generates
+ * parity outputs that corresponds to an erasure code scheme, like XOR and
+ * Reed-Solomon.
+ *
+ * It extends the {@link RawErasureCoder} interface.
+ */
+public interface RawErasureEncoder extends RawErasureCoder {
+
+  /**
+   * Encode with inputs and generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs);
+
+  /**
+   * Encode with inputs and generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void encode(byte[][] inputs, byte[][] outputs);
+
+  /**
+   * Encode with inputs and generates outputs
+   * @param inputs
+   * @param outputs
+   */
+  public void encode(ECChunk[] inputs, ECChunk[] outputs);
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java
new file mode 100644
index 0000000000000..98307a7b3c2f6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawDecoder.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw decoder in XOR code scheme in pure Java, adapted from HDFS-RAID.
+ */
+public class XorRawDecoder extends AbstractRawErasureDecoder {
+
+  @Override
+  protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+                          ByteBuffer[] outputs) {
+    assert(erasedIndexes.length == outputs.length);
+    assert(erasedIndexes.length <= 1);
+
+    int bufSize = inputs[0].remaining();
+    int erasedIdx = erasedIndexes[0];
+
+    // Set the output to zeros.
+    for (int j = 0; j < bufSize; j++) {
+      outputs[0].put(j, (byte) 0);
+    }
+
+    // Process the inputs.
+    for (int i = 0; i < inputs.length; i++) {
+      // Skip the erased location.
+      if (i == erasedIdx) {
+        continue;
+      }
+
+      for (int j = 0; j < bufSize; j++) {
+        outputs[0].put(j, (byte) (outputs[0].get(j) ^ inputs[i].get(j)));
+      }
+    }
+  }
+
+  @Override
+  protected void doDecode(byte[][] inputs, int[] erasedIndexes,
+                          byte[][] outputs) {
+    assert(erasedIndexes.length == outputs.length);
+    assert(erasedIndexes.length <= 1);
+
+    int bufSize = inputs[0].length;
+    int erasedIdx = erasedIndexes[0];
+
+    // Set the output to zeros.
+    for (int j = 0; j < bufSize; j++) {
+      outputs[0][j] = 0;
+    }
+
+    // Process the inputs.
+    for (int i = 0; i < inputs.length; i++) {
+      // Skip the erased location.
+      if (i == erasedIdx) {
+        continue;
+      }
+
+      for (int j = 0; j < bufSize; j++) {
+        outputs[0][j] ^= inputs[i][j];
+      }
+    }
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java
new file mode 100644
index 0000000000000..99b20b92e7ad4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XorRawEncoder.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw encoder in XOR code scheme in pure Java, adapted from HDFS-RAID.
+ */
+public class XorRawEncoder extends AbstractRawErasureEncoder {
+
+  @Override
+  protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+    int bufSize = inputs[0].remaining();
+
+    // Get the first buffer's data.
+    for (int j = 0; j < bufSize; j++) {
+      outputs[0].put(j, inputs[0].get(j));
+    }
+
+    // XOR with everything else.
+    for (int i = 1; i < inputs.length; i++) {
+      for (int j = 0; j < bufSize; j++) {
+        outputs[0].put(j, (byte) (outputs[0].get(j) ^ inputs[i].get(j)));
+      }
+    }
+  }
+
+  @Override
+  protected void doEncode(byte[][] inputs, byte[][] outputs) {
+    int bufSize = inputs[0].length;
+
+    // Get the first buffer's data.
+    for (int j = 0; j < bufSize; j++) {
+      outputs[0][j] = inputs[0][j];
+    }
+
+    // XOR with everything else.
+    for (int i = 1; i < inputs.length; i++) {
+      for (int j = 0; j < bufSize; j++) {
+        outputs[0][j] ^= inputs[i][j];
+      }
+    }
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
index c11678dd5d0dc..f17be1ab90d3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
@@ -2341,7 +2341,7 @@ public void write(DataOutput out) throws IOException {
    *          A list of TFile paths.
    */
   public static void main(String[] args) {
-    System.out.printf("TFile Dumper (TFile %s, BCFile %s)\n", TFile.API_VERSION
+    System.out.printf("TFile Dumper (TFile %s, BCFile %s)%n", TFile.API_VERSION
         .toString(), BCFile.API_VERSION.toString());
     if (args.length == 0) {
       System.out
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFileDumper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFileDumper.java
index 829a1c68c45e9..aabdf57a26661 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFileDumper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFileDumper.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -176,7 +177,7 @@ static public void dumpInfo(String file, PrintStream out, Configuration conf)
       for (Iterator<Map.Entry<String, String>> it = entrySet.iterator(); it
           .hasNext();) {
         Map.Entry<String, String> e = it.next();
-        out.printf("%s : %s\n", Align.format(e.getKey(), maxKeyLength,
+        out.printf("%s : %s%n", Align.format(e.getKey(), maxKeyLength,
             Align.LEFT), e.getValue());
       }
       out.println();
@@ -200,7 +201,7 @@ static public void dumpInfo(String file, PrintStream out, Configuration conf)
         String endKey = "End-Key";
         int endKeyWidth = Math.max(endKey.length(), maxKeySampleLen * 2 + 5);
 
-        out.printf("%s %s %s %s %s %s\n", Align.format(blkID, blkIDWidth,
+        out.printf("%s %s %s %s %s %s%n", Align.format(blkID, blkIDWidth,
             Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
             Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(
                 rawSize, rawSizeWidth, Align.CENTER), Align.format(records,
@@ -233,7 +234,7 @@ static public void dumpInfo(String file, PrintStream out, Configuration conf)
               out.printf("%X", b);
             }
           } else {
-            out.print(new String(key, 0, sampleLen));
+            out.print(new String(key, 0, sampleLen, Charsets.UTF_8));
           }
           if (sampleLen < key.length) {
             out.print("...");
@@ -267,7 +268,7 @@ static public void dumpInfo(String file, PrintStream out, Configuration conf)
                 * 10);
         String compression = "Compression";
         int compressionWidth = compression.length();
-        out.printf("%s %s %s %s %s\n", Align.format(name, nameWidth,
+        out.printf("%s %s %s %s %s%n", Align.format(name, nameWidth,
             Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
             Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(
                 rawSize, rawSizeWidth, Align.CENTER), Align.format(compression,
@@ -280,7 +281,7 @@ static public void dumpInfo(String file, PrintStream out, Configuration conf)
           BlockRegion region = e.getValue().getRegion();
           String blkCompression =
               e.getValue().getCompressionAlgorithm().getName();
-          out.printf("%s %s %s %s %s\n", Align.format(blkName, nameWidth,
+          out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth,
               Align.LEFT), Align.format(region.getOffset(), offsetWidth,
               Align.LEFT), Align.format(region.getCompressedSize(),
               blkLenWidth, Align.LEFT), Align.format(region.getRawSize(),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 4a1ae7a6b7078..bc6e62ae40275 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -508,11 +508,63 @@ public static class Windows {
     
     public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L;
 
+    /**
+     * Create a directory with permissions set to the specified mode.  By setting
+     * permissions at creation time, we avoid issues related to the user lacking
+     * WRITE_DAC rights on subsequent chmod calls.  One example where this can
+     * occur is writing to an SMB share where the user does not have Full Control
+     * rights, and therefore WRITE_DAC is denied.
+     *
+     * @param path directory to create
+     * @param mode permissions of new directory
+     * @throws IOException if there is an I/O error
+     */
+    public static void createDirectoryWithMode(File path, int mode)
+        throws IOException {
+      createDirectoryWithMode0(path.getAbsolutePath(), mode);
+    }
+
+    /** Wrapper around CreateDirectory() on Windows */
+    private static native void createDirectoryWithMode0(String path, int mode)
+        throws NativeIOException;
+
     /** Wrapper around CreateFile() on Windows */
     public static native FileDescriptor createFile(String path,
         long desiredAccess, long shareMode, long creationDisposition)
         throws IOException;
 
+    /**
+     * Create a file for write with permissions set to the specified mode.  By
+     * setting permissions at creation time, we avoid issues related to the user
+     * lacking WRITE_DAC rights on subsequent chmod calls.  One example where
+     * this can occur is writing to an SMB share where the user does not have
+     * Full Control rights, and therefore WRITE_DAC is denied.
+     *
+     * This method mimics the semantics implemented by the JDK in
+     * {@link java.io.FileOutputStream}.  The file is opened for truncate or
+     * append, the sharing mode allows other readers and writers, and paths
+     * longer than MAX_PATH are supported.  (See io_util_md.c in the JDK.)
+     *
+     * @param path file to create
+     * @param append if true, then open file for append
+     * @param mode permissions of new directory
+     * @return FileOutputStream of opened file
+     * @throws IOException if there is an I/O error
+     */
+    public static FileOutputStream createFileOutputStreamWithMode(File path,
+        boolean append, int mode) throws IOException {
+      long desiredAccess = GENERIC_WRITE;
+      long shareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
+      long creationDisposition = append ? OPEN_ALWAYS : CREATE_ALWAYS;
+      return new FileOutputStream(createFileWithMode0(path.getAbsolutePath(),
+          desiredAccess, shareMode, creationDisposition, mode));
+    }
+
+    /** Wrapper around CreateFile() with security descriptor on Windows */
+    private static native FileDescriptor createFileWithMode0(String path,
+        long desiredAccess, long shareMode, long creationDisposition, int mode)
+        throws NativeIOException;
+
     /** Wrapper around SetFilePointer() on Windows */
     public static native long setFilePointer(FileDescriptor fd,
         long distanceToMove, long moveMethod) throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 45a46603fac06..bdcb96c74a67a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -88,7 +88,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
-import org.htrace.Trace;
+import org.apache.htrace.Trace;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -849,6 +849,12 @@ private void handleConnectionFailure(int curRetries, IOException ioe
         throw ioe;
       }
 
+      // Throw the exception if the thread is interrupted
+      if (Thread.currentThread().isInterrupted()) {
+        LOG.warn("Interrupted while trying for connection");
+        throw ioe;
+      }
+
       try {
         Thread.sleep(action.delayMillis);
       } catch (InterruptedException e) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index ddda55cb404cc..e75de1581fe7c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -49,9 +49,8 @@
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.ProtoUtil;
 import org.apache.hadoop.util.Time;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingService;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 40f6515e4a04d..8ada0fff9801f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -412,11 +412,18 @@ public static <T> ProtocolProxy<T> waitForProtocolProxy(Class<T> protocol,
         throw ioe;
       }
 
+      if (Thread.currentThread().isInterrupted()) {
+        // interrupted during some IO; this may not have been caught
+        throw new InterruptedIOException("Interrupted waiting for the proxy");
+      }
+
       // wait for retry
       try {
         Thread.sleep(1000);
       } catch (InterruptedException ie) {
-        // IGNORE
+        Thread.currentThread().interrupt();
+        throw (IOException) new InterruptedIOException(
+            "Interrupted waiting for the proxy").initCause(ioe);
       }
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
index c457500e902b5..d5e795b92f17c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 
 @InterfaceAudience.Private
@@ -53,7 +54,8 @@ private RpcConstants() {
   /**
    * The first four bytes of Hadoop RPC connections
    */
-  public static final ByteBuffer HEADER = ByteBuffer.wrap("hrpc".getBytes());
+  public static final ByteBuffer HEADER = ByteBuffer.wrap("hrpc".getBytes
+      (Charsets.UTF_8));
   public static final int HEADER_LEN_AFTER_HRPC_PART = 3; // 3 bytes that follow
   
   // 1 : Introduce ping and server does not throw away RPCs
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 021e03537b4e5..475fb1101dc88 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -69,6 +69,7 @@
 import javax.security.sasl.SaslException;
 import javax.security.sasl.SaslServer;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -116,10 +117,10 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceInfo;
-import org.htrace.TraceScope;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceInfo;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.ByteString;
@@ -182,7 +183,7 @@ boolean isTerse(Class<?> t) {
    * and send back a nicer response.
    */
   private static final ByteBuffer HTTP_GET_BYTES = ByteBuffer.wrap(
-      "GET ".getBytes());
+      "GET ".getBytes(Charsets.UTF_8));
   
   /**
    * An HTTP response to send back if we detect an HTTP request to our IPC
@@ -665,7 +666,7 @@ void shutdown() {
         assert !running;
         readSelector.wakeup();
         try {
-          join();
+          super.join();
         } catch (InterruptedException ie) {
           Thread.currentThread().interrupt();
         }
@@ -748,6 +749,13 @@ void doAccept(SelectionKey key) throws InterruptedException, IOException,  OutOf
         
         Reader reader = getReader();
         Connection c = connectionManager.register(channel);
+        // If the connectionManager can't take it, close the connection.
+        if (c == null) {
+          if (channel.isOpen()) {
+            IOUtils.cleanup(null, channel);
+          }
+          continue;
+        }
         key.attach(c);  // so closeCurrentConnection can get the object
         reader.addConnection(c);
       }
@@ -1119,7 +1127,8 @@ public class Connection {
     private ByteBuffer data;
     private ByteBuffer dataLengthBuffer;
     private LinkedList<Call> responseQueue;
-    private volatile int rpcCount = 0; // number of outstanding rpcs
+    // number of outstanding rpcs
+    private AtomicInteger rpcCount = new AtomicInteger();
     private long lastContact;
     private int dataLength;
     private Socket socket;
@@ -1207,17 +1216,17 @@ public long getLastContact() {
 
     /* Return true if the connection has no outstanding rpc */
     private boolean isIdle() {
-      return rpcCount == 0;
+      return rpcCount.get() == 0;
     }
     
     /* Decrement the outstanding RPC count */
     private void decRpcCount() {
-      rpcCount--;
+      rpcCount.decrementAndGet();
     }
     
     /* Increment the outstanding RPC count */
     private void incRpcCount() {
-      rpcCount++;
+      rpcCount.incrementAndGet();
     }
     
     private UserGroupInformation getAuthorizedUgi(String authorizedId)
@@ -1708,7 +1717,7 @@ private void setupBadVersionResponse(int clientVersion) throws IOException {
     private void setupHttpRequestOnIpcPortResponse() throws IOException {
       Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null, this);
       fakeCall.setResponse(ByteBuffer.wrap(
-          RECEIVED_HTTP_REQ_RESPONSE.getBytes()));
+          RECEIVED_HTTP_REQ_RESPONSE.getBytes(Charsets.UTF_8)));
       responder.doRespond(fakeCall);
     }
 
@@ -2068,9 +2077,9 @@ private synchronized void close() {
         LOG.debug("Ignoring socket shutdown exception", e);
       }
       if (channel.isOpen()) {
-        try {channel.close();} catch(Exception e) {}
+        IOUtils.cleanup(null, channel);
       }
-      try {socket.close();} catch(Exception e) {}
+      IOUtils.cleanup(null, socket);
     }
   }
 
@@ -2729,6 +2738,7 @@ private class ConnectionManager {
     final private int idleScanInterval;
     final private int maxIdleTime;
     final private int maxIdleToClose;
+    final private int maxConnections;
     
     ConnectionManager() {
       this.idleScanTimer = new Timer(
@@ -2745,6 +2755,9 @@ private class ConnectionManager {
       this.maxIdleToClose = conf.getInt(
           CommonConfigurationKeysPublic.IPC_CLIENT_KILL_MAX_KEY,
           CommonConfigurationKeysPublic.IPC_CLIENT_KILL_MAX_DEFAULT);
+      this.maxConnections = conf.getInt(
+          CommonConfigurationKeysPublic.IPC_SERVER_MAX_CONNECTIONS_KEY,
+          CommonConfigurationKeysPublic.IPC_SERVER_MAX_CONNECTIONS_DEFAULT);
       // create a set with concurrency -and- a thread-safe iterator, add 2
       // for listener and idle closer threads
       this.connections = Collections.newSetFromMap(
@@ -2772,11 +2785,19 @@ int size() {
       return count.get();
     }
 
+    boolean isFull() {
+      // The check is disabled when maxConnections <= 0.
+      return ((maxConnections > 0) && (size() >= maxConnections));
+    }
+
     Connection[] toArray() {
       return connections.toArray(new Connection[0]);
     }
 
     Connection register(SocketChannel channel) {
+      if (isFull()) {
+        return null;
+      }
       Connection connection = new Connection(channel, Time.now());
       add(connection);
       if (LOG.isDebugEnabled()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index 94a6b1d593bf4..cea881c25da43 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -42,8 +42,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.*;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 /** An RpcEngine implementation for Writable data. */
 @InterfaceStability.Evolving
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
index f775dd71beb32..9ade62f27ab9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
@@ -17,12 +17,11 @@
 
 package org.apache.hadoop.jmx;
 
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.management.ManagementFactory;
-import java.lang.reflect.Array;
-import java.util.Iterator;
-import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.http.HttpServer2;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
 
 import javax.management.AttributeNotFoundException;
 import javax.management.InstanceNotFoundException;
@@ -43,12 +42,12 @@
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpServer2;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Array;
+import java.util.Iterator;
+import java.util.Set;
 
 /*
  * This servlet is based off of the JMXProxyServlet from Tomcat 7.0.14. It has
@@ -114,16 +113,16 @@
  *  
  *  The bean's name and modelerType will be returned for all beans.
  *
- *  Optional paramater "callback" should be used to deliver JSONP response.
- *  
  */
 public class JMXJsonServlet extends HttpServlet {
   private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
+  static final String ACCESS_CONTROL_ALLOW_METHODS =
+      "Access-Control-Allow-Methods";
+  static final String ACCESS_CONTROL_ALLOW_ORIGIN =
+      "Access-Control-Allow-Origin";
 
   private static final long serialVersionUID = 1L;
 
-  private static final String CALLBACK_PARAM = "callback";
-
   /**
    * MBean server.
    */
@@ -164,19 +163,13 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) {
         return;
       }
       JsonGenerator jg = null;
-      String jsonpcb = null;
       PrintWriter writer = null;
       try {
         writer = response.getWriter();
  
-        // "callback" parameter implies JSONP outpout
-        jsonpcb = request.getParameter(CALLBACK_PARAM);
-        if (jsonpcb != null) {
-          response.setContentType("application/javascript; charset=utf8");
-          writer.write(jsonpcb + "(");
-        } else {
-          response.setContentType("application/json; charset=utf8");
-        }
+        response.setContentType("application/json; charset=utf8");
+        response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, "GET");
+        response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 
         jg = jsonFactory.createJsonGenerator(writer);
         jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
@@ -209,9 +202,6 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) {
         if (jg != null) {
           jg.close();
         }
-        if (jsonpcb != null) {
-           writer.write(");");
-        }
         if (writer != null) {
           writer.close();
         }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
index 77f74cc404911..4749ce19a65f8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -24,6 +24,7 @@
 import javax.servlet.*;
 import javax.servlet.http.*;
 
+import com.google.common.base.Charsets;
 import org.apache.commons.logging.*;
 import org.apache.commons.logging.impl.*;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -66,7 +67,7 @@ private static void process(String urlstring) {
       connection.connect();
 
       BufferedReader in = new BufferedReader(new InputStreamReader(
-          connection.getInputStream()));
+          connection.getInputStream(), Charsets.UTF_8));
       for(String line; (line = in.readLine()) != null; )
         if (line.startsWith(MARKER)) {
           System.out.println(TAG.matcher(line).replaceAll(""));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/file/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/file/FileContext.java
deleted file mode 100644
index fcbe7c4828750..0000000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/file/FileContext.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * FileContext.java
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics.file;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-
-/**
- * Metrics context for writing metrics to a file.<p/>
- *
- * This class is configured by setting ContextFactory attributes which in turn
- * are usually configured through a properties file.  All the attributes are
- * prefixed by the contextName. For example, the properties file might contain:
- * <pre>
- * myContextName.fileName=/tmp/metrics.log
- * myContextName.period=5
- * </pre>
- * @see org.apache.hadoop.metrics2.sink.FileSink for metrics 2.0.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-@Deprecated
-public class FileContext extends AbstractMetricsContext {
-    
-  /* Configuration attribute names */
-  @InterfaceAudience.Private
-  protected static final String FILE_NAME_PROPERTY = "fileName";
-  @InterfaceAudience.Private
-  protected static final String PERIOD_PROPERTY = "period";
-    
-  private File file = null;              // file for metrics to be written to
-  private PrintWriter writer = null;
-    
-  /** Creates a new instance of FileContext */
-  @InterfaceAudience.Private
-  public FileContext() {}
-    
-  @Override
-  @InterfaceAudience.Private
-  public void init(String contextName, ContextFactory factory) {
-    super.init(contextName, factory);
-        
-    String fileName = getAttribute(FILE_NAME_PROPERTY);
-    if (fileName != null) {
-      file = new File(fileName);
-    }
-        
-    parseAndSetPeriod(PERIOD_PROPERTY);
-  }
-
-  /**
-   * Returns the configured file name, or null.
-   */
-  @InterfaceAudience.Private
-  public String getFileName() {
-    if (file == null) {
-      return null;
-    } else {
-      return file.getName();
-    }
-  }
-    
-  /**
-   * Starts or restarts monitoring, by opening in append-mode, the
-   * file specified by the <code>fileName</code> attribute,
-   * if specified. Otherwise the data will be written to standard
-   * output.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void startMonitoring()
-    throws IOException 
-  {
-    if (file == null) {
-      writer = new PrintWriter(new BufferedOutputStream(System.out));
-    } else {
-      writer = new PrintWriter(new FileWriter(file, true));
-    }
-    super.startMonitoring();
-  }
-    
-  /**
-   * Stops monitoring, closing the file.
-   * @see #close()
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void stopMonitoring() {
-    super.stopMonitoring();
-        
-    if (writer != null) {
-      writer.close();
-      writer = null;
-    }
-  }
-    
-  /**
-   * Emits a metrics record to a file.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void emitRecord(String contextName, String recordName, OutputRecord outRec) {
-    writer.print(contextName);
-    writer.print(".");
-    writer.print(recordName);
-    String separator = ": ";
-    for (String tagName : outRec.getTagNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(tagName);
-      writer.print("=");
-      writer.print(outRec.getTag(tagName));
-    }
-    for (String metricName : outRec.getMetricNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(metricName);
-      writer.print("=");
-      writer.print(outRec.getMetric(metricName));
-    }
-    writer.println();
-  }
-    
-  /**
-   * Flushes the output writer, forcing updates to disk.
-   */
-  @Override
-  @InterfaceAudience.Private
-  public void flush() {
-    writer.flush();
-  }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
index 841874fc08e34..5ed2652b11c0f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
@@ -21,14 +21,12 @@
 package org.apache.hadoop.metrics.ganglia;
 
 import java.io.IOException;
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.SocketAddress;
-import java.net.SocketException;
+import java.net.*;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -53,13 +51,16 @@ public class GangliaContext extends AbstractMetricsContext {
   private static final String SLOPE_PROPERTY = "slope";
   private static final String TMAX_PROPERTY = "tmax";
   private static final String DMAX_PROPERTY = "dmax";
-    
+  private static final String MULTICAST_PROPERTY = "multicast";
+  private static final String MULTICAST_TTL_PROPERTY = "multicast.ttl";
+
   private static final String DEFAULT_UNITS = "";
   private static final String DEFAULT_SLOPE = "both";
   private static final int DEFAULT_TMAX = 60;
   private static final int DEFAULT_DMAX = 0;
   private static final int DEFAULT_PORT = 8649;
   private static final int BUFFER_SIZE = 1500;       // as per libgmond.c
+  private static final int DEFAULT_MULTICAST_TTL = 1;
 
   private final Log LOG = LogFactory.getLog(this.getClass());    
 
@@ -82,6 +83,8 @@ public class GangliaContext extends AbstractMetricsContext {
   private Map<String,String> slopeTable;
   private Map<String,String> tmaxTable;
   private Map<String,String> dmaxTable;
+  private boolean multicastEnabled;
+  private int multicastTtl;
     
   protected DatagramSocket datagramSocket;
     
@@ -103,11 +106,26 @@ public void init(String contextName, ContextFactory factory) {
     slopeTable = getAttributeTable(SLOPE_PROPERTY);
     tmaxTable  = getAttributeTable(TMAX_PROPERTY);
     dmaxTable  = getAttributeTable(DMAX_PROPERTY);
+    multicastEnabled = Boolean.parseBoolean(getAttribute(MULTICAST_PROPERTY));
+    String multicastTtlValue = getAttribute(MULTICAST_TTL_PROPERTY);
+    if (multicastEnabled) {
+      if (multicastTtlValue == null) {
+        multicastTtl = DEFAULT_MULTICAST_TTL;
+      } else {
+        multicastTtl = Integer.parseInt(multicastTtlValue);
+      }
+    }
         
     try {
-      datagramSocket = new DatagramSocket();
-    } catch (SocketException se) {
-      se.printStackTrace();
+      if (multicastEnabled) {
+        LOG.info("Enabling multicast for Ganglia with TTL " + multicastTtl);
+        datagramSocket = new MulticastSocket();
+        ((MulticastSocket) datagramSocket).setTimeToLive(multicastTtl);
+      } else {
+        datagramSocket = new DatagramSocket();
+      }
+    } catch (IOException e) {
+      LOG.error(e);
     }
   }
 
@@ -225,7 +243,7 @@ protected int getDmax(String metricName) {
    * a multiple of 4.
    */
   protected void xdr_string(String s) {
-    byte[] bytes = s.getBytes();
+    byte[] bytes = s.getBytes(Charsets.UTF_8);
     int len = bytes.length;
     xdr_int(len);
     System.arraycopy(bytes, 0, buffer, offset, len);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext31.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext31.java
index 39509f0c89a81..f35ad1816f2b5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext31.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext31.java
@@ -86,11 +86,6 @@ protected void emitMetric(String name, String type,  String value)
       value + " from hostname" + hostName);
 
     String units = getUnits(name);
-    if (units == null) {
-      LOG.warn("Metric name " + name + ", value " + value
-        + " had 'null' units");
-      units = "";
-    }
     int slope = getSlope(name);
     int tmax = getTmax(name);
     int dmax = getDmax(name);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/package.html b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/package.html
index 87598e503325c..b9acfaea69976 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/package.html
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/package.html
@@ -54,6 +54,12 @@
     <dt><i>contextName</i>.period</dt>
     <dd>The period in seconds on which the metric data is sent to the
     server(s).</dd>
+
+    <dt><i>contextName</i>.multicast</dt>
+    <dd>Enable multicast for Ganglia</dd>
+
+    <dt><i>contextName</i>.multicast.ttl</dt>
+    <dd>TTL for multicast packets</dd>
     
     <dt><i>contextName</i>.units.<i>recordName</i>.<i>metricName</i></dt>
     <dd>The units for the specified metric in the specified record.</dd>
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/CompositeContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/CompositeContext.java
index 60f5fec44aa65..ad494040de7e6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/CompositeContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/spi/CompositeContext.java
@@ -55,7 +55,7 @@ public void init(String contextName, ContextFactory factory) {
     int nKids;
     try {
       String sKids = getAttribute(ARITY_LABEL);
-      nKids = Integer.valueOf(sKids);
+      nKids = Integer.parseInt(sKids);
     } catch (Exception e) {
       LOG.error("Unable to initialize composite metric " + contextName +
                 ": could not init arity", e);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java
index be442edb1b85b..5345c1baf88fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java
@@ -69,7 +69,8 @@ public Iterator<MetricsRecordBuilderImpl> iterator() {
     return rbs.iterator();
   }
 
-  void clear() { rbs.clear(); }
+  @InterfaceAudience.Private
+  public void clear() { rbs.clear(); }
 
   MetricsCollectorImpl setRecordFilter(MetricsFilter rf) {
     recordFilter = rf;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
index e4b5580536bb5..167205e93e3c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsConfig.java
@@ -269,14 +269,14 @@ public String toString() {
 
   static String toString(Configuration c) {
     ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-    PrintStream ps = new PrintStream(buffer);
-    PropertiesConfiguration tmp = new PropertiesConfiguration();
-    tmp.copy(c);
     try {
+      PrintStream ps = new PrintStream(buffer, false, "UTF-8");
+      PropertiesConfiguration tmp = new PropertiesConfiguration();
+      tmp.copy(c);
       tmp.save(ps);
+      return buffer.toString("UTF-8");
     } catch (Exception e) {
       throw new MetricsConfigException(e);
     }
-    return buffer.toString();
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index baa75ab8cfdab..32b00f3fdfa37 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -384,7 +384,7 @@ synchronized void onTimerEvent() {
    * Requests an immediate publish of all metrics from sources to sinks.
    */
   @Override
-  public void publishMetricsNow() {
+  public synchronized void publishMetricsNow() {
     if (sinks.size() > 0) {
       publishMetrics(sampleMetrics(), true);
     }    
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
index 1c0d30e234f84..4b561f2fb5649 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java
@@ -363,6 +363,20 @@ Collection<MutableMetric> metrics() {
   }
 
   private void checkMetricName(String name) {
+    // Check for invalid characters in metric name
+    boolean foundWhitespace = false;
+    for (int i = 0; i < name.length(); i++) {
+      char c = name.charAt(i);
+      if (Character.isWhitespace(c)) {
+        foundWhitespace = true;
+        break;
+      }
+    }
+    if (foundWhitespace) {
+      throw new MetricsException("Metric name '"+ name +
+          "' contains illegal whitespace character");
+    }
+    // Check if name has already been registered
     if (metricsMap.containsKey(name)) {
       throw new MetricsException("Metric name "+ name +" already exists!");
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java
index b0fb0d4709f0f..77139e2e96fda 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java
@@ -23,23 +23,24 @@
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * A mutable int counter for implementing metrics sources
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class MutableCounterInt extends MutableCounter {
-  private volatile int value;
+  private AtomicInteger value = new AtomicInteger();
 
   MutableCounterInt(MetricsInfo info, int initValue) {
     super(info);
-    this.value = initValue;
+    this.value.set(initValue);
   }
 
   @Override
-  public synchronized void incr() {
-    ++value;
-    setChanged();
+  public void incr() {
+    incr(1);
   }
 
   /**
@@ -47,18 +48,18 @@ public synchronized void incr() {
    * @param delta of the increment
    */
   public synchronized void incr(int delta) {
-    value += delta;
+    value.addAndGet(delta);
     setChanged();
   }
 
   public int value() {
-    return value;
+    return value.get();
   }
 
   @Override
   public void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
-      builder.addCounter(info(), value);
+      builder.addCounter(info(), value());
       clearChanged();
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
index 43ea4906bbafc..03a6043375f1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java
@@ -23,6 +23,8 @@
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * A mutable long counter
  */
@@ -30,36 +32,35 @@
 @InterfaceStability.Evolving
 public class MutableCounterLong extends MutableCounter {
 
-  private volatile long value;
+  private AtomicLong value = new AtomicLong();
 
   MutableCounterLong(MetricsInfo info, long initValue) {
     super(info);
-    this.value = initValue;
+    this.value.set(initValue);
   }
 
   @Override
-  public synchronized void incr() {
-    ++value;
-    setChanged();
+  public void incr() {
+    incr(1);
   }
 
   /**
    * Increment the value by a delta
    * @param delta of the increment
    */
-  public synchronized void incr(long delta) {
-    value += delta;
+  public void incr(long delta) {
+    value.addAndGet(delta);
     setChanged();
   }
 
   public long value() {
-    return value;
+    return value.get();
   }
 
   @Override
   public void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
-      builder.addCounter(info(), value);
+      builder.addCounter(info(), value());
       clearChanged();
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java
index 1a3a15cf6865c..cce45286ce675 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java
@@ -23,6 +23,8 @@
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * A mutable int gauge
  */
@@ -30,44 +32,42 @@
 @InterfaceStability.Evolving
 public class MutableGaugeInt extends MutableGauge {
 
-  private volatile int value;
+  private AtomicInteger value = new AtomicInteger();
 
   MutableGaugeInt(MetricsInfo info, int initValue) {
     super(info);
-    this.value = initValue;
+    this.value.set(initValue);
   }
 
   public int value() {
-    return value;
+    return value.get();
   }
 
   @Override
-  public synchronized void incr() {
-    ++value;
-    setChanged();
+  public void incr() {
+    incr(1);
   }
 
   /**
    * Increment by delta
    * @param delta of the increment
    */
-  public synchronized void incr(int delta) {
-    value += delta;
+  public void incr(int delta) {
+    value.addAndGet(delta);
     setChanged();
   }
 
   @Override
-  public synchronized void decr() {
-    --value;
-    setChanged();
+  public void decr() {
+    decr(1);
   }
 
   /**
    * decrement by delta
    * @param delta of the decrement
    */
-  public synchronized void decr(int delta) {
-    value -= delta;
+  public void decr(int delta) {
+    value.addAndGet(-delta);
     setChanged();
   }
 
@@ -76,14 +76,14 @@ public synchronized void decr(int delta) {
    * @param value to set
    */
   public void set(int value) {
-    this.value = value;
+    this.value.set(value);
     setChanged();
   }
 
   @Override
   public void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
-      builder.addGauge(info(), value);
+      builder.addGauge(info(), value());
       clearChanged();
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java
index 69df4304e9b7b..a2a8632ad3c38 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java
@@ -23,6 +23,8 @@
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * A mutable long gauge
  */
@@ -30,44 +32,42 @@
 @InterfaceStability.Evolving
 public class MutableGaugeLong extends MutableGauge {
 
-  private volatile long value;
+  private AtomicLong value = new AtomicLong();
 
   MutableGaugeLong(MetricsInfo info, long initValue) {
     super(info);
-    this.value = initValue;
+    this.value.set(initValue);
   }
 
   public long value() {
-    return value;
+    return value.get();
   }
 
   @Override
-  public synchronized void incr() {
-    ++value;
-    setChanged();
+  public void incr() {
+    incr(1);
   }
 
   /**
    * Increment by delta
    * @param delta of the increment
    */
-  public synchronized void incr(long delta) {
-    value += delta;
+  public void incr(long delta) {
+    value.addAndGet(delta);
     setChanged();
   }
 
   @Override
-  public synchronized void decr() {
-    --value;
-    setChanged();
+  public void decr() {
+    decr(1);
   }
 
   /**
    * decrement by delta
    * @param delta of the decrement
    */
-  public synchronized void decr(long delta) {
-    value -= delta;
+  public void decr(long delta) {
+    value.addAndGet(-delta);
     setChanged();
   }
 
@@ -76,14 +76,14 @@ public synchronized void decr(long delta) {
    * @param value to set
    */
   public void set(long value) {
-    this.value = value;
+    this.value.set(value);
     setChanged();
   }
 
   @Override
   public void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
-      builder.addGauge(info(), value);
+      builder.addGauge(info(), value());
       clearChanged();
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/FileSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/FileSink.java
index d1364160e2d67..ab121bcf67f6b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/FileSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/FileSink.java
@@ -20,9 +20,9 @@
 
 import java.io.Closeable;
 import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 
 import org.apache.commons.configuration.SubsetConfiguration;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -40,15 +40,15 @@
 @InterfaceStability.Evolving
 public class FileSink implements MetricsSink, Closeable {
   private static final String FILENAME_KEY = "filename";
-  private PrintWriter writer;
+  private PrintStream writer;
 
   @Override
   public void init(SubsetConfiguration conf) {
     String filename = conf.getString(FILENAME_KEY);
     try {
-      writer = filename == null
-          ? new PrintWriter(System.out)
-          : new PrintWriter(new FileWriter(new File(filename), true));
+      writer = filename == null ? System.out
+          : new PrintStream(new FileOutputStream(new File(filename)),
+                            true, "UTF-8");
     } catch (Exception e) {
       throw new MetricsException("Error creating "+ filename, e);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
index 9bc3f15d97e91..e46a654e823e8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
@@ -18,24 +18,24 @@
 
 package org.apache.hadoop.metrics2.sink;
 
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.io.Closeable;
-import java.net.Socket;
-
 import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.metrics2.AbstractMetric;
 import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsRecord;
 import org.apache.hadoop.metrics2.MetricsSink;
 import org.apache.hadoop.metrics2.MetricsTag;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.Socket;
+
 /**
  * A metrics sink that writes to a Graphite server
  */
@@ -46,29 +46,22 @@ public class GraphiteSink implements MetricsSink, Closeable {
     private static final String SERVER_HOST_KEY = "server_host";
     private static final String SERVER_PORT_KEY = "server_port";
     private static final String METRICS_PREFIX = "metrics_prefix";
-    private Writer writer = null;
     private String metricsPrefix = null;
-    private Socket socket = null;
+    private Graphite graphite = null;
 
     @Override
     public void init(SubsetConfiguration conf) {
         // Get Graphite host configurations.
-        String serverHost = conf.getString(SERVER_HOST_KEY);
-        Integer serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY));
+        final String serverHost = conf.getString(SERVER_HOST_KEY);
+        final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY));
 
         // Get Graphite metrics graph prefix.
         metricsPrefix = conf.getString(METRICS_PREFIX);
         if (metricsPrefix == null)
             metricsPrefix = "";
 
-        try {
-            // Open an connection to Graphite server.
-            socket = new Socket(serverHost, serverPort);
-            writer = new OutputStreamWriter(socket.getOutputStream());
-        } catch (Exception e) {
-            throw new MetricsException("Error creating connection, "
-                    + serverHost + ":" + serverPort, e);
-        }
+        graphite = new Graphite(serverHost, serverPort);
+        graphite.connect();
     }
 
     @Override
@@ -102,39 +95,111 @@ public void putMetrics(MetricsRecord record) {
         }
 
         try {
-            if(writer != null){
-              writer.write(lines.toString());
-            } else {
-              throw new MetricsException("Writer in GraphiteSink is null!");
-            }
+          graphite.write(lines.toString());
         } catch (Exception e) {
-            throw new MetricsException("Error sending metrics", e);
+          LOG.warn("Error sending metrics to Graphite", e);
+          try {
+            graphite.close();
+          } catch (Exception e1) {
+            throw new MetricsException("Error closing connection to Graphite", e1);
+          }
         }
     }
 
     @Override
     public void flush() {
+      try {
+        graphite.flush();
+      } catch (Exception e) {
+        LOG.warn("Error flushing metrics to Graphite", e);
         try {
-            writer.flush();
-        } catch (Exception e) {
-            throw new MetricsException("Error flushing metrics", e);
+          graphite.close();
+        } catch (Exception e1) {
+          throw new MetricsException("Error closing connection to Graphite", e1);
         }
+      }
     }
 
     @Override
     public void close() throws IOException {
-      try {
-        IOUtils.closeStream(writer);
-        writer = null;
-        LOG.info("writer in GraphiteSink is closed!");
-      } catch (Throwable e){
-        throw new MetricsException("Error closing writer", e);
-      } finally {
-        if (socket != null && !socket.isClosed()) {
-          socket.close();
+      graphite.close();
+    }
+
+    public static class Graphite {
+      private final static int MAX_CONNECTION_FAILURES = 5;
+
+      private String serverHost;
+      private int serverPort;
+      private Writer writer = null;
+      private Socket socket = null;
+      private int connectionFailures = 0;
+
+      public Graphite(String serverHost, int serverPort) {
+        this.serverHost = serverHost;
+        this.serverPort = serverPort;
+      }
+
+      public void connect() {
+        if (isConnected()) {
+          throw new MetricsException("Already connected to Graphite");
+        }
+        if (tooManyConnectionFailures()) {
+          // return silently (there was ERROR in logs when we reached limit for the first time)
+          return;
+        }
+        try {
+          // Open a connection to Graphite server.
+          socket = new Socket(serverHost, serverPort);
+          writer = new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8);
+        } catch (Exception e) {
+          connectionFailures++;
+          if (tooManyConnectionFailures()) {
+            // first time when connection limit reached, report to logs
+            LOG.error("Too many connection failures, would not try to connect again.");
+          }
+          throw new MetricsException("Error creating connection, "
+              + serverHost + ":" + serverPort, e);
+        }
+      }
+
+      public void write(String msg) throws IOException {
+        if (!isConnected()) {
+          connect();
+        }
+        if (isConnected()) {
+          writer.write(msg);
+        }
+      }
+
+      public void flush() throws IOException {
+        if (isConnected()) {
+          writer.flush();
+        }
+      }
+
+      public boolean isConnected() {
+        return socket != null && socket.isConnected() && !socket.isClosed();
+      }
+
+      public void close() throws IOException {
+        try {
+          if (writer != null) {
+            writer.close();
+          }
+        } catch (IOException ex) {
+          if (socket != null) {
+            socket.close();
+          }
+        } finally {
           socket = null;
-          LOG.info("socket in GraphiteSink is closed!");
+          writer = null;
         }
       }
+
+      private boolean tooManyConnectionFailures() {
+        return connectionFailures > MAX_CONNECTION_FAILURES;
+      }
+
     }
+
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
index b3581f9e8d9bd..c9df0ffcc3be8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
@@ -19,16 +19,13 @@
 package org.apache.hadoop.metrics2.sink.ganglia;
 
 import java.io.IOException;
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.SocketAddress;
-import java.net.SocketException;
-import java.net.UnknownHostException;
+import java.net.*;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.metrics2.MetricsSink;
@@ -62,7 +59,11 @@ public abstract class AbstractGangliaSink implements MetricsSink {
   public static final int DEFAULT_DMAX = 0;
   public static final GangliaSlope DEFAULT_SLOPE = GangliaSlope.both;
   public static final int DEFAULT_PORT = 8649;
+  public static final boolean DEFAULT_MULTICAST_ENABLED = false;
+  public static final int DEFAULT_MULTICAST_TTL = 1;
   public static final String SERVERS_PROPERTY = "servers";
+  public static final String MULTICAST_ENABLED_PROPERTY = "multicast";
+  public static final String MULTICAST_TTL_PROPERTY = "multicast.ttl";
   public static final int BUFFER_SIZE = 1500; // as per libgmond.c
   public static final String SUPPORT_SPARSE_METRICS_PROPERTY = "supportsparse";
   public static final boolean SUPPORT_SPARSE_METRICS_DEFAULT = false;
@@ -71,6 +72,8 @@ public abstract class AbstractGangliaSink implements MetricsSink {
   private String hostName = "UNKNOWN.example.com";
   private DatagramSocket datagramSocket;
   private List<? extends SocketAddress> metricsServers;
+  private boolean multicastEnabled;
+  private int multicastTtl;
   private byte[] buffer = new byte[BUFFER_SIZE];
   private int offset;
   private boolean supportSparseMetrics = SUPPORT_SPARSE_METRICS_DEFAULT;
@@ -132,6 +135,9 @@ public void init(SubsetConfiguration conf) {
     // load the gannglia servers from properties
     metricsServers = Servers.parse(conf.getString(SERVERS_PROPERTY),
         DEFAULT_PORT);
+    multicastEnabled = conf.getBoolean(MULTICAST_ENABLED_PROPERTY,
+            DEFAULT_MULTICAST_ENABLED);
+    multicastTtl = conf.getInt(MULTICAST_TTL_PROPERTY, DEFAULT_MULTICAST_TTL);
 
     // extract the Ganglia conf per metrics
     gangliaConfMap = new HashMap<String, GangliaConf>();
@@ -141,9 +147,15 @@ public void init(SubsetConfiguration conf) {
     loadGangliaConf(GangliaConfType.slope);
 
     try {
-      datagramSocket = new DatagramSocket();
-    } catch (SocketException se) {
-      LOG.error(se);
+      if (multicastEnabled) {
+        LOG.info("Enabling multicast for Ganglia with TTL " + multicastTtl);
+        datagramSocket = new MulticastSocket();
+        ((MulticastSocket) datagramSocket).setTimeToLive(multicastTtl);
+      } else {
+        datagramSocket = new DatagramSocket();
+      }
+    } catch (IOException e) {
+      LOG.error(e);
     }
 
     // see if sparseMetrics is supported. Default is false
@@ -223,7 +235,7 @@ protected String getHostName() {
    * @param s the string to be written to buffer at offset location
    */
   protected void xdr_string(String s) {
-    byte[] bytes = s.getBytes();
+    byte[] bytes = s.getBytes(Charsets.UTF_8);
     int len = bytes.length;
     xdr_int(len);
     System.arraycopy(bytes, 0, buffer, offset, len);
@@ -256,6 +268,12 @@ protected void xdr_int(int i) {
   protected void emitToGangliaHosts() throws IOException {
     try {
       for (SocketAddress socketAddress : metricsServers) {
+        if (socketAddress == null || !(socketAddress instanceof InetSocketAddress))
+          throw new IllegalArgumentException("Unsupported Address type");
+        InetSocketAddress inetAddress = (InetSocketAddress)socketAddress;
+        if(inetAddress.isUnresolved()) {
+          throw new UnknownHostException("Unresolved host: " + inetAddress);
+        }
         DatagramPacket packet =
           new DatagramPacket(buffer, offset, socketAddress);
         datagramSocket.send(packet);
@@ -287,4 +305,12 @@ protected boolean isSupportSparseMetrics() {
   void setDatagramSocket(DatagramSocket datagramSocket) {
     this.datagramSocket = datagramSocket;
   }
+
+  /**
+   * Used only by unit tests
+   * @return the datagramSocket for this sink
+   */
+  DatagramSocket getDatagramSocket() {
+    return datagramSocket;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index b535ddaee2f7a..ef1092bb2f757 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -287,8 +287,8 @@ private static String canonicalizeHost(String host) {
     if (fqHost == null) {
       try {
         fqHost = SecurityUtil.getByName(host).getHostName();
-        // slight race condition, but won't hurt 
-        canonicalizedHostCache.put(host, fqHost);
+        // slight race condition, but won't hurt
+        canonicalizedHostCache.putIfAbsent(host, fqHost);
       } catch (UnknownHostException e) {
         fqHost = host;
       }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
index 8a0a0033fd2d6..086650bd7d31a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
@@ -171,8 +171,7 @@ public List<String> getDependency(String name) {
 
     @Override
     public String toString() {
-      return super.toString() + ", " + dependencyScriptName != null ?
-          ("dependency script " + dependencyScriptName) : NO_SCRIPT;
+      return "dependency script " + dependencyScriptName;
     }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
index ed12b3c6be027..b50f7e936ba9b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
@@ -338,6 +338,12 @@ int select(SelectableChannel channel, int ops, long timeout)
             return ret;
           }
           
+          if (Thread.currentThread().isInterrupted()) {
+            throw new InterruptedIOException("Interrupted while waiting for "
+                + "IO on channel " + channel + ". " + timeout
+                + " millis timeout left.");
+          }
+
           /* Sometimes select() returns 0 much before timeout for 
            * unknown reasons. So select again if required.
            */
@@ -348,12 +354,6 @@ int select(SelectableChannel channel, int ops, long timeout)
             }
           }
           
-          if (Thread.currentThread().isInterrupted()) {
-            throw new InterruptedIOException("Interruped while waiting for " +
-                                             "IO on channel " + channel +
-                                             ". " + timeout + 
-                                             " millis timeout left.");
-          }
         }
       } finally {
         if (key != null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
index 2662108124d60..59c0ca96750a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
@@ -20,13 +20,16 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY;
 
 import java.io.BufferedReader;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -96,9 +99,10 @@ private Map<String, String> load() {
         return null;
       }
   
-      BufferedReader reader = null;
-      try {
-        reader = new BufferedReader(new FileReader(filename));
+
+      try (BufferedReader reader =
+               new BufferedReader(new InputStreamReader(
+                   new FileInputStream(filename), Charsets.UTF_8))) {
         String line = reader.readLine();
         while (line != null) {
           line = line.trim();
@@ -115,15 +119,6 @@ private Map<String, String> load() {
       } catch (Exception e) {
         LOG.warn(filename + " cannot be read.", e);
         return null;
-      } finally {
-        if (reader != null) {
-          try {
-            reader.close();
-          } catch (IOException e) {
-            LOG.warn(filename + " cannot be read.", e);
-            return null;
-          }
-        }
       }
       return loadMap;
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
index 4fb9e45614133..43d1b66d44f6d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.security;
 
+import com.google.common.base.Charsets;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.conf.Configuration;
@@ -24,8 +25,10 @@
 import org.apache.hadoop.http.FilterInitializer;
 import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.util.HashMap;
 import java.util.Map;
@@ -78,10 +81,10 @@ public void initFilter(FilterContainer container, Configuration conf) {
     if (signatureSecretFile == null) {
       throw new RuntimeException("Undefined property: " + SIGNATURE_SECRET_FILE);      
     }
-    
-    try {
-      StringBuilder secret = new StringBuilder();
-      Reader reader = new FileReader(signatureSecretFile);
+
+    StringBuilder secret = new StringBuilder();
+    try (Reader reader = new InputStreamReader(
+        new FileInputStream(signatureSecretFile), Charsets.UTF_8)) {
       int c = reader.read();
       while (c > -1) {
         secret.append((char)c);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index b81e810f191c7..e6b8722c3558c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -217,7 +218,8 @@ public void readTokenStorageStream(DataInputStream in) throws IOException {
     readFields(in);
   }
   
-  private static final byte[] TOKEN_STORAGE_MAGIC = "HDTS".getBytes();
+  private static final byte[] TOKEN_STORAGE_MAGIC =
+      "HDTS".getBytes(Charsets.UTF_8);
   private static final byte TOKEN_STORAGE_VERSION = 0;
   
   public void writeTokenStorageToStream(DataOutputStream os)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index c5004197e5638..9fd39b09abac0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,8 +23,16 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Ticker;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -52,21 +60,21 @@ public class Groups {
   private static final Log LOG = LogFactory.getLog(Groups.class);
   
   private final GroupMappingServiceProvider impl;
-  
-  private final Map<String, CachedGroups> userToGroupsMap = 
-    new ConcurrentHashMap<String, CachedGroups>();
-  private final Map<String, List<String>> staticUserToGroupsMap = 
+
+  private final LoadingCache<String, List<String>> cache;
+  private final Map<String, List<String>> staticUserToGroupsMap =
       new HashMap<String, List<String>>();
   private final long cacheTimeout;
   private final long negativeCacheTimeout;
   private final long warningDeltaMs;
   private final Timer timer;
+  private Set<String> negativeCache;
 
   public Groups(Configuration conf) {
     this(conf, new Timer());
   }
 
-  public Groups(Configuration conf, Timer timer) {
+  public Groups(Configuration conf, final Timer timer) {
     impl = 
       ReflectionUtils.newInstance(
           conf.getClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, 
@@ -86,12 +94,30 @@ public Groups(Configuration conf, Timer timer) {
     parseStaticMapping(conf);
 
     this.timer = timer;
+    this.cache = CacheBuilder.newBuilder()
+      .refreshAfterWrite(cacheTimeout, TimeUnit.MILLISECONDS)
+      .ticker(new TimerToTickerAdapter(timer))
+      .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
+      .build(new GroupCacheLoader());
+
+    if(negativeCacheTimeout > 0) {
+      Cache<String, Boolean> tempMap = CacheBuilder.newBuilder()
+        .expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+        .ticker(new TimerToTickerAdapter(timer))
+        .build();
+      negativeCache = Collections.newSetFromMap(tempMap.asMap());
+    }
 
     if(LOG.isDebugEnabled())
       LOG.debug("Group mapping impl=" + impl.getClass().getName() + 
           "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" +
           warningDeltaMs);
   }
+  
+  @VisibleForTesting
+  Set<String> getNegativeCache() {
+    return negativeCache;
+  }
 
   /*
    * Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -123,78 +149,106 @@ private void parseStaticMapping(Configuration conf) {
     }
   }
 
-  /**
-   * Determine whether the CachedGroups is expired.
-   * @param groups cached groups for one user.
-   * @return true if groups is expired from useToGroupsMap.
-   */
-  private boolean hasExpired(CachedGroups groups, long startMs) {
-    if (groups == null) {
-      return true;
-    }
-    long timeout = cacheTimeout;
-    if (isNegativeCacheEnabled() && groups.getGroups().isEmpty()) {
-      // This CachedGroups is in the negative cache, thus it should expire
-      // sooner.
-      timeout = negativeCacheTimeout;
-    }
-    return groups.getTimestamp() + timeout <= startMs;
-  }
-  
   private boolean isNegativeCacheEnabled() {
     return negativeCacheTimeout > 0;
   }
 
+  private IOException noGroupsForUser(String user) {
+    return new IOException("No groups found for user " + user);
+  }
+
   /**
    * Get the group memberships of a given user.
+   * If the user's group is not cached, this method may block.
    * @param user User's name
    * @return the group memberships of the user
-   * @throws IOException
+   * @throws IOException if user does not exist
    */
-  public List<String> getGroups(String user) throws IOException {
+  public List<String> getGroups(final String user) throws IOException {
     // No need to lookup for groups of static users
     List<String> staticMapping = staticUserToGroupsMap.get(user);
     if (staticMapping != null) {
       return staticMapping;
     }
-    // Return cached value if available
-    CachedGroups groups = userToGroupsMap.get(user);
-    long startMs = timer.monotonicNow();
-    if (!hasExpired(groups, startMs)) {
-      if(LOG.isDebugEnabled()) {
-        LOG.debug("Returning cached groups for '" + user + "'");
-      }
-      if (groups.getGroups().isEmpty()) {
-        // Even with enabling negative cache, getGroups() has the same behavior
-        // that throws IOException if the groups for the user is empty.
-        throw new IOException("No groups found for user " + user);
+
+    // Check the negative cache first
+    if (isNegativeCacheEnabled()) {
+      if (negativeCache.contains(user)) {
+        throw noGroupsForUser(user);
       }
-      return groups.getGroups();
     }
 
-    // Create and cache user's groups
-    List<String> groupList = impl.getGroups(user);
-    long endMs = timer.monotonicNow();
-    long deltaMs = endMs - startMs ;
-    UserGroupInformation.metrics.addGetGroups(deltaMs);
-    if (deltaMs > warningDeltaMs) {
-      LOG.warn("Potential performance problem: getGroups(user=" + user +") " +
-          "took " + deltaMs + " milliseconds.");
+    try {
+      return cache.get(user);
+    } catch (ExecutionException e) {
+      throw (IOException)e.getCause();
     }
-    groups = new CachedGroups(groupList, endMs);
-    if (groups.getGroups().isEmpty()) {
-      if (isNegativeCacheEnabled()) {
-        userToGroupsMap.put(user, groups);
+  }
+
+  /**
+   * Convert millisecond times from hadoop's timer to guava's nanosecond ticker.
+   */
+  private static class TimerToTickerAdapter extends Ticker {
+    private Timer timer;
+
+    public TimerToTickerAdapter(Timer timer) {
+      this.timer = timer;
+    }
+
+    @Override
+    public long read() {
+      final long NANOSECONDS_PER_MS = 1000000;
+      return timer.monotonicNow() * NANOSECONDS_PER_MS;
+    }
+  }
+
+  /**
+   * Deals with loading data into the cache.
+   */
+  private class GroupCacheLoader extends CacheLoader<String, List<String>> {
+    /**
+     * This method will block if a cache entry doesn't exist, and
+     * any subsequent requests for the same user will wait on this
+     * request to return. If a user already exists in the cache,
+     * this will be run in the background.
+     * @param user key of cache
+     * @return List of groups belonging to user
+     * @throws IOException to prevent caching negative entries
+     */
+    @Override
+    public List<String> load(String user) throws Exception {
+      List<String> groups = fetchGroupList(user);
+
+      if (groups.isEmpty()) {
+        if (isNegativeCacheEnabled()) {
+          negativeCache.add(user);
+        }
+
+        // We throw here to prevent Cache from retaining an empty group
+        throw noGroupsForUser(user);
       }
-      throw new IOException("No groups found for user " + user);
+
+      return groups;
     }
-    userToGroupsMap.put(user, groups);
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Returning fetched groups for '" + user + "'");
+
+    /**
+     * Queries impl for groups belonging to the user. This could involve I/O and take awhile.
+     */
+    private List<String> fetchGroupList(String user) throws IOException {
+      long startMs = timer.monotonicNow();
+      List<String> groupList = impl.getGroups(user);
+      long endMs = timer.monotonicNow();
+      long deltaMs = endMs - startMs ;
+      UserGroupInformation.metrics.addGetGroups(deltaMs);
+      if (deltaMs > warningDeltaMs) {
+        LOG.warn("Potential performance problem: getGroups(user=" + user +") " +
+          "took " + deltaMs + " milliseconds.");
+      }
+
+      return groupList;
     }
-    return groups.getGroups();
   }
-  
+
   /**
    * Refresh all user-to-groups mappings.
    */
@@ -205,7 +259,10 @@ public void refresh() {
     } catch (IOException e) {
       LOG.warn("Error refreshing groups cache", e);
     }
-    userToGroupsMap.clear();
+    cache.invalidateAll();
+    if(isNegativeCacheEnabled()) {
+      negativeCache.clear();
+    }
   }
 
   /**
@@ -221,40 +278,6 @@ public void cacheGroupsAdd(List<String> groups) {
     }
   }
 
-  /**
-   * Class to hold the cached groups
-   */
-  private static class CachedGroups {
-    final long timestamp;
-    final List<String> groups;
-    
-    /**
-     * Create and initialize group cache
-     */
-    CachedGroups(List<String> groups, long timestamp) {
-      this.groups = groups;
-      this.timestamp = timestamp;
-    }
-
-    /**
-     * Returns time of last cache update
-     *
-     * @return time of last cache update
-     */
-    public long getTimestamp() {
-      return timestamp;
-    }
-
-    /**
-     * Get list of cached groups
-     *
-     * @return cached groups
-     */
-    public List<String> getGroups() {
-      return groups;
-    }
-  }
-
   private static Groups GROUPS = null;
   
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 76f53804b33ff..d463ac7a2be3f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -17,8 +17,10 @@
  */
 package org.apache.hadoop.security;
 
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Hashtable;
@@ -34,6 +36,7 @@
 import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -338,6 +341,8 @@ public synchronized void setConf(Configuration conf) {
 
     int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
     SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
+    // Limit the attributes returned to only those required to speed up the search. See HADOOP-10626 for more details.
+    SEARCH_CONTROLS.setReturningAttributes(new String[] {groupNameAttr});
 
     this.conf = conf;
   }
@@ -366,11 +371,10 @@ String extractPassword(String pwFile) {
       // an anonymous bind
       return "";
     }
-    
-    Reader reader = null;
-    try {
-      StringBuilder password = new StringBuilder();
-      reader = new FileReader(pwFile);
+
+    StringBuilder password = new StringBuilder();
+    try (Reader reader = new InputStreamReader(
+        new FileInputStream(pwFile), Charsets.UTF_8)) {
       int c = reader.read();
       while (c > -1) {
         password.append((char)c);
@@ -379,8 +383,6 @@ String extractPassword(String pwFile) {
       return password.toString().trim();
     } catch (IOException ioe) {
       throw new RuntimeException("Could not read password file: " + pwFile, ioe);
-    } finally {
-      IOUtils.cleanup(LOG, reader);
     }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
index bd9c448da7f62..4495a66c4322f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
@@ -17,11 +17,11 @@
  */
 package org.apache.hadoop.security;
 
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.HashSet;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -36,14 +36,9 @@
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Unstable
 public class NetgroupCache {
-  private static boolean netgroupToUsersMapUpdated = true;
-  private static Map<String, Set<String>> netgroupToUsersMap =
-    new ConcurrentHashMap<String, Set<String>>();
-
-  private static Map<String, Set<String>> userToNetgroupsMap =
+  private static ConcurrentHashMap<String, Set<String>> userToNetgroupsMap =
     new ConcurrentHashMap<String, Set<String>>();
 
-
   /**
    * Get netgroups for a given user
    *
@@ -52,21 +47,11 @@ public class NetgroupCache {
    */
   public static void getNetgroups(final String user,
       List<String> groups) {
-    if(netgroupToUsersMapUpdated) {
-      netgroupToUsersMapUpdated = false; // at the beginning to avoid race
-      //update userToNetgroupsMap
-      for(String netgroup : netgroupToUsersMap.keySet()) {
-        for(String netuser : netgroupToUsersMap.get(netgroup)) {
-          // add to userToNetgroupsMap
-          if(!userToNetgroupsMap.containsKey(netuser)) {
-            userToNetgroupsMap.put(netuser, new HashSet<String>());
-          }
-          userToNetgroupsMap.get(netuser).add(netgroup);
-        }
-      }
-    }
-    if(userToNetgroupsMap.containsKey(user)) {
-      groups.addAll(userToNetgroupsMap.get(user));
+    Set<String> userGroups = userToNetgroupsMap.get(user);
+    //ConcurrentHashMap does not allow null values; 
+    //So null value check can be used to check if the key exists
+    if (userGroups != null) {
+      groups.addAll(userGroups);
     }
   }
 
@@ -76,7 +61,15 @@ public static void getNetgroups(final String user,
    * @return list of cached groups
    */
   public static List<String> getNetgroupNames() {
-    return new LinkedList<String>(netgroupToUsersMap.keySet());
+    return new LinkedList<String>(getGroups());
+  }
+
+  private static Set<String> getGroups() {
+    Set<String> allGroups = new HashSet<String> ();
+    for (Set<String> userGroups : userToNetgroupsMap.values()) {
+      allGroups.addAll(userGroups);
+    }
+    return allGroups;
   }
 
   /**
@@ -86,14 +79,13 @@ public static List<String> getNetgroupNames() {
    * @return true if group is cached, false otherwise
    */
   public static boolean isCached(String group) {
-    return netgroupToUsersMap.containsKey(group);
+    return getGroups().contains(group);
   }
 
   /**
    * Clear the cache
    */
   public static void clear() {
-    netgroupToUsersMap.clear();
     userToNetgroupsMap.clear();
   }
 
@@ -104,7 +96,20 @@ public static void clear() {
    * @param users list of users for a given group
    */
   public static void add(String group, List<String> users) {
-    netgroupToUsersMap.put(group, new HashSet<String>(users));
-    netgroupToUsersMapUpdated = true; // at the end to avoid race
+    for (String user : users) {
+      Set<String> userGroups = userToNetgroupsMap.get(user);
+      // ConcurrentHashMap does not allow null values; 
+      // So null value check can be used to check if the key exists
+      if (userGroups == null) {
+        //Generate a ConcurrentHashSet (backed by the keyset of the ConcurrentHashMap)
+        userGroups =
+            Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+        Set<String> currentSet = userToNetgroupsMap.putIfAbsent(user, userGroups);
+        if (currentSet != null) {
+          userGroups = currentSet;
+        }
+      }
+      userGroups.add(group);
+    }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
index c4fc965a95664..0b49cfbc4462b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.security;
 
 import java.net.InetAddress;
+import java.util.Locale;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -65,7 +66,7 @@ public void setConf(Configuration conf) {
         CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION,
         QualityOfProtection.AUTHENTICATION.toString());
     for (int i=0; i < qop.length; i++) {
-      qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase()).getSaslQop();
+      qop[i] = QualityOfProtection.valueOf(qop[i].toUpperCase(Locale.ENGLISH)).getSaslQop();
     }
     properties.put(Sasl.QOP, StringUtils.join(",", qop));
     properties.put(Sasl.SERVER_AUTH, "true");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index dfb0898a44923..4a1a397ed6f45 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -573,17 +573,15 @@ public int read(byte b[]) throws IOException {
     }
 
     @Override
-    public int read(byte[] buf, int off, int len) throws IOException {
-      synchronized(unwrappedRpcBuffer) {
-        // fill the buffer with the next RPC message
-        if (unwrappedRpcBuffer.remaining() == 0) {
-          readNextRpcPacket();
-        }
-        // satisfy as much of the request as possible
-        int readLen = Math.min(len, unwrappedRpcBuffer.remaining());
-        unwrappedRpcBuffer.get(buf, off, readLen);
-        return readLen;
+    public synchronized int read(byte[] buf, int off, int len) throws IOException {
+      // fill the buffer with the next RPC message
+      if (unwrappedRpcBuffer.remaining() == 0) {
+        readNextRpcPacket();
       }
+      // satisfy as much of the request as possible
+      int readLen = Math.min(len, unwrappedRpcBuffer.remaining());
+      unwrappedRpcBuffer.get(buf, off, readLen);
+      return readLen;
     }
     
     // all messages must be RPC SASL wrapped, else an exception is thrown
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
index 83f46efd6e456..f2b21e851bbc2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
@@ -44,6 +44,7 @@
 import javax.security.sasl.SaslServerFactory;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -184,11 +185,11 @@ public static void init(Configuration conf) {
   }
   
   static String encodeIdentifier(byte[] identifier) {
-    return new String(Base64.encodeBase64(identifier));
+    return new String(Base64.encodeBase64(identifier), Charsets.UTF_8);
   }
 
   static byte[] decodeIdentifier(String identifier) {
-    return Base64.decodeBase64(identifier.getBytes());
+    return Base64.decodeBase64(identifier.getBytes(Charsets.UTF_8));
   }
 
   public static <T extends TokenIdentifier> T getIdentifier(String id,
@@ -206,7 +207,8 @@ public static <T extends TokenIdentifier> T getIdentifier(String id,
   }
 
   static char[] encodePassword(byte[] password) {
-    return new String(Base64.encodeBase64(password)).toCharArray();
+    return new String(Base64.encodeBase64(password),
+                      Charsets.UTF_8).toCharArray();
   }
 
   /** Splitting fully qualified Kerberos name into parts */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 768294d707b18..fd362d038d3d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -22,11 +22,13 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -73,6 +75,10 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
 
   private final File staticMappingFile;
   private StaticMapping staticMapping = null;
+  // Last time the static map was modified, measured time difference in
+  // milliseconds since midnight, January 1, 1970 UTC
+  private long lastModificationTimeStaticMap = 0;
+  
   private boolean constructFullMapAtInit = false;
 
   // Used for parsing the static mapping file.
@@ -86,7 +92,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   // Maps for id to name map. Guarded by this object monitor lock
   private BiMap<Integer, String> uidNameMap = HashBiMap.create();
   private BiMap<Integer, String> gidNameMap = HashBiMap.create();
-
   private long lastUpdateTime = 0; // Last time maps were updated
 
   /*
@@ -116,7 +121,7 @@ public ShellBasedIdMapping(Configuration conf,
         conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
             IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
     staticMappingFile = new File(staticFilePath);
-
+    updateStaticMapping();
     updateMaps();
   }
 
@@ -181,7 +186,7 @@ private static void reportDuplicateEntry(final String header,
       final Integer key, final String value,
       final Integer ekey, final String evalue) {    
       LOG.warn("\n" + header + String.format(
-          "new entry (%d, %s), existing entry: (%d, %s).\n%s\n%s",
+          "new entry (%d, %s), existing entry: (%d, %s).%n%s%n%s",
           key, value, ekey, evalue,
           "The new entry is to be ignored for the following reason.",
           DUPLICATE_NAME_ID_DEBUG_INFO));
@@ -217,7 +222,9 @@ public static boolean updateMapInternal(BiMap<Integer, String> map,
     try {
       Process process = Runtime.getRuntime().exec(
           new String[] { "bash", "-c", command });
-      br = new BufferedReader(new InputStreamReader(process.getInputStream()));
+      br = new BufferedReader(
+          new InputStreamReader(process.getInputStream(),
+                                Charset.defaultCharset()));
       String line = null;
       while ((line = br.readLine()) != null) {
         String[] nameId = line.split(regex);
@@ -286,20 +293,42 @@ private static boolean isInteger(final String s) {
     return true;
   }
 
-  private void initStaticMapping() throws IOException {
-    staticMapping = new StaticMapping(
-        new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
+  private synchronized void updateStaticMapping() throws IOException {
+    final boolean init = (staticMapping == null);
+    //
+    // if the static mapping file
+    //   - was modified after last update, load the map again;
+    //   - did not exist but was added since last update, load the map;
+    //   - existed before but deleted since last update, clear the map
+    //
     if (staticMappingFile.exists()) {
-      LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
-      staticMapping = parseStaticMap(staticMappingFile);
+      // check modification time, reload the file if the last modification
+      // time changed since prior load.
+      long lmTime = staticMappingFile.lastModified();
+      if (lmTime != lastModificationTimeStaticMap) {
+        LOG.info(init? "Using " : "Reloading " + "'" + staticMappingFile
+            + "' for static UID/GID mapping...");
+        lastModificationTimeStaticMap = lmTime;
+        staticMapping = parseStaticMap(staticMappingFile);        
+      }
     } else {
-      LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
-          + "' does not exist.");
+      if (init) {
+        staticMapping = new StaticMapping(new HashMap<Integer, Integer>(),
+            new HashMap<Integer, Integer>());
+      }
+      if (lastModificationTimeStaticMap != 0 || init) {
+        // print the following log at initialization or when the static
+        // mapping file was deleted after prior load
+        LOG.info("Not doing static UID/GID mapping because '"
+            + staticMappingFile + "' does not exist.");
+      }
+      lastModificationTimeStaticMap = 0;
+      staticMapping.clear();
     }
-  }  
+  }
 
   /*
-   * Reset the maps to empty.
+   * Refresh static map, and reset the other maps to empty.
    * For testing code, a full map may be re-constructed here when the object
    * was created with constructFullMapAtInit being set to true.
    */
@@ -310,15 +339,16 @@ synchronized public void updateMaps() throws IOException {
 
     if (constructFullMapAtInit) {
       loadFullMaps();
+      // set constructFullMapAtInit to false to allow testing code to
+      // do incremental update to maps after initial construction
+      constructFullMapAtInit = false;
     } else {
+      updateStaticMapping();
       clearNameMaps();
     }
   }
   
   synchronized private void loadFullUserMap() throws IOException {
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
     BiMap<Integer, String> uMap = HashBiMap.create();
     if (OS.startsWith("Mac")) {
       updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
@@ -332,9 +362,6 @@ synchronized private void loadFullUserMap() throws IOException {
   }
 
   synchronized private void loadFullGroupMap() throws IOException {
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
     BiMap<Integer, String> gMap = HashBiMap.create();
 
     if (OS.startsWith("Mac")) {
@@ -349,7 +376,6 @@ synchronized private void loadFullGroupMap() throws IOException {
   }
 
   synchronized private void loadFullMaps() throws IOException {
-    initStaticMapping();
     loadFullUserMap();
     loadFullGroupMap();
   }
@@ -439,9 +465,7 @@ synchronized private void updateMapIncr(final String name,
     }
 
     boolean updated = false;
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
+    updateStaticMapping();
 
     if (OS.startsWith("Linux")) {
       if (isGrp) {
@@ -477,9 +501,7 @@ synchronized private void updateMapIncr(final int id,
     }
     
     boolean updated = false;
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
+    updateStaticMapping();
 
     if (OS.startsWith("Linux")) {
       if (isGrp) {
@@ -543,6 +565,15 @@ public StaticMapping(Map<Integer, Integer> uidMapping,
       this.uidMapping = new PassThroughMap<Integer>(uidMapping);
       this.gidMapping = new PassThroughMap<Integer>(gidMapping);
     }
+
+    public void clear() {
+      uidMapping.clear();
+      gidMapping.clear();
+    }
+
+    public boolean isNonEmpty() {
+      return uidMapping.size() > 0 || gidMapping.size() > 0;
+    }
   }
   
   static StaticMapping parseStaticMap(File staticMapFile)
@@ -552,7 +583,7 @@ static StaticMapping parseStaticMap(File staticMapFile)
     Map<Integer, Integer> gidMapping = new HashMap<Integer, Integer>();
     
     BufferedReader in = new BufferedReader(new InputStreamReader(
-        new FileInputStream(staticMapFile)));
+        new FileInputStream(staticMapFile), Charsets.UTF_8));
     
     try {
       String line = null;
@@ -574,8 +605,8 @@ static StaticMapping parseStaticMap(File staticMapFile)
         // We know the line is fine to parse without error checking like this
         // since it matched the regex above.
         String firstComponent = lineMatcher.group(1);
-        int remoteId = Integer.parseInt(lineMatcher.group(2));
-        int localId = Integer.parseInt(lineMatcher.group(3));
+        int remoteId = parseId(lineMatcher.group(2));
+        int localId = parseId(lineMatcher.group(3));
         if (firstComponent.equals("uid")) {
           uidMapping.put(localId, remoteId);
         } else {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 0541f9d9cd00d..4b0b5f305fec1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -44,9 +44,9 @@
 
 import javax.security.auth.Subject;
 import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.kerberos.KerberosKey;
 import javax.security.auth.kerberos.KerberosPrincipal;
 import javax.security.auth.kerberos.KerberosTicket;
+import javax.security.auth.kerberos.KeyTab;
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
 import javax.security.auth.login.LoginContext;
@@ -610,20 +610,6 @@ private void setLogin(LoginContext login) {
     user.setLogin(login);
   }
 
-  private static Class<?> KEY_TAB_CLASS = KerberosKey.class;
-  static {
-    try {
-      // We use KEY_TAB_CLASS to determine if the UGI is logged in from
-      // keytab. In JDK6 and JDK7, if useKeyTab and storeKey are specified
-      // in the Krb5LoginModule, then some number of KerberosKey objects
-      // are added to the Subject's private credentials. However, in JDK8,
-      // a KeyTab object is added instead. More details in HADOOP-10786.
-      KEY_TAB_CLASS = Class.forName("javax.security.auth.kerberos.KeyTab");
-    } catch (ClassNotFoundException cnfe) {
-      // Ignore. javax.security.auth.kerberos.KeyTab does not exist in JDK6.
-    }
-  }
-
   /**
    * Create a UserGroupInformation for the given subject.
    * This does not change the subject or acquire new credentials.
@@ -632,7 +618,7 @@ private void setLogin(LoginContext login) {
   UserGroupInformation(Subject subject) {
     this.subject = subject;
     this.user = subject.getPrincipals(User.class).iterator().next();
-    this.isKeytab = !subject.getPrivateCredentials(KEY_TAB_CLASS).isEmpty();
+    this.isKeytab = !subject.getPrivateCredentials(KeyTab.class).isEmpty();
     this.isKrbTkt = !subject.getPrivateCredentials(KerberosTicket.class).isEmpty();
   }
   
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index 6d9c6af26310c..f39740309ea4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -44,7 +44,7 @@ public class CredentialShell extends Configured implements Tool {
       "   [" + DeleteCommand.USAGE + "]\n" +
       "   [" + ListCommand.USAGE + "]\n";
 
-  private boolean interactive = false;
+  private boolean interactive = true;
   private Command command = null;
 
   /** allows stdout to be captured if necessary */
@@ -116,8 +116,8 @@ protected int init(String[] args) throws IOException {
         userSuppliedProvider = true;
         getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, 
             args[++i]);
-      } else if (args[i].equals("-i") || (args[i].equals("-interactive"))) {
-        interactive = true;
+      } else if (args[i].equals("-f") || (args[i].equals("-force"))) {
+        interactive = false;
       } else if (args[i].equals("-v") || (args[i].equals("-value"))) {
         value = args[++i];
       } else if (args[i].equals("-help")) {
@@ -236,11 +236,13 @@ public String getUsage() {
   }
 
   private class DeleteCommand extends Command {
-    public static final String USAGE = "delete <alias> [-provider] [-help]";
+    public static final String USAGE =
+        "delete <alias> [-provider] [-f] [-help]";
     public static final String DESC =
-        "The delete subcommand deletes the credenital\n" +
+        "The delete subcommand deletes the credential\n" +
         "specified as the <alias> argument from within the provider\n" +
-        "indicated through the -provider argument";
+        "indicated through the -provider argument. The command asks for\n" +
+        "confirmation unless the -f option is specified.";
 
     String alias = null;
     boolean cont = true;
@@ -267,9 +269,9 @@ public boolean validate() {
       if (interactive) {
         try {
           cont = ToolRunner
-              .confirmPrompt("You are about to DELETE the credential: " + 
+              .confirmPrompt("You are about to DELETE the credential " +
                   alias + " from CredentialProvider " + provider.toString() +
-                  ". Continue?:");
+                  ". Continue? ");
           if (!cont) {
             out.println("Nothing has been deleted.");
           }
@@ -293,7 +295,7 @@ public void execute() throws IOException {
           provider.flush();
           printProviderWritten();
         } catch (IOException e) {
-          out.println(alias + "has NOT been deleted.");
+          out.println(alias + " has NOT been deleted.");
           throw e;
         }
       }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 5dc2abfd13ff6..05958a058a3c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.security.alias;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -165,7 +166,7 @@ public CredentialEntry getCredentialEntry(String alias) throws IOException {
   }
   
   public static char[] bytesToChars(byte[] bytes) {
-    String pass = new String(bytes);
+    String pass = new String(bytes, Charsets.UTF_8);
     return pass.toCharArray();
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java
index 262cbadd71ae7..127ccf005d8ca 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
@@ -56,7 +57,8 @@ public synchronized CredentialEntry getCredentialEntry(String alias) {
     if (bytes == null) {
       return null;
     }
-    return new CredentialEntry(alias, new String(bytes).toCharArray());
+    return new CredentialEntry(
+        alias, new String(bytes, Charsets.UTF_8).toCharArray());
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
index 272538a90fd57..5d2951686006e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.MachineList;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -44,6 +45,7 @@
 @InterfaceStability.Evolving
 public class ServiceAuthorizationManager {
   static final String BLOCKED = ".blocked";
+  static final String HOSTS = ".hosts";
 
   private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml";
 
@@ -51,6 +53,10 @@ public class ServiceAuthorizationManager {
   // and second ACL specifies blocked entries.
   private volatile Map<Class<?>, AccessControlList[]> protocolToAcls =
     new IdentityHashMap<Class<?>, AccessControlList[]>();
+  // For each class, first MachineList in the array specifies the allowed entries
+  // and second MachineList specifies blocked entries.
+  private volatile Map<Class<?>, MachineList[]> protocolToMachineLists =
+    new IdentityHashMap<Class<?>, MachineList[]>();
   
   /**
    * Configuration key for controlling service-level authorization for Hadoop.
@@ -85,7 +91,8 @@ public void authorize(UserGroupInformation user,
                                InetAddress addr
                                ) throws AuthorizationException {
     AccessControlList[] acls = protocolToAcls.get(protocol);
-    if (acls == null) {
+    MachineList[] hosts = protocolToMachineLists.get(protocol);
+    if (acls == null || hosts == null) {
       throw new AuthorizationException("Protocol " + protocol + 
                                        " is not known.");
     }
@@ -115,6 +122,16 @@ public void authorize(UserGroupInformation user,
           " is not authorized for protocol " + protocol + 
           ", expected client Kerberos principal is " + clientPrincipal);
     }
+    if (addr != null) {
+      String hostAddress = addr.getHostAddress();
+      if (hosts.length != 2 || !hosts[0].includes(hostAddress) ||
+          hosts[1].includes(hostAddress)) {
+        AUDITLOG.warn(AUTHZ_FAILED_FOR + " for protocol=" + protocol
+            + " from host = " +  hostAddress);
+        throw new AuthorizationException("Host " + hostAddress +
+            " is not authorized for protocol " + protocol) ;
+      }
+    }
     AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol);
   }
 
@@ -135,6 +152,8 @@ public void refreshWithLoadedConfiguration(Configuration conf,
       PolicyProvider provider) {
     final Map<Class<?>, AccessControlList[]> newAcls =
       new IdentityHashMap<Class<?>, AccessControlList[]>();
+    final Map<Class<?>, MachineList[]> newMachineLists =
+      new IdentityHashMap<Class<?>, MachineList[]>();
     
     String defaultAcl = conf.get(
         CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL,
@@ -143,6 +162,13 @@ public void refreshWithLoadedConfiguration(Configuration conf,
     String defaultBlockedAcl = conf.get(
       CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL, "");
 
+    String defaultServiceHostsKey = getHostKey(
+      CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL);
+    String defaultMachineList = conf.get(defaultServiceHostsKey,
+      MachineList.WILDCARD_VALUE);
+    String defaultBlockedMachineList= conf.get(
+     defaultServiceHostsKey+ BLOCKED, "");
+
     // Parse the config file
     Service[] services = provider.getServices();
     if (services != null) {
@@ -157,11 +183,26 @@ public void refreshWithLoadedConfiguration(Configuration conf,
            conf.get(service.getServiceKey() + BLOCKED,
            defaultBlockedAcl));
         newAcls.put(service.getProtocol(), new AccessControlList[] {acl, blockedAcl});
+        String serviceHostsKey = getHostKey(service.getServiceKey());
+        MachineList machineList = new MachineList (conf.get(serviceHostsKey, defaultMachineList));
+        MachineList blockedMachineList = new MachineList(
+          conf.get(serviceHostsKey + BLOCKED, defaultBlockedMachineList));
+        newMachineLists.put(service.getProtocol(),
+            new MachineList[] {machineList, blockedMachineList});
       }
     }
 
     // Flip to the newly parsed permissions
     protocolToAcls = newAcls;
+    protocolToMachineLists = newMachineLists;
+  }
+
+  private String getHostKey(String serviceKey) {
+    int endIndex = serviceKey.lastIndexOf(".");
+    if (endIndex != -1) {
+      return serviceKey.substring(0, endIndex)+ HOSTS;
+    }
+    return serviceKey;
   }
 
   @VisibleForTesting
@@ -178,4 +219,19 @@ public AccessControlList getProtocolsAcls(Class<?> className) {
   public AccessControlList getProtocolsBlockedAcls(Class<?> className) {
     return protocolToAcls.get(className)[1];
   }
+
+  @VisibleForTesting
+  public Set<Class<?>> getProtocolsWithMachineLists() {
+    return protocolToMachineLists.keySet();
+  }
+
+  @VisibleForTesting
+  public MachineList getProtocolsMachineList(Class<?> className) {
+    return protocolToMachineLists.get(className)[0];
+  }
+
+  @VisibleForTesting
+  public MachineList getProtocolsBlockedMachineList(Class<?> className) {
+    return protocolToMachineLists.get(className)[1];
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index d6bc99535a986..ec522dcff89bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -45,6 +45,7 @@
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
 import org.apache.curator.framework.recipes.shared.SharedCount;
+import org.apache.curator.framework.recipes.shared.VersionedValue;
 import org.apache.curator.retry.RetryNTimes;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -58,7 +59,6 @@
 import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
-import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -109,10 +109,10 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
       "ZKDelegationTokenSecretManagerClient";
 
   private static final String ZK_DTSM_NAMESPACE = "ZKDTSMRoot";
-  private static final String ZK_DTSM_SEQNUM_ROOT = "ZKDTSMSeqNumRoot";
-  private static final String ZK_DTSM_KEYID_ROOT = "ZKDTSMKeyIdRoot";
-  private static final String ZK_DTSM_TOKENS_ROOT = "ZKDTSMTokensRoot";
-  private static final String ZK_DTSM_MASTER_KEY_ROOT = "ZKDTSMMasterKeyRoot";
+  private static final String ZK_DTSM_SEQNUM_ROOT = "/ZKDTSMSeqNumRoot";
+  private static final String ZK_DTSM_KEYID_ROOT = "/ZKDTSMKeyIdRoot";
+  private static final String ZK_DTSM_TOKENS_ROOT = "/ZKDTSMTokensRoot";
+  private static final String ZK_DTSM_MASTER_KEY_ROOT = "/ZKDTSMMasterKeyRoot";
 
   private static final String DELEGATION_KEY_PREFIX = "DK_";
   private static final String DELEGATION_TOKEN_PREFIX = "DT_";
@@ -505,11 +505,20 @@ protected int getDelegationTokenSeqNum() {
     return delTokSeqCounter.getCount();
   }
 
+  private void incrSharedCount(SharedCount sharedCount) throws Exception {
+    while (true) {
+      // Loop until we successfully increment the counter
+      VersionedValue<Integer> versionedValue = sharedCount.getVersionedValue();
+      if (sharedCount.trySetCount(versionedValue, versionedValue.getValue() + 1)) {
+        break;
+      }
+    }
+  }
+
   @Override
   protected int incrementDelegationTokenSeqNum() {
     try {
-      while (!delTokSeqCounter.trySetCount(delTokSeqCounter.getCount() + 1)) {
-      }
+      incrSharedCount(delTokSeqCounter);
     } catch (InterruptedException e) {
       // The ExpirationThread is just finishing.. so dont do anything..
       LOG.debug("Thread interrupted while performing token counter increment", e);
@@ -537,8 +546,7 @@ protected int getCurrentKeyId() {
   @Override
   protected int incrementCurrentKeyId() {
     try {
-      while (!keyIdSeqCounter.trySetCount(keyIdSeqCounter.getCount() + 1)) {
-      }
+      incrSharedCount(keyIdSeqCounter);
     } catch (InterruptedException e) {
       // The ExpirationThread is just finishing.. so dont do anything..
       LOG.debug("Thread interrupted while performing keyId increment", e);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
index 81993e9af2b81..01ba76d849a15 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
@@ -25,23 +25,24 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
+import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.ShutdownHookManager;
-import org.htrace.HTraceConfiguration;
-import org.htrace.SpanReceiver;
-import org.htrace.Trace;
+import org.apache.htrace.SpanReceiver;
+import org.apache.htrace.SpanReceiverBuilder;
+import org.apache.htrace.Trace;
 
 /**
  * This class provides functions for reading the names of SpanReceivers from
@@ -100,7 +101,8 @@ private static String getUniqueLocalTraceFileName() {
       // out of /proc/self/stat.  (There isn't any portable way to get the
       // process ID from Java.)
       reader = new BufferedReader(
-          new InputStreamReader(new FileInputStream("/proc/self/stat")));
+          new InputStreamReader(new FileInputStream("/proc/self/stat"),
+                                Charsets.UTF_8));
       String line = reader.readLine();
       if (line == null) {
         throw new EOFException();
@@ -154,60 +156,13 @@ public synchronized void loadSpanReceivers(Configuration conf) {
 
   private synchronized SpanReceiver loadInstance(String className,
       List<ConfigurationPair> extraConfig) throws IOException {
-    className = className.trim();
-    if (!className.contains(".")) {
-      className = "org.htrace.impl." + className;
-    }
-    Class<?> implClass = null;
-    SpanReceiver impl;
-    try {
-      implClass = Class.forName(className);
-      Object o = ReflectionUtils.newInstance(implClass, config);
-      impl = (SpanReceiver)o;
-      impl.configure(wrapHadoopConf(config, extraConfig));
-    } catch (ClassCastException e) {
-      throw new IOException("Class " + className +
-          " does not implement SpanReceiver.");
-    } catch (ClassNotFoundException e) {
-      throw new IOException("Class " + className + " cannot be found.");
-    } catch (SecurityException e) {
-      throw new IOException("Got SecurityException while loading " +
-          "SpanReceiver " + className);
-    } catch (IllegalArgumentException e) {
-      throw new IOException("Got IllegalArgumentException while loading " +
-          "SpanReceiver " + className, e);
-    } catch (RuntimeException e) {
-      throw new IOException("Got RuntimeException while loading " +
-          "SpanReceiver " + className, e);
-    }
-    return impl;
-  }
-
-  private static HTraceConfiguration wrapHadoopConf(final Configuration conf,
-          List<ConfigurationPair> extraConfig) {
-    final HashMap<String, String> extraMap = new HashMap<String, String>();
-    for (ConfigurationPair pair : extraConfig) {
-      extraMap.put(pair.getKey(), pair.getValue());
+    SpanReceiverBuilder builder =
+        new SpanReceiverBuilder(TraceUtils.wrapHadoopConf(config, extraConfig));
+    SpanReceiver rcvr = builder.spanReceiverClass(className.trim()).build();
+    if (rcvr == null) {
+      throw new IOException("Failed to load SpanReceiver " + className);
     }
-    return new HTraceConfiguration() {
-      public static final String HTRACE_CONF_PREFIX = "hadoop.htrace.";
-
-      @Override
-      public String get(String key) {
-        if (extraMap.containsKey(key)) {
-          return extraMap.get(key);
-        }
-        return conf.get(HTRACE_CONF_PREFIX + key);
-      }
-
-      @Override
-      public String get(String key, String defaultValue) {
-        if (extraMap.containsKey(key)) {
-          return extraMap.get(key);
-        }
-        return conf.get(HTRACE_CONF_PREFIX + key, defaultValue);
-      }
-    };
+    return rcvr;
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdmin.java
index 4ae5aedccfa6c..5fdfbfadd2d92 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdmin.java
@@ -25,6 +25,7 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -91,7 +92,7 @@ private int addSpanReceiver(List<String> args) throws IOException {
       return 1;
     }
     ByteArrayOutputStream configStream = new ByteArrayOutputStream();
-    PrintStream configsOut = new PrintStream(configStream);
+    PrintStream configsOut = new PrintStream(configStream, false, "UTF-8");
     SpanReceiverInfoBuilder factory = new SpanReceiverInfoBuilder(className);
     String prefix = "";
     for (int i = 0; i < args.size(); ++i) {
@@ -113,13 +114,15 @@ private int addSpanReceiver(List<String> args) throws IOException {
       configsOut.print(prefix + key + " = " + value);
       prefix = ", ";
     }
+
+    String configStreamStr = configStream.toString("UTF-8");
     try {
       long id = remote.addSpanReceiver(factory.build());
       System.out.println("Added trace span receiver " + id +
-          " with configuration " + configStream.toString());
+          " with configuration " + configStreamStr);
     } catch (IOException e) {
       System.out.println("addSpanReceiver error with configuration " +
-          configStream.toString());
+                             configStreamStr);
       throw e;
     }
     return 0;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceSamplerFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceSamplerFactory.java
deleted file mode 100644
index 54bcb81b2cda5..0000000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceSamplerFactory.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.tracing;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.htrace.Sampler;
-import org.htrace.impl.ProbabilitySampler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@InterfaceAudience.Private
-public class TraceSamplerFactory {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TraceSamplerFactory.class);
-
-  public static Sampler<?> createSampler(Configuration conf) {
-    String samplerStr = conf.get(CommonConfigurationKeys.HADOOP_TRACE_SAMPLER,
-        CommonConfigurationKeys.HADOOP_TRACE_SAMPLER_DEFAULT);
-    if (samplerStr.equals("NeverSampler")) {
-      LOG.debug("HTrace is OFF for all spans.");
-      return Sampler.NEVER;
-    } else if (samplerStr.equals("AlwaysSampler")) {
-      LOG.info("HTrace is ON for all spans.");
-      return Sampler.ALWAYS;
-    } else if (samplerStr.equals("ProbabilitySampler")) {
-      double percentage =
-          conf.getDouble("htrace.probability.sampler.percentage", 0.01d);
-      LOG.info("HTrace is ON for " + percentage + "% of top-level spans.");
-      return new ProbabilitySampler(percentage / 100.0d);
-    } else {
-      throw new RuntimeException("Can't create sampler " + samplerStr +
-          ".  Available samplers are NeverSampler, AlwaysSampler, " +
-          "and ProbabilitySampler.");
-    }
-  }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
new file mode 100644
index 0000000000000..11797e69b4af5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceUtils.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tracing;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
+import org.apache.htrace.HTraceConfiguration;
+
+/**
+ * This class provides utility functions for tracing.
+ */
+@InterfaceAudience.Private
+public class TraceUtils {
+  public static final String HTRACE_CONF_PREFIX = "hadoop.htrace.";
+  private static List<ConfigurationPair> EMPTY = Collections.emptyList();
+
+  public static HTraceConfiguration wrapHadoopConf(final Configuration conf) {
+    return wrapHadoopConf(conf, EMPTY);
+  }
+
+  public static HTraceConfiguration wrapHadoopConf(final Configuration conf,
+          List<ConfigurationPair> extraConfig) {
+    final HashMap<String, String> extraMap = new HashMap<String, String>();
+    for (ConfigurationPair pair : extraConfig) {
+      extraMap.put(pair.getKey(), pair.getValue());
+    }
+    return new HTraceConfiguration() {
+      @Override
+      public String get(String key) {
+        if (extraMap.containsKey(key)) {
+          return extraMap.get(key);
+        }
+        return conf.get(HTRACE_CONF_PREFIX + key, "");
+      }
+
+      @Override
+      public String get(String key, String defaultValue) {
+        if (extraMap.containsKey(key)) {
+          return extraMap.get(key);
+        }
+        return conf.get(HTRACE_CONF_PREFIX + key, defaultValue);
+      }
+    };
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index d2ab015567d86..6d37c28774e69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -67,10 +67,8 @@ public boolean accept(File dir, String name) {
   };
 
   static {
-    InputStream is = null;
-    try {
-      is = ApplicationClassLoader.class.getClassLoader().
-          getResourceAsStream(PROPERTIES_FILE);
+    try (InputStream is = ApplicationClassLoader.class.getClassLoader()
+        .getResourceAsStream(PROPERTIES_FILE);) {
       if (is == null) {
         throw new ExceptionInInitializerError("properties file " +
             PROPERTIES_FILE + " is not found");
@@ -216,28 +214,43 @@ protected synchronized Class<?> loadClass(String name, boolean resolve)
     return c;
   }
 
+  /**
+   * Checks if a class should be included as a system class.
+   *
+   * A class is a system class if and only if it matches one of the positive
+   * patterns and none of the negative ones.
+   *
+   * @param name the class name to check
+   * @param systemClasses a list of system class configurations.
+   * @return true if the class is a system class
+   */
   public static boolean isSystemClass(String name, List<String> systemClasses) {
+    boolean result = false;
     if (systemClasses != null) {
       String canonicalName = name.replace('/', '.');
       while (canonicalName.startsWith(".")) {
         canonicalName=canonicalName.substring(1);
       }
       for (String c : systemClasses) {
-        boolean result = true;
+        boolean shouldInclude = true;
         if (c.startsWith("-")) {
           c = c.substring(1);
-          result = false;
+          shouldInclude = false;
         }
         if (canonicalName.startsWith(c)) {
           if (   c.endsWith(".")                                   // package
               || canonicalName.length() == c.length()              // class
               ||    canonicalName.length() > c.length()            // nested
                  && canonicalName.charAt(c.length()) == '$' ) {
-            return result;
+            if (shouldInclude) {
+              result = true;
+            } else {
+              return false;
+            }
           }
         }
       }
     }
-    return false;
+    return result;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java
similarity index 83%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java
rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java
index 89a0db6eb47ee..84ddc32f88c1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ChunkedArrayList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ChunkedArrayList.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hdfs.util;
+package org.apache.hadoop.util;
 
 import java.util.AbstractList;
 import java.util.Iterator;
@@ -110,11 +110,33 @@ public ChunkedArrayList(int initialChunkCapacity, int maxChunkSize) {
 
   @Override
   public Iterator<T> iterator() {
-    return Iterables.concat(chunks).iterator();
+    final Iterator<T> it = Iterables.concat(chunks).iterator();
+
+    return new Iterator<T>() {
+      @Override
+      public boolean hasNext() {
+        return it.hasNext();
+      }
+
+      @Override
+      public T next() {
+        return it.next();
+      }
+
+      @Override
+      public void remove() {
+        it.remove();
+        size--;
+      }
+    };
   }
 
   @Override
   public boolean add(T e) {
+    if (size == Integer.MAX_VALUE) {
+      throw new RuntimeException("Can't add an additional element to the " +
+          "list; list already has INT_MAX elements.");
+    }
     if (lastChunk == null) {
       addChunk(initialChunkCapacity);
     } else if (lastChunk.size() >= lastChunkCapacity) {
@@ -164,8 +186,20 @@ int getMaxChunkSize() {
   }
 
   @Override
-  public T get(int arg0) {
-    throw new UnsupportedOperationException(
-        this.getClass().getName() + " does not support random access");
+  public T get(int idx) {
+    if (idx < 0) {
+      throw new IndexOutOfBoundsException();
+    }
+    int base = 0;
+    Iterator<List<T>> it = chunks.iterator();
+    while (it.hasNext()) {
+      List<T> list = it.next();
+      int size = list.size();
+      if (idx < base + size) {
+        return list.get(idx - base);
+      }
+      base += size;
+    }
+    throw new IndexOutOfBoundsException();
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
index a57342fa889db..65d85f79f8dad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ComparableVersion.java
@@ -9,8 +9,6 @@
 //      to
 //        package org.apache.hadoop.util;
 //   2. Removed author tags to clear hadoop author tag warning
-//        author <a href="mailto:kenney@apache.org">Kenney Westerhof</a>
-//        author <a href="mailto:hboutemy@apache.org">Hervé Boutemy</a>
 //
 package org.apache.hadoop.util;
 
@@ -195,6 +193,8 @@ public StringItem( String value, boolean followedByDigit )
                     case 'm':
                         value = "milestone";
                         break;
+                    default:
+                        break;
                 }
             }
             this.value = ALIASES.getProperty( value , value );
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
index 8bfb5d93aef88..b0c12be731c8a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
@@ -19,24 +19,28 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * FileBasedIPList loads a list of subnets in CIDR format and ip addresses from a file.
+ * FileBasedIPList loads a list of subnets in CIDR format and ip addresses from
+ * a file.
  *
- * Given an ip address, isIn  method returns true if ip belongs to one of the subnets.
+ * Given an ip address, isIn  method returns true if ip belongs to one of the
+ * subnets.
  *
  * Thread safe.
  */
-
 public class FileBasedIPList implements IPList {
 
   private static final Log LOG = LogFactory.getLog(FileBasedIPList.class);
@@ -46,7 +50,12 @@ public class FileBasedIPList implements IPList {
 
   public FileBasedIPList(String fileName) {
     this.fileName = fileName;
-    String[] lines = readLines(fileName);
+    String[] lines;
+    try {
+      lines = readLines(fileName);
+    } catch (IOException e) {
+      lines = null;
+    }
     if (lines != null) {
       addressList = new MachineList(new HashSet<String>(Arrays.asList(lines)));
     } else {
@@ -67,35 +76,39 @@ public  boolean isIn(String ipAddress) {
   }
 
   /**
-   * reads the lines in a file.
+   * Reads the lines in a file.
    * @param fileName
    * @return lines in a String array; null if the file does not exist or if the
    * file name is null
    * @throws IOException
    */
-  private static String[] readLines(String fileName) {
+  private static String[] readLines(String fileName) throws IOException {
     try {
       if (fileName != null) {
         File file = new File (fileName);
         if (file.exists()) {
-          FileReader fileReader = new FileReader(file);
-          BufferedReader bufferedReader = new BufferedReader(fileReader);
-          List<String> lines = new ArrayList<String>();
-          String line = null;
-          while ((line = bufferedReader.readLine()) != null) {
-            lines.add(line);
+          try (
+              Reader fileReader = new InputStreamReader(
+                  new FileInputStream(file), Charsets.UTF_8);
+              BufferedReader bufferedReader = new BufferedReader(fileReader)) {
+            List<String> lines = new ArrayList<String>();
+            String line = null;
+            while ((line = bufferedReader.readLine()) != null) {
+              lines.add(line);
+            }
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Loaded IP list of size = " + lines.size() +
+                  " from file = " + fileName);
+            }
+            return (lines.toArray(new String[lines.size()]));
           }
-          bufferedReader.close();
-          LOG.debug("Loaded IP list of size = " + lines.size() +" from file = " + fileName);
-          return(lines.toArray(new String[lines.size()]));
-        }
-        else {
+        } else {
           LOG.debug("Missing ip list file : "+ fileName);
         }
       }
-    }
-    catch (Throwable t) {
-      LOG.error(t);
+    } catch (IOException ioe) {
+      LOG.error(ioe);
+      throw ioe;
     }
     return null;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
new file mode 100644
index 0000000000000..b7feb22d34d21
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.net.URL;
+import java.security.CodeSource;
+
+/**
+ * This entry point exists for diagnosing classloader problems:
+ * is a class or resource present -and if so, where?
+ *
+ * <p/>
+ * Actions
+ * <p/>
+ * <ul>
+ *   <li><pre>load</pre>: load a class but do not attempt to create it </li>
+ *   <li><pre>create</pre>: load and create a class, print its string value</li>
+ *   <li><pre>printresource</pre>: load a resource then print it to stdout</li>
+ *   <li><pre>resource</pre>: load a resource then print the URL of that
+ *   resource</li>
+ * </ul>
+ *
+ * It returns an error code if a class/resource cannot be loaded/found
+ * -and optionally a class may be requested as being loaded.
+ * The latter action will call the class's constructor -it must support an
+ * empty constructor); any side effects from the
+ * constructor or static initializers will take place.
+ *
+ * All error messages are printed to {@link System#out}; errors
+ * to {@link System#err}.
+ * 
+ */
+@SuppressWarnings("UseOfSystemOutOrSystemErr")
+public final class FindClass extends Configured implements Tool {
+
+  /**
+   * create command: {@value}
+   */
+  public static final String A_CREATE = "create";
+
+  /**
+   * Load command: {@value}
+   */
+  public static final String A_LOAD = "load";
+
+  /**
+   * Command to locate a resource: {@value}
+   */
+  public static final String A_RESOURCE = "locate";
+
+  /**
+   * Command to locate and print a resource: {@value}
+   */
+  public static final String A_PRINTRESOURCE = "print";
+
+  /**
+   * Exit code when the operation succeeded: {@value}
+   */
+  public static final int SUCCESS = 0;
+
+  /**
+   * generic error {@value}
+   */
+  protected static final int E_GENERIC = 1;
+
+  /**
+   * usage error -bad arguments or similar {@value}
+   */
+  protected static final int E_USAGE = 2;
+
+  /**
+   * class or resource not found {@value}
+   */
+  protected static final int E_NOT_FOUND = 3;
+
+  /**
+   * class load failed {@value}
+   */
+  protected static final int E_LOAD_FAILED = 4;
+
+  /**
+   * class creation failed {@value}
+   */
+  protected static final int E_CREATE_FAILED = 5;
+
+  /**
+   * Output stream. Defaults to {@link System#out}
+   */
+  private static PrintStream stdout = System.out;
+  
+  /**
+   * Error stream. Defaults to {@link System#err}
+   */
+  private static PrintStream stderr = System.err;
+
+  /**
+   * Empty constructor; passes a new Configuration
+   * object instance to its superclass's constructor
+   */
+  public FindClass() {
+    super(new Configuration());
+  }
+
+  /**
+   * Create a class with a specified configuration
+   * @param conf configuration
+   */
+  public FindClass(Configuration conf) {
+    super(conf);
+  }
+
+  /**
+   * Change the output streams to be something other than the 
+   * System.out and System.err streams
+   * @param out new stdout stream
+   * @param err new stderr stream
+   */
+  @VisibleForTesting
+  public static void setOutputStreams(PrintStream out, PrintStream err) {
+    stdout = out;
+    stderr = err;
+  }
+
+  /**
+   * Get a class fromt the configuration
+   * @param name the class name
+   * @return the class
+   * @throws ClassNotFoundException if the class was not found
+   * @throws Error on other classloading problems
+   */
+  private Class getClass(String name) throws ClassNotFoundException {
+    return getConf().getClassByName(name);
+  }
+
+  /**
+   * Get the resource
+   * @param name resource name
+   * @return URL or null for not found
+   */
+  private URL getResource(String name) {
+    return getConf().getResource(name);
+  }
+
+  /**
+   * Load a resource
+   * @param name resource name
+   * @return the status code
+   */
+  private int loadResource(String name) {
+    URL url = getResource(name);
+    if (url == null) {
+      err("Resource not found: %s", name);
+      return E_NOT_FOUND;
+    }
+    out("%s: %s", name, url);
+    return SUCCESS;
+  }
+
+  /**
+   * Dump a resource to out
+   * @param name resource name
+   * @return the status code
+   */
+  @SuppressWarnings("NestedAssignment")
+  private int dumpResource(String name) {
+    URL url = getResource(name);
+    if (url == null) {
+      err("Resource not found:" + name);
+      return E_NOT_FOUND;
+    }
+    try {
+      //open the resource
+      InputStream instream = url.openStream();
+      //read it in and print
+      int data;
+      while (-1 != (data = instream.read())) {
+        stdout.print((char) data);
+      }
+      //end of file
+      stdout.print('\n');
+      return SUCCESS;
+    } catch (IOException e) {
+      printStack(e, "Failed to read resource %s at URL %s", name, url);
+      return E_LOAD_FAILED;
+    }
+  }
+
+  /**
+   * print something to stderr
+   * @param s string to print
+   */
+  private static void err(String s, Object... args) {
+    stderr.format(s, args);
+    stderr.print('\n');
+  }
+
+  /**
+   * print something to stdout
+   * @param s string to print
+   */
+  private static void out(String s, Object... args) {
+    stdout.format(s, args);
+    stdout.print('\n');
+  }
+
+  /**
+   * print a stack trace with text
+   * @param e the exception to print
+   * @param text text to print
+   */
+  private static void printStack(Throwable e, String text, Object... args) {
+    err(text, args);
+    e.printStackTrace(stderr);
+  }
+
+  /**
+   * Loads the class of the given name
+   * @param name classname
+   * @return outcome code
+   */
+  private int loadClass(String name) {
+    try {
+      Class clazz = getClass(name);
+      loadedClass(name, clazz);
+      return SUCCESS;
+    } catch (ClassNotFoundException e) {
+      printStack(e, "Class not found " + name);
+      return E_NOT_FOUND;
+    } catch (Exception e) {
+      printStack(e, "Exception while loading class " + name);
+      return E_LOAD_FAILED;
+    } catch (Error e) {
+      printStack(e, "Error while loading class " + name);
+      return E_LOAD_FAILED;
+    }
+  }
+
+  /**
+   * Log that a class has been loaded, and where from.
+   * @param name classname
+   * @param clazz class
+   */
+  private void loadedClass(String name, Class clazz) {
+    out("Loaded %s as %s", name, clazz);
+    CodeSource source = clazz.getProtectionDomain().getCodeSource();
+    URL url = source.getLocation();
+    out("%s: %s", name, url);
+  }
+
+  /**
+   * Create an instance of a class
+   * @param name classname
+   * @return the outcome
+   */
+  private int createClassInstance(String name) {
+    try {
+      Class clazz = getClass(name);
+      loadedClass(name, clazz);
+      Object instance = clazz.newInstance();
+      try {
+        //stringify
+        out("Created instance " + instance.toString());
+      } catch (Exception e) {
+        //catch those classes whose toString() method is brittle, but don't fail the probe
+        printStack(e,
+                   "Created class instance but the toString() operator failed");
+      }
+      return SUCCESS;
+    } catch (ClassNotFoundException e) {
+      printStack(e, "Class not found " + name);
+      return E_NOT_FOUND;
+    } catch (Exception e) {
+      printStack(e, "Exception while creating class " + name);
+      return E_CREATE_FAILED;
+    } catch (Error e) {
+      printStack(e, "Exception while creating class " + name);
+      return E_CREATE_FAILED;
+    }
+  }
+
+  /**
+   * Run the class/resource find or load operation
+   * @param args command specific arguments.
+   * @return the outcome
+   * @throws Exception if something went very wrong
+   */
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length != 2) {
+      return usage(args);
+    }
+    String action = args[0];
+    String name = args[1];
+    int result;
+    if (A_LOAD.equals(action)) {
+      result = loadClass(name);
+    } else if (A_CREATE.equals(action)) {
+      //first load to separate load errors from create
+      result = loadClass(name);
+      if (result == SUCCESS) {
+        //class loads, so instantiate it
+        result = createClassInstance(name);
+      }
+    } else if (A_RESOURCE.equals(action)) {
+      result = loadResource(name);
+    } else if (A_PRINTRESOURCE.equals(action)) {
+      result = dumpResource(name);
+    } else {
+      result = usage(args);
+    }
+    return result;
+  }
+
+  /**
+   * Print a usage message
+   * @param args the command line arguments
+   * @return an exit code
+   */
+  private int usage(String[] args) {
+    err(
+      "Usage : [load | create] <classname>");
+    err(
+      "        [locate | print] <resourcename>]");
+    err("The return codes are:");
+    explainResult(SUCCESS,
+                  "The operation was successful");
+    explainResult(E_GENERIC,
+                  "Something went wrong");
+    explainResult(E_USAGE,
+                  "This usage message was printed");
+    explainResult(E_NOT_FOUND,
+                  "The class or resource was not found");
+    explainResult(E_LOAD_FAILED,
+                  "The class was found but could not be loaded");
+    explainResult(E_CREATE_FAILED,
+                  "The class was loaded, but an instance of it could not be created");
+    return E_USAGE;
+  }
+
+  /**
+   * Explain an error code as part of the usage
+   * @param errorcode error code returned
+   * @param text error text
+   */
+  private void explainResult(int errorcode, String text) {
+    err(" %2d -- %s ", errorcode , text);
+  }
+
+  /**
+   * Main entry point. 
+   * Runs the class via the {@link ToolRunner}, then
+   * exits with an appropriate exit code. 
+   * @param args argument list
+   */
+  public static void main(String[] args) {
+    try {
+      int result = ToolRunner.run(new FindClass(), args);
+      System.exit(result);
+    } catch (Exception e) {
+      printStack(e, "Running FindClass");
+      System.exit(E_GENERIC);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index d0e765529c7b5..0a46a7a81caf9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -284,6 +284,17 @@ private void processGeneralOptions(Configuration conf,
         conf.addResource(new Path(value));
       }
     }
+
+    if (line.hasOption('D')) {
+      String[] property = line.getOptionValues('D');
+      for(String prop : property) {
+        String[] keyval = prop.split("=", 2);
+        if (keyval.length == 2) {
+          conf.set(keyval[0], keyval[1], "from command line");
+        }
+      }
+    }
+
     if (line.hasOption("libjars")) {
       conf.set("tmpjars", 
                validateFiles(line.getOptionValue("libjars"), conf),
@@ -307,15 +318,6 @@ private void processGeneralOptions(Configuration conf,
                 validateFiles(line.getOptionValue("archives"), conf),
                 "from -archives command line option");
     }
-    if (line.hasOption('D')) {
-      String[] property = line.getOptionValues('D');
-      for(String prop : property) {
-        String[] keyval = prop.split("=", 2);
-        if (keyval.length == 2) {
-          conf.set(keyval[0], keyval[1], "from command line");
-        }
-      }
-    }
     conf.setBoolean("mapreduce.client.genericoptionsparser.used", true);
     
     // tokensFile
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index b012add42c5fb..ae77e6c33335f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -22,6 +22,7 @@
 import java.util.Set;
 import java.util.HashSet;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.Log;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -72,7 +73,8 @@ public static void readFileToSetWithFileInputStream(String type,
       throws IOException {
     BufferedReader reader = null;
     try {
-      reader = new BufferedReader(new InputStreamReader(fileInputStream));
+      reader = new BufferedReader(
+          new InputStreamReader(fileInputStream, Charsets.UTF_8));
       String line;
       while ((line = reader.readLine()) != null) {
         String[] nodes = line.split("[ \t\n\f\r]+");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index e8af45e746234..1fe77964514d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -30,7 +31,6 @@
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
@@ -172,7 +172,7 @@ public String toString() {
   private class Monitor implements Runnable {
     @Override
     public void run() {
-      Stopwatch sw = new Stopwatch();
+      StopWatch sw = new StopWatch();
       Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes();
       while (shouldRun) {
         sw.reset().start();
@@ -181,7 +181,7 @@ public void run() {
         } catch (InterruptedException ie) {
           return;
         }
-        long extraSleepTime = sw.elapsedMillis() - SLEEP_INTERVAL_MS;
+        long extraSleepTime = sw.now(TimeUnit.MILLISECONDS) - SLEEP_INTERVAL_MS;
         Map<String, GcTimes> gcTimesAfterSleep = getGcTimes();
 
         if (extraSleepTime > warnThresholdMs) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LogAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LogAdapter.java
new file mode 100644
index 0000000000000..6ef9093a28248
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LogAdapter.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.commons.logging.Log;
+import org.slf4j.Logger;
+
+class LogAdapter {
+  private Log LOG;
+  private Logger LOGGER;
+
+  private LogAdapter(Log LOG) {
+    this.LOG = LOG;
+  }
+
+  private LogAdapter(Logger LOGGER) {
+    this.LOGGER = LOGGER;
+  }
+
+  public static LogAdapter create(Log LOG) {
+    return new LogAdapter(LOG);
+  }
+
+  public static LogAdapter create(Logger LOGGER) {
+    return new LogAdapter(LOGGER);
+  }
+
+  public void info(String msg) {
+    if (LOG != null) {
+      LOG.info(msg);
+    } else if (LOGGER != null) {
+      LOGGER.info(msg);
+    }
+  }
+
+  public void warn(String msg, Throwable t) {
+    if (LOG != null) {
+      LOG.warn(msg, t);
+    } else if (LOGGER != null) {
+      LOGGER.warn(msg, t);
+    }
+  }
+
+  public void debug(Throwable t) {
+    if (LOG != null) {
+      LOG.debug(t);
+    } else if (LOGGER != null) {
+      LOGGER.debug("", t);
+    }
+  }
+
+  public void error(String msg) {
+    if (LOG != null) {
+      LOG.error(msg);
+    } else if (LOGGER != null) {
+      LOGGER.error(msg);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index d1a0870f679c2..d60d08387e863 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -45,6 +45,7 @@
 public class MachineList {
   
   public static final Log LOG = LogFactory.getLog(MachineList.class);
+  public static final String WILDCARD_VALUE = "*";
 
   /**
    * InetAddressFactory is used to obtain InetAddress from host.
@@ -91,7 +92,7 @@ public MachineList(Collection<String> hostEntries) {
   public MachineList(Collection<String> hostEntries, InetAddressFactory addressFactory) {
     this.addressFactory = addressFactory;
     if (hostEntries != null) {
-      if ((hostEntries.size() == 1) && (hostEntries.contains("*"))) {
+      if ((hostEntries.size() == 1) && (hostEntries.contains(WILDCARD_VALUE))) {
         all = true; 
         ipAddresses = null; 
         hostNames = null; 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index 641635542fa42..81448ab2d4dbb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -108,14 +108,14 @@ public static void main(String[] args) {
     }
 
     System.out.println("Native library checking:");
-    System.out.printf("hadoop:  %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
-    System.out.printf("zlib:    %b %s\n", zlibLoaded, zlibLibraryName);
-    System.out.printf("snappy:  %b %s\n", snappyLoaded, snappyLibraryName);
-    System.out.printf("lz4:     %b %s\n", lz4Loaded, lz4LibraryName);
-    System.out.printf("bzip2:   %b %s\n", bzip2Loaded, bzip2LibraryName);
-    System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
+    System.out.printf("hadoop:  %b %s%n", nativeHadoopLoaded, hadoopLibraryName);
+    System.out.printf("zlib:    %b %s%n", zlibLoaded, zlibLibraryName);
+    System.out.printf("snappy:  %b %s%n", snappyLoaded, snappyLibraryName);
+    System.out.printf("lz4:     %b %s%n", lz4Loaded, lz4LibraryName);
+    System.out.printf("bzip2:   %b %s%n", bzip2Loaded, bzip2LibraryName);
+    System.out.printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
     if (Shell.WINDOWS) {
-      System.out.printf("winutils: %b %s\n", winutilsExists, winutilsPath);
+      System.out.printf("winutils: %b %s%n", winutilsExists, winutilsPath);
     }
 
     if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
index efa4de3ea2043..df571f35e2fcd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
@@ -34,16 +34,13 @@ public class PrintJarMainClass {
    * @param args
    */
   public static void main(String[] args) {
-    try {
-      JarFile jar_file = new JarFile(args[0]);
-      if (jar_file != null) {
-        Manifest manifest = jar_file.getManifest();
-        if (manifest != null) {
-          String value = manifest.getMainAttributes().getValue("Main-Class");
-          if (value != null) {
-            System.out.println(value.replaceAll("/", "."));
-            return;
-          }
+    try (JarFile jar_file = new JarFile(args[0])) {
+      Manifest manifest = jar_file.getManifest();
+      if (manifest != null) {
+        String value = manifest.getMainAttributes().getValue("Main-Class");
+        if (value != null) {
+          System.out.println(value.replaceAll("/", "."));
+          return;
         }
       }
     } catch (Throwable e) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
index 36b5ff11bc8c2..4b3b7efbf75ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
@@ -27,8 +27,8 @@
 import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.*;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.htrace.Span;
-import org.htrace.Trace;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
 
 import com.google.protobuf.ByteString;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
index 3977e60287a4a..d9a73263d8521 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
@@ -20,13 +20,16 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -154,7 +157,7 @@ private static String getTaskName(long id, String name) {
    * @param stream the stream to
    * @param title a string title for the stack trace
    */
-  public synchronized static void printThreadInfo(PrintWriter stream,
+  public synchronized static void printThreadInfo(PrintStream stream,
                                      String title) {
     final int STACK_DEPTH = 20;
     boolean contention = threadBean.isThreadContentionMonitoringEnabled();
@@ -215,9 +218,12 @@ public static void logThreadInfo(Log log,
         }
       }
       if (dumpStack) {
-        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-        printThreadInfo(new PrintWriter(buffer), title);
-        log.info(buffer.toString());
+        try {
+          ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+          printThreadInfo(new PrintStream(buffer, false, "UTF-8"), title);
+          log.info(buffer.toString(Charset.defaultCharset().name()));
+        } catch (UnsupportedEncodingException ignored) {
+        }
       }
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
index 6a8ca0f9938e5..2fd9b5589dc81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
@@ -70,14 +70,14 @@ public static long parseLongParam(ServletRequest request, String param)
       throw new IOException("Invalid request has no " + param + " parameter");
     }
     
-    return Long.valueOf(paramStr);
+    return Long.parseLong(paramStr);
   }
 
   public static final String HTML_TAIL = "<hr />\n"
-    + "<a href='http://hadoop.apache.org/core'>Hadoop</a>, " 
+    + "<a href='http://hadoop.apache.org/core'>Hadoop</a>, "
     + Calendar.getInstance().get(Calendar.YEAR) + ".\n"
     + "</body></html>";
-  
+
   /**
    * HTML footer to be added in the jsps.
    * @return the HTML footer.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index a44e99212674d..f0100d440ab35 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Timer;
@@ -493,11 +494,11 @@ private void runCommand() throws IOException {
       timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
     }
     final BufferedReader errReader = 
-            new BufferedReader(new InputStreamReader(process
-                                                     .getErrorStream()));
+            new BufferedReader(new InputStreamReader(
+                process.getErrorStream(), Charset.defaultCharset()));
     BufferedReader inReader = 
-            new BufferedReader(new InputStreamReader(process
-                                                     .getInputStream()));
+            new BufferedReader(new InputStreamReader(
+                process.getInputStream(), Charset.defaultCharset()));
     final StringBuffer errMsg = new StringBuffer();
     
     // read error and input streams as this would free up the buffers
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java
index c8667386fefc7..62338c930369c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java
@@ -42,10 +42,10 @@ public enum SignalLogger {
    * Our signal handler.
    */
   private static class Handler implements SignalHandler {
-    final private org.apache.commons.logging.Log LOG;
+    final private LogAdapter LOG;
     final private SignalHandler prevHandler;
 
-    Handler(String name, Log LOG) {
+    Handler(String name, LogAdapter LOG) {
       this.LOG = LOG;
       prevHandler = Signal.handle(new Signal(name), this);
     }
@@ -69,6 +69,10 @@ public void handle(Signal signal) {
    * @param LOG        The log4j logfile to use in the signal handlers.
    */
   public void register(final Log LOG) {
+    register(LogAdapter.create(LOG));
+  }
+
+  void register(final LogAdapter LOG) {
     if (registered) {
       throw new IllegalStateException("Can't re-install the signal handlers.");
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
new file mode 100644
index 0000000000000..b9d0d0b6646d8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.Closeable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A simplified StopWatch implementation which can measure times in nanoseconds.
+ */
+public class StopWatch implements Closeable {
+  private boolean isStarted;
+  private long startNanos;
+  private long currentElapsedNanos;
+
+  public StopWatch() {
+  }
+
+  /**
+   * The method is used to find out if the StopWatch is started.
+   * @return boolean If the StopWatch is started.
+   */
+  public boolean isRunning() {
+    return isStarted;
+  }
+
+  /**
+   * Start to measure times and make the state of stopwatch running.
+   * @return this instance of StopWatch.
+   */
+  public StopWatch start() {
+    if (isStarted) {
+      throw new IllegalStateException("StopWatch is already running");
+    }
+    isStarted = true;
+    startNanos = System.nanoTime();
+    return this;
+  }
+
+  /**
+   * Stop elapsed time and make the state of stopwatch stop.
+   * @return this instance of StopWatch.
+   */
+  public StopWatch stop() {
+    if (!isStarted) {
+      throw new IllegalStateException("StopWatch is already stopped");
+    }
+    long now = System.nanoTime();
+    isStarted = false;
+    currentElapsedNanos += now - startNanos;
+    return this;
+  }
+
+  /**
+   * Reset elapsed time to zero and make the state of stopwatch stop.
+   * @return this instance of StopWatch.
+   */
+  public StopWatch reset() {
+    currentElapsedNanos = 0;
+    isStarted = false;
+    return this;
+  }
+
+  /**
+   * @return current elapsed time in specified timeunit.
+   */
+  public long now(TimeUnit timeUnit) {
+    return timeUnit.convert(now(), TimeUnit.NANOSECONDS);
+
+  }
+
+  /**
+   * @return current elapsed time in nanosecond.
+   */
+  public long now() {
+    return isStarted ?
+        System.nanoTime() - startNanos + currentElapsedNanos :
+        currentElapsedNanos;
+  }
+
+  @Override
+  public String toString() {
+    return String.valueOf(now());
+  }
+
+  @Override
+  public void close() {
+    if (isStarted) {
+      stop();
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 4e2783df88db5..ff8edc389dfec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -628,6 +628,22 @@ private static String toStartupShutdownString(String prefix, String [] msg) {
    */
   public static void startupShutdownMessage(Class<?> clazz, String[] args,
                                      final org.apache.commons.logging.Log LOG) {
+    startupShutdownMessage(clazz, args, LogAdapter.create(LOG));
+  }
+
+  /**
+   * Print a log message for starting up and shutting down
+   * @param clazz the class of the server
+   * @param args arguments
+   * @param LOG the target log object
+   */
+  public static void startupShutdownMessage(Class<?> clazz, String[] args,
+                                     final org.slf4j.Logger LOG) {
+    startupShutdownMessage(clazz, args, LogAdapter.create(LOG));
+  }
+
+  static void startupShutdownMessage(Class<?> clazz, String[] args,
+                                     final LogAdapter LOG) { 
     final String hostname = NetUtils.getHostname();
     final String classname = clazz.getSimpleName();
     LOG.info(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
index e2dea6d368dd6..f8b95192c82f9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/BloomFilter.java
@@ -157,7 +157,7 @@ public boolean membershipTest(Key key) {
 
   @Override
   public void not() {
-    bits.flip(0, vectorSize - 1);
+    bits.flip(0, vectorSize);
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/exception.c b/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
index 228af11bd8e19..fc072e8002bf2 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/exception.c
@@ -110,9 +110,15 @@ jthrowable newIOException(JNIEnv* env, const char *fmt, ...)
 
 const char* terror(int errnum)
 {
+
+#if defined(__sun)
+// MT-Safe under Solaris which doesn't support sys_errlist/sys_nerr
+  return strerror(errnum); 
+#else
   if ((errnum < 0) || (errnum >= sys_nerr)) {
     return "unknown error.";
   }
   return sys_errlist[errnum];
+#endif
 }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/exception.h b/hadoop-common-project/hadoop-common/src/main/native/src/exception.h
index 1ec47a653c8ef..afa08e913f8d9 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/exception.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/exception.h
@@ -19,6 +19,19 @@
 
 #include <jni.h> /* for jthrowable */
 #include <stdarg.h> /* for va_list */
+#include "org_apache_hadoop.h"
+
+#ifdef WINDOWS
+/*
+ * gcc-style type-checked format arguments are not supported on Windows, so just
+ * stub this macro.
+ */
+#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs)
+# else
+/* Use gcc type-checked format arguments. */
+#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \
+  __attribute__((format(printf, formatArg, varArgs)))
+#endif
 
 /**
  * Create a new Exception.
@@ -48,7 +61,7 @@ jthrowable newExceptionV(JNIEnv* env, const char *name,
  * @return              The RuntimeException
  */
 jthrowable newException(JNIEnv* env, const char *name, const char *fmt, ...)
-    __attribute__((format(printf, 3, 4)));
+    TYPE_CHECKED_PRINTF_FORMAT(3, 4);
 
 /**
  * Create a new RuntimeException.
@@ -62,7 +75,7 @@ jthrowable newException(JNIEnv* env, const char *name, const char *fmt, ...)
  * @return              The RuntimeException
  */
 jthrowable newRuntimeException(JNIEnv* env, const char *fmt, ...)
-    __attribute__((format(printf, 2, 3)));
+    TYPE_CHECKED_PRINTF_FORMAT(2, 3);
 
 /**
  * Create a new IOException.
@@ -77,7 +90,7 @@ jthrowable newRuntimeException(JNIEnv* env, const char *fmt, ...)
  *                      to create the NativeIOException.
  */
 jthrowable newIOException(JNIEnv* env, const char *fmt, ...)
-    __attribute__((format(printf, 2, 3)));
+    TYPE_CHECKED_PRINTF_FORMAT(2, 3);
 
 /**
  * Thread-safe strerror alternative.
@@ -87,4 +100,5 @@ jthrowable newIOException(JNIEnv* env, const char *fmt, ...)
  */
 const char* terror(int errnum);
 
+#undef TYPE_CHECKED_PRINTF_FORMAT
 #endif
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c
index d4cd6dfbd24b7..ef81bea2ce009 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c
@@ -43,15 +43,25 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_initIDs(
                                  JNIEnv *env, jclass class, jstring libname)
 {
-    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
-    if (strcmp(bzlib_name, "system-native") == 0)
-      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    const char *bzlib_name = NULL;
+    const char *java_lib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (java_lib_name == NULL) {
+        // Java code will get OutOfMemoryException thrown by GetStringUTFChars
+        goto cleanup;
+    }
+
+    if (strcmp(java_lib_name, "system-native") == 0) {
+        bzlib_name = HADOOP_BZIP2_LIBRARY;
+    } else {
+        bzlib_name = java_lib_name;
+    }
+
     // Load the native library.
     void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
     if (!libbz2) {
         THROW(env, "java/lang/UnsatisfiedLinkError",
               "Cannot load bzip2 native library");
-        return;
+        goto cleanup;
     }
 
     // Locate the requisite symbols from libbz2.so.
@@ -83,6 +93,11 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_initIDs(
                                                      "Ljava/nio/Buffer;");
     Bzip2Compressor_directBufferSize = (*env)->GetFieldID(env, class, 
                                                   "directBufferSize", "I");
+ cleanup:
+    if(java_lib_name != NULL) {
+        (*env)->ReleaseStringUTFChars(env,libname,java_lib_name);
+        java_lib_name = NULL;
+    }
 }
 
 JNIEXPORT jlong JNICALL
@@ -234,9 +249,10 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
 {
     if (dlsym_BZ2_bzCompressEnd(BZSTREAM(stream)) != BZ_OK) {
         THROW(env, "java/lang/InternalError", NULL);
-    } else {
-        free(BZSTREAM(stream));
     }
+
+    free(BZSTREAM(stream));
+
 }
 
 JNIEXPORT jstring JNICALL
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c
index b6c5213524611..ad9bcb72c6c91 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c
@@ -42,15 +42,25 @@ JNIEXPORT void JNICALL
 Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_initIDs(
                                  JNIEnv *env, jclass class, jstring libname)
 {
-    const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL);
-    if (strcmp(bzlib_name, "system-native") == 0)
-      bzlib_name = HADOOP_BZIP2_LIBRARY;
+    const char *bzlib_name = NULL;
+    const char *java_lib_name = (*env)->GetStringUTFChars(env, libname, NULL);
+    if (java_lib_name == NULL) {
+        // Java code will get OutOfMemoryException thrown by GetStringUTFChars
+        goto cleanup;
+    }
+
+    if (strcmp(java_lib_name, "system-native") == 0) {
+        bzlib_name = HADOOP_BZIP2_LIBRARY;
+    } else {
+        bzlib_name = java_lib_name;
+    }
+
     // Load the native library.
     void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL);
     if (!libbz2) {
         THROW(env, "java/lang/UnsatisfiedLinkError",
               "Cannot load bzip2 native library");
-        return;
+        goto cleanup;
     }
 
     // Locate the requisite symbols from libbz2.so.
@@ -80,6 +90,11 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_initIDs(
                                                 "Ljava/nio/Buffer;");
     Bzip2Decompressor_directBufferSize = (*env)->GetFieldID(env, class, 
                                                 "directBufferSize", "I");
+cleanup:
+    if(java_lib_name != NULL) {
+         (*env)->ReleaseStringUTFChars(env,libname,java_lib_name);
+         java_lib_name = NULL;
+    }
 }
 
 JNIEXPORT jlong JNICALL
@@ -237,9 +252,10 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_end(
 {
     if (dlsym_BZ2_bzDecompressEnd(BZSTREAM(stream)) != BZ_OK) {
         THROW(env, "java/lang/InternalError", 0);
-    } else {
-        free(BZSTREAM(stream));
     }
+
+    free(BZSTREAM(stream));
+
 }
 
 /**
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index f0f9ebcb7f458..071d83000262b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -19,6 +19,7 @@
 #include "org_apache_hadoop.h"
 #include "org_apache_hadoop_io_nativeio_NativeIO.h"
 #include "org_apache_hadoop_io_nativeio_NativeIO_POSIX.h"
+#include "exception.h"
 
 #ifdef UNIX
 #include <assert.h>
@@ -514,6 +515,86 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_open(
 #endif
 }
 
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    createDirectoryWithMode0
+ * Signature: (Ljava/lang/String;I)V
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT void JNICALL
+  Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_createDirectoryWithMode0
+  (JNIEnv *env, jclass clazz, jstring j_path, jint mode)
+{
+#ifdef WINDOWS
+  DWORD dwRtnCode = ERROR_SUCCESS;
+
+  LPCWSTR path = (LPCWSTR) (*env)->GetStringChars(env, j_path, NULL);
+  if (!path) {
+    goto done;
+  }
+
+  dwRtnCode = CreateDirectoryWithMode(path, mode);
+
+done:
+  if (path) {
+    (*env)->ReleaseStringChars(env, j_path, (const jchar*) path);
+  }
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioe(env, dwRtnCode);
+  }
+#else
+  THROW(env, "java/io/IOException",
+    "The function Windows.createDirectoryWithMode0() is not supported on this platform");
+#endif
+}
+
+/*
+ * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
+ * Method:    createFileWithMode0
+ * Signature: (Ljava/lang/String;JJJI)Ljava/io/FileDescriptor;
+ *
+ * The "00024" in the function name is an artifact of how JNI encodes
+ * special characters. U+0024 is '$'.
+ */
+JNIEXPORT jobject JNICALL
+  Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_createFileWithMode0
+  (JNIEnv *env, jclass clazz, jstring j_path,
+  jlong desiredAccess, jlong shareMode, jlong creationDisposition, jint mode)
+{
+#ifdef WINDOWS
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  HANDLE hFile = INVALID_HANDLE_VALUE;
+  jobject fd = NULL;
+
+  LPCWSTR path = (LPCWSTR) (*env)->GetStringChars(env, j_path, NULL);
+  if (!path) {
+    goto done;
+  }
+
+  dwRtnCode = CreateFileWithMode(path, desiredAccess, shareMode,
+      creationDisposition, mode, &hFile);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  fd = fd_create(env, (long) hFile);
+
+done:
+  if (path) {
+    (*env)->ReleaseStringChars(env, j_path, (const jchar*) path);
+  }
+  if (dwRtnCode != ERROR_SUCCESS) {
+    throw_ioe(env, dwRtnCode);
+  }
+  return fd;
+#else
+  THROW(env, "java/io/IOException",
+    "The function Windows.createFileWithMode0() is not supported on this platform");
+#endif
+}
+
 /*
  * Class:     org_apache_hadoop_io_nativeio_NativeIO_Windows
  * Method:    createFile
@@ -813,11 +894,7 @@ void throw_ioe(JNIEnv* env, int errnum)
   char message[80];
   jstring jstr_message;
 
-  if ((errnum >= 0) && (errnum < sys_nerr)) {
-    snprintf(message, sizeof(message), "%s", sys_errlist[errnum]);
-  } else {
-    snprintf(message, sizeof(message), "Unknown error %d", errnum);
-  }
+  snprintf(message,sizeof(message),"%s",terror(errnum));
 
   jobject errno_obj = errno_to_enum(env, errnum);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e7a382d9bc9bd..f31db3154f79c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -719,7 +719,7 @@ for ldap providers in the same way as above does.
 
 <property>
   <name>fs.s3a.connection.timeout</name>
-  <value>5000</value>
+  <value>50000</value>
   <description>Socket connection timeout in seconds.</description>
 </property>
 
@@ -1653,4 +1653,27 @@ for ldap providers in the same way as above does.
   </description>
 </property>
 
+<property>
+  <name>hadoop.htrace.spanreceiver.classes</name>
+  <value></value>
+  <description>
+    A comma separated list of the fully-qualified class name of classes 
+    implementing SpanReceiver. The tracing system works by collecting 
+    information in structs called 'Spans'. It is up to you to choose 
+    how you want to receive this information by implementing the 
+    SpanReceiver interface.
+  </description>
+</property>
+
+ <property>
+  <name>ipc.server.max.connections</name>
+  <value>0</value>
+  <description>The maximum number of concurrent connections a server is allowed
+    to accept. If this limit is exceeded, incoming connections will first fill
+    the listen queue and then may go to an OS-specific listen overflow queue. 
+    The client may fail or timeout, but the server can avoid running out of file
+    descriptors using this feature. 0 means no limit.
+  </description>
+</property>
+
 </configuration>
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
index 77fc58637ff54..0ac9adb83b11f 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
@@ -165,6 +165,12 @@ DWORD JunctionPointCheck(__in LPCWSTR pathName, __out LPBOOL result);
 
 DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode);
 
+DWORD CreateDirectoryWithMode(__in LPCWSTR path, __in INT mode);
+
+DWORD CreateFileWithMode(__in LPCWSTR lpPath, __in DWORD dwDesiredAccess,
+    __in DWORD dwShareMode, __in DWORD dwCreationDisposition, __in INT mode,
+    __out_opt PHANDLE pHFile);
+
 DWORD GetLocalGroupsForUser(__in LPCWSTR user,
   __out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
index 933c177b463f8..5e775df016c27 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
@@ -138,15 +138,17 @@ DWORD GetFileInformationByName(
 // Function: IsLongWindowsPath
 //
 // Description:
-//  Checks if the path is longer than MAX_PATH in which case it needs to be
-//  prepended with \\?\ for Windows OS to understand it.
+//  Checks if the path is longer than (MAX_PATH - 13) in which case it needs to
+//  be prepended with \\?\ for Windows OS to understand it.  The -13 is to
+//  account for an additional constraint for directories that it must be possible
+//  to append an additional path separator followed by an 8.3 file name.
 //
 // Returns:
 //  TRUE long path
 //  FALSE otherwise
 static BOOL IsLongWindowsPath(__in PCWSTR path)
 {
-  return (wcslen(path) + 1) > MAX_PATH;
+  return (wcslen(path) + 1) > (MAX_PATH - 13);
 }
 
 //----------------------------------------------------------------------------
@@ -1451,6 +1453,265 @@ DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode)
   return ret;
 }
 
+//----------------------------------------------------------------------------
+// Function: GetTokenInformationByClass
+//
+// Description:
+//  Gets a class of information from a token.  On success, this function has
+//  dynamically allocated memory and set the ppTokenInformation parameter to
+//  point to it.  The caller owns this memory and is reponsible for releasing it
+//  by calling LocalFree.
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+static DWORD GetTokenInformationByClass(__in HANDLE hToken,
+    __in TOKEN_INFORMATION_CLASS class, __out_opt LPVOID *ppTokenInformation) {
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  LPVOID pTokenInformation = NULL;
+  DWORD dwSize = 0;
+
+  // Call GetTokenInformation first time to get the required buffer size.
+  if (!GetTokenInformation(hToken, class, NULL, 0, &dwSize)) {
+    dwRtnCode = GetLastError();
+    if (dwRtnCode != ERROR_INSUFFICIENT_BUFFER) {
+      return dwRtnCode;
+    }
+  }
+
+  // Allocate memory.
+  pTokenInformation = LocalAlloc(LPTR, dwSize);
+  if (!pTokenInformation) {
+    return GetLastError();
+  }
+
+  // Call GetTokenInformation second time to fill our buffer with data.
+  if (!GetTokenInformation(hToken, class, pTokenInformation, dwSize, &dwSize)) {
+    LocalFree(pTokenInformation);
+    return GetLastError();
+  }
+
+  *ppTokenInformation = pTokenInformation;
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
+// Function: GetWindowsDACLsForCreate
+//
+// Description:
+//  Get the Windows discretionary access control list equivalent to the given
+//  mode, suitable for creating a new file or directory.  Ownership is assumed
+//  to be the current process owner and primary group.  On success, this function
+//  has dynamically allocated memory and set the ppDACL parameter to point to it.
+//  The caller owns this memory and is reponsible for releasing it by calling
+//  LocalFree.
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+static DWORD GetWindowsDACLsForCreate(__in INT mode, __out PACL *ppDACL) {
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  HANDLE hToken = NULL;
+  DWORD dwSize = 0;
+  PTOKEN_OWNER pTokenOwner = NULL;
+  PTOKEN_PRIMARY_GROUP pTokenPrimaryGroup = NULL;
+  PSID pOwnerSid = NULL, pGroupSid = NULL;
+  PACL pDACL = NULL;
+
+  if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken)) {
+    dwRtnCode = GetLastError();
+    goto done;
+  }
+
+  dwRtnCode = GetTokenInformationByClass(hToken, TokenOwner, &pTokenOwner);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+  pOwnerSid = pTokenOwner->Owner;
+
+  dwRtnCode = GetTokenInformationByClass(hToken, TokenPrimaryGroup,
+      &pTokenPrimaryGroup);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+  pGroupSid = pTokenPrimaryGroup->PrimaryGroup;
+
+  dwRtnCode = GetWindowsDACLs(mode, pOwnerSid, pGroupSid, &pDACL);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  *ppDACL = pDACL;
+
+done:
+  if (hToken) {
+    CloseHandle(hToken);
+  }
+  LocalFree(pTokenOwner);
+  LocalFree(pTokenPrimaryGroup);
+  return dwRtnCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateSecurityDescriptorForCreate
+//
+// Description:
+//  Creates a security descriptor with the given DACL, suitable for creating a
+//  new file or directory.  On success, this function has dynamically allocated
+//  memory and set the ppSD parameter to point to it.  The caller owns this
+//  memory and is reponsible for releasing it by calling LocalFree.
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+static DWORD CreateSecurityDescriptorForCreate(__in PACL pDACL,
+    __out PSECURITY_DESCRIPTOR *ppSD) {
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  PSECURITY_DESCRIPTOR pSD = NULL;
+
+  pSD = LocalAlloc(LPTR, SECURITY_DESCRIPTOR_MIN_LENGTH);
+  if (!pSD) {
+    dwRtnCode = GetLastError();
+    goto done;
+  }
+
+  if (!InitializeSecurityDescriptor(pSD, SECURITY_DESCRIPTOR_REVISION)) {
+    dwRtnCode = GetLastError();
+    goto done;
+  }
+
+  if (!SetSecurityDescriptorDacl(pSD, TRUE, pDACL, FALSE)) {
+    dwRtnCode = GetLastError();
+    goto done;
+  }
+
+  *ppSD = pSD;
+
+done:
+  if (dwRtnCode != ERROR_SUCCESS) {
+    LocalFree(pSD);
+  }
+  return dwRtnCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateDirectoryWithMode
+//
+// Description:
+//  Create a directory with initial security descriptor containing a
+//  discretionary access control list equivalent to the given mode.
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+// Notes:
+//  This function is long path safe, i.e. the path will be converted to long
+//  path format if not already converted. So the caller does not need to do
+//  the conversion before calling the method.
+//
+DWORD CreateDirectoryWithMode(__in LPCWSTR lpPath, __in INT mode) {
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  LPWSTR lpLongPath = NULL;
+  PACL pDACL = NULL;
+  PSECURITY_DESCRIPTOR pSD = NULL;
+  SECURITY_ATTRIBUTES sa;
+
+  dwRtnCode = ConvertToLongPath(lpPath, &lpLongPath);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  dwRtnCode = GetWindowsDACLsForCreate(mode, &pDACL);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  dwRtnCode = CreateSecurityDescriptorForCreate(pDACL, &pSD);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  sa.nLength = sizeof(SECURITY_ATTRIBUTES);
+  sa.lpSecurityDescriptor = pSD;
+  sa.bInheritHandle = FALSE;
+
+  if (!CreateDirectoryW(lpLongPath, &sa)) {
+    dwRtnCode = GetLastError();
+  }
+
+done:
+  LocalFree(lpLongPath);
+  LocalFree(pDACL);
+  LocalFree(pSD);
+  return dwRtnCode;
+}
+
+//----------------------------------------------------------------------------
+// Function: CreateFileWithMode
+//
+// Description:
+//  Create a file with initial security descriptor containing a discretionary
+//  access control list equivalent to the given mode.
+//
+// Returns:
+//  ERROR_SUCCESS: on success
+//  Error code: otherwise
+//
+// Notes:
+//  This function is long path safe, i.e. the path will be converted to long
+//  path format if not already converted. So the caller does not need to do
+//  the conversion before calling the method.
+//
+DWORD CreateFileWithMode(__in LPCWSTR lpPath, __in DWORD dwDesiredAccess,
+    __in DWORD dwShareMode, __in DWORD dwCreationDisposition, __in INT mode,
+    __out PHANDLE pHFile) {
+  DWORD dwRtnCode = ERROR_SUCCESS;
+  LPWSTR lpLongPath = NULL;
+  PACL pDACL = NULL;
+  PSECURITY_DESCRIPTOR pSD = NULL;
+  SECURITY_ATTRIBUTES sa;
+  DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+  HANDLE hFile = INVALID_HANDLE_VALUE;
+
+  dwRtnCode = ConvertToLongPath(lpPath, &lpLongPath);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  dwRtnCode = GetWindowsDACLsForCreate(mode, &pDACL);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  dwRtnCode = CreateSecurityDescriptorForCreate(pDACL, &pSD);
+  if (dwRtnCode != ERROR_SUCCESS) {
+    goto done;
+  }
+
+  sa.nLength = sizeof(SECURITY_ATTRIBUTES);
+  sa.lpSecurityDescriptor = pSD;
+  sa.bInheritHandle = FALSE;
+
+  hFile = CreateFileW(lpLongPath, dwDesiredAccess, dwShareMode, &sa,
+    dwCreationDisposition, dwFlagsAndAttributes, NULL);
+  if (hFile == INVALID_HANDLE_VALUE) {
+    dwRtnCode = GetLastError();
+    goto done;
+  }
+
+  *pHFile = hFile;
+
+done:
+  LocalFree(lpLongPath);
+  LocalFree(pDACL);
+  LocalFree(pSD);
+  return dwRtnCode;
+}
+
 //----------------------------------------------------------------------------
 // Function: GetAccntNameFromSid
 //
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
index f5f1deb0b894a..52b05528a4cc7 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
@@ -11,83 +11,81 @@
 ~~ limitations under the License. See accompanying LICENSE file.
 
   ---
-  Hadoop Map Reduce Next Generation-${project.version} - Cluster Setup
+  Hadoop ${project.version} - Cluster Setup
   ---
   ---
   ${maven.build.timestamp}
 
 %{toc|section=1|fromDepth=0}
 
-Hadoop MapReduce Next Generation - Cluster Setup
+Hadoop Cluster Setup
 
 * {Purpose}
 
-  This document describes how to install, configure and manage non-trivial
+  This document describes how to install and configure
   Hadoop clusters ranging from a few nodes to extremely large clusters
-  with thousands of nodes.
+  with thousands of nodes.  To play with Hadoop, you may first want to
+  install it on a single machine (see {{{./SingleCluster.html}Single Node Setup}}).
 
-  To play with Hadoop, you may first want to install it on a single
-  machine (see {{{./SingleCluster.html}Single Node Setup}}).
+  This document does not cover advanced topics such as {{{./SecureMode.html}Security}} or
+  High Availability.
 
 * {Prerequisites}
 
-  Download a stable version of Hadoop from Apache mirrors.
+  * Install Java. See the {{{http://wiki.apache.org/hadoop/HadoopJavaVersions}Hadoop Wiki}} for known good versions.
+  * Download a stable version of Hadoop from Apache mirrors.
 
 * {Installation}
 
   Installing a Hadoop cluster typically involves unpacking the software on all
-  the machines in the cluster or installing RPMs.
+  the machines in the cluster or installing it via a packaging system as
+  appropriate for your operating system.  It is important to divide up the hardware
+  into functions.
 
   Typically one machine in the cluster is designated as the NameNode and
-  another machine the as ResourceManager, exclusively. These are the masters.
+  another machine the as ResourceManager, exclusively. These are the masters. Other
+  services (such as Web App Proxy Server and MapReduce Job History server) are usually
+  run either on dedicated hardware or on shared infrastrucutre, depending upon the load.
 
   The rest of the machines in the cluster act as both DataNode and NodeManager.
   These are the slaves.
 
-* {Running Hadoop in Non-Secure Mode}
+* {Configuring Hadoop in Non-Secure Mode}
 
-  The following sections describe how to configure a Hadoop cluster.
-
-  {Configuration Files}
-
-    Hadoop configuration is driven by two types of important configuration files:
+    Hadoop's Java configuration is driven by two types of important configuration files:
 
       * Read-only default configuration - <<<core-default.xml>>>,
         <<<hdfs-default.xml>>>, <<<yarn-default.xml>>> and
         <<<mapred-default.xml>>>.
 
-      * Site-specific configuration - <<conf/core-site.xml>>,
-        <<conf/hdfs-site.xml>>, <<conf/yarn-site.xml>> and
-        <<conf/mapred-site.xml>>.
-
+      * Site-specific configuration - <<<etc/hadoop/core-site.xml>>>,
+        <<<etc/hadoop/hdfs-site.xml>>>, <<<etc/hadoop/yarn-site.xml>>> and
+        <<<etc/hadoop/mapred-site.xml>>>.
 
-    Additionally, you can control the Hadoop scripts found in the bin/
-    directory of the distribution, by setting site-specific values via the
-    <<conf/hadoop-env.sh>> and <<yarn-env.sh>>.
 
-  {Site Configuration}
+  Additionally, you can control the Hadoop scripts found in the bin/
+  directory of the distribution, by setting site-specific values via the
+  <<<etc/hadoop/hadoop-env.sh>>> and <<<etc/hadoop/yarn-env.sh>>>.
 
   To configure the Hadoop cluster you will need to configure the
   <<<environment>>> in which the Hadoop daemons execute as well as the
   <<<configuration parameters>>> for the Hadoop daemons.
 
-  The Hadoop daemons are NameNode/DataNode and ResourceManager/NodeManager.
+  HDFS daemons are NameNode, SecondaryNameNode, and DataNode.  YARN damones
+  are ResourceManager, NodeManager, and WebAppProxy.  If MapReduce is to be
+  used, then the MapReduce Job History Server will also be running.  For
+  large installations, these are generally running on separate hosts.
 
 
 ** {Configuring Environment of Hadoop Daemons}
 
-  Administrators should use the <<conf/hadoop-env.sh>> and
-  <<conf/yarn-env.sh>> script to do site-specific customization of the
-  Hadoop daemons' process environment.
+  Administrators should use the <<<etc/hadoop/hadoop-env.sh>>> and optionally the
+  <<<etc/hadoop/mapred-env.sh>>> and <<<etc/hadoop/yarn-env.sh>>> scripts to do
+  site-specific customization of the Hadoop daemons' process environment.
 
-  At the very least you should specify the <<<JAVA_HOME>>> so that it is
+  At the very least, you must specify the <<<JAVA_HOME>>> so that it is
   correctly defined on each remote node.
 
-  In most cases you should also specify <<<HADOOP_PID_DIR>>> and
-  <<<HADOOP_SECURE_DN_PID_DIR>>> to point to directories that can only be
-  written to by the users that are going to run the hadoop daemons.
-  Otherwise there is the potential for a symlink attack.
-
   Administrators can configure individual daemons using the configuration
   options shown below in the table:
 
@@ -114,20 +112,42 @@ Hadoop MapReduce Next Generation - Cluster Setup
   statement should be added in hadoop-env.sh :
 
 ----
-  export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC ${HADOOP_NAMENODE_OPTS}"
+  export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
 ----
 
+  See <<<etc/hadoop/hadoop-env.sh>>> for other examples.
+
   Other useful configuration parameters that you can customize include:
 
-    * <<<HADOOP_LOG_DIR>>> / <<<YARN_LOG_DIR>>> - The directory where the
-      daemons' log files are stored. They are automatically created if they
-      don't exist.
+    * <<<HADOOP_PID_DIR>>> - The directory where the
+      daemons' process id files are stored.
+
+    * <<<HADOOP_LOG_DIR>>> - The directory where the
+      daemons' log files are stored. Log files are automatically created
+      if they don't exist.
+
+    * <<<HADOOP_HEAPSIZE_MAX>>> - The maximum amount of
+      memory to use for the Java heapsize.  Units supported by the JVM
+      are also supported here.  If no unit is present, it will be assumed
+      the number is in megabytes. By default, Hadoop will let the JVM
+      determine how much to use. This value can be overriden on
+      a per-daemon basis using the appropriate <<<_OPTS>>> variable listed above.
+      For example, setting <<<HADOOP_HEAPSIZE_MAX=1g>>> and
+      <<<HADOOP_NAMENODE_OPTS="-Xmx5g">>>  will configure the NameNode with 5GB heap.
+
+  In most cases, you should specify the <<<HADOOP_PID_DIR>>> and
+  <<<HADOOP_LOG_DIR>>> directories such that they can only be
+  written to by the users that are going to run the hadoop daemons.
+  Otherwise there is the potential for a symlink attack.
+
+  It is also traditional to configure <<<HADOOP_PREFIX>>> in the system-wide
+  shell environment configuration.  For example, a simple script inside
+  <<</etc/profile.d>>>:
 
-    * <<<HADOOP_HEAPSIZE>>> / <<<YARN_HEAPSIZE>>> - The maximum amount of
-      heapsize to use, in MB e.g. if the varibale is set to 1000 the heap
-      will be set to 1000MB.  This is used to configure the heap
-      size for the daemon. By default, the value is 1000.  If you want to
-      configure the values separately for each deamon you can use.
+---
+  HADOOP_PREFIX=/path/to/hadoop
+  export HADOOP_PREFIX
+---
 
 *--------------------------------------+--------------------------------------+
 || Daemon                              || Environment Variable                |
@@ -141,12 +161,12 @@ Hadoop MapReduce Next Generation - Cluster Setup
 | Map Reduce Job History Server        | HADOOP_JOB_HISTORYSERVER_HEAPSIZE    |
 *--------------------------------------+--------------------------------------+
 
-** {Configuring the Hadoop Daemons in Non-Secure Mode}
+** {Configuring the Hadoop Daemons}
 
     This section deals with important parameters to be specified in
     the given configuration files:
 
-    * <<<conf/core-site.xml>>>
+    * <<<etc/hadoop/core-site.xml>>>
 
 *-------------------------+-------------------------+------------------------+
 || Parameter              || Value                  || Notes                 |
@@ -157,7 +177,7 @@ Hadoop MapReduce Next Generation - Cluster Setup
 | | | Size of read/write buffer used in SequenceFiles. |
 *-------------------------+-------------------------+------------------------+
 
-    * <<<conf/hdfs-site.xml>>>
+    * <<<etc/hadoop/hdfs-site.xml>>>
 
       * Configurations for NameNode:
 
@@ -195,7 +215,7 @@ Hadoop MapReduce Next Generation - Cluster Setup
 | | | stored in all named directories, typically on different devices. |
 *-------------------------+-------------------------+------------------------+
 
-    * <<<conf/yarn-site.xml>>>
+    * <<<etc/hadoop/yarn-site.xml>>>
 
       * Configurations for ResourceManager and NodeManager:
 
@@ -341,9 +361,7 @@ Hadoop MapReduce Next Generation - Cluster Setup
 | | | Be careful, set this too small and you will spam the name node. |
 *-------------------------+-------------------------+------------------------+
 
-
-
-    * <<<conf/mapred-site.xml>>>
+    * <<<etc/hadoop/mapred-site.xml>>>
 
       * Configurations for MapReduce Applications:
 
@@ -395,22 +413,6 @@ Hadoop MapReduce Next Generation - Cluster Setup
 | | | Directory where history files are managed by the MR JobHistory Server. |
 *-------------------------+-------------------------+------------------------+
 
-* {Hadoop Rack Awareness}
-
-    The HDFS and the YARN components are rack-aware.
-
-    The NameNode and the ResourceManager obtains the rack information of the
-    slaves in the cluster by invoking an API <resolve> in an administrator
-    configured module.
-
-    The API resolves the DNS name (also IP address) to a rack id.
-
-    The site-specific module to use can be configured using the configuration
-    item <<<topology.node.switch.mapping.impl>>>. The default implementation
-    of the same runs a script/command configured using
-    <<<topology.script.file.name>>>. If <<<topology.script.file.name>>> is
-    not set, the rack id </default-rack> is returned for any passed IP address.
-
 * {Monitoring Health of NodeManagers}
 
     Hadoop provides a mechanism by which administrators can configure the
@@ -433,7 +435,7 @@ Hadoop MapReduce Next Generation - Cluster Setup
     node was healthy is also displayed on the web interface.
 
     The following parameters can be used to control the node health
-    monitoring script in <<<conf/yarn-site.xml>>>.
+    monitoring script in <<<etc/hadoop/yarn-site.xml>>>.
 
 *-------------------------+-------------------------+------------------------+
 || Parameter              || Value                  || Notes                 |
@@ -465,224 +467,170 @@ Hadoop MapReduce Next Generation - Cluster Setup
   disk is either raided or a failure in the boot disk is identified by the
   health checker script.
 
-* {Slaves file}
+* {Slaves File}
 
-  Typically you choose one machine in the cluster to act as the NameNode and
-  one machine as to act as the ResourceManager, exclusively. The rest of the
-  machines act as both a DataNode and NodeManager and are referred to as
-  <slaves>.
+  List all slave hostnames or IP addresses in your <<<etc/hadoop/slaves>>>
+  file, one per line.  Helper scripts (described below) will use the
+  <<<etc/hadoop/slaves>>> file to run commands on many hosts at once.  It is not
+  used for any of the Java-based Hadoop configuration.  In order
+  to use this functionality, ssh trusts (via either passphraseless ssh or
+  some other means, such as Kerberos) must be established for the accounts
+  used to run Hadoop.
 
-  List all slave hostnames or IP addresses in your <<<conf/slaves>>> file,
-  one per line.
+* {Hadoop Rack Awareness}
+
+  Many Hadoop components are rack-aware and take advantage of the
+  network topology for performance and safety. Hadoop daemons obtain the
+  rack information of the slaves in the cluster by invoking an administrator
+  configured module.  See the {{{./RackAwareness.html}Rack Awareness}}
+  documentation for more specific information.
+
+  It is highly recommended configuring rack awareness prior to starting HDFS.
 
 * {Logging}
 
-  Hadoop uses the Apache log4j via the Apache Commons Logging framework for
-  logging. Edit the <<<conf/log4j.properties>>> file to customize the
+  Hadoop uses the {{{http://logging.apache.org/log4j/2.x/}Apache log4j}} via the Apache Commons Logging framework for
+  logging. Edit the <<<etc/hadoop/log4j.properties>>> file to customize the
   Hadoop daemons' logging configuration (log-formats and so on).
 
 * {Operating the Hadoop Cluster}
 
   Once all the necessary configuration is complete, distribute the files to the
-  <<<HADOOP_CONF_DIR>>> directory on all the machines.
+  <<<HADOOP_CONF_DIR>>> directory on all the machines.  This should be the
+  same directory on all machines.
+
+  In general, it is recommended that HDFS and YARN run as separate users.
+  In the majority of installations, HDFS processes execute as 'hdfs'.  YARN
+  is typically using the 'yarn' account.
 
 ** Hadoop Startup
 
-  To start a Hadoop cluster you will need to start both the HDFS and YARN
-  cluster.
+    To start a Hadoop cluster you will need to start both the HDFS and YARN
+    cluster.
 
-  Format a new distributed filesystem:
+    The first time you bring up HDFS, it must be formatted.  Format a new
+    distributed filesystem as <hdfs>:
 
 ----
-$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
 ----
 
-  Start the HDFS with the following command, run on the designated NameNode:
+    Start the HDFS NameNode with the following command on the
+    designated node as <hdfs>:
 
 ----
-$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
-----    	
-
-  Run a script to start DataNodes on all slaves:
-
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode
 ----
-$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
-----    	
 
-  Start the YARN with the following command, run on the designated
-  ResourceManager:
+    Start a HDFS DataNode with the following command on each
+    designated node as <hdfs>:
 
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
-----    	
-
-  Run a script to start NodeManagers on all slaves:
-
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start datanode
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
-----    	
 
-  Start a standalone WebAppProxy server.  If multiple servers
-  are used with load balancing it should be run on each of them:
+    If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
+    (see {{{./SingleCluster.html}Single Node Setup}}), all of the
+    HDFS processes can be started with a utility script.  As <hdfs>:
 
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh start proxyserver --config $HADOOP_CONF_DIR
+[hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
 ----
 
-  Start the MapReduce JobHistory Server with the following command, run on the
-  designated server:
+    Start the YARN with the following command, run on the designated
+    ResourceManager as <yarn>:
 
 ----
-$ $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver --config $HADOOP_CONF_DIR
-----    	
-
-** Hadoop Shutdown
-
-  Stop the NameNode with the following command, run on the designated
-  NameNode:
-
+[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start resourcemanager
 ----
-$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode
-----    	
 
-  Run a script to stop DataNodes on all slaves:
+    Run a script to start a NodeManager on each designated host as <yarn>:
 
 ----
-$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode
-----    	
-
-  Stop the ResourceManager with the following command, run on the designated
-  ResourceManager:
-
+[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start nodemanager
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
-----    	
 
-  Run a script to stop NodeManagers on all slaves:
+    Start a standalone WebAppProxy server. Run on the WebAppProxy
+    server as <yarn>.  If multiple servers are used with load balancing
+    it should be run on each of them:
 
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
-----    	
+[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start proxyserver
+----
 
-  Stop the WebAppProxy server. If multiple servers are used with load
-  balancing it should be run on each of them:
+    If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
+    (see {{{./SingleCluster.html}Single Node Setup}}), all of the
+    YARN processes can be started with a utility script.  As <yarn>:
 
 ----
-$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh stop proxyserver --config $HADOOP_CONF_DIR
+[yarn]$ $HADOOP_PREFIX/sbin/start-yarn.sh
 ----
 
-
-  Stop the MapReduce JobHistory Server with the following command, run on the
-  designated server:
+    Start the MapReduce JobHistory Server with the following command, run
+    on the designated server as <mapred>:
 
 ----
-$ $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh stop historyserver --config $HADOOP_CONF_DIR
-----    	
-
-
-* {Operating the Hadoop Cluster}
-
-  Once all the necessary configuration is complete, distribute the files to the
-  <<<HADOOP_CONF_DIR>>> directory on all the machines.
-
-  This section also describes the various Unix users who should be starting the
-  various components and uses the same Unix accounts and groups used previously:
-
-** Hadoop Startup
+[mapred]$ $HADOOP_PREFIX/bin/mapred --daemon start historyserver
+----
 
-    To start a Hadoop cluster you will need to start both the HDFS and YARN
-    cluster.
+** Hadoop Shutdown
 
-    Format a new distributed filesystem as <hdfs>:
+  Stop the NameNode with the following command, run on the designated NameNode
+  as <hdfs>:
 
 ----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop namenode
 ----
 
-    Start the HDFS with the following command, run on the designated NameNode
-    as <hdfs>:
+  Run a script to stop a DataNode as <hdfs>:
 
 ----
-[hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
-----    	
-
-    Run a script to start DataNodes on all slaves as <root> with a special
-    environment variable <<<HADOOP_SECURE_DN_USER>>> set to <hdfs>:
-
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop datanode
 ----
-[root]$ HADOOP_SECURE_DN_USER=hdfs $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
-----    	
 
-    Start the YARN with the following command, run on the designated
-    ResourceManager as <yarn>:
+    If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
+    (see {{{./SingleCluster.html}Single Node Setup}}), all of the
+    HDFS processes may be stopped with a utility script.  As <hdfs>:
 
 ----
-[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
-----    	
-
-    Run a script to start NodeManagers on all slaves as <yarn>:
-
+[hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
 ----
-[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
-----    	
 
-    Start a standalone WebAppProxy server. Run on the WebAppProxy
-    server as <yarn>.  If multiple servers are used with load balancing
-    it should be run on each of them:
+  Stop the ResourceManager with the following command, run on the designated
+  ResourceManager as <yarn>:
 
 ----
-[yarn]$ $HADOOP_YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
-----    	
-
-    Start the MapReduce JobHistory Server with the following command, run on the
-    designated server as <mapred>:
-
+[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop resourcemanager
 ----
-[mapred]$ $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver --config $HADOOP_CONF_DIR
-----    	
 
-** Hadoop Shutdown
-
-  Stop the NameNode with the following command, run on the designated NameNode
-  as <hdfs>:
+  Run a script to stop a NodeManager on a slave as <yarn>:
 
 ----
-[hdfs]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode
-----    	
-
-  Run a script to stop DataNodes on all slaves as <root>:
-
+[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop nodemanager
 ----
-[root]$ $HADOOP_PREFIX/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode
-----    	
 
-  Stop the ResourceManager with the following command, run on the designated
-  ResourceManager as <yarn>:
+    If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
+    (see {{{./SingleCluster.html}Single Node Setup}}), all of the
+    YARN processes can be stopped with a utility script.  As <yarn>:
 
 ----
-[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
-----    	
-
-  Run a script to stop NodeManagers on all slaves as <yarn>:
-
+[yarn]$ $HADOOP_PREFIX/sbin/stop-yarn.sh
 ----
-[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
-----    	
 
   Stop the WebAppProxy server. Run on the WebAppProxy  server as
   <yarn>.  If multiple servers are used with load balancing it
   should be run on each of them:
 
 ----
-[yarn]$ $HADOOP_YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
+[yarn]$ $HADOOP_PREFIX/bin/yarn stop proxyserver
 ----
 
   Stop the MapReduce JobHistory Server with the following command, run on the
   designated server as <mapred>:
 
 ----
-[mapred]$ $HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh stop historyserver --config $HADOOP_CONF_DIR
-----    	
+[mapred]$ $HADOOP_PREFIX/bin/mapred --daemon stop historyserver
+----
 
 * {Web Interfaces}
 
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
index 6d2fd5e2b7e6b..67c8bc37b317c 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
@@ -21,102 +21,161 @@
 
 %{toc}
 
-Overview
+Hadoop Commands Guide
 
-   All hadoop commands are invoked by the <<<bin/hadoop>>> script. Running the
-   hadoop script without any arguments prints the description for all
-   commands.
+* Overview
 
-   Usage: <<<hadoop [--config confdir] [--loglevel loglevel] [COMMAND]
-             [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
+   All of the Hadoop commands and subprojects follow the same basic structure:
 
-   Hadoop has an option parsing framework that employs parsing generic
-   options as well as running classes.
+   Usage: <<<shellcommand [SHELL_OPTIONS] [COMMAND] [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
 
+*--------+---------+
+|| FIELD || Description 
 *-----------------------+---------------+
-|| COMMAND_OPTION       || Description
+| shellcommand | The command of the project being invoked.  For example,
+               | Hadoop common uses <<<hadoop>>>, HDFS uses <<<hdfs>>>, 
+               | and YARN uses <<<yarn>>>.
+*---------------+-------------------+
+| SHELL_OPTIONS | Options that the shell processes prior to executing Java.
 *-----------------------+---------------+
-| <<<--config confdir>>>| Overwrites the default Configuration directory.  Default is <<<${HADOOP_HOME}/conf>>>.
+| COMMAND | Action to perform.
 *-----------------------+---------------+
-| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
-|                       | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
-|                       | Default is INFO.
+| GENERIC_OPTIONS       | The common set of options supported by 
+                        | multiple commands.
 *-----------------------+---------------+
-| GENERIC_OPTIONS       | The common set of options supported by multiple commands.
-| COMMAND_OPTIONS       | Various commands with their options are described in the following sections. The commands have been grouped into User Commands and Administration Commands.
+| COMMAND_OPTIONS       | Various commands with their options are 
+                        | described in this documention for the 
+                        | Hadoop common sub-project.  HDFS and YARN are
+                        | covered in other documents.
 *-----------------------+---------------+
 
-Generic Options
+** {Shell Options}
 
-   The following options are supported by {{dfsadmin}}, {{fs}}, {{fsck}},
-   {{job}} and {{fetchdt}}. Applications should implement 
-   {{{../../api/org/apache/hadoop/util/Tool.html}Tool}} to support
-   GenericOptions.
+   All of the shell commands will accept a common set of options.  For some commands,
+   these options are ignored. For example, passing <<<---hostnames>>> on a
+   command that only executes on a single host will be ignored.
+
+*-----------------------+---------------+
+|| SHELL_OPTION       || Description
+*-----------------------+---------------+
+| <<<--buildpaths>>>    | Enables developer versions of jars.
+*-----------------------+---------------+
+| <<<--config confdir>>> | Overwrites the default Configuration 
+                         | directory.  Default is <<<${HADOOP_PREFIX}/conf>>>.
+*-----------------------+----------------+
+| <<<--daemon mode>>>   | If the command supports daemonization (e.g.,
+                        | <<<hdfs namenode>>>), execute in the appropriate
+                        | mode. Supported modes are <<<start>>> to start the
+                        | process in daemon mode, <<<stop>>> to stop the
+                        | process, and <<<status>>> to determine the active
+                        | status of the process.  <<<status>>> will return
+                        | an {{{http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html}LSB-compliant}} result code. 
+                        | If no option is provided, commands that support
+                        | daemonization will run in the foreground.   
+*-----------------------+---------------+
+| <<<--debug>>>         | Enables shell level configuration debugging information
+*-----------------------+---------------+
+| <<<--help>>>          | Shell script usage information.
+*-----------------------+---------------+
+| <<<--hostnames>>> | A space delimited list of hostnames where to execute 
+                    | a multi-host subcommand. By default, the content of
+                    | the <<<slaves>>> file is used.  
+*-----------------------+----------------+
+| <<<--hosts>>> | A file that contains a list of hostnames where to execute
+                | a multi-host subcommand. By default, the content of the
+                | <<<slaves>>> file is used.  
+*-----------------------+----------------+
+| <<<--loglevel loglevel>>> | Overrides the log level. Valid log levels are
+|                           | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
+|                           | Default is INFO.
+*-----------------------+---------------+
+
+** {Generic Options}
+
+   Many subcommands honor a common set of configuration options to alter their behavior:
 
 *------------------------------------------------+-----------------------------+
 ||            GENERIC_OPTION                     ||            Description
 *------------------------------------------------+-----------------------------+
+|<<<-archives \<comma separated list of archives\> >>> | Specify comma separated
+                                                 | archives to be unarchived on
+                                                 | the compute machines. Applies
+                                                 | only to job.
+*------------------------------------------------+-----------------------------+
 |<<<-conf \<configuration file\> >>>             | Specify an application
                                                  | configuration file.
 *------------------------------------------------+-----------------------------+
 |<<<-D \<property\>=\<value\> >>>                | Use value for given property.
 *------------------------------------------------+-----------------------------+
-|<<<-jt \<local\> or \<resourcemanager:port\>>>> | Specify a ResourceManager.
-                                                 | Applies only to job.
-*------------------------------------------------+-----------------------------+
 |<<<-files \<comma separated list of files\> >>> | Specify comma separated files
                                                  | to be copied to the map
                                                  | reduce cluster.  Applies only
                                                  | to job.
 *------------------------------------------------+-----------------------------+
+|<<<-jt \<local\> or \<resourcemanager:port\>>>> | Specify a ResourceManager.
+                                                 | Applies only to job.
+*------------------------------------------------+-----------------------------+
 |<<<-libjars \<comma seperated list of jars\> >>>| Specify comma separated jar
                                                  | files to include in the
                                                  | classpath. Applies only to
                                                  | job.
 *------------------------------------------------+-----------------------------+
-|<<<-archives \<comma separated list of archives\> >>> | Specify comma separated
-                                                 | archives to be unarchived on
-                                                 | the compute machines. Applies
-                                                 | only to job.
-*------------------------------------------------+-----------------------------+
 
-User Commands
+Hadoop Common Commands
 
-   Commands useful for users of a hadoop cluster.
+  All of these commands are executed from the <<<hadoop>>> shell command.  They
+  have been broken up into {{User Commands}} and 
+  {{Admininistration Commands}}.
+
+* User Commands
 
-* <<<archive>>>
+   Commands useful for users of a hadoop cluster.
 
+** <<<archive>>>
+    
    Creates a hadoop archive. More information can be found at
-   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html}
+  {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html}
    Hadoop Archives Guide}}.
 
-* <<<credential>>>
+** <<<checknative>>>
 
-   Command to manage credentials, passwords and secrets within credential providers.
+    Usage: <<<hadoop checknative [-a] [-h] >>>
 
-   The CredentialProvider API in Hadoop allows for the separation of applications
-   and how they store their required passwords/secrets. In order to indicate
-   a particular provider type and location, the user must provide the
-   <hadoop.security.credential.provider.path> configuration element in core-site.xml
-   or use the command line option <<<-provider>>> on each of the following commands.
-   This provider path is a comma-separated list of URLs that indicates the type and
-   location of a list of providers that should be consulted.
-   For example, the following path:
+*-----------------+-----------------------------------------------------------+
+|| COMMAND_OPTION || Description
+*-----------------+-----------------------------------------------------------+
+| -a              | Check all libraries are available.
+*-----------------+-----------------------------------------------------------+
+| -h              | print help
+*-----------------+-----------------------------------------------------------+
 
-   <<<user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks>>>
+    This command checks the availability of the Hadoop native code.  See
+    {{{NativeLibraries.html}}} for more information.  By default, this command 
+    only checks the availability of libhadoop.
 
-   indicates that the current user's credentials file should be consulted through
-   the User Provider, that the local file located at <<</tmp/test.jceks>>> is a Java Keystore
-   Provider and that the file located within HDFS at <<<nn1.example.com/my/path/test.jceks>>>
-   is also a store for a Java Keystore Provider.
+** <<<classpath>>>
 
-   When utilizing the credential command it will often be for provisioning a password
-   or secret to a particular credential store provider. In order to explicitly
-   indicate which provider store to use the <<<-provider>>> option should be used. Otherwise,
-   given a path of multiple providers, the first non-transient provider will be used.
-   This may or may not be the one that you intended.
+   Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>>
 
-   Example: <<<-provider jceks://file/tmp/test.jceks>>>
+*-----------------+-----------------------------------------------------------+
+|| COMMAND_OPTION || Description
+*-----------------+-----------------------------------------------------------+
+| --glob          | expand wildcards
+*-----------------+-----------------------------------------------------------+
+| --jar <path>    | write classpath as manifest in jar named <path>
+*-----------------+-----------------------------------------------------------+
+| -h, --help      | print help
+*-----------------+-----------------------------------------------------------+
+
+   Prints the class path needed to get the Hadoop jar and the required
+   libraries.  If called without arguments, then prints the classpath set up by
+   the command scripts, which is likely to contain wildcards in the classpath
+   entries.  Additional options print the classpath after wildcard expansion or
+   write the classpath into the manifest of a jar file.  The latter is useful in
+   environments where wildcards cannot be used and the expanded classpath exceeds
+   the maximum supported command line length.
+
+** <<<credential>>>
 
    Usage: <<<hadoop credential <subcommand> [options]>>>
 
@@ -143,109 +202,96 @@ User Commands
                     | indicated.
 *-------------------+-------------------------------------------------------+
 
-* <<<distcp>>>
+   Command to manage credentials, passwords and secrets within credential providers.
 
-   Copy file or directories recursively. More information can be found at
-   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html}
-   Hadoop DistCp Guide}}.
+   The CredentialProvider API in Hadoop allows for the separation of applications
+   and how they store their required passwords/secrets. In order to indicate
+   a particular provider type and location, the user must provide the
+   <hadoop.security.credential.provider.path> configuration element in core-site.xml
+   or use the command line option <<<-provider>>> on each of the following commands.
+   This provider path is a comma-separated list of URLs that indicates the type and
+   location of a list of providers that should be consulted. For example, the following path:
+   <<<user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks>>>
 
-* <<<fs>>>
+   indicates that the current user's credentials file should be consulted through
+   the User Provider, that the local file located at <<</tmp/test.jceks>>> is a Java Keystore
+   Provider and that the file located within HDFS at <<<nn1.example.com/my/path/test.jceks>>>
+   is also a store for a Java Keystore Provider.
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#dfs}<<<hdfs dfs>>>}}
-   instead.
+   When utilizing the credential command it will often be for provisioning a password
+   or secret to a particular credential store provider. In order to explicitly
+   indicate which provider store to use the <<<-provider>>> option should be used. Otherwise,
+   given a path of multiple providers, the first non-transient provider will be used.
+   This may or may not be the one that you intended.
 
-* <<<fsck>>>
+   Example: <<<-provider jceks://file/tmp/test.jceks>>>
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#fsck}<<<hdfs fsck>>>}}
-   instead.
+** <<<distch>>>
 
-* <<<fetchdt>>>
+  Usage: <<<hadoop distch [-f urilist_url] [-i] [-log logdir] path:owner:group:permissions>>>
+  
+*-------------------+-------------------------------------------------------+
+||COMMAND_OPTION    ||                   Description
+*-------------------+-------------------------------------------------------+
+| -f | List of objects to change
+*----+------------+
+| -i | Ignore failures
+*----+------------+
+| -log | Directory to log output
+*-----+---------+
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#fetchdt}
-   <<<hdfs fetchdt>>>}} instead.
+  Change the ownership and permissions on many files at once.
 
-* <<<jar>>>
+** <<<distcp>>>
 
-   Runs a jar file. Users can bundle their Map Reduce code in a jar file and
-   execute it using this command.
+   Copy file or directories recursively. More information can be found at
+   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html}
+   Hadoop DistCp Guide}}.
 
-   Usage: <<<hadoop jar <jar> [mainClass] args...>>>
+** <<<fs>>>
 
-   The streaming jobs are run via this command. Examples can be referred from
-   Streaming examples
+   This command is documented in the {{{./FileSystemShell.html}File System Shell Guide}}.  It is a synonym for <<<hdfs dfs>>> when HDFS is in use.
 
-   Word count example is also run using jar command. It can be referred from
-   Wordcount example
+** <<<jar>>>
 
-   Use {{{../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar}<<<yarn jar>>>}}
-   to launch YARN applications instead.
+  Usage: <<<hadoop jar <jar> [mainClass] args...>>>
 
-* <<<job>>>
+  Runs a jar file. 
+  
+  Use {{{../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar}<<<yarn jar>>>}}
+  to launch YARN applications instead.
 
-   Deprecated. Use
-   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#job}
-   <<<mapred job>>>}} instead.
+** <<<jnipath>>>
 
-* <<<pipes>>>
+    Usage: <<<hadoop jnipath>>>
 
-   Deprecated. Use
-   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#pipes}
-   <<<mapred pipes>>>}} instead.
+    Print the computed java.library.path.
 
-* <<<queue>>>
+** <<<key>>>
 
-   Deprecated. Use
-   {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#queue}
-   <<<mapred queue>>>}} instead.
+    Manage keys via the KeyProvider.
 
-* <<<version>>>
+** <<<trace>>>
 
-   Prints the version.
+    View and modify Hadoop tracing settings.   See the {{{./Tracing.html}Tracing Guide}}.
+
+** <<<version>>>
 
    Usage: <<<hadoop version>>>
 
-* <<<CLASSNAME>>>
+   Prints the version.
 
-   hadoop script can be used to invoke any class.
+** <<<CLASSNAME>>>
 
    Usage: <<<hadoop CLASSNAME>>>
 
-   Runs the class named <<<CLASSNAME>>>.
-
-* <<<classpath>>>
-
-   Prints the class path needed to get the Hadoop jar and the required
-   libraries.  If called without arguments, then prints the classpath set up by
-   the command scripts, which is likely to contain wildcards in the classpath
-   entries.  Additional options print the classpath after wildcard expansion or
-   write the classpath into the manifest of a jar file.  The latter is useful in
-   environments where wildcards cannot be used and the expanded classpath exceeds
-   the maximum supported command line length.
+   Runs the class named <<<CLASSNAME>>>.  The class must be part of a package.
 
-   Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| --glob          | expand wildcards
-*-----------------+-----------------------------------------------------------+
-| --jar <path>    | write classpath as manifest in jar named <path>
-*-----------------+-----------------------------------------------------------+
-| -h, --help      | print help
-*-----------------+-----------------------------------------------------------+
-
-Administration Commands
+* {Administration Commands}
 
    Commands useful for administrators of a hadoop cluster.
 
-* <<<balancer>>>
-
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#balancer}
-   <<<hdfs balancer>>>}} instead.
-
-* <<<daemonlog>>>
-
-   Get/Set the log level for each daemon.
+** <<<daemonlog>>>
 
    Usage: <<<hadoop daemonlog -getlevel <host:port> <name> >>>
    Usage: <<<hadoop daemonlog -setlevel <host:port> <name> <level> >>>
@@ -262,22 +308,20 @@ Administration Commands
                                | connects to http://<host:port>/logLevel?log=<name>
 *------------------------------+-----------------------------------------------------------+
 
-* <<<datanode>>>
+   Get/Set the log level for each daemon.
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#datanode}
-   <<<hdfs datanode>>>}} instead.
+* Files
 
-* <<<dfsadmin>>>
+** <<etc/hadoop/hadoop-env.sh>>
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#dfsadmin}
-   <<<hdfs dfsadmin>>>}} instead.
+    This file stores the global settings used by all Hadoop shell commands.
 
-* <<<namenode>>>
+** <<etc/hadoop/hadoop-user-functions.sh>>
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#namenode}
-   <<<hdfs namenode>>>}} instead.
+    This file allows for advanced users to override some shell functionality.
 
-* <<<secondarynamenode>>>
+** <<~/.hadooprc>>
 
-   Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#secondarynamenode}
-   <<<hdfs secondarynamenode>>>}} instead.
+    This stores the personal environment for an individual user.  It is
+    processed after the hadoop-env.sh and hadoop-user-functions.sh files
+    and can contain the same settings.
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
index 1a9618c95e359..3fd56fcb3dfb0 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
@@ -45,46 +45,62 @@ bin/hadoop fs <args>
    Differences are described with each of the commands. Error information is
    sent to stderr and the output is sent to stdout.
 
-appendToFile
+   If HDFS is being used, <<<hdfs dfs>>> is a synonym.
 
-      Usage: <<<hdfs dfs -appendToFile <localsrc> ... <dst> >>>
+   See the {{{./CommandsManual.html}Commands Manual}} for generic shell options.
+
+* appendToFile
+
+      Usage: <<<hadoop fs -appendToFile <localsrc> ... <dst> >>>
 
       Append single src, or multiple srcs from local file system to the
       destination file system. Also reads input from stdin and appends to
       destination file system.
 
-        * <<<hdfs dfs -appendToFile localfile /user/hadoop/hadoopfile>>>
+        * <<<hadoop fs -appendToFile localfile /user/hadoop/hadoopfile>>>
 
-        * <<<hdfs dfs -appendToFile localfile1 localfile2 /user/hadoop/hadoopfile>>>
+        * <<<hadoop fs -appendToFile localfile1 localfile2 /user/hadoop/hadoopfile>>>
 
-        * <<<hdfs dfs -appendToFile localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
+        * <<<hadoop fs -appendToFile localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
 
-        * <<<hdfs dfs -appendToFile - hdfs://nn.example.com/hadoop/hadoopfile>>>
+        * <<<hadoop fs -appendToFile - hdfs://nn.example.com/hadoop/hadoopfile>>>
           Reads the input from stdin.
 
       Exit Code:
 
       Returns 0 on success and 1 on error.
 
-cat
+* cat
 
-   Usage: <<<hdfs dfs -cat URI [URI ...]>>>
+   Usage: <<<hadoop fs -cat URI [URI ...]>>>
 
    Copies source paths to stdout.
 
    Example:
 
-     * <<<hdfs dfs -cat hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
+     * <<<hadoop fs -cat hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
 
-     * <<<hdfs dfs -cat file:///file3 /user/hadoop/file4>>>
+     * <<<hadoop fs -cat file:///file3 /user/hadoop/file4>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-chgrp
+* checksum
+
+  Usage: <<<hadoop fs -checksum URI>>>
+
+  Returns the checksum information of a file.
+
+  Example:
+
+    * <<<hadoop fs -checksum hdfs://nn1.example.com/file1>>>
 
-   Usage: <<<hdfs dfs -chgrp [-R] GROUP URI [URI ...]>>>
+    * <<<hadoop fs -checksum file:///etc/hosts>>>
+
+* chgrp
+
+   Usage: <<<hadoop fs -chgrp [-R] GROUP URI [URI ...]>>>
 
    Change group association of files. The user must be the owner of files, or
    else a super-user. Additional information is in the
@@ -94,9 +110,9 @@ chgrp
 
      * The -R option will make the change recursively through the directory structure.
 
-chmod
+* chmod
 
-   Usage: <<<hdfs dfs -chmod [-R] <MODE[,MODE]... | OCTALMODE> URI [URI ...]>>>
+   Usage: <<<hadoop fs -chmod [-R] <MODE[,MODE]... | OCTALMODE> URI [URI ...]>>>
 
    Change the permissions of files. With -R, make the change recursively
    through the directory structure. The user must be the owner of the file, or
@@ -107,9 +123,9 @@ chmod
 
      * The -R option will make the change recursively through the directory structure.
 
-chown
+* chown
 
-   Usage: <<<hdfs dfs -chown [-R] [OWNER][:[GROUP]] URI [URI ]>>>
+   Usage: <<<hadoop fs -chown [-R] [OWNER][:[GROUP]] URI [URI ]>>>
 
    Change the owner of files. The user must be a super-user. Additional information
    is in the {{{../hadoop-hdfs/HdfsPermissionsGuide.html}Permissions Guide}}.
@@ -118,9 +134,9 @@ chown
 
      * The -R option will make the change recursively through the directory structure.
 
-copyFromLocal
+* copyFromLocal
 
-   Usage: <<<hdfs dfs -copyFromLocal <localsrc> URI>>>
+   Usage: <<<hadoop fs -copyFromLocal <localsrc> URI>>>
 
    Similar to put command, except that the source is restricted to a local
    file reference.
@@ -129,41 +145,45 @@ copyFromLocal
 
      * The -f option will overwrite the destination if it already exists.
 
-copyToLocal
+* copyToLocal
 
-   Usage: <<<hdfs dfs -copyToLocal [-ignorecrc] [-crc] URI <localdst> >>>
+   Usage: <<<hadoop fs -copyToLocal [-ignorecrc] [-crc] URI <localdst> >>>
 
    Similar to get command, except that the destination is restricted to a
    local file reference.
 
-count
+* count
 
-   Usage: <<<hdfs dfs -count [-q] [-h] <paths> >>>
+   Usage: <<<hadoop fs -count [-q] [-h] [-v] <paths> >>>
 
    Count the number of directories, files and bytes under the paths that match
    the specified file pattern.  The output columns with -count are: DIR_COUNT,
-   FILE_COUNT, CONTENT_SIZE FILE_NAME
+   FILE_COUNT, CONTENT_SIZE PATHNAME
 
    The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
-   REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
+   REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME
 
    The -h option shows sizes in human readable format.
 
+   The -v option displays a header line.
+
    Example:
 
-     * <<<hdfs dfs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
+     * <<<hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
 
-     * <<<hdfs dfs -count -q hdfs://nn1.example.com/file1>>>
+     * <<<hadoop fs -count -q hdfs://nn1.example.com/file1>>>
 
-     * <<<hdfs dfs -count -q -h hdfs://nn1.example.com/file1>>>
+     * <<<hadoop fs -count -q -h hdfs://nn1.example.com/file1>>>
+
+     * <<<hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-cp
+* cp
 
-   Usage: <<<hdfs dfs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> >>>
+   Usage: <<<hadoop fs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> >>>
 
    Copy files from source to destination. This command allows multiple sources
    as well in which case the destination must be a directory.
@@ -177,7 +197,7 @@ cp
     Options:
 
       * The -f option will overwrite the destination if it already exists.
-      
+
       * The -p option will preserve file attributes [topx] (timestamps,
         ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
         then preserves timestamps, ownership, permission. If -pa is specified,
@@ -187,17 +207,41 @@ cp
 
    Example:
 
-     * <<<hdfs dfs -cp /user/hadoop/file1 /user/hadoop/file2>>>
+     * <<<hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2>>>
 
-     * <<<hdfs dfs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir>>>
+     * <<<hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-du
+* createSnapshot
+
+  See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
 
-   Usage: <<<hdfs dfs -du [-s] [-h] URI [URI ...]>>>
+
+* deleteSnapshot
+
+  See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
+
+* df
+
+   Usage: <<<hadoop fs -df [-h] URI [URI ...]>>>
+
+   Displays free space.
+
+   Options:
+
+     * The -h option will format file sizes in a "human-readable" fashion (e.g
+       64.0m instead of 67108864)
+
+   Example:
+
+     * <<<hadoop dfs -df /user/hadoop/dir1>>>
+
+* du
+
+   Usage: <<<hadoop fs -du [-s] [-h] URI [URI ...]>>>
 
    Displays sizes of files and directories contained in the given directory or
    the length of a file in case its just a file.
@@ -212,29 +256,29 @@ du
 
    Example:
 
-    * hdfs dfs -du /user/hadoop/dir1 /user/hadoop/file1 hdfs://nn.example.com/user/hadoop/dir1
+    * <<<hadoop fs -du /user/hadoop/dir1 /user/hadoop/file1 hdfs://nn.example.com/user/hadoop/dir1>>>
 
    Exit Code:
    Returns 0 on success and -1 on error.
 
-dus
+* dus
 
-   Usage: <<<hdfs dfs -dus <args> >>>
+   Usage: <<<hadoop fs -dus <args> >>>
 
    Displays a summary of file lengths.
 
-   <<Note:>> This command is deprecated. Instead use <<<hdfs dfs -du -s>>>.
+   <<Note:>> This command is deprecated. Instead use <<<hadoop fs -du -s>>>.
 
-expunge
+* expunge
 
-   Usage: <<<hdfs dfs -expunge>>>
+   Usage: <<<hadoop fs -expunge>>>
 
    Empty the Trash. Refer to the {{{../hadoop-hdfs/HdfsDesign.html}
    HDFS Architecture Guide}} for more information on the Trash feature.
 
-find
+* find
 
-   Usage: <<<hdfs dfs -find <path> ... <expression> ... >>>
+   Usage: <<<hadoop fs -find <path> ... <expression> ... >>>
 
    Finds all files that match the specified expression and applies selected
    actions to them. If no <path> is specified then defaults to the current
@@ -269,15 +313,15 @@ find
 
    Example:
 
-   <<<hdfs dfs -find / -name test -print>>>
+   <<<hadoop fs -find / -name test -print>>>
 
    Exit Code:
 
      Returns 0 on success and -1 on error.
 
-get
+* get
 
-   Usage: <<<hdfs dfs -get [-ignorecrc] [-crc] <src> <localdst> >>>
+   Usage: <<<hadoop fs -get [-ignorecrc] [-crc] <src> <localdst> >>>
 
    Copy files to the local file system. Files that fail the CRC check may be
    copied with the -ignorecrc option. Files and CRCs may be copied using the
@@ -285,17 +329,17 @@ get
 
    Example:
 
-     * <<<hdfs dfs -get /user/hadoop/file localfile>>>
+     * <<<hadoop fs -get /user/hadoop/file localfile>>>
 
-     * <<<hdfs dfs -get hdfs://nn.example.com/user/hadoop/file localfile>>>
+     * <<<hadoop fs -get hdfs://nn.example.com/user/hadoop/file localfile>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-getfacl
+* getfacl
 
-   Usage: <<<hdfs dfs -getfacl [-R] <path> >>>
+   Usage: <<<hadoop fs -getfacl [-R] <path> >>>
 
    Displays the Access Control Lists (ACLs) of files and directories. If a
    directory has a default ACL, then getfacl also displays the default ACL.
@@ -308,17 +352,17 @@ getfacl
 
    Examples:
 
-     * <<<hdfs dfs -getfacl /file>>>
+     * <<<hadoop fs -getfacl /file>>>
 
-     * <<<hdfs dfs -getfacl -R /dir>>>
+     * <<<hadoop fs -getfacl -R /dir>>>
 
    Exit Code:
 
    Returns 0 on success and non-zero on error.
 
-getfattr
+* getfattr
 
-   Usage: <<<hdfs dfs -getfattr [-R] {-n name | -d} [-e en] <path> >>>
+   Usage: <<<hadoop fs -getfattr [-R] {-n name | -d} [-e en] <path> >>>
 
    Displays the extended attribute names and values (if any) for a file or
    directory.
@@ -337,26 +381,32 @@ getfattr
 
    Examples:
 
-     * <<<hdfs dfs -getfattr -d /file>>>
+     * <<<hadoop fs -getfattr -d /file>>>
 
-     * <<<hdfs dfs -getfattr -R -n user.myAttr /dir>>>
+     * <<<hadoop fs -getfattr -R -n user.myAttr /dir>>>
 
    Exit Code:
 
    Returns 0 on success and non-zero on error.
 
-getmerge
+* getmerge
 
-   Usage: <<<hdfs dfs -getmerge <src> <localdst> [addnl]>>>
+   Usage: <<<hadoop fs -getmerge <src> <localdst> [addnl]>>>
 
    Takes a source directory and a destination file as input and concatenates
    files in src into the destination local file. Optionally addnl can be set to
    enable adding a newline character at the
    end of each file.
 
-ls
+* help
+
+   Usage: <<<hadoop fs -help>>>
+
+   Return usage output.
+
+* ls
 
-   Usage: <<<hdfs dfs -ls [-R] <args> >>>
+   Usage: <<<hadoop fs -ls [-R] <args> >>>
 
    Options:
 
@@ -377,23 +427,23 @@ permissions userid groupid modification_date modification_time dirname
 
    Example:
 
-     * <<<hdfs dfs -ls /user/hadoop/file1>>>
+     * <<<hadoop fs -ls /user/hadoop/file1>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-lsr
+* lsr
 
-   Usage: <<<hdfs dfs -lsr <args> >>>
+   Usage: <<<hadoop fs -lsr <args> >>>
 
    Recursive version of ls.
 
-   <<Note:>> This command is deprecated. Instead use <<<hdfs dfs -ls -R>>>
+   <<Note:>> This command is deprecated. Instead use <<<hadoop fs -ls -R>>>
 
-mkdir
+* mkdir
 
-   Usage: <<<hdfs dfs -mkdir [-p] <paths> >>>
+   Usage: <<<hadoop fs -mkdir [-p] <paths> >>>
 
    Takes path uri's as argument and creates directories.
 
@@ -403,30 +453,30 @@ mkdir
 
    Example:
 
-     * <<<hdfs dfs -mkdir /user/hadoop/dir1 /user/hadoop/dir2>>>
+     * <<<hadoop fs -mkdir /user/hadoop/dir1 /user/hadoop/dir2>>>
 
-     * <<<hdfs dfs -mkdir hdfs://nn1.example.com/user/hadoop/dir hdfs://nn2.example.com/user/hadoop/dir>>>
+     * <<<hadoop fs -mkdir hdfs://nn1.example.com/user/hadoop/dir hdfs://nn2.example.com/user/hadoop/dir>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-moveFromLocal
+* moveFromLocal
 
-   Usage: <<<hdfs dfs -moveFromLocal <localsrc> <dst> >>>
+   Usage: <<<hadoop fs -moveFromLocal <localsrc> <dst> >>>
 
    Similar to put command, except that the source localsrc is deleted after
    it's copied.
 
-moveToLocal
+* moveToLocal
 
-   Usage: <<<hdfs dfs -moveToLocal [-crc] <src> <dst> >>>
+   Usage: <<<hadoop fs -moveToLocal [-crc] <src> <dst> >>>
 
    Displays a "Not implemented yet" message.
 
-mv
+* mv
 
-   Usage: <<<hdfs dfs -mv URI [URI ...] <dest> >>>
+   Usage: <<<hadoop fs -mv URI [URI ...] <dest> >>>
 
    Moves files from source to destination. This command allows multiple sources
    as well in which case the destination needs to be a directory. Moving files
@@ -434,38 +484,42 @@ mv
 
    Example:
 
-     * <<<hdfs dfs -mv /user/hadoop/file1 /user/hadoop/file2>>>
+     * <<<hadoop fs -mv /user/hadoop/file1 /user/hadoop/file2>>>
 
-     * <<<hdfs dfs -mv hdfs://nn.example.com/file1 hdfs://nn.example.com/file2 hdfs://nn.example.com/file3 hdfs://nn.example.com/dir1>>>
+     * <<<hadoop fs -mv hdfs://nn.example.com/file1 hdfs://nn.example.com/file2 hdfs://nn.example.com/file3 hdfs://nn.example.com/dir1>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-put
+* put
 
-   Usage: <<<hdfs dfs -put <localsrc> ... <dst> >>>
+   Usage: <<<hadoop fs -put <localsrc> ... <dst> >>>
 
    Copy single src, or multiple srcs from local file system to the destination
    file system. Also reads input from stdin and writes to destination file
    system.
 
-     * <<<hdfs dfs -put localfile /user/hadoop/hadoopfile>>>
+     * <<<hadoop fs -put localfile /user/hadoop/hadoopfile>>>
 
-     * <<<hdfs dfs -put localfile1 localfile2 /user/hadoop/hadoopdir>>>
+     * <<<hadoop fs -put localfile1 localfile2 /user/hadoop/hadoopdir>>>
 
-     * <<<hdfs dfs -put localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
+     * <<<hadoop fs -put localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
 
-     * <<<hdfs dfs -put - hdfs://nn.example.com/hadoop/hadoopfile>>>
+     * <<<hadoop fs -put - hdfs://nn.example.com/hadoop/hadoopfile>>>
        Reads the input from stdin.
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-rm
+* renameSnapshot
 
-   Usage: <<<hdfs dfs -rm [-f] [-r|-R] [-skipTrash] URI [URI ...]>>>
+  See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
+
+* rm
+
+   Usage: <<<hadoop fs -rm [-f] [-r|-R] [-skipTrash] URI [URI ...]>>>
 
    Delete files specified as args.
 
@@ -484,23 +538,37 @@ rm
 
    Example:
 
-     * <<<hdfs dfs -rm hdfs://nn.example.com/file /user/hadoop/emptydir>>>
+     * <<<hadoop fs -rm hdfs://nn.example.com/file /user/hadoop/emptydir>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-rmr
+* rmdir
+
+   Usage: <<<hadoop fs -rmdir [--ignore-fail-on-non-empty] URI [URI ...]>>>
+
+   Delete a directory.
+
+   Options:
 
-   Usage: <<<hdfs dfs -rmr [-skipTrash] URI [URI ...]>>>
+     * --ignore-fail-on-non-empty: When using wildcards, do not fail if a directory still contains files.
+
+   Example:
+
+     * <<<hadoop fs -rmdir /user/hadoop/emptydir>>>
+
+* rmr
+
+   Usage: <<<hadoop fs -rmr [-skipTrash] URI [URI ...]>>>
 
    Recursive version of delete.
 
-   <<Note:>> This command is deprecated. Instead use <<<hdfs dfs -rm -r>>>
+   <<Note:>> This command is deprecated. Instead use <<<hadoop fs -rm -r>>>
 
-setfacl
+* setfacl
 
-   Usage: <<<hdfs dfs -setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>] >>>
+   Usage: <<<hadoop fs -setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>] >>>
 
    Sets Access Control Lists (ACLs) of files and directories.
 
@@ -528,27 +596,27 @@ setfacl
 
    Examples:
 
-      * <<<hdfs dfs -setfacl -m user:hadoop:rw- /file>>>
+      * <<<hadoop fs -setfacl -m user:hadoop:rw- /file>>>
 
-      * <<<hdfs dfs -setfacl -x user:hadoop /file>>>
+      * <<<hadoop fs -setfacl -x user:hadoop /file>>>
 
-      * <<<hdfs dfs -setfacl -b /file>>>
+      * <<<hadoop fs -setfacl -b /file>>>
 
-      * <<<hdfs dfs -setfacl -k /dir>>>
+      * <<<hadoop fs -setfacl -k /dir>>>
 
-      * <<<hdfs dfs -setfacl --set user::rw-,user:hadoop:rw-,group::r--,other::r-- /file>>>
+      * <<<hadoop fs -setfacl --set user::rw-,user:hadoop:rw-,group::r--,other::r-- /file>>>
 
-      * <<<hdfs dfs -setfacl -R -m user:hadoop:r-x /dir>>>
+      * <<<hadoop fs -setfacl -R -m user:hadoop:r-x /dir>>>
 
-      * <<<hdfs dfs -setfacl -m default:user:hadoop:r-x /dir>>>
+      * <<<hadoop fs -setfacl -m default:user:hadoop:r-x /dir>>>
 
    Exit Code:
 
    Returns 0 on success and non-zero on error.
 
-setfattr
+* setfattr
 
-   Usage: <<<hdfs dfs -setfattr {-n name [-v value] | -x name} <path> >>>
+   Usage: <<<hadoop fs -setfattr {-n name [-v value] | -x name} <path> >>>
 
    Sets an extended attribute name and value for a file or directory.
 
@@ -566,19 +634,19 @@ setfattr
 
    Examples:
 
-      * <<<hdfs dfs -setfattr -n user.myAttr -v myValue /file>>>
+      * <<<hadoop fs -setfattr -n user.myAttr -v myValue /file>>>
 
-      * <<<hdfs dfs -setfattr -n user.noValue /file>>>
+      * <<<hadoop fs -setfattr -n user.noValue /file>>>
 
-      * <<<hdfs dfs -setfattr -x user.myAttr /file>>>
+      * <<<hadoop fs -setfattr -x user.myAttr /file>>>
 
    Exit Code:
 
    Returns 0 on success and non-zero on error.
 
-setrep
+* setrep
 
-   Usage: <<<hdfs dfs -setrep [-R] [-w] <numReplicas> <path> >>>
+   Usage: <<<hadoop fs -setrep [-R] [-w] <numReplicas> <path> >>>
 
    Changes the replication factor of a file. If <path> is a directory then
    the command recursively changes the replication factor of all files under
@@ -593,28 +661,33 @@ setrep
 
    Example:
 
-     * <<<hdfs dfs -setrep -w 3 /user/hadoop/dir1>>>
+     * <<<hadoop fs -setrep -w 3 /user/hadoop/dir1>>>
 
    Exit Code:
 
    Returns 0 on success and -1 on error.
 
-stat
+* stat
 
-   Usage: <<<hdfs dfs -stat URI [URI ...]>>>
+   Usage: <<<hadoop fs -stat [format] \<path\> ...>>>
 
-   Returns the stat information on the path.
+   Print statistics about the file/directory at \<path\> in the specified
+   format. Format accepts filesize in blocks (%b), type (%F), group name of
+   owner (%g), name (%n), block size (%o), replication (%r), user name of
+   owner(%u), and modification date (%y, %Y). %y shows UTC date as
+   "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC.
+   If the format is not specified, %y is used by default.
 
    Example:
 
-     * <<<hdfs dfs -stat path>>>
+     * <<<hadoop fs -stat "%F %u:%g %b %y %n" /file>>>
 
    Exit Code:
    Returns 0 on success and -1 on error.
 
-tail
+* tail
 
-   Usage: <<<hdfs dfs -tail [-f] URI>>>
+   Usage: <<<hadoop fs -tail [-f] URI>>>
 
    Displays last kilobyte of the file to stdout.
 
@@ -624,43 +697,54 @@ tail
 
    Example:
 
-     * <<<hdfs dfs -tail pathname>>>
+     * <<<hadoop fs -tail pathname>>>
 
    Exit Code:
    Returns 0 on success and -1 on error.
 
-test
+* test
 
-   Usage: <<<hdfs dfs -test -[ezd] URI>>>
+   Usage: <<<hadoop fs -test -[defsz] URI>>>
 
    Options:
 
-     * The -e option will check to see if the file exists, returning 0 if true.
+     * -d: f the path is a directory, return 0.
+
+     * -e: if the path exists, return 0.
 
-     * The -z option will check to see if the file is zero length, returning 0 if true.
+     * -f: if the path is a file, return 0.
 
-     * The -d option will check to see if the path is directory, returning 0 if true.
+     * -s: if the path is not empty, return 0.
+
+     * -z: if the file is zero length, return 0.
 
    Example:
 
-     * <<<hdfs dfs -test -e filename>>>
+     * <<<hadoop fs -test -e filename>>>
 
-text
+* text
 
-   Usage: <<<hdfs dfs -text <src> >>>
+   Usage: <<<hadoop fs -text <src> >>>
 
    Takes a source file and outputs the file in text format. The allowed formats
    are zip and TextRecordInputStream.
 
-touchz
+* touchz
 
-   Usage: <<<hdfs dfs -touchz URI [URI ...]>>>
+   Usage: <<<hadoop fs -touchz URI [URI ...]>>>
 
    Create a file of zero length.
 
    Example:
 
-     * <<<hdfs dfs -touchz pathname>>>
+     * <<<hadoop fs -touchz pathname>>>
 
    Exit Code:
    Returns 0 on success and -1 on error.
+
+
+* usage
+
+   Usage: <<<hadoop fs -usage command>>>
+
+   Return the help for an individual command.
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm
new file mode 100644
index 0000000000000..dbd8d92820f83
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm
@@ -0,0 +1,140 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~   http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+  ---
+  Hadoop ${project.version} - Rack Awareness
+  ---
+  ---
+  ${maven.build.timestamp}
+
+%{toc|section=1|fromDepth=0}
+
+Rack Awareness
+
+  Hadoop components are rack-aware.  For example, HDFS block placement will
+  use rack awareness for fault tolerance by placing one block replica on a 
+  different rack.  This provides data availability in the event of a network
+  switch failure or partition within the cluster.  
+
+  Hadoop master daemons obtain the rack id of the cluster slaves by invoking
+  either an external script or java class as specified by configuration files.
+  Using either the java class or external script for topology, output must
+  adhere to the java <<org.apache.hadoop.net.DNSToSwitchMapping>>
+  interface.  The interface expects a one-to-one correspondence to be
+  maintained and the topology information in the format of '/myrack/myhost',
+  where '/' is the topology delimiter, 'myrack' is the rack identifier, and
+  'myhost' is the individual host.  Assuming a single /24 subnet per rack, 
+  one could use the format of '/192.168.100.0/192.168.100.5' as a 
+  unique rack-host topology mapping.
+
+  To use the java class for topology mapping, the class name is specified by
+  the <<topology.node.switch.mapping.impl>> parameter in the configuration
+  file. An example, NetworkTopology.java, is included with the hadoop
+  distribution and can be customized by the Hadoop administrator. Using a 
+  Java class instead of an external script has a performance benefit in
+  that Hadoop doesn't need to fork an external process when a new slave node
+  registers itself.  
+
+  If implementing an external script, it will be specified with the
+  <<topology.script.file.name>> parameter in the configuration files.  Unlike
+  the java class, the external topology script is not included with the Hadoop
+  distribution and is provided by the administrator.  Hadoop will send 
+  multiple IP addresses to ARGV when forking the topology script.  The  
+  number of IP addresses sent to the topology script is controlled with
+  <<net.topology.script.number.args>> and defaults to 100. If
+  <<net.topology.script.number.args>> was changed to 1, a topology script
+  would get forked for each IP submitted by DataNodes and/or NodeManagers.
+
+  If <<topology.script.file.name>> or <<topology.node.switch.mapping.impl>> is 
+  not set, the rack id '/default-rack' is returned for any passed IP address.
+  While this behavior appears desirable, it can cause issues with HDFS block
+  replication as  default behavior is to write one replicated block off rack
+  and is unable to do so as there is only a single rack named '/default-rack'.
+
+  An additional configuration setting is
+  <<mapreduce.jobtracker.taskcache.levels>> which determines the number of
+  levels (in the network topology) of caches MapReduce will use. So, for
+  example, if it is the default value of 2, two levels of caches will be
+  constructed - one for hosts (host -> task mapping) and another for racks
+  (rack -> task mapping). Giving us our one-to-one mapping of '/myrack/myhost'.
+  
+* {python Example}
+
++-------------------------------+
+  #!/usr/bin/python
+  # this script makes assumptions about the physical environment.
+  #  1) each rack is its own layer 3 network with a /24 subnet, which 
+  # could be typical where each rack has its own
+  #     switch with uplinks to a central core router.
+  #     
+  #             +-----------+
+  #             |core router|
+  #             +-----------+
+  #            /             \
+  #   +-----------+        +-----------+
+  #   |rack switch|        |rack switch|
+  #   +-----------+        +-----------+
+  #   | data node |        | data node |
+  #   +-----------+        +-----------+
+  #   | data node |        | data node |
+  #   +-----------+        +-----------+
+  #
+  # 2) topology script gets list of IP's as input, calculates network address, and prints '/network_address/ip'.
+
+  import netaddr
+  import sys             
+  sys.argv.pop(0)                                                  # discard name of topology script from argv list as we just want IP addresses
+
+  netmask = '255.255.255.0'                                        # set netmask to what's being used in your environment.  The example uses a /24
+
+  for ip in sys.argv:                                              # loop over list of datanode IP's
+   address = '{0}/{1}'.format(ip, netmask)                      # format address string so it looks like 'ip/netmask' to make netaddr work
+   try:
+       network_address = netaddr.IPNetwork(address).network     # calculate and print network address
+       print "/{0}".format(network_address)                     
+   except:
+       print "/rack-unknown"                                    # print catch-all value if unable to calculate network address
++-------------------------------+
+
+* {bash Example}
+
++-------------------------------+
+  #!/bin/bash
+  # Here's a bash example to show just how simple these scripts can be
+  # Assuming we have flat network with everything on a single switch, we can fake a rack topology. 
+  # This could occur in a lab environment where we have limited nodes,like 2-8 physical machines on a unmanaged switch. 
+  # This may also apply to multiple virtual machines running on the same physical hardware.  
+  # The number of machines isn't important, but that we are trying to fake a network topology when there isn't one. 
+  #
+  #       +----------+    +--------+
+  #       |jobtracker|    |datanode| 
+  #       +----------+    +--------+
+  #              \        /
+  #  +--------+  +--------+  +--------+
+  #  |datanode|--| switch |--|datanode|
+  #  +--------+  +--------+  +--------+
+  #              /        \
+  #       +--------+    +--------+
+  #       |datanode|    |namenode| 
+  #       +--------+    +--------+
+  #
+  # With this network topology, we are treating each host as a rack.  This is being done by taking the last octet 
+  # in the datanode's IP and prepending it with the word '/rack-'.  The advantage for doing this is so HDFS
+  # can create its 'off-rack' block copy.
+  # 1) 'echo $@' will echo all ARGV values to xargs.  
+  # 2) 'xargs' will enforce that we print a single argv value per line
+  # 3) 'awk' will split fields on dots and append the last field to the string '/rack-'. If awk 
+  #    fails to split on four dots, it will still print '/rack-' last field value
+
+  echo $@ | xargs -n 1 | awk -F '.' '{print "/rack-"$NF}'
++-------------------------------+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
index 8e17e8981d952..02352197e5ffe 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
@@ -202,58 +202,7 @@ KVNO Timestamp         Principal
 
   Some products such as Apache Oozie which access the services of Hadoop
   on behalf of end users need to be able to impersonate end users.
-  You can configure proxy user using properties
-  <<<hadoop.proxyuser.${superuser}.hosts>>> along with either or both of 
-  <<<hadoop.proxyuser.${superuser}.groups>>>
-  and <<<hadoop.proxyuser.${superuser}.users>>>.
-
-  For example, by specifying as below in core-site.xml,
-  user named <<<oozie>>> accessing from any host
-  can impersonate any user belonging to any group.
-
-----
-  <property>
-    <name>hadoop.proxyuser.oozie.hosts</name>
-    <value>*</value>
-  </property>
-  <property>
-    <name>hadoop.proxyuser.oozie.groups</name>
-    <value>*</value>
-  </property>
-----
-
-  User named <<<oozie>>> accessing from any host
-  can impersonate user1 and user2 by specifying as below in core-site.xml.
-
-----
-  <property>
-    <name>hadoop.proxyuser.oozie.hosts</name>
-    <value>*</value>
-  </property>
-  <property>
-    <name>hadoop.proxyuser.oozie.users</name>
-    <value>user1,user2</value>
-  </property>
-----
-
-  The <<<hadoop.proxyuser.${superuser}.hosts>>> accepts list of ip addresses,
-  ip address ranges in CIDR format and/or host names.
-  
-  For example, by specifying as below in core-site.xml,
-  user named <<<oozie>>> accessing from hosts in the range 
-  10.222.0.0-15 and 10.113.221.221
-  can impersonate any user belonging to any group.
-  
-----
-  <property>
-    <name>hadoop.proxyuser.oozie.hosts</name>
-    <value>10.222.0.0/16,10.113.221.221</value>
-  </property>
-  <property>
-    <name>hadoop.proxyuser.oozie.groups</name>
-    <value>*</value>
-  </property>
-----
+  See {{{./Superusers.html}the doc of proxy user}} for details.
 
 ** Secure DataNode
 
@@ -699,7 +648,7 @@ Configuration for  <<<conf/yarn-site.xml>>>
 | | | required for validating the secure access of the <container-executor> |
 | | | binary. |
 *-------------------------+-------------------------+------------------------+
-| <<<banned.users>>> | hfds,yarn,mapred,bin | Banned users. |
+| <<<banned.users>>> | hdfs,yarn,mapred,bin | Banned users. |
 *-------------------------+-------------------------+------------------------+
 | <<<allowed.system.users>>> | foo,bar | Allowed system users. |
 *-------------------------+-------------------------+------------------------+
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
index 6f714545ffe17..86fb3d6cb2240 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
@@ -159,6 +159,31 @@ security.ha.service.protocol.acl      | ACL for HAService protocol used by HAAdm
    the ability to refresh the service-level authorization configuration to
    certain users/groups.
 
+  ** Access Control using list of ip addresses, host names and ip ranges
+
+   Access to a service can be controlled based on the ip address of the client accessing
+   the service. It is possible to restrict access to a service from a set of machines by
+   specifying a list of ip addresses, host names and ip ranges. The property name for each service
+   is derived from the corresponding acl's property name. If the property name of acl is
+   security.client.protocol.acl, property name for the hosts list will be
+   security.client.protocol.hosts.
+
+   If hosts list is not defined for a service, the value of
+   <<<security.service.authorization.default.hosts>>> is applied. If
+   <<<security.service.authorization.default.hosts>>> is not defined, <<<*>>>  is applied.
+
+   It is possible to specify a blocked list of hosts. Only those machines which are in the
+   hosts list, but not in the blocked hosts list will be granted access to the service. The property
+   name is derived by suffixing with ".blocked".
+
+   Example: The property name of blocked hosts list for <<<security.client.protocol.hosts>>
+   will be <<<security.client.protocol.hosts.blocked>>>
+
+   If blocked hosts list is not defined for a service, the value of
+   <<<security.service.authorization.default.hosts.blocked>>> is applied. If
+   <<<security.service.authorization.default.hosts.blocked>>> is not defined,
+   empty blocked hosts list is applied.
+
 ** Examples
 
    Allow only users <<<alice>>>, <<<bob>>> and users in the <<<mapreduce>>> group to submit
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
index ef7532a9e520f..eb9c88af39c91 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
@@ -11,12 +11,12 @@
 ~~ limitations under the License. See accompanying LICENSE file.
 
   ---
-  Hadoop MapReduce Next Generation ${project.version} - Setting up a Single Node Cluster.
+  Hadoop ${project.version} - Setting up a Single Node Cluster.
   ---
   ---
   ${maven.build.timestamp}
 
-Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
+Hadoop - Setting up a Single Node Cluster.
 
 %{toc|section=1|fromDepth=0}
 
@@ -46,7 +46,9 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
          HadoopJavaVersions}}.
 
    [[2]] ssh must be installed and sshd must be running to use the Hadoop
-         scripts that manage remote Hadoop daemons.
+         scripts that manage remote Hadoop daemons if the optional start
+         and stop scripts are to be used.  Additionally, it is recommmended that
+         pdsh also be installed for better ssh resource management.
 
 ** Installing Software
 
@@ -57,7 +59,7 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
 
 ----
   $ sudo apt-get install ssh
-  $ sudo apt-get install rsync
+  $ sudo apt-get install pdsh
 ----
 
 * Download
@@ -75,9 +77,6 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
 ----
   # set to the root of your Java installation
   export JAVA_HOME=/usr/java/latest
-
-  # Assuming your installation directory is /usr/local/hadoop
-  export HADOOP_PREFIX=/usr/local/hadoop
 ----
 
   Try the following command:
@@ -158,6 +157,7 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
 ----
   $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
   $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
+  $ chmod 0700 ~/.ssh/authorized_keys
 ----
 
 ** Execution
@@ -228,7 +228,7 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
   $ sbin/stop-dfs.sh
 ----
 
-** YARN on Single Node
+** YARN on a Single Node
 
   You can run a MapReduce job on YARN in a pseudo-distributed mode by setting
   a few parameters and running ResourceManager daemon and NodeManager daemon
@@ -239,7 +239,7 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
 
   [[1]] Configure parameters as follows:
 
-        etc/hadoop/mapred-site.xml:
+        <<<etc/hadoop/mapred-site.xml>>>:
 
 +---+
 <configuration>
@@ -250,7 +250,7 @@ Hadoop MapReduce Next Generation - Setting up a Single Node Cluster.
 </configuration>
 +---+
 
-        etc/hadoop/yarn-site.xml:
+        <<<etc/hadoop/yarn-site.xml>>>:
 
 +---+
 <configuration>
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
index f9408846435ef..78ed9a49c9097 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
@@ -11,19 +11,19 @@
 ~~ limitations under the License. See accompanying LICENSE file.
 
   ---
-  Superusers Acting On Behalf Of Other Users
+  Proxy user - Superusers Acting On Behalf Of Other Users
   ---
   ---
   ${maven.build.timestamp}
 
-Superusers Acting On Behalf Of Other Users
+Proxy user - Superusers Acting On Behalf Of Other Users
 
 %{toc|section=1|fromDepth=0}
 
 * Introduction
 
    This document describes how a superuser can submit jobs or access hdfs
-   on behalf of another user in a secured way.
+   on behalf of another user.
 
 * Use Case
 
@@ -38,9 +38,12 @@ Superusers Acting On Behalf Of Other Users
    on a connection authenticated with super's kerberos credentials. In
    other words super is impersonating the user joe.
 
+   Some products such as Apache Oozie need this.
+
+
 * Code example
 
-   In this example super's kerberos credentials are used for login and a
+   In this example super's credentials are used for login and a
    proxy user ugi object is created for joe. The operations are performed
    within the doAs method of this proxy user ugi object.
 
@@ -63,21 +66,26 @@ Superusers Acting On Behalf Of Other Users
 
 * Configurations
 
-   The superuser must be configured on namenode and jobtracker to be
-   allowed to impersonate another user. Following configurations are
-   required.
+   You can configure proxy user using properties
+   <<<hadoop.proxyuser.${superuser}.hosts>>> along with either or both of 
+   <<<hadoop.proxyuser.${superuser}.groups>>>
+   and <<<hadoop.proxyuser.${superuser}.users>>>.
+
+   By specifying as below in core-site.xml,
+   the superuser named <<<super>>> can connect
+   only from <<<host1>>> and <<<host2>>>
+   to impersonate a user belonging to <<<group1>>> and <<<group2>>>.
 
 ----
-   <property>
-     <name>hadoop.proxyuser.super.groups</name>
-     <value>group1,group2</value>
-     <description>Allow the superuser super to impersonate any members of the group group1 and group2</description>
-   </property>
    <property>
      <name>hadoop.proxyuser.super.hosts</name>
      <value>host1,host2</value>
-     <description>The superuser can connect only from host1 and host2 to impersonate a user</description>
    </property>
+   <property>
+     <name>hadoop.proxyuser.super.groups</name>
+     <value>group1,group2</value>
+   </property>
+
 ----
 
    If these configurations are not present, impersonation will not be
@@ -85,11 +93,47 @@ Superusers Acting On Behalf Of Other Users
 
    If more lax security is preferred, the wildcard value * may be used to
    allow impersonation from any host or of any user.
+   For example, by specifying as below in core-site.xml,
+   user named <<<oozie>>> accessing from any host
+   can impersonate any user belonging to any group.
+
+----
+  <property>
+    <name>hadoop.proxyuser.oozie.hosts</name>
+    <value>*</value>
+  </property>
+  <property>
+    <name>hadoop.proxyuser.oozie.groups</name>
+    <value>*</value>
+  </property>
+----
+
+   The <<<hadoop.proxyuser.${superuser}.hosts>>> accepts list of ip addresses,
+   ip address ranges in CIDR format and/or host names.
+   For example, by specifying as below,
+   user named <<<super>>> accessing from hosts in the range 
+   <<<10.222.0.0-15>>> and <<<10.113.221.221>>> can impersonate
+   <<<user1>>> and <<<user2>>>.
+      
+----
+   <property>
+     <name>hadoop.proxyuser.super.hosts</name>
+     <value>10.222.0.0/16,10.113.221.221</value>
+   </property>
+   <property>
+     <name>hadoop.proxyuser.super.users</name>
+     <value>user1,user2</value>
+   </property>
+----
+
 
 * Caveats
 
-   The superuser must have kerberos credentials to be able to impersonate
-   another user. It cannot use delegation tokens for this feature. It
+   If the cluster is running in {{{./SecureMode.html}Secure Mode}},
+   the superuser must have kerberos credentials to be able to impersonate
+   another user.
+
+   It cannot use delegation tokens for this feature. It
    would be wrong if superuser adds its own delegation token to the proxy
    user ugi, as it will allow the proxy user to connect to the service
    with the privileges of the superuser.
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
index 9eda2208ba593..c51037b169bcb 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
@@ -16,19 +16,32 @@
   ---
   ${maven.build.timestamp}
 
-Enabling Dapper-like Tracing in HDFS
+Enabling Dapper-like Tracing in Hadoop
 
 %{toc|section=1|fromDepth=0}
 
-* {Dapper-like Tracing in HDFS}
+* {Dapper-like Tracing in Hadoop}
 
 ** HTrace
 
   {{{https://issues.apache.org/jira/browse/HDFS-5274}HDFS-5274}}
   added support for tracing requests through HDFS,
-  using the open source tracing library, {{{https://github.com/cloudera/htrace}HTrace}}.
+  using the open source tracing library, {{{https://git-wip-us.apache.org/repos/asf/incubator-htrace.git}Apache HTrace}}.
   Setting up tracing is quite simple, however it requires some very minor changes to your client code.
 
+** Samplers
+  Configure the samplers in <<<core-site.xml>>> property: <<<hadoop.htrace.sampler>>>.
+  The value can be NeverSampler, AlwaysSampler or ProbabilitySampler. NeverSampler: HTrace is OFF 
+  for all spans; AlwaysSampler: HTrace is ON for all spans; ProbabilitySampler: HTrace is ON for 
+  some percentage% of top-level spans.
+
++----
+  <property>
+    <name>hadoop.htrace.sampler</name>
+    <value>NeverSampler</value>
+  </property>
++----
+
 ** SpanReceivers
 
   The tracing system works by collecting information in structs called 'Spans'.
@@ -42,12 +55,12 @@ public void receiveSpan(Span span);
   Configure what SpanReceivers you'd like to use
   by putting a comma separated list of the fully-qualified class name of
   classes implementing SpanReceiver
-  in <<<hdfs-site.xml>>> property: <<<hadoop.htrace.spanreceiver.classes>>>.
+  in <<<core-site.xml>>> property: <<<hadoop.htrace.spanreceiver.classes>>>.
 
 +----
   <property>
     <name>hadoop.htrace.spanreceiver.classes</name>
-    <value>org.htrace.impl.LocalFileSpanReceiver</value>
+    <value>org.apache.htrace.impl.LocalFileSpanReceiver</value>
   </property>
   <property>
     <name>hadoop.htrace.local-file-span-receiver.path</name>
@@ -83,11 +96,11 @@ public void receiveSpan(Span span);
   $ git clone https://github.com/cloudera/htrace
   $ cd htrace/htrace-zipkin
   $ mvn compile assembly:single
-  $ cp target/htrace-zipkin-*-jar-with-dependencies.jar $HADOOP_HOME/share/hadoop/hdfs/lib/
+  $ cp target/htrace-zipkin-*-jar-with-dependencies.jar $HADOOP_HOME/share/hadoop/common/lib/
 +----
 
   The sample configuration for <<<ZipkinSpanReceiver>>> is shown below.
-  By adding these to <<<hdfs-site.xml>>> of NameNode and DataNodes,
+  By adding these to <<<core-site.xml>>> of NameNode and DataNodes,
   <<<ZipkinSpanReceiver>>> is initialized on the startup.
   You also need this configuration on the client node in addition to the servers.
 
@@ -118,11 +131,11 @@ public void receiveSpan(Span span);
 +----
   $ hadoop trace -list -host 192.168.56.2:9000
   ID  CLASS
-  1   org.htrace.impl.LocalFileSpanReceiver
+  1   org.apache.htrace.impl.LocalFileSpanReceiver
 
   $ hadoop trace -list -host 192.168.56.2:50020
   ID  CLASS
-  1   org.htrace.impl.LocalFileSpanReceiver
+  1   org.apache.htrace.impl.LocalFileSpanReceiver
 +----
 
   <<<hadoop trace -remove>>> removes span receiver from server.
@@ -143,7 +156,7 @@ public void receiveSpan(Span span);
 
   $ hadoop trace -list -host 192.168.56.2:9000
   ID  CLASS
-  2   org.htrace.impl.LocalFileSpanReceiver
+  2   org.apache.htrace.impl.LocalFileSpanReceiver
 +----
 
 
@@ -159,9 +172,9 @@ public void receiveSpan(Span span);
 +----
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.tracing.SpanReceiverHost;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 ...
 
@@ -187,9 +200,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.tracing.SpanReceiverHost;
 import org.apache.hadoop.util.ToolRunner;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 public class TracingFsShell {
   public static void main(String argv[]) throws Exception {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index d21500f2b5704..b84045d25ebe9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -209,6 +209,31 @@ public void testFinalParam() throws IOException {
     assertNull("my var is not final", conf2.get("my.var"));
   }
 
+  public void testCompactFormat() throws IOException {
+    out=new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    appendCompactFormatProperty("a", "b");
+    appendCompactFormatProperty("c", "d", true);
+    appendCompactFormatProperty("e", "f", false, "g");
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    Configuration conf = new Configuration(false);
+    conf.addResource(fileResource);
+
+    assertEquals("b", conf.get("a"));
+
+    assertEquals("d", conf.get("c"));
+    Set<String> s = conf.getFinalParameters();
+    assertEquals(1, s.size());
+    assertTrue(s.contains("c"));
+
+    assertEquals("f", conf.get("e"));
+    String[] sources = conf.getPropertySources("e");
+    assertEquals(2, sources.length);
+    assertEquals("g", sources[0]);
+    assertEquals(fileResource.toString(), sources[1]);
+  }
+
   public static void assertEq(Object a, Object b) {
     System.out.println("assertEq: " + a + ", " + b);
     assertEquals(a, b);
@@ -264,6 +289,36 @@ void appendProperty(String name, String val, boolean isFinal,
     out.write("</property>\n");
   }
   
+  void appendCompactFormatProperty(String name, String val) throws IOException {
+    appendCompactFormatProperty(name, val, false);
+  }
+
+  void appendCompactFormatProperty(String name, String val, boolean isFinal)
+    throws IOException {
+    appendCompactFormatProperty(name, val, isFinal, null);
+  }
+
+  void appendCompactFormatProperty(String name, String val, boolean isFinal,
+      String source)
+    throws IOException {
+    out.write("<property ");
+    out.write("name=\"");
+    out.write(name);
+    out.write("\" ");
+    out.write("value=\"");
+    out.write(val);
+    out.write("\" ");
+    if (isFinal) {
+      out.write("final=\"true\" ");
+    }
+    if (source != null) {
+      out.write("source=\"");
+      out.write(source);
+      out.write("\" ");
+    }
+    out.write("/>\n");
+  }
+
   public void testOverlay() throws IOException{
     out=new BufferedWriter(new FileWriter(CONFIG));
     startConfig();
@@ -1216,12 +1271,56 @@ public void testSettingKeyNull() throws Exception {
   }
 
   public void testInvalidSubstitutation() {
+    final Configuration configuration = new Configuration(false);
+
+    // 2-var loops
+    //
+    final String key = "test.random.key";
+    for (String keyExpression : Arrays.asList(
+        "${" + key + "}",
+        "foo${" + key + "}",
+        "foo${" + key + "}bar",
+        "${" + key + "}bar")) {
+      configuration.set(key, keyExpression);
+      assertEquals("Unexpected value", keyExpression, configuration.get(key));
+    }
+
+    //
+    // 3-variable loops
+    //
+
+    final String expVal1 = "${test.var2}";
+    String testVar1 = "test.var1";
+    configuration.set(testVar1, expVal1);
+    configuration.set("test.var2", "${test.var3}");
+    configuration.set("test.var3", "${test.var1}");
+    assertEquals("Unexpected value", expVal1, configuration.get(testVar1));
+
+    // 3-variable loop with non-empty value prefix/suffix
+    //
+    final String expVal2 = "foo2${test.var2}bar2";
+    configuration.set(testVar1, expVal2);
+    configuration.set("test.var2", "foo3${test.var3}bar3");
+    configuration.set("test.var3", "foo1${test.var1}bar1");
+    assertEquals("Unexpected value", expVal2, configuration.get(testVar1));
+  }
+
+  public void testIncompleteSubbing() {
+    Configuration configuration = new Configuration(false);
     String key = "test.random.key";
-    String keyExpression = "${" + key + "}";
-    Configuration configuration = new Configuration();
-    configuration.set(key, keyExpression);
-    String value = configuration.get(key);
-    assertTrue("Unexpected value " + value, value.equals(keyExpression));
+    for (String keyExpression : Arrays.asList(
+        "{}",
+        "${}",
+        "{" + key,
+        "${" + key,
+        "foo${" + key,
+        "foo${" + key + "bar",
+        "foo{" + key + "}bar",
+        "${" + key + "bar")) {
+      configuration.set(key, keyExpression);
+      String value = configuration.get(key);
+      assertTrue("Unexpected value " + value, value.equals(keyExpression));
+    }
   }
 
   public void testBoolean() {
@@ -1308,6 +1407,52 @@ public void testGetFinalParameters() throws Exception {
     assertTrue("my.var is not final", finalParameters.contains("my.var"));
   }
 
+  /**
+   * A test to check whether this thread goes into infinite loop because of
+   * destruction of data structure by resize of Map. This problem was reported
+   * by SPARK-2546.
+   * @throws Exception
+   */
+  public void testConcurrentAccesses() throws Exception {
+    out = new BufferedWriter(new FileWriter(CONFIG));
+    startConfig();
+    declareProperty("some.config", "xyz", "xyz", false);
+    endConfig();
+    Path fileResource = new Path(CONFIG);
+    Configuration conf = new Configuration();
+    conf.addResource(fileResource);
+
+    class ConfigModifyThread extends Thread {
+      final private Configuration config;
+      final private String prefix;
+
+      public ConfigModifyThread(Configuration conf, String prefix) {
+        config = conf;
+        this.prefix = prefix;
+      }
+
+      @Override
+      public void run() {
+        for (int i = 0; i < 100000; i++) {
+          config.set("some.config.value-" + prefix + i, "value");
+        }
+      }
+    }
+
+    ArrayList<ConfigModifyThread> threads = new ArrayList<>();
+    for (int i = 0; i < 100; i++) {
+      threads.add(new ConfigModifyThread(conf, String.valueOf(i)));
+    }
+    for (Thread t: threads) {
+      t.start();
+    }
+    for (Thread t: threads) {
+      t.join();
+    }
+    // If this test completes without going into infinite loop,
+    // it's expected behaviour.
+  }
+
   public static void main(String[] argv) throws Exception {
     junit.textui.TestRunner.main(new String[]{
       TestConfiguration.class.getName()
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
new file mode 100644
index 0000000000000..c3fe3a39f5bf4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -0,0 +1,417 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import java.lang.Class;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Base class for comparing fields in one or more Configuration classes
+ * against a corresponding .xml file.  Usage is intended as follows:
+ * <p></p>
+ * <ol>
+ * <li> Create a subclass to TestConfigurationFieldsBase
+ * <li> Define <code>initializeMemberVariables</code> method in the
+ *      subclass.  In this class, do the following:
+ * <p></p>
+ *   <ol>
+ *   <li> <b>Required</b> Set the variable <code>xmlFilename</code> to
+ *        the appropriate xml definition file
+ *   <li> <b>Required</b> Set the variable <code>configurationClasses</code>
+ *        to an array of the classes which define the constants used by the
+ *        code corresponding to the xml files
+ *   <li> <b>Optional</b> Set <code>errorIfMissingConfigProps</code> if the
+ *        subclass should throw an error in the method
+ *        <code>testCompareXmlAgainstConfigurationClass</code>
+ *   <li> <b>Optional</b> Set <code>errorIfMissingXmlProps</code> if the
+ *        subclass should throw an error in the method
+ *        <code>testCompareConfigurationClassAgainstXml</code>
+ *   <li> <b>Optional</b> Instantiate and populate strings into one or
+ *        more of the following variables:
+ *        <br><code>configurationPropsToSkipCompare</code>
+ *        <br><code>configurationPrefixToSkipCompare</code>
+ *        <br><code>xmlPropsToSkipCompare</code>
+ *        <br><code>xmlPrefixToSkipCompare</code>
+ *        <br>
+ *        in order to get comparisons clean
+ *   </ol>
+ * </ol>
+ * <p></p>
+ * The tests to do class-to-file and file-to-class should automatically
+ * run.  This class (and its subclasses) are mostly not intended to be
+ * overridden, but to do a very specific form of comparison testing.
+ */
+@Ignore
+public abstract class TestConfigurationFieldsBase {
+
+  /**
+   * Member variable for storing xml filename.
+   */
+  protected String xmlFilename = null;
+
+  /**
+   * Member variable for storing all related Configuration classes.
+   */
+  protected Class[] configurationClasses = null;
+
+  /**
+   * Throw error during comparison if missing configuration properties.
+   * Intended to be set by subclass.
+   */
+  protected boolean errorIfMissingConfigProps = false;
+
+  /**
+   * Throw error during comparison if missing xml properties.  Intended
+   * to be set by subclass.
+   */
+  protected boolean errorIfMissingXmlProps = false;
+
+  /**
+   * Set of properties to skip extracting (and thus comparing later) in 
+   * extractMemberVariablesFromConfigurationFields.
+   */
+  protected Set<String> configurationPropsToSkipCompare = null;
+
+  /**
+   * Set of property prefixes to skip extracting (and thus comparing later)
+   * in * extractMemberVariablesFromConfigurationFields.
+   */
+  protected Set<String> configurationPrefixToSkipCompare = null;
+
+  /**
+   * Set of properties to skip extracting (and thus comparing later) in 
+   * extractPropertiesFromXml.
+   */
+  protected Set<String> xmlPropsToSkipCompare = null;
+
+  /**
+   * Set of property prefixes to skip extracting (and thus comparing later)
+   * in extractPropertiesFromXml.
+   */
+  protected Set<String> xmlPrefixToSkipCompare = null;
+
+  /**
+   * Member variable to store Configuration variables for later comparison.
+   */
+  private Map<String,String> configurationMemberVariables = null;
+
+  /**
+   * Member variable to store XML properties for later comparison.
+   */
+  private Map<String,String> xmlKeyValueMap = null;
+
+  /**
+   * Member variable to store Configuration variables that are not in the
+   * corresponding XML file.
+   */
+  private Set<String> configurationFieldsMissingInXmlFile = null;
+
+  /**
+   * Member variable to store XML variables that are not in the
+   * corresponding Configuration class(es).
+   */
+  private Set<String> xmlFieldsMissingInConfiguration = null;
+
+  /**
+   * Abstract method to be used by subclasses for initializing base
+   * members.
+   */
+  public abstract void initializeMemberVariables();
+ 
+  /**
+   * Utility function to extract &quot;public static final&quot; member
+   * variables from a Configuration type class.
+   *
+   * @param fields The class member variables
+   * @return HashMap containing <StringValue,MemberVariableName> entries
+   */
+  private HashMap<String,String>
+      extractMemberVariablesFromConfigurationFields(Field[] fields) {
+    // Sanity Check
+    if (fields==null)
+      return null;
+
+    HashMap<String,String> retVal = new HashMap<String,String>();
+
+    // Setup regexp for valid properties
+    String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$";
+    Pattern p = Pattern.compile(propRegex);
+
+    // Iterate through class member variables
+    int totalFields = 0;
+    String value;
+    for (Field f : fields) {
+      // Filter out anything that isn't "public static final"
+      if (!Modifier.isStatic(f.getModifiers()) ||
+          !Modifier.isPublic(f.getModifiers()) ||
+          !Modifier.isFinal(f.getModifiers())) {
+        continue;
+      }
+      // Filter out anything that isn't a string.  int/float are generally
+      // default values
+      if (!f.getType().getName().equals("java.lang.String")) {
+        continue;
+      }
+      // Convert found member into String
+      try {
+        value = (String) f.get(null);
+      } catch (IllegalAccessException iaException) {
+        continue;
+      }
+      // Special Case: Detect and ignore partial properties (ending in x)
+      //               or file properties (ending in .xml)
+      if (value.endsWith(".xml") ||
+          value.endsWith(".")    ||
+          value.endsWith("-"))
+        continue;
+      // Ignore known configuration props
+      if (configurationPropsToSkipCompare != null) {
+        if (configurationPropsToSkipCompare.contains(value)) {
+          continue;
+        }
+      }
+      // Ignore known configuration prefixes
+      boolean skipPrefix = false;
+      if (configurationPrefixToSkipCompare != null) {
+        for (String cfgPrefix : configurationPrefixToSkipCompare) {
+	  if (value.startsWith(cfgPrefix)) {
+            skipPrefix = true;
+            break;
+	  }
+	}
+      }
+      if (skipPrefix) {
+        continue;
+      }
+      // Positive Filter: Look only for property values.  Expect it to look
+      //                  something like: blah.blah2(.blah3.blah4...)
+      Matcher m = p.matcher(value);
+      if (!m.find()) {
+        continue;
+      }
+
+      // Save member variable/value as hash
+      retVal.put(value,f.getName());
+    }
+
+    return retVal;
+  }
+
+  /**
+   * Pull properties and values from filename.
+   *
+   * @param filename XML filename
+   * @return HashMap containing <Property,Value> entries from XML file
+   */
+  private HashMap<String,String> extractPropertiesFromXml
+      (String filename) {
+    if (filename==null) {
+      return null;
+    }
+
+    // Iterate through XML file for name/value pairs
+    Configuration conf = new Configuration(false);
+    conf.setAllowNullValueProperties(true);
+    conf.addResource(filename);
+
+    HashMap<String,String> retVal = new HashMap<String,String>();
+    Iterator<Map.Entry<String,String>> kvItr = conf.iterator();
+    while (kvItr.hasNext()) {
+      Map.Entry<String,String> entry = kvItr.next();
+      String key = entry.getKey();
+      // Ignore known xml props
+      if (xmlPropsToSkipCompare != null) {
+        if (xmlPropsToSkipCompare.contains(key)) {
+          continue;
+        }
+      }
+      // Ignore known xml prefixes
+      boolean skipPrefix = false;
+      if (xmlPrefixToSkipCompare != null) {
+        for (String xmlPrefix : xmlPrefixToSkipCompare) {
+	  if (key.startsWith(xmlPrefix)) {
+	    skipPrefix = true;
+            break;
+	  }
+	}
+      }
+      if (skipPrefix) {
+        continue;
+      }
+      if (conf.onlyKeyExists(key)) {
+        retVal.put(key,null);
+      } else {
+        String value = conf.get(key);
+        if (value!=null) {
+          retVal.put(key,entry.getValue());
+        }
+      }
+      kvItr.remove();
+    }
+    return retVal;
+  }
+
+  /**
+   * Perform set difference operation on keyMap2 from keyMap1.
+   *
+   * @param keyMap1 The initial set
+   * @param keyMap2 The set to subtract
+   * @return Returns set operation keyMap1-keyMap2
+   */
+  private static Set<String> compareConfigurationToXmlFields(Map<String,String> keyMap1, Map<String,String> keyMap2) {
+    Set<String> retVal = new HashSet<String>(keyMap1.keySet());
+    retVal.removeAll(keyMap2.keySet());
+    return retVal;
+  }
+
+  /**
+   * Initialize the four variables corresponding the Configuration
+   * class and the XML properties file.
+   */
+  @Before
+  public void setupTestConfigurationFields() throws Exception {
+    initializeMemberVariables();
+
+    // Error if subclass hasn't set class members
+    assertTrue(xmlFilename!=null);
+    assertTrue(configurationClasses!=null);
+
+    // Create class member/value map
+    configurationMemberVariables = new HashMap<String,String>();
+    for (Class c : configurationClasses) {
+      Field[] fields = c.getDeclaredFields();
+      Map<String,String> memberMap =
+          extractMemberVariablesFromConfigurationFields(fields);
+      if (memberMap!=null) {
+        configurationMemberVariables.putAll(memberMap);
+      }
+    }
+
+    // Create XML key/value map
+    xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+
+    // Find class members not in the XML file
+    configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
+        (configurationMemberVariables, xmlKeyValueMap);
+
+    // Find XML properties not in the class
+    xmlFieldsMissingInConfiguration = compareConfigurationToXmlFields
+        (xmlKeyValueMap, configurationMemberVariables);
+  }
+
+  /**
+   * Compares the properties that are in the Configuration class, but not
+   * in the XML properties file.
+   */
+  @Test
+  public void testCompareConfigurationClassAgainstXml() {
+    // Error if subclass hasn't set class members
+    assertTrue(xmlFilename!=null);
+    assertTrue(configurationClasses!=null);
+
+    final int missingXmlSize = configurationFieldsMissingInXmlFile.size();
+
+    for (Class c : configurationClasses) {
+      System.out.println(c);
+    }
+    System.out.println("  (" + configurationMemberVariables.size() + " member variables)");
+    System.out.println();
+    StringBuffer xmlErrorMsg = new StringBuffer();
+    for (Class c : configurationClasses) {
+      xmlErrorMsg.append(c);
+      xmlErrorMsg.append(" ");
+    }
+    xmlErrorMsg.append("has ");
+    xmlErrorMsg.append(missingXmlSize);
+    xmlErrorMsg.append(" variables missing in ");
+    xmlErrorMsg.append(xmlFilename);
+    System.out.println(xmlErrorMsg.toString());
+    System.out.println();
+    if (missingXmlSize==0) {
+      System.out.println("  (None)");
+    } else {
+      for (String missingField : configurationFieldsMissingInXmlFile) {
+        System.out.println("  " + missingField);
+      }
+    }
+    System.out.println();
+    System.out.println("=====");
+    System.out.println();
+    if (errorIfMissingXmlProps) {
+      assertTrue(xmlErrorMsg.toString(), missingXmlSize==0);
+    }
+  }
+
+  /**
+   * Compares the properties that are in the XML properties file, but not
+   * in the Configuration class.
+   */
+  @Test
+  public void testCompareXmlAgainstConfigurationClass() {
+    // Error if subclass hasn't set class members
+    assertTrue(xmlFilename!=null);
+    assertTrue(configurationClasses!=null);
+
+    final int missingConfigSize = xmlFieldsMissingInConfiguration.size();
+
+    System.out.println("File " + xmlFilename + " (" + xmlKeyValueMap.size() + " properties)");
+    System.out.println();
+    StringBuffer configErrorMsg = new StringBuffer();
+    configErrorMsg.append(xmlFilename);
+    configErrorMsg.append(" has ");
+    configErrorMsg.append(missingConfigSize);
+    configErrorMsg.append(" properties missing in");
+    for (Class c : configurationClasses) {
+      configErrorMsg.append("  " + c);
+    }
+    System.out.println(configErrorMsg.toString());
+    System.out.println();
+    if (missingConfigSize==0) {
+      System.out.println("  (None)");
+    } else {
+      for (String missingField : xmlFieldsMissingInConfiguration) {
+        System.out.println("  " + missingField);
+      }
+    }
+    System.out.println();
+    System.out.println("=====");
+    System.out.println();
+    if ( errorIfMissingConfigProps ) {
+      assertTrue(configErrorMsg.toString(), missingConfigSize==0);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 08231f98cbc02..6e2ceaf1b8aac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -41,16 +41,15 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.primitives.Longs;
 
 public class TestCryptoCodec {
   private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class);
-  private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 
-    0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
-  private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 
-    0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+  private static byte[] key = new byte[16];
+  private static byte[] iv = new byte[16];
   private static final int bufferSize = 4096;
   
   private Configuration conf = new Configuration();
@@ -61,6 +60,13 @@ public class TestCryptoCodec {
   private final String opensslCodecClass = 
       "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec";
   
+  @Before
+  public void setUp() throws IOException {
+    Random random = new SecureRandom();
+    random.nextBytes(key);
+    random.nextBytes(iv);
+  }
+
   @Test(timeout=120000)
   public void testJceAesCtrCryptoCodec() throws Exception {
     if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
@@ -72,9 +78,15 @@ public void testJceAesCtrCryptoCodec() throws Exception {
       Assume.assumeTrue(false);
     }
     Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
-    cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
-    cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
-    cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass);
+    cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
+    // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
+    for(int i = 0; i < 8; i++) {
+      iv[8 + i] = (byte) 0xff;
+    }
+    cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, jceCodecClass, opensslCodecClass, iv);
   }
   
   @Test(timeout=120000)
@@ -88,13 +100,19 @@ public void testOpensslAesCtrCryptoCodec() throws Exception {
       Assume.assumeTrue(false);
     }
     Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
-    cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass);
-    cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass);
-    cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass);
+    cryptoCodecTest(conf, seed, 0, opensslCodecClass, opensslCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
+    // Overflow test, IV: xx xx xx xx xx xx xx xx ff ff ff ff ff ff ff ff 
+    for(int i = 0; i < 8; i++) {
+      iv[8 + i] = (byte) 0xff;
+    }
+    cryptoCodecTest(conf, seed, count, opensslCodecClass, opensslCodecClass, iv);
+    cryptoCodecTest(conf, seed, count, opensslCodecClass, jceCodecClass, iv);
   }
   
   private void cryptoCodecTest(Configuration conf, int seed, int count, 
-      String encCodecClass, String decCodecClass) throws IOException, 
+      String encCodecClass, String decCodecClass, byte[] iv) throws IOException, 
       GeneralSecurityException {
     CryptoCodec encCodec = null;
     try {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
index 998cd6fe1225e..ef09d94739643 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
@@ -42,6 +42,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 public class TestKeyProviderFactory {
 
@@ -430,4 +431,51 @@ public void testGetProviderViaURI() throws Exception {
     Assert.assertNull(kp);
 
   }
+
+  @Test
+  public void testJksProviderWithKeytoolKeys() throws Exception {
+    final Configuration conf = new Configuration();
+    final String keystoreDirAbsolutePath =
+        conf.getResource("hdfs7067.keystore").getPath();
+    final String ourUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file@/" +
+        keystoreDirAbsolutePath;
+
+    conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
+
+    final KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+
+    // Sanity check that we are using the right keystore
+    @SuppressWarnings("unused")
+    final KeyProvider.KeyVersion keyVersion =
+            provider.getKeyVersion("testkey5@0");
+    try {
+      @SuppressWarnings("unused")
+      final KeyProvider.KeyVersion keyVersionWrongKeyNameFormat =
+          provider.getKeyVersion("testkey2");
+      fail("should have thrown an exception");
+    } catch (IOException e) {
+      // No version in key path testkey2/
+      GenericTestUtils.assertExceptionContains("No version in key path", e);
+    }
+    try {
+      @SuppressWarnings("unused")
+      final KeyProvider.KeyVersion keyVersionCurrentKeyNotWrongKeyNameFormat =
+          provider.getCurrentKey("testkey5@0");
+      fail("should have thrown an exception getting testkey5@0");
+    } catch (IOException e) {
+      // javax.crypto.spec.SecretKeySpec cannot be cast to
+      // org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata
+      GenericTestUtils.assertExceptionContains("other non-Hadoop method", e);
+    }
+    try {
+      @SuppressWarnings("unused")
+      KeyProvider.KeyVersion keyVersionCurrentKeyNotReally =
+          provider.getCurrentKey("testkey2");
+      fail("should have thrown an exception getting testkey2");
+    } catch (IOException e) {
+      // javax.crypto.spec.SecretKeySpec cannot be cast to
+      // org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata
+      GenericTestUtils.assertExceptionContains("other non-Hadoop method", e);
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
index 3407eb7cecfe2..fe887182a23fa 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
@@ -75,7 +75,8 @@ public void cleanUp() throws Exception {
   private void deleteKey(KeyShell ks, String keyName) throws Exception {
     int rc;
     outContent.reset();
-    final String[] delArgs = {"delete", keyName, "-provider", jceksProvider};
+    final String[] delArgs =
+        {"delete", keyName, "-f", "-provider", jceksProvider};
     rc = ks.run(delArgs);
     assertEquals(0, rc);
     assertTrue(outContent.toString().contains(keyName + " has been " +
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
index 50a0031c9fa7f..6c2e5b88bcc6d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
@@ -137,18 +137,4 @@ public void testRefillReservoir() throws Exception {
     }
     random.close();
   }
-  
-  @Test(timeout=120000)
-  public void testOsSecureRandomSetConf() throws IOException {
-    Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
-    OsSecureRandom random = new OsSecureRandom();
-    for(int n = 0; n < 10; ++n) {
-      random.setConf(new Configuration());
-      String[] scmd = new String[] {"/bin/sh", "-c", "lsof | wc -l"};
-      ShellCommandExecutor sce = new ShellCommandExecutor(scmd);
-      sce.execute(); 
-      System.out.println("==lsof result " + n + ":");
-      System.out.println(sce.getOutput());
-    }
-  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
index bd1f0aedadba0..d91091f07cc3a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
@@ -28,6 +28,7 @@
 import org.junit.Test;
 import static org.apache.hadoop.fs.FileContextTestHelper.*;
 import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.test.GenericTestUtils;
 
 /**
  * <p>
@@ -54,14 +55,8 @@ public abstract class FileContextCreateMkdirBaseTest {
   protected final FileContextTestHelper fileContextTestHelper;
   protected static FileContext fc;
       
-  {
-    try {
-      ((Log4JLogger)FileSystem.LOG).getLogger().setLevel(Level.DEBUG);
-    }
-    catch(Exception e) {
-      System.out.println("Cannot change log level\n"
-          + StringUtils.stringifyException(e));
-    }
+  static {
+    GenericTestUtils.setLogLevel(FileSystem.LOG, Level.DEBUG);
   }
   
   public FileContextCreateMkdirBaseTest() {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
index a5d8403c66ae3..4a88c51ccecdd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
@@ -22,7 +22,6 @@
 import java.net.URI;
 import java.util.Random;
 
-
 import org.apache.commons.lang.RandomStringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.token.Token;
@@ -127,28 +126,36 @@ public Path getDefaultWorkingDirectory(FileSystem fSys)
    */
   public static long createFile(FileSystem fSys, Path path, int numBlocks,
       int blockSize, short numRepl, boolean createParent) throws IOException {
-    FSDataOutputStream out = 
-      fSys.create(path, false, 4096, numRepl, blockSize );
+    return createFile(fSys, path, getFileData(numBlocks, blockSize),
+        blockSize, numRepl);
+  }
 
-    byte[] data = getFileData(numBlocks, blockSize);
-    out.write(data, 0, data.length);
-    out.close();
+  public static long createFile(FileSystem fSys, Path path, byte[] data,
+      int blockSize, short numRepl) throws IOException {
+    FSDataOutputStream out = 
+        fSys.create(path, false, 4096, numRepl, blockSize);
+    try {
+      out.write(data, 0, data.length);
+    } finally {
+      out.close();
+    }
     return data.length;
   }
 
-
   public static long createFile(FileSystem fSys, Path path, int numBlocks,
       int blockSize, boolean createParent) throws IOException {
-      return createFile(fSys, path, numBlocks, blockSize, fSys.getDefaultReplication(path), true);
+    return createFile(fSys, path, numBlocks, blockSize,
+        fSys.getDefaultReplication(path), true);
   }
 
   public static long createFile(FileSystem fSys, Path path, int numBlocks,
       int blockSize) throws IOException {
-      return createFile(fSys, path, numBlocks, blockSize, true);
+    return createFile(fSys, path, numBlocks, blockSize, true);
   }
 
   public static long createFile(FileSystem fSys, Path path) throws IOException {
-    return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, DEFAULT_NUM_REPL, true);
+    return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE,
+        DEFAULT_NUM_REPL, true);
   }
 
   public long createFile(FileSystem fSys, String name) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index 1f5b863f2321c..9fe2cd7f12f02 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -577,7 +577,8 @@ public void testCreateLinkUsingPartQualPath2() throws IOException {
     } catch (IOException e) {
       // Expected
       if (wrapper instanceof FileContextTestWrapper) {
-        assertEquals("No AbstractFileSystem for scheme: null", e.getMessage());
+        GenericTestUtils.assertExceptionContains(
+            AbstractFileSystem.NO_ABSTRACT_FS_ERROR, e);
       } else if (wrapper instanceof FileSystemTestWrapper) {
         assertEquals("No FileSystem for scheme: null", e.getMessage());
       }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
index 9c8a8a4e06716..5db0de33943b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
@@ -137,15 +137,15 @@ public void testReadFields() throws IOException {
   // check the header with quotas
   @Test
   public void testGetHeaderWithQuota() {
-    String header = "  name quota  rem name quota     space quota "
-        + "rem space quota  directories        files              bytes ";
+    String header = "       QUOTA       REM_QUOTA     SPACE_QUOTA "
+        + "REM_SPACE_QUOTA    DIR_COUNT   FILE_COUNT       CONTENT_SIZE ";
     assertEquals(header, ContentSummary.getHeader(true));
   }
 
   // check the header without quotas
   @Test
   public void testGetHeaderNoQuota() {
-    String header = " directories        files              bytes ";
+    String header = "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE ";
     assertEquals(header, ContentSummary.getHeader(false));
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java
new file mode 100644
index 0000000000000..584ca40a3af81
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContext.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+public class TestFileContext {
+  private static final Log LOG = LogFactory.getLog(TestFileContext.class);
+
+  @Test
+  public void testDefaultURIWithoutScheme() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "/");
+    try {
+      FileContext.getFileContext(conf);
+      fail(UnsupportedFileSystemException.class + " not thrown!");
+    } catch (UnsupportedFileSystemException ufse) {
+      LOG.info("Expected exception: ", ufse);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
index 512ba0b365c15..c68861c55cf2a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
@@ -295,91 +295,11 @@ public void testCreateHardLinkMultEmptyList() throws IOException {
     String[] emptyList = {};
     
     //test the case of empty file list
-    int callCount = createHardLinkMult(src, emptyList, tgt_mult, 
-        getMaxAllowedCmdArgLength());
-    //check no exec calls were made
-    assertEquals(0, callCount);
+    createHardLinkMult(src, emptyList, tgt_mult);
     //check nothing changed in the directory tree
     validateSetup();
   }
-  
-  /**
-   * Test createHardLinkMult(), again, this time with the "too long list" 
-   * case where the total size of the command line arguments exceed the 
-   * allowed maximum.  In this case, the list should be automatically 
-   * broken up into chunks, each chunk no larger than the max allowed.
-   * 
-   * We use an extended version of the method call, specifying the
-   * size limit explicitly, to simulate the "too long" list with a 
-   * relatively short list.
-   */
-  @Test
-  public void testCreateHardLinkMultOversizeAndEmpty() throws IOException {
-    
-    // prep long filenames - each name takes 10 chars in the arg list
-    // (9 actual chars plus terminal null or delimeter blank)
-    String name1 = "x11111111";
-    String name2 = "x22222222";
-    String name3 = "x33333333";
-    File x1_long = new File(src, name1);
-    File x2_long = new File(src, name2);
-    File x3_long = new File(src, name3);
-    //set up source files with long file names
-    x1.renameTo(x1_long);
-    x2.renameTo(x2_long);
-    x3.renameTo(x3_long);
-    //validate setup
-    assertTrue(x1_long.exists());
-    assertTrue(x2_long.exists());
-    assertTrue(x3_long.exists());
-    assertFalse(x1.exists());
-    assertFalse(x2.exists());
-    assertFalse(x3.exists());
-    
-    //prep appropriate length information to construct test case for
-    //oversize filename list
-    int callCount;
-    String[] emptyList = {};
-    String[] fileNames = src.list();
-    //get fixed size of arg list without any filenames
-    int overhead = getLinkMultArgLength(src, emptyList, tgt_mult);
-    //select a maxLength that is slightly too short to hold 3 filenames
-    int maxLength = overhead + (int)(2.5 * (float)(1 + name1.length())); 
-    
-    //now test list of three filenames when there is room for only 2.5
-    callCount = createHardLinkMult(src, fileNames, tgt_mult, maxLength);
-    //check the request was completed in exactly two "chunks"
-    assertEquals(2, callCount);
-    String[] tgt_multNames = tgt_mult.list();
-    //sort directory listings before comparsion
-    Arrays.sort(fileNames);
-    Arrays.sort(tgt_multNames);
-    //and check the results were as expected in the dir tree
-    assertArrayEquals(fileNames, tgt_multNames);
-    
-    //Test the case where maxlength is too small even for one filename.
-    //It should go ahead and try the single files.
-    
-    //Clear the test dir tree
-    FileUtil.fullyDelete(tgt_mult);
-    assertFalse(tgt_mult.exists());
-    tgt_mult.mkdirs();
-    assertTrue(tgt_mult.exists() && tgt_mult.list().length == 0);
-    //set a limit size much smaller than a single filename
-    maxLength = overhead + (int)(0.5 * (float)(1 + name1.length()));
-    //attempt the method call
-    callCount = createHardLinkMult(src, fileNames, tgt_mult, 
-        maxLength);
-    //should go ahead with each of the three single file names
-    assertEquals(3, callCount);
-    tgt_multNames = tgt_mult.list();
-    //sort directory listings before comparsion
-    Arrays.sort(fileNames);
-    Arrays.sort(tgt_multNames);
-    //and check the results were as expected in the dir tree
-    assertArrayEquals(fileNames, tgt_multNames);
-  }
-  
+
   /*
    * Assume that this test won't usually be run on a Windows box.
    * This test case allows testing of the correct syntax of the Windows
@@ -392,18 +312,13 @@ public void testCreateHardLinkMultOversizeAndEmpty() throws IOException {
    */
   @Test
   public void testWindowsSyntax() {
-    class win extends HardLinkCGWin {};
+    class win extends HardLinkCGWin {}
 
     //basic checks on array lengths
-    assertEquals(5, win.hardLinkCommand.length); 
-    assertEquals(7, win.hardLinkMultPrefix.length);
-    assertEquals(7, win.hardLinkMultSuffix.length);
     assertEquals(4, win.getLinkCountCommand.length);
 
-    assertTrue(win.hardLinkMultPrefix[4].equals("%f"));
     //make sure "%f" was not munged
     assertEquals(2, ("%f").length()); 
-    assertTrue(win.hardLinkMultDir.equals("\\%f"));
     //make sure "\\%f" was munged correctly
     assertEquals(3, ("\\%f").length()); 
     assertTrue(win.getLinkCountCommand[1].equals("hardlink"));
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
index 994b9b2506d42..ca01702217791 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
@@ -19,10 +19,12 @@
 package org.apache.hadoop.fs.loadGenerator;
 
 import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
@@ -36,10 +38,11 @@
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Options.CreateOpts;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Tool;
@@ -48,8 +51,11 @@
 import com.google.common.base.Preconditions;
 
 /** The load generator is a tool for testing NameNode behavior under
- * different client loads.
- * It allows the user to generate different mixes of read, write,
+ * different client loads. Note there is a subclass of this clas that lets 
+ * you run a the load generator as a MapReduce job (see LoadGeneratorMR in the 
+ * MapReduce project.
+ * 
+ * The loadGenerator allows the user to generate different mixes of read, write,
  * and list requests by specifying the probabilities of read and
  * write. The user controls the intensity of the load by
  * adjusting parameters for the number of worker threads and the delay
@@ -58,15 +64,24 @@
  * generator exits, it print some NameNode statistics like the average
  * execution time of each kind of operations and the NameNode
  * throughput.
+ *
+ * The program can run in one of two forms. As a regular single process command
+ * that runs multiple threads to generate load on the NN or as a Map Reduce
+ * program that runs multiple (multi-threaded) map tasks that generate load
+ * on the NN; the results summary is generated by a single reduce task.
+ * 
  * 
  * The user may either specify constant duration, read and write 
  * probabilities via the command line, or may specify a text file
  * that acts as a script of which read and write probabilities to
- * use for specified durations.
+ * use for specified durations. If no duration is specified the program
+ * runs till killed (duration required if run as MapReduce).
  * 
  * The script takes the form of lines of duration in seconds, read
  * probability and write probability, each separated by white space.
- * Blank lines and lines starting with # (comments) are ignored.
+ * Blank lines and lines starting with # (comments) are ignored. If load
+ * generator is run as a MapReduce program then the script file needs to be
+ * accessible on the the Map task as a HDFS file.
  * 
  * After command line argument parsing and data initialization,
  * the load generator spawns the number of worker threads 
@@ -116,31 +131,43 @@
 public class LoadGenerator extends Configured implements Tool {
   public static final Log LOG = LogFactory.getLog(LoadGenerator.class);
   
-  private volatile boolean shouldRun = true;
-  private Path root = DataGenerator.DEFAULT_ROOT;
-  private FileContext fc;
-  private int maxDelayBetweenOps = 0;
-  private int numOfThreads = 200;
-  private long [] durations = {0};
-  private double [] readProbs = {0.3333};
-  private double [] writeProbs = {0.3333};
-  private volatile int currentIndex = 0;
-  long totalTime = 0;
-  private long startTime = Time.now()+10000;
+  private volatile static boolean shouldRun = true;
+  protected static Path root = DataGenerator.DEFAULT_ROOT;
+  private static FileContext fc;
+  protected static int maxDelayBetweenOps = 0;
+  protected static int numOfThreads = 200;
+  protected static long [] durations = {0};
+  protected static double [] readProbs = {0.3333};
+  protected static double [] writeProbs = {0.3333};
+  private static volatile int currentIndex = 0;
+  protected static long totalTime = 0;
+  protected static long startTime = Time.now()+10000;
   final static private int BLOCK_SIZE = 10;
-  private ArrayList<String> files = new ArrayList<String>();  // a table of file names
-  private ArrayList<String> dirs = new ArrayList<String>(); // a table of directory names
-  private Random r = null;
-  final private static String USAGE = "java LoadGenerator\n" +
-  	"-readProbability <read probability>\n" +
-    "-writeProbability <write probability>\n" +
-    "-root <root>\n" +
-    "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n" +
-    "-numOfThreads <numOfThreads>\n" +
-    "-elapsedTime <elapsedTimeInSecs>\n" +
-    "-startTime <startTimeInMillis>\n" +
-    "-scriptFile <filename>";
-  final private String hostname;
+  private static ArrayList<String> files = new ArrayList<String>();  // a table of file names
+  private static ArrayList<String> dirs = new ArrayList<String>(); // a table of directory names
+  protected static Random r = null;
+  protected static long seed = 0;
+  protected static String scriptFile = null;
+  protected static final String FLAGFILE_DEFAULT = "/tmp/flagFile";
+  protected static Path flagFile = new Path(FLAGFILE_DEFAULT);
+  protected String hostname;
+  final private static String USAGE_CMD = "java LoadGenerator\n";
+  final protected static String USAGE_ARGS = 
+	  "-readProbability <read probability>\n" +
+      "-writeProbability <write probability>\n" +
+      "-root <root>\n" +
+      "-maxDelayBetweenOps <maxDelayBetweenOpsInMillis>\n" +
+      "-numOfThreads <numOfThreads>\n" +
+      "-elapsedTime <elapsedTimeInSecs>\n" +
+      "-startTime <startTimeInMillis>\n" +
+      "-scriptFile <filename>\n" +
+      "-flagFile <filename>";
+  final private static String USAGE = USAGE_CMD + USAGE_ARGS;
+  
+
+  
+
+
   private final byte[] WRITE_CONTENTS = new byte[4096];
 
   private static final int ERR_TEST_FAILED = 2;
@@ -151,15 +178,21 @@ public LoadGenerator() throws IOException, UnknownHostException {
     hostname = addr.getHostName();
     Arrays.fill(WRITE_CONTENTS, (byte) 'a');
   }
+  
+  public LoadGenerator(Configuration conf) throws IOException, UnknownHostException {
+    this();
+    setConf(conf);
+  }
 
-  private final static int OPEN = 0;
-  private final static int LIST = 1;
-  private final static int CREATE = 2;
-  private final static int WRITE_CLOSE = 3;
-  private final static int DELETE = 4;
-  private final static int TOTAL_OP_TYPES =5;
-  private long [] executionTime = new long[TOTAL_OP_TYPES];
-  private long [] totalNumOfOps = new long[TOTAL_OP_TYPES];
+  protected final static int OPEN = 0;
+  protected final static int LIST = 1;
+  protected final static int CREATE = 2;
+  protected final static int WRITE_CLOSE = 3;
+  protected final static int DELETE = 4;
+  protected final static int TOTAL_OP_TYPES =5;
+  protected static long [] executionTime = new long[TOTAL_OP_TYPES];
+  protected static long [] numOfOps = new long[TOTAL_OP_TYPES];
+  protected static long totalOps = 0; // across all of types
   
   /** A thread sends a stream of requests to the NameNode.
    * At each iteration, it first decides if it is going to read a file,
@@ -192,7 +225,7 @@ private DFSClientThread(int id) {
       this.id = id;
     }
     
-    /** Main loop
+    /** Main loop for each thread
      * Each iteration decides what's the next operation and then pauses.
      */
     @Override
@@ -295,7 +328,7 @@ private void genFile(Path file, long fileSize) throws IOException {
             CreateOpts.createParent(), CreateOpts.bufferSize(4096),
             CreateOpts.repFac((short) 3));
         executionTime[CREATE] += (Time.now() - startTime);
-        totalNumOfOps[CREATE]++;
+        numOfOps[CREATE]++;
 
         long i = fileSize;
         while (i > 0) {
@@ -306,28 +339,67 @@ private void genFile(Path file, long fileSize) throws IOException {
 
         startTime = Time.now();
         executionTime[WRITE_CLOSE] += (Time.now() - startTime);
-        totalNumOfOps[WRITE_CLOSE]++;
+        numOfOps[WRITE_CLOSE]++;
       } finally {
         IOUtils.cleanup(LOG, out);
       }
     }
   }
   
-  /** Main function:
+  /** Main function called by tool runner.
    * It first initializes data by parsing the command line arguments.
-   * It then starts the number of DFSClient threads as specified by
-   * the user.
-   * It stops all the threads when the specified elapsed time is passed.
-   * Before exiting, it prints the average execution for 
-   * each operation and operation throughput.
+   * It then calls the loadGenerator
    */
   @Override
   public int run(String[] args) throws Exception {
-    int exitCode = init(args);
+    int exitCode = parseArgs(false, args);
     if (exitCode != 0) {
       return exitCode;
     }
+    System.out.println("Running LoadGenerator against fileSystem: " + 
+    FileContext.getFileContext().getDefaultFileSystem().getUri());
+    exitCode = generateLoadOnNN();
+    printResults(System.out);
+    return exitCode;
+  }
+    
+  boolean stopFileCreated() {
+    try {
+      fc.getFileStatus(flagFile);
+    } catch (FileNotFoundException e) {
+      return false;
+    } catch (IOException e) {
+      LOG.error("Got error when checking if file exists:" + flagFile, e);
+    }
+    LOG.info("Flag file was created. Stopping the test.");
+    return true;
+  }
+  
+ /**
+  * This is the main function - run threads to generate load on NN
+  * It starts the number of DFSClient threads as specified by
+  * the user.
+  * It stops all the threads when the specified elapsed time is passed.
+  */
+  protected int generateLoadOnNN() throws InterruptedException {
+    int hostHashCode = hostname.hashCode();
+    if (seed == 0) {
+      r = new Random(System.currentTimeMillis()+hostHashCode);
+    } else {
+      r = new Random(seed+hostHashCode);
+    }
+    try {
+      fc = FileContext.getFileContext(getConf());
+    } catch (IOException ioe) {
+      System.err.println("Can not initialize the file system: " + 
+          ioe.getLocalizedMessage());
+      return -1;
+    }
     
+    int status = initFileDirTables();
+    if (status != 0) {
+      return status;
+    }
     barrier();
     
     DFSClientThread[] threads = new DFSClientThread[numOfThreads];
@@ -337,91 +409,99 @@ public int run(String[] args) throws Exception {
     }
     
     if (durations[0] > 0) {
-      while(shouldRun) {
-        Thread.sleep(durations[currentIndex] * 1000);
-        totalTime += durations[currentIndex];
-        
-        // Are we on the final line of the script?
-        if( (currentIndex + 1) == durations.length) {
-          shouldRun = false;
-        } else {
-          if(LOG.isDebugEnabled()) {
-            LOG.debug("Moving to index " + currentIndex + ": r = "
-                + readProbs[currentIndex] + ", w = " + writeProbs
-                + " for duration " + durations[currentIndex]);
+      if (durations.length == 1) {// There is a fixed run time
+        while (shouldRun) {
+          Thread.sleep(2000);
+          totalTime += 2;
+          if (totalTime >= durations[0] || stopFileCreated()) {
+            shouldRun = false;
+          }
+        }
+      } else {
+        // script run
+
+        while (shouldRun) {
+          Thread.sleep(durations[currentIndex] * 1000);
+          totalTime += durations[currentIndex];
+          // Are we on the final line of the script?
+          if ((currentIndex + 1) == durations.length || stopFileCreated()) {
+            shouldRun = false;
+          } else {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Moving to index " + currentIndex + ": r = "
+                  + readProbs[currentIndex] + ", w = " + writeProbs
+                  + " for duration " + durations[currentIndex]);
+            }
+            currentIndex++;
           }
-          currentIndex++;
         }
       }
-    } 
+    }
     
     if(LOG.isDebugEnabled()) {
       LOG.debug("Done with testing.  Waiting for threads to finish.");
     }
-
+    
     boolean failed = false;
     for (DFSClientThread thread : threads) {
       thread.join();
       for (int i=0; i<TOTAL_OP_TYPES; i++) {
         executionTime[i] += thread.executionTime[i];
-        totalNumOfOps[i] += thread.totalNumOfOps[i];
+        numOfOps[i] += thread.totalNumOfOps[i];
       }
       failed = failed || thread.failed;
     }
-
+    int exitCode = 0;
     if (failed) {
       exitCode = -ERR_TEST_FAILED;
     }
 
-    long totalOps = 0;
+    totalOps = 0;
     for (int i=0; i<TOTAL_OP_TYPES; i++) {
-      totalOps += totalNumOfOps[i];
+      totalOps += numOfOps[i];
     }
-    
-    if (totalNumOfOps[OPEN] != 0) {
-      System.out.println("Average open execution time: " + 
-          (double)executionTime[OPEN]/totalNumOfOps[OPEN] + "ms");
-    }
-    if (totalNumOfOps[LIST] != 0) {
-      System.out.println("Average list execution time: " + 
-          (double)executionTime[LIST]/totalNumOfOps[LIST] + "ms");
-    }
-    if (totalNumOfOps[DELETE] != 0) {
-      System.out.println("Average deletion execution time: " + 
-          (double)executionTime[DELETE]/totalNumOfOps[DELETE] + "ms");
-      System.out.println("Average create execution time: " + 
-          (double)executionTime[CREATE]/totalNumOfOps[CREATE] + "ms");
-      System.out.println("Average write_close execution time: " + 
-          (double)executionTime[WRITE_CLOSE]/totalNumOfOps[WRITE_CLOSE] + "ms");
-    }
-    if (durations[0] != 0) { 
-      System.out.println("Average operations per second: " + 
+    return exitCode;
+  }
+  
+  protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
+    out.println("Result of running LoadGenerator against fileSystem: " + 
+    FileContext.getFileContext().getDefaultFileSystem().getUri());
+    if (numOfOps[OPEN] != 0) {
+      out.println("Average open execution time: " + 
+          (double)executionTime[OPEN]/numOfOps[OPEN] + "ms");
+    }
+    if (numOfOps[LIST] != 0) {
+      out.println("Average list execution time: " + 
+          (double)executionTime[LIST]/numOfOps[LIST] + "ms");
+    }
+    if (numOfOps[DELETE] != 0) {
+      out.println("Average deletion execution time: " + 
+          (double)executionTime[DELETE]/numOfOps[DELETE] + "ms");
+      out.println("Average create execution time: " + 
+          (double)executionTime[CREATE]/numOfOps[CREATE] + "ms");
+      out.println("Average write_close execution time: " + 
+          (double)executionTime[WRITE_CLOSE]/numOfOps[WRITE_CLOSE] + "ms");
+    }
+    if (totalTime != 0) { 
+      out.println("Average operations per second: " + 
           (double)totalOps/totalTime +"ops/s");
     }
-    System.out.println();
-    return exitCode;
+    out.println();
   }
+    
 
   /** Parse the command line arguments and initialize the data */
-  private int init(String[] args) throws IOException {
-    try {
-      fc = FileContext.getFileContext(getConf());
-    } catch (IOException ioe) {
-      System.err.println("Can not initialize the file system: " + 
-          ioe.getLocalizedMessage());
-      return -1;
-    }
-    int hostHashCode = hostname.hashCode();
-    boolean scriptSpecified = false;
-    
-    try {
+  protected int parseArgs(boolean runAsMapReduce, String[] args) throws IOException {
+   try {
       for (int i = 0; i < args.length; i++) { // parse command line
         if (args[i].equals("-scriptFile")) {
-          if(loadScriptFile(args[++i]) == -1)
+          scriptFile = args[++i];
+          if (durations[0] > 0)  {
+            System.err.println("Can't specify elapsedTime and use script.");
             return -1;
-          scriptSpecified = true;
+          }
         } else if (args[i].equals("-readProbability")) {
-          if(scriptSpecified) {
+          if (scriptFile != null) {
             System.err.println("Can't specify probabilities and use script.");
             return -1;
           }
@@ -432,7 +512,7 @@ private int init(String[] args) throws IOException {
             return -1;
           }
         } else if (args[i].equals("-writeProbability")) {
-          if(scriptSpecified) {
+          if (scriptFile != null) {
             System.err.println("Can't specify probabilities and use script.");
             return -1;
           }
@@ -456,14 +536,18 @@ private int init(String[] args) throws IOException {
         } else if (args[i].equals("-startTime")) {
           startTime = Long.parseLong(args[++i]);
         } else if (args[i].equals("-elapsedTime")) {
-          if(scriptSpecified) {
+          if (scriptFile != null) {
             System.err.println("Can't specify elapsedTime and use script.");
             return -1;
           }
           durations[0] = Long.parseLong(args[++i]);
         } else if (args[i].equals("-seed")) {
-          r = new Random(Long.parseLong(args[++i])+hostHashCode);
-        } else {
+          seed = Long.parseLong(args[++i]);
+          r = new Random(seed);
+        }  else if (args[i].equals("-flagFile")) {
+          LOG.info("got flagFile:" + flagFile);
+          flagFile = new Path(args[++i]);
+        }else { 
           System.err.println(USAGE);
           ToolRunner.printGenericCommandUsage(System.err);
           return -1;
@@ -475,6 +559,12 @@ private int init(String[] args) throws IOException {
       return -1;
     }
     
+    // Load Script File if not MR; for MR scriptFile is loaded by Mapper
+    if (!runAsMapReduce && scriptFile != null) { 
+      if(loadScriptFile(scriptFile, true) == -1)
+        return -1;
+    }
+    
     for(int i = 0; i < readProbs.length; i++) {
       if (readProbs[i] + writeProbs[i] <0 || readProbs[i]+ writeProbs[i] > 1) {
         System.err.println(
@@ -483,12 +573,7 @@ private int init(String[] args) throws IOException {
         return -1;
       }
     }
-    
-    if (r==null) {
-      r = new Random(Time.now()+hostHashCode);
-    }
-    
-    return initFileDirTables();
+    return 0;
   }
 
   private static void parseScriptLine(String line, ArrayList<Long> duration,
@@ -527,9 +612,25 @@ private static void parseScriptLine(String line, ArrayList<Long> duration,
    * @return 0 if successful, -1 if not
    * @throws IOException if errors with file IO
    */
-  private int loadScriptFile(String filename) throws IOException  {
-    FileReader fr = new FileReader(new File(filename));
-    BufferedReader br = new BufferedReader(fr);
+  protected static int loadScriptFile(String filename, boolean readLocally) throws IOException {
+    
+    FileContext fc;
+    if (readLocally) { // read locally - program is run without MR
+      fc = FileContext.getLocalFSFileContext();
+    } else {
+      fc = FileContext.getFileContext(); // use default file system
+    }
+    DataInputStream in = null;
+    try {
+      in = fc.open(new Path(filename));
+    } catch (IOException e) {
+      System.err.println("Unable to open scriptFile: " + filename);
+
+      System.exit(-1);
+    } 
+    InputStreamReader inr = new InputStreamReader(in);
+    
+    BufferedReader br = new BufferedReader(inr);
     ArrayList<Long> duration  = new ArrayList<Long>();
     ArrayList<Double> readProb  = new ArrayList<Double>();
     ArrayList<Double> writeProb = new ArrayList<Double>();
@@ -619,7 +720,7 @@ private void initFileDirTables(Path path) throws IOException {
    * This allows multiple instances of this program, running on clock
    * synchronized nodes, to start at roughly the same time.
    */
-  private void barrier() {
+  private static void barrier() {
     long sleepTime;
     while ((sleepTime = startTime - Time.now()) > 0) {
       try {
@@ -635,9 +736,7 @@ private void barrier() {
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
-    int res = ToolRunner.run(new Configuration(),
-        new LoadGenerator(), args);
+    int res = ToolRunner.run(new Configuration(), new LoadGenerator(), args);
     System.exit(res);
   }
-
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 6c753c6a8f43b..1f2f2d4804632 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
 import org.junit.Test;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -47,7 +48,6 @@ public class TestCount {
   private static Configuration conf;
   private static FileSystem mockFs;
   private static FileStatus fileStat;
-  private static ContentSummary mockCs;
 
   @BeforeClass
   public static void setup() {
@@ -55,7 +55,6 @@ public static void setup() {
     conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
     mockFs = mock(FileSystem.class);
     fileStat = mock(FileStatus.class);
-    mockCs = mock(ContentSummary.class);
     when(fileStat.isFile()).thenReturn(true);
   }
 
@@ -87,6 +86,85 @@ public void processOptionsAll() {
     assertTrue(count.isHumanReadable());
   }
 
+  // check no options is handled correctly
+  @Test
+  public void processOptionsNoOptions() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("dummy");
+    Count count = new Count();
+    count.processOptions(options);
+    assertFalse(count.isShowQuotas());
+  }
+
+  // check -q is handled correctly
+  @Test
+  public void processOptionsShowQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-q");
+    options.add("dummy");
+    Count count = new Count();
+    count.processOptions(options);
+    assertTrue(count.isShowQuotas());
+  }
+
+  // check missing arguments is handled correctly
+  @Test
+  public void processOptionsMissingArgs() {
+    LinkedList<String> options = new LinkedList<String>();
+    Count count = new Count();
+    try {
+      count.processOptions(options);
+      fail("Count.processOptions - NotEnoughArgumentsException not thrown");
+    } catch (NotEnoughArgumentsException e) {
+    }
+    assertFalse(count.isShowQuotas());
+  }
+
+  // check the correct header is produced with no quotas (-v)
+  @Test
+  public void processOptionsHeaderNoQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-v");
+    options.add("dummy");
+
+    PrintStream out = mock(PrintStream.class);
+
+    Count count = new Count();
+    count.out = out;
+
+    count.processOptions(options);
+
+    String noQuotasHeader =
+    // <----12----> <----12----> <-------18------->
+      "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME";
+    verify(out).println(noQuotasHeader);
+    verifyNoMoreInteractions(out);
+  }
+
+  // check the correct header is produced with quotas (-q -v)
+  @Test
+  public void processOptionsHeaderWithQuotas() {
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-q");
+    options.add("-v");
+    options.add("dummy");
+
+    PrintStream out = mock(PrintStream.class);
+
+    Count count = new Count();
+    count.out = out;
+
+    count.processOptions(options);
+
+    String withQuotasHeader =
+    // <----12----> <-----15------> <-----15------> <-----15------>
+      "       QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA " +
+    // <----12----> <----12----> <-------18------->
+      "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME";
+    verify(out).println(withQuotasHeader);
+    verifyNoMoreInteractions(out);
+  }
+
   // check quotas are reported correctly
   @Test
   public void processPathShowQuotas() throws Exception {
@@ -211,29 +289,48 @@ public void getName() {
   public void getUsage() {
     Count count = new Count();
     String actual = count.getUsage();
-    String expected = "-count [-q] [-h] <path> ...";
+    String expected = "-count [-q] [-h] [-v] <path> ...";
     assertEquals("Count.getUsage", expected, actual);
   }
 
+  // check the correct description is returned
+  @Test
+  public void getDescription() {
+    Count count = new Count();
+    String actual = count.getDescription();
+    String expected =
+        "Count the number of directories, files and bytes under the paths\n"
+        + "that match the specified file pattern.  The output columns are:\n"
+        + "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+        + "or, with the -q option:\n"
+        + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+        + "      DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+        + "The -h option shows file sizes in human readable format.\n"
+        + "The -v option displays a header line.";
+
+    assertEquals("Count.getDescription", expected, actual);
+  }
+
 
   // mock content system
   static class MockContentSummary extends ContentSummary {
-    
-    public MockContentSummary() {}
+
+    public MockContentSummary() {
+    }
 
     @Override
     public String toString(boolean qOption, boolean hOption) {
       if (qOption) {
         if (hOption) {
-          return(HUMAN + WITH_QUOTAS);
+          return (HUMAN + WITH_QUOTAS);
         } else {
-          return(BYTES + WITH_QUOTAS);
+          return (BYTES + WITH_QUOTAS);
         }
       } else {
         if (hOption) {
-          return(HUMAN + NO_QUOTAS);
+          return (HUMAN + NO_QUOTAS);
         } else {
-          return(BYTES + NO_QUOTAS);
+          return (BYTES + NO_QUOTAS);
         }
       }
     }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java
index 1db792453a297..5aee61166ff34 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/MiniZKFCCluster.java
@@ -155,11 +155,16 @@ public void setUnreachable(int idx, boolean unreachable) {
 
   /**
    * Wait for the given HA service to enter the given HA state.
+   * This is based on the state of ZKFC, not the state of HA service.
+   * There could be difference between the two. For example,
+   * When the service becomes unhealthy, ZKFC will quit ZK election and
+   * transition to HAServiceState.INITIALIZING and remain in that state
+   * until the service becomes healthy.
    */
   public void waitForHAState(int idx, HAServiceState state)
       throws Exception {
-    DummyHAService svc = getService(idx);
-    while (svc.state != state) {
+    DummyZKFC svc = getZkfc(idx);
+    while (svc.getServiceState() != state) {
       ctx.checkException();
       Thread.sleep(50);
     }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java
index 83a29dd11d9ea..d8271c5059de4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverController.java
@@ -211,8 +211,8 @@ public void testAutoFailoverOnBadHealth() throws Exception {
       LOG.info("Faking svc0 unhealthy, should failover to svc1");
       cluster.setHealthy(0, false);
       
-      LOG.info("Waiting for svc0 to enter standby state");
-      cluster.waitForHAState(0, HAServiceState.STANDBY);
+      LOG.info("Waiting for svc0 to enter initializing state");
+      cluster.waitForHAState(0, HAServiceState.INITIALIZING);
       cluster.waitForHAState(1, HAServiceState.ACTIVE);
   
       LOG.info("Allowing svc0 to be healthy again, making svc1 unreachable " +
@@ -332,7 +332,7 @@ public void testBecomingActiveFails() throws Exception {
       Mockito.verify(svc1.proxy, Mockito.timeout(2000).atLeastOnce())
         .transitionToActive(Mockito.<StateChangeRequestInfo>any());
 
-      cluster.waitForHAState(0, HAServiceState.STANDBY);
+      cluster.waitForHAState(0, HAServiceState.INITIALIZING);
       cluster.waitForHAState(1, HAServiceState.STANDBY);
       
       LOG.info("Faking svc0 healthy again, should go back to svc0");
@@ -587,12 +587,12 @@ public void testOneOfEverything() throws Exception {
 
       // Failover by bad health
       cluster.setHealthy(0, false);
-      cluster.waitForHAState(0, HAServiceState.STANDBY);
+      cluster.waitForHAState(0, HAServiceState.INITIALIZING);
       cluster.waitForHAState(1, HAServiceState.ACTIVE);
       cluster.setHealthy(1, true);
       cluster.setHealthy(0, false);
       cluster.waitForHAState(1, HAServiceState.ACTIVE);
-      cluster.waitForHAState(0, HAServiceState.STANDBY);
+      cluster.waitForHAState(0, HAServiceState.INITIALIZING);
       cluster.setHealthy(0, true);
       
       cluster.waitForHealthState(0, State.SERVICE_HEALTHY);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
index ecf2d0f4446a7..4a4de41100b36 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
@@ -28,15 +28,32 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URL;
 import java.net.MalformedURLException;
 
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
 /**
  * This is a base class for functional tests of the {@link HttpServer2}.
  * The methods are static for other classes to import statically.
  */
 public class HttpServerFunctionalTest extends Assert {
+  @SuppressWarnings("serial")
+  public static class LongHeaderServlet extends HttpServlet {
+    @Override
+    public void doGet(HttpServletRequest request,
+                      HttpServletResponse response
+    ) throws ServletException, IOException {
+      Assert.assertEquals(63 * 1024, request.getHeader("longheader").length());
+      response.setStatus(HttpServletResponse.SC_OK);
+    }
+  }
+
   /** JVM property for the webapp test dir : {@value} */
   public static final String TEST_BUILD_WEBAPPS = "test.build.webapps";
   /** expected location of the test.build.webapps dir: {@value} */
@@ -44,6 +61,7 @@ public class HttpServerFunctionalTest extends Assert {
   
   /** name of the test webapp: {@value} */
   private static final String TEST = "test";
+  protected static URL baseUrl;
 
   /**
    * Create but do not start the test webapp server. The test webapp dir is
@@ -227,4 +245,18 @@ protected static String readOutput(URL url) throws IOException {
     }
     return out.toString();
   }
+
+  /**
+   *  Test that verifies headers can be up to 64K long.
+   *  The test adds a 63K header leaving 1K for other headers.
+   *  This is because the header buffer setting is for ALL headers,
+   *  names and values included. */
+  protected void testLongHeader(HttpURLConnection conn) throws IOException {
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0 ; i < 63 * 1024; i++) {
+      sb.append("a");
+    }
+    conn.setRequestProperty("longheader", sb.toString());
+    assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index ac03968c2d4fb..5b202da0eb41c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -17,36 +17,6 @@
  */
 package org.apache.hadoop.http;
 
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletContext;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
-import javax.servlet.http.HttpServletResponse;
-
-import org.junit.Assert;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -59,6 +29,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -66,12 +37,36 @@
 import org.mortbay.jetty.Connector;
 import org.mortbay.util.ajax.JSON;
 
-import static org.mockito.Mockito.*;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
 
 public class TestHttpServer extends HttpServerFunctionalTest {
   static final Log LOG = LogFactory.getLog(TestHttpServer.class);
   private static HttpServer2 server;
-  private static URL baseUrl;
   private static final int MAX_THREADS = 10;
   
   @SuppressWarnings("serial")
@@ -124,17 +119,6 @@ public void doGet(HttpServletRequest request,
     }    
   }
 
-  @SuppressWarnings("serial")
-  public static class LongHeaderServlet extends HttpServlet {
-    @Override
-    public void doGet(HttpServletRequest request,
-                      HttpServletResponse response
-    ) throws ServletException, IOException {
-      Assert.assertEquals(63 * 1024, request.getHeader("longheader").length());
-      response.setStatus(HttpServletResponse.SC_OK);
-    }
-  }
-
   @SuppressWarnings("serial")
   public static class HtmlContentServlet extends HttpServlet {
     @Override
@@ -214,20 +198,10 @@ public void run() {
                  readOutput(new URL(baseUrl, "/echomap?a=b&c<=d&a=>")));
   }
 
-  /** 
-   *  Test that verifies headers can be up to 64K long. 
-   *  The test adds a 63K header leaving 1K for other headers.
-   *  This is because the header buffer setting is for ALL headers,
-   *  names and values included. */
   @Test public void testLongHeader() throws Exception {
     URL url = new URL(baseUrl, "/longheader");
     HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-    StringBuilder sb = new StringBuilder();
-    for (int i = 0 ; i < 63 * 1024; i++) {
-      sb.append("a");
-    }
-    conn.setRequestProperty("longheader", sb.toString());
-    assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+    testLongHeader(conn);
   }
 
   @Test public void testContentTypes() throws Exception {
@@ -426,8 +400,9 @@ public void testRequestQuoterWithNull() throws Exception {
     Mockito.doReturn(null).when(request).getParameterValues("dummy");
     RequestQuoter requestQuoter = new RequestQuoter(request);
     String[] parameterValues = requestQuoter.getParameterValues("dummy");
-    Assert.assertEquals("It should return null "
-        + "when there are no values for the parameter", null, parameterValues);
+    Assert.assertNull(
+        "It should return null " + "when there are no values for the parameter",
+        parameterValues);
   }
 
   @Test
@@ -547,8 +522,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort)
       // not bound, ephemeral should return requested port (0 for ephemeral)
       List<?> listeners = (List<?>) Whitebox.getInternalState(server,
           "listeners");
-      Connector listener = (Connector) Whitebox.getInternalState(
-          listeners.get(0), "listener");
+      Connector listener = (Connector) listeners.get(0);
 
       assertEquals(port, listener.getPort());
       // verify hostname is what was given
@@ -582,16 +556,4 @@ public void testNoCacheHeader() throws Exception {
     assertNotNull(conn.getHeaderField("Date"));
     assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
   }
-
-  /**
-   * HTTPServer.Builder should proceed if a external connector is available.
-   */
-  @Test
-  public void testHttpServerBuilderWithExternalConnector() throws Exception {
-    Connector c = mock(Connector.class);
-    doReturn("localhost").when(c).getHost();
-    HttpServer2 s = new HttpServer2.Builder().setName("test").setConnector(c)
-        .build();
-    s.stop();
-  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
index 3d5d8b63f97f8..70fea872e0f96 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestSSLHttpServer.java
@@ -49,7 +49,6 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
   private static final Log LOG = LogFactory.getLog(TestSSLHttpServer.class);
   private static Configuration conf;
   private static HttpServer2 server;
-  private static URL baseUrl;
   private static String keystoresDir;
   private static String sslConfDir;
   private static SSLFactory clientSslFactory;
@@ -85,6 +84,7 @@ public static void setup() throws Exception {
             sslConf.get("ssl.server.truststore.password"),
             sslConf.get("ssl.server.truststore.type", "jks")).build();
     server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class);
+    server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
     server.start();
     baseUrl = new URL("https://"
         + NetUtils.getHostPortString(server.getConnectorAddress(0)));
@@ -106,6 +106,19 @@ public void testEcho() throws Exception {
         "/echo?a=b&c<=d&e=>")));
   }
 
+  /**
+   *  Test that verifies headers can be up to 64K long.
+   *  The test adds a 63K header leaving 1K for other headers.
+   *  This is because the header buffer setting is for ALL headers,
+   *  names and values included. */
+  @Test
+  public void testLongHeader() throws Exception {
+    URL url = new URL(baseUrl, "/longheader");
+    HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
+    conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
+    testLongHeader(conn);
+  }
+
   private static String readOut(URL url) throws Exception {
     HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
     conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
index 5db3c91a482d9..ac4fb483f67c5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
@@ -24,14 +24,21 @@
 import java.io.ByteArrayInputStream;
 import java.io.EOFException;
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
 
@@ -245,4 +252,41 @@ public void testSkipFully() throws IOException {
       in.close();
     }
   }
+
+  private static enum NoEntry3Filter implements FilenameFilter {
+    INSTANCE;
+
+    @Override
+    public boolean accept(File dir, String name) {
+      return !name.equals("entry3");
+    }
+  }
+
+  @Test
+  public void testListDirectory() throws IOException {
+    File dir = new File("testListDirectory");
+    Files.createDirectory(dir.toPath());
+    try {
+      Set<String> entries = new HashSet<String>();
+      entries.add("entry1");
+      entries.add("entry2");
+      entries.add("entry3");
+      for (String entry : entries) {
+        Files.createDirectory(new File(dir, entry).toPath());
+      }
+      List<String> list = IOUtils.listDirectory(dir,
+          NoEntry3Filter.INSTANCE);
+      for (String entry : list) {
+        Assert.assertTrue(entries.remove(entry));
+      }
+      Assert.assertTrue(entries.contains("entry3"));
+      list = IOUtils.listDirectory(dir, null);
+      for (String entry : list) {
+        entries.remove(entry);
+      }
+      Assert.assertTrue(entries.isEmpty());
+    } finally {
+      FileUtils.deleteDirectory(dir);
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
new file mode 100644
index 0000000000000..9482b4348254c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Test base of common utilities for tests not only raw coders but also block
+ * coders.
+ */
+public abstract class TestCoderBase {
+  protected static Random RAND = new Random();
+
+  protected int numDataUnits;
+  protected int numParityUnits;
+  protected int chunkSize = 16 * 1024;
+
+  // Indexes of erased data units. Will also support test of erasing
+  // parity units
+  protected int[] erasedDataIndexes = new int[] {0};
+
+  // Data buffers are either direct or on-heap, for performance the two cases
+  // may go to different coding implementations.
+  protected boolean usingDirectBuffer = true;
+
+  /**
+   * Compare and verify if erased chunks are equal to recovered chunks
+   * @param erasedChunks
+   * @param recoveredChunks
+   */
+  protected void compareAndVerify(ECChunk[] erasedChunks,
+                                  ECChunk[] recoveredChunks) {
+    byte[][] erased = ECChunk.toArray(erasedChunks);
+    byte[][] recovered = ECChunk.toArray(recoveredChunks);
+    for (int i = 0; i < erasedChunks.length; ++i) {
+      assertArrayEquals("Decoding and comparing failed.", erased[i],
+          recovered[i]);
+    }
+  }
+
+  /**
+   * Adjust and return erased indexes based on the array of the input chunks (
+   * parity chunks + data chunks).
+   * @return
+   */
+  protected int[] getErasedIndexesForDecoding() {
+    int[] erasedIndexesForDecoding = new int[erasedDataIndexes.length];
+    for (int i = 0; i < erasedDataIndexes.length; ++i) {
+      erasedIndexesForDecoding[i] = erasedDataIndexes[i] + numParityUnits;
+    }
+    return erasedIndexesForDecoding;
+  }
+
+  /**
+   * Return input chunks for decoding, which is parityChunks + dataChunks.
+   * @param dataChunks
+   * @param parityChunks
+   * @return
+   */
+  protected ECChunk[] prepareInputChunksForDecoding(ECChunk[] dataChunks,
+                                                  ECChunk[] parityChunks) {
+    ECChunk[] inputChunks = new ECChunk[numParityUnits + numDataUnits];
+    
+    int idx = 0;
+    for (int i = 0; i < numParityUnits; i++) {
+      inputChunks[idx ++] = parityChunks[i];
+    }
+    for (int i = 0; i < numDataUnits; i++) {
+      inputChunks[idx ++] = dataChunks[i];
+    }
+    
+    return inputChunks;
+  }
+
+  /**
+   * Have a copy of the data chunks that's to be erased thereafter. The copy
+   * will be used to compare and verify with the to be recovered chunks.
+   * @param dataChunks
+   * @return
+   */
+  protected ECChunk[] copyDataChunksToErase(ECChunk[] dataChunks) {
+    ECChunk[] copiedChunks = new ECChunk[erasedDataIndexes.length];
+
+    int j = 0;
+    for (int i = 0; i < erasedDataIndexes.length; ++i) {
+      copiedChunks[j ++] = cloneChunkWithData(dataChunks[erasedDataIndexes[i]]);
+    }
+
+    return copiedChunks;
+  }
+
+  /**
+   * Erase some data chunks to test the recovering of them
+   * @param dataChunks
+   */
+  protected void eraseSomeDataBlocks(ECChunk[] dataChunks) {
+    for (int i = 0; i < erasedDataIndexes.length; ++i) {
+      eraseDataFromChunk(dataChunks[erasedDataIndexes[i]]);
+    }
+  }
+
+  /**
+   * Erase data from the specified chunks, putting ZERO bytes to the buffers.
+   * @param chunks
+   */
+  protected void eraseDataFromChunks(ECChunk[] chunks) {
+    for (int i = 0; i < chunks.length; ++i) {
+      eraseDataFromChunk(chunks[i]);
+    }
+  }
+
+  /**
+   * Erase data from the specified chunk, putting ZERO bytes to the buffer.
+   * @param chunk
+   */
+  protected void eraseDataFromChunk(ECChunk chunk) {
+    ByteBuffer chunkBuffer = chunk.getBuffer();
+    // erase the data
+    chunkBuffer.position(0);
+    for (int i = 0; i < chunkSize; ++i) {
+      chunkBuffer.put((byte) 0);
+    }
+    chunkBuffer.flip();
+  }
+
+  /**
+   * Clone chunks along with copying the associated data. It respects how the
+   * chunk buffer is allocated, direct or non-direct. It avoids affecting the
+   * original chunk buffers.
+   * @param chunks
+   * @return
+   */
+  protected static ECChunk[] cloneChunksWithData(ECChunk[] chunks) {
+    ECChunk[] results = new ECChunk[chunks.length];
+    for (int i = 0; i < chunks.length; ++i) {
+      results[i] = cloneChunkWithData(chunks[i]);
+    }
+
+    return results;
+  }
+
+  /**
+   * Clone chunk along with copying the associated data. It respects how the
+   * chunk buffer is allocated, direct or non-direct. It avoids affecting the
+   * original chunk.
+   * @param chunk
+   * @return a new chunk
+   */
+  protected static ECChunk cloneChunkWithData(ECChunk chunk) {
+    ByteBuffer srcBuffer = chunk.getBuffer();
+    ByteBuffer destBuffer;
+
+    byte[] bytesArr = new byte[srcBuffer.remaining()];
+    srcBuffer.mark();
+    srcBuffer.get(bytesArr);
+    srcBuffer.reset();
+
+    if (srcBuffer.hasArray()) {
+      destBuffer = ByteBuffer.wrap(bytesArr);
+    } else {
+      destBuffer = ByteBuffer.allocateDirect(srcBuffer.remaining());
+      destBuffer.put(bytesArr);
+      destBuffer.flip();
+    }
+
+    return new ECChunk(destBuffer);
+  }
+
+  /**
+   * Allocate a chunk for output or writing.
+   * @return
+   */
+  protected ECChunk allocateOutputChunk() {
+    ByteBuffer buffer = allocateOutputBuffer();
+
+    return new ECChunk(buffer);
+  }
+
+  /**
+   * Allocate a buffer for output or writing.
+   * @return
+   */
+  protected ByteBuffer allocateOutputBuffer() {
+    ByteBuffer buffer = usingDirectBuffer ?
+        ByteBuffer.allocateDirect(chunkSize) : ByteBuffer.allocate(chunkSize);
+
+    return buffer;
+  }
+
+  /**
+   * Prepare data chunks for each data unit, by generating random data.
+   * @return
+   */
+  protected ECChunk[] prepareDataChunksForEncoding() {
+    ECChunk[] chunks = new ECChunk[numDataUnits];
+    for (int i = 0; i < chunks.length; i++) {
+      chunks[i] = generateDataChunk();
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Generate data chunk by making random data.
+   * @return
+   */
+  protected ECChunk generateDataChunk() {
+    ByteBuffer buffer = allocateOutputBuffer();
+    for (int i = 0; i < chunkSize; i++) {
+      buffer.put((byte) RAND.nextInt(256));
+    }
+    buffer.flip();
+
+    return new ECChunk(buffer);
+  }
+
+  /**
+   * Prepare parity chunks for encoding, each chunk for each parity unit.
+   * @return
+   */
+  protected ECChunk[] prepareParityChunksForEncoding() {
+    ECChunk[] chunks = new ECChunk[numParityUnits];
+    for (int i = 0; i < chunks.length; i++) {
+      chunks[i] = allocateOutputChunk();
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Prepare output chunks for decoding, each output chunk for each erased
+   * chunk.
+   * @return
+   */
+  protected ECChunk[] prepareOutputChunksForDecoding() {
+    ECChunk[] chunks = new ECChunk[erasedDataIndexes.length];
+    for (int i = 0; i < chunks.length; i++) {
+      chunks[i] = allocateOutputChunk();
+    }
+
+    return chunks;
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
new file mode 100644
index 0000000000000..9119211641fa4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.TestCoderBase;
+
+/**
+ * Raw coder test base with utilities.
+ */
+public abstract class TestRawCoderBase extends TestCoderBase {
+  protected Class<? extends RawErasureEncoder> encoderClass;
+  protected Class<? extends RawErasureDecoder> decoderClass;
+
+  /**
+   * Generating source data, encoding, recovering and then verifying.
+   * RawErasureCoder mainly uses ECChunk to pass input and output data buffers,
+   * it supports two kinds of ByteBuffers, one is array backed, the other is
+   * direct ByteBuffer. Have usingDirectBuffer to indicate which case to test.
+   * @param usingDirectBuffer
+   */
+  protected void testCoding(boolean usingDirectBuffer) {
+    // Generate data and encode
+    ECChunk[] dataChunks = prepareDataChunksForEncoding();
+    ECChunk[] parityChunks = prepareParityChunksForEncoding();
+    RawErasureEncoder encoder = createEncoder();
+
+    // Backup all the source chunks for later recovering because some coders
+    // may affect the source data.
+    ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
+    // Make a copy of a strip for later comparing
+    ECChunk[] toEraseDataChunks = copyDataChunksToErase(clonedDataChunks);
+
+    encoder.encode(dataChunks, parityChunks);
+    // Erase the copied sources
+    eraseSomeDataBlocks(clonedDataChunks);
+
+    //Decode
+    ECChunk[] inputChunks = prepareInputChunksForDecoding(clonedDataChunks,
+        parityChunks);
+    ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
+    RawErasureDecoder decoder = createDecoder();
+    decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+
+    //Compare
+    compareAndVerify(toEraseDataChunks, recoveredChunks);
+  }
+
+  /**
+   * Create the raw erasure encoder to test
+   * @return
+   */
+  protected RawErasureEncoder createEncoder() {
+    RawErasureEncoder encoder;
+    try {
+      encoder = encoderClass.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to create encoder", e);
+    }
+
+    encoder.initialize(numDataUnits, numParityUnits, chunkSize);
+    return encoder;
+  }
+
+  /**
+   * create the raw erasure decoder to test
+   * @return
+   */
+  protected RawErasureDecoder createDecoder() {
+    RawErasureDecoder decoder;
+    try {
+      decoder = decoderClass.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to create decoder", e);
+    }
+
+    decoder.initialize(numDataUnits, numParityUnits, chunkSize);
+    return decoder;
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXorRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXorRawCoder.java
new file mode 100644
index 0000000000000..8e59b8a2c3ca7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXorRawCoder.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Random;
+
+/**
+ * Test XOR encoding and decoding.
+ */
+public class TestXorRawCoder extends TestRawCoderBase {
+  private static Random RAND = new Random();
+
+  @Before
+  public void setup() {
+    this.encoderClass = XorRawEncoder.class;
+    this.decoderClass = XorRawDecoder.class;
+
+    this.numDataUnits = 10;
+    this.numParityUnits = 1;
+
+    this.erasedDataIndexes = new int[] {0};
+  }
+
+  @Test
+  public void testCodingNoDirectBuffer() {
+    testCoding(false);
+  }
+
+  @Test
+  public void testCodingDirectBuffer() {
+    testCoding(true);
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index 02516a183aa6f..04a74120c86ea 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -1184,6 +1184,57 @@ public void run() {
     }
   }
 
+  @Test
+  public void testMaxConnections() throws Exception {
+    conf.setInt("ipc.server.max.connections", 5);
+    Server server = null;
+    Thread connectors[] = new Thread[10];
+
+    try {
+      server = new TestServer(3, false);
+      final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+      server.start();
+      assertEquals(0, server.getNumOpenConnections());
+
+      for (int i = 0; i < 10; i++) {
+        connectors[i] = new Thread() {
+          @Override
+          public void run() {
+            Socket sock = null;
+            try {
+              sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
+              NetUtils.connect(sock, addr, 3000);
+              try {
+                Thread.sleep(4000);
+              } catch (InterruptedException ie) { }
+            } catch (IOException ioe) {
+            } finally {
+              if (sock != null) {
+                try {
+                  sock.close();
+                } catch (IOException ioe) { }
+              }
+            }
+          }
+        };
+        connectors[i].start();
+      }
+
+      Thread.sleep(1000);
+      // server should only accept up to 5 connections
+      assertEquals(5, server.getNumOpenConnections());
+
+      for (int i = 0; i < 10; i++) {
+        connectors[i].join();
+      }
+    } finally {
+      if (server != null) {
+        server.stop();
+      }
+      conf.setInt("ipc.server.max.connections", 0);
+    }
+  }
+
   private void assertRetriesOnSocketTimeouts(Configuration conf,
       int maxTimeoutRetries) throws IOException {
     SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
new file mode 100644
index 0000000000000..5807998a15785
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc;
+
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedByInterruptException;
+
+/**
+ * tests that the proxy can be interrupted
+ */
+public class TestRPCWaitForProxy extends Assert {
+  private static final String ADDRESS = "0.0.0.0";
+  private static final Logger
+      LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
+
+  private static final Configuration conf = new Configuration();
+
+  /**
+   * This tests that the time-bounded wait for a proxy operation works, and
+   * times out.
+   *
+   * @throws Throwable any exception other than that which was expected
+   */
+  @Test(timeout = 10000)
+  public void testWaitForProxy() throws Throwable {
+    RpcThread worker = new RpcThread(0);
+    worker.start();
+    worker.join();
+    Throwable caught = worker.getCaught();
+    assertNotNull("No exception was raised", caught);
+    if (!(caught instanceof ConnectException)) {
+      throw caught;
+    }
+  }
+
+  /**
+   * This test sets off a blocking thread and then interrupts it, before
+   * checking that the thread was interrupted
+   *
+   * @throws Throwable any exception other than that which was expected
+   */
+  @Test(timeout = 10000)
+  public void testInterruptedWaitForProxy() throws Throwable {
+    RpcThread worker = new RpcThread(100);
+    worker.start();
+    Thread.sleep(1000);
+    assertTrue("worker hasn't started", worker.waitStarted);
+    worker.interrupt();
+    worker.join();
+    Throwable caught = worker.getCaught();
+    assertNotNull("No exception was raised", caught);
+    // looking for the root cause here, which can be wrapped
+    // as part of the NetUtils work. Having this test look
+    // a the type of exception there would be brittle to improvements
+    // in exception diagnostics.
+    Throwable cause = caught.getCause();
+    if (cause == null) {
+      // no inner cause, use outer exception as root cause.
+      cause = caught;
+    }
+    if (!(cause instanceof InterruptedIOException)
+        && !(cause instanceof ClosedByInterruptException)) {
+      throw caught;
+    }
+  }
+
+  /**
+   * This thread waits for a proxy for the specified timeout, and retains any
+   * throwable that was raised in the process
+   */
+
+  private class RpcThread extends Thread {
+    private Throwable caught;
+    private int connectRetries;
+    private volatile boolean waitStarted = false;
+
+    private RpcThread(int connectRetries) {
+      this.connectRetries = connectRetries;
+    }
+    @Override
+    public void run() {
+      try {
+        Configuration config = new Configuration(conf);
+        config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
+            connectRetries);
+        config.setInt(
+            IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+            connectRetries);
+        waitStarted = true;
+        TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
+            TestProtocol.versionID,
+            new InetSocketAddress(ADDRESS, 20),
+            config,
+            15000L);
+        proxy.echo("");
+      } catch (Throwable throwable) {
+        caught = throwable;
+      }
+    }
+
+    public Throwable getCaught() {
+      return caught;
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
index a119f2c29ceae..cf7014ddc6ba2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
@@ -18,20 +18,21 @@
 package org.apache.hadoop.jmx;
 
 
-import java.net.URL;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.http.HttpServerFunctionalTest;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.jmx.JMXJsonServlet.ACCESS_CONTROL_ALLOW_METHODS;
+import static org.apache.hadoop.jmx.JMXJsonServlet.ACCESS_CONTROL_ALLOW_ORIGIN;
+
 public class TestJMXJsonServlet extends HttpServerFunctionalTest {
-  private   static final Log LOG = LogFactory.getLog(TestJMXJsonServlet.class);
   private static HttpServer2 server;
   private static URL baseUrl;
 
@@ -53,54 +54,31 @@ public static void assertReFind(String re, String value) {
   
   @Test public void testQuery() throws Exception {
     String result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Runtime"));
-    LOG.info("/jmx?qry=java.lang:type=Runtime RESULT: "+result);
     assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Runtime\"", result);
     assertReFind("\"modelerType\"", result);
     
     result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory"));
-    LOG.info("/jmx?qry=java.lang:type=Memory RESULT: "+result);
     assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
     assertReFind("\"modelerType\"", result);
     
     result = readOutput(new URL(baseUrl, "/jmx"));
-    LOG.info("/jmx RESULT: "+result);
     assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
     
     // test to get an attribute of a mbean
     result = readOutput(new URL(baseUrl, 
         "/jmx?get=java.lang:type=Memory::HeapMemoryUsage"));
-    LOG.info("/jmx RESULT: "+result);
     assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
     assertReFind("\"committed\"\\s*:", result);
     
     // negative test to get an attribute of a mbean
     result = readOutput(new URL(baseUrl, 
         "/jmx?get=java.lang:type=Memory::"));
-    LOG.info("/jmx RESULT: "+result);
-    assertReFind("\"ERROR\"", result);
-
-    // test to get JSONP result
-    result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory&callback=mycallback1"));
-    LOG.info("/jmx?qry=java.lang:type=Memory&callback=mycallback RESULT: "+result);
-    assertReFind("^mycallback1\\(\\{", result);
-    assertReFind("\\}\\);$", result);
-
-    // negative test to get an attribute of a mbean as JSONP
-    result = readOutput(new URL(baseUrl,
-        "/jmx?get=java.lang:type=Memory::&callback=mycallback2"));
-    LOG.info("/jmx RESULT: "+result);
-    assertReFind("^mycallback2\\(\\{", result);
     assertReFind("\"ERROR\"", result);
-    assertReFind("\\}\\);$", result);
-
-    // test to get an attribute of a mbean as JSONP
-    result = readOutput(new URL(baseUrl,
-        "/jmx?get=java.lang:type=Memory::HeapMemoryUsage&callback=mycallback3"));
-    LOG.info("/jmx RESULT: "+result);
-    assertReFind("^mycallback3\\(\\{", result);
-    assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
-    assertReFind("\"committed\"\\s*:", result);
-    assertReFind("\\}\\);$", result);
 
+    // test to CORS headers
+    HttpURLConnection conn = (HttpURLConnection)
+        new URL(baseUrl, "/jmx?qry=java.lang:type=Memory").openConnection();
+    assertEquals("GET", conn.getHeaderField(ACCESS_CONTROL_ALLOW_METHODS));
+    assertNotNull(conn.getHeaderField(ACCESS_CONTROL_ALLOW_ORIGIN));
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics/ganglia/TestGangliaContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics/ganglia/TestGangliaContext.java
index deb8231154cd8..8637f8ceea4b6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics/ganglia/TestGangliaContext.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics/ganglia/TestGangliaContext.java
@@ -22,13 +22,54 @@
 package org.apache.hadoop.metrics.ganglia;
 
 import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.hadoop.metrics.ContextFactory;
 import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
 
+import java.net.MulticastSocket;
+
 public class TestGangliaContext {
+  @Test
+  public void testShouldCreateDatagramSocketByDefault() throws Exception {
+    GangliaContext context = new GangliaContext();
+    context.init("gangliaContext", ContextFactory.getFactory());
+    assertFalse("Created MulticastSocket", context.datagramSocket instanceof MulticastSocket);
+  }
+
+  @Test
+  public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception {
+    GangliaContext context = new GangliaContext();
+    ContextFactory factory = ContextFactory.getFactory();
+    factory.setAttribute("gangliaContext.multicast", "false");
+    context.init("gangliaContext", factory);
+    assertFalse("Created MulticastSocket", context.datagramSocket instanceof MulticastSocket);
+  }
+
+  @Test
+  public void testShouldCreateMulticastSocket() throws Exception {
+    GangliaContext context = new GangliaContext();
+    ContextFactory factory = ContextFactory.getFactory();
+    factory.setAttribute("gangliaContext.multicast", "true");
+    context.init("gangliaContext", factory);
+    assertTrue("Did not create MulticastSocket", context.datagramSocket instanceof MulticastSocket);
+    MulticastSocket multicastSocket = (MulticastSocket) context.datagramSocket;
+    assertEquals("Did not set default TTL", multicastSocket.getTimeToLive(), 1);
+  }
+
+  @Test
+  public void testShouldSetMulticastSocketTtl() throws Exception {
+    GangliaContext context = new GangliaContext();
+    ContextFactory factory = ContextFactory.getFactory();
+    factory.setAttribute("gangliaContext.multicast", "true");
+    factory.setAttribute("gangliaContext.multicast.ttl", "10");
+    context.init("gangliaContext", factory);
+    MulticastSocket multicastSocket = (MulticastSocket) context.datagramSocket;
+    assertEquals("Did not set TTL", multicastSocket.getTimeToLive(), 10);
+  }
   
   @Test
   public void testCloseShouldCloseTheSocketWhichIsCreatedByInit() throws Exception {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/MetricsRecords.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/MetricsRecords.java
new file mode 100644
index 0000000000000..3c0999ed5dfe2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/MetricsRecords.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.impl;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Utility class mainly for tests
+ */
+public class MetricsRecords {
+
+  public static void assertTag(MetricsRecord record, String tagName,
+      String expectedValue) {
+    MetricsTag processIdTag = getFirstTagByName(record,
+        tagName);
+    assertNotNull(processIdTag);
+    assertEquals(expectedValue, processIdTag.value());
+  }
+
+  public static void assertMetric(MetricsRecord record,
+      String metricName,
+      Number expectedValue) {
+    AbstractMetric resourceLimitMetric = getFirstMetricByName(
+        record, metricName);
+    assertNotNull(resourceLimitMetric);
+    assertEquals(expectedValue, resourceLimitMetric.value());
+  }
+
+  private static MetricsTag getFirstTagByName(MetricsRecord record, String name) {
+    return Iterables.getFirst(Iterables.filter(record.tags(),
+        new MetricsTagPredicate(name)), null);
+  }
+
+  private static AbstractMetric getFirstMetricByName(
+      MetricsRecord record, String name) {
+    return Iterables.getFirst(
+        Iterables.filter(record.metrics(), new AbstractMetricPredicate(name)),
+        null);
+  }
+
+  private static class MetricsTagPredicate implements Predicate<MetricsTag> {
+    private String tagName;
+
+    public MetricsTagPredicate(String tagName) {
+
+      this.tagName = tagName;
+    }
+
+    @Override
+    public boolean apply(MetricsTag input) {
+      return input.name().equals(tagName);
+    }
+  }
+
+  private static class AbstractMetricPredicate
+      implements Predicate<AbstractMetric> {
+    private String metricName;
+
+    public AbstractMetricPredicate(
+        String metricName) {
+      this.metricName = metricName;
+    }
+
+    @Override
+    public boolean apply(AbstractMetric input) {
+      return input.name().equals(metricName);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
index 09f0081276f7c..5fac41e02781a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
@@ -18,23 +18,7 @@
 
 package org.apache.hadoop.metrics2.impl;
 
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsRecord;
 import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.sink.GraphiteSink;
@@ -42,6 +26,23 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.internal.util.reflection.Whitebox;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.reset;
+
+
 public class TestGraphiteMetrics {
     private AbstractMetric makeMetric(String name, Number value) {
         AbstractMetric metric = mock(AbstractMetric.class);
@@ -50,6 +51,12 @@ private AbstractMetric makeMetric(String name, Number value) {
         return metric;
     }
 
+    private GraphiteSink.Graphite makeGraphite() {
+      GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class);
+      when(mockGraphite.isConnected()).thenReturn(true);
+      return mockGraphite;
+    }
+
     @Test
     public void testPutMetrics() {
         GraphiteSink sink = new GraphiteSink();
@@ -61,18 +68,18 @@ public void testPutMetrics() {
         metrics.add(makeMetric("foo2", 2.25));
         MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
 
-        OutputStreamWriter mockWriter = mock(OutputStreamWriter.class);
         ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
-        Whitebox.setInternalState(sink, "writer", mockWriter);
+        final GraphiteSink.Graphite mockGraphite = makeGraphite();
+        Whitebox.setInternalState(sink, "graphite", mockGraphite);
         sink.putMetrics(record);
 
         try {
-            verify(mockWriter).write(argument.capture());
+          verify(mockGraphite).write(argument.capture());
         } catch (IOException e) {
-            e.printStackTrace();
+          e.printStackTrace();
         }
 
-        String result = argument.getValue().toString();
+        String result = argument.getValue();
 
         assertEquals(true,
             result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
@@ -86,24 +93,25 @@ public void testPutMetrics2() {
         GraphiteSink sink = new GraphiteSink();
         List<MetricsTag> tags = new ArrayList<MetricsTag>();
         tags.add(new MetricsTag(MsInfo.Context, "all"));
-        tags.add(new MetricsTag(MsInfo.Hostname, null));
+      tags.add(new MetricsTag(MsInfo.Hostname, null));
         Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
         metrics.add(makeMetric("foo1", 1));
         metrics.add(makeMetric("foo2", 2));
         MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
 
-        OutputStreamWriter mockWriter = mock(OutputStreamWriter.class);
+
         ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
-        Whitebox.setInternalState(sink, "writer", mockWriter);
+        final GraphiteSink.Graphite mockGraphite = makeGraphite();
+        Whitebox.setInternalState(sink, "graphite", mockGraphite);
         sink.putMetrics(record);
 
         try {
-            verify(mockWriter).write(argument.capture());
+            verify(mockGraphite).write(argument.capture());
         } catch (IOException e) {
             e.printStackTrace();
         }
 
-        String result = argument.getValue().toString();
+        String result = argument.getValue();
 
         assertEquals(true,
             result.equals("null.all.Context.Context=all.foo1 1 10\n" + 
@@ -120,8 +128,8 @@ public void testPutMetrics3() {
 
       // setup GraphiteSink
       GraphiteSink sink = new GraphiteSink();
-      ByteArrayOutputStream out = new ByteArrayOutputStream();
-      Whitebox.setInternalState(sink, "writer", new OutputStreamWriter(out));
+      final GraphiteSink.Graphite mockGraphite = makeGraphite();
+      Whitebox.setInternalState(sink, "graphite", mockGraphite);
 
       // given two metrics records with timestamps 1000 milliseconds apart.
       List<MetricsTag> tags = Collections.emptyList();
@@ -141,15 +149,16 @@ public void testPutMetrics3() {
       }
 
       // then the timestamps in the graphite stream should differ by one second.
-      String expectedOutput
-        = "null.default.Context.foo1 1 1000000000\n"
-        + "null.default.Context.foo1 1 1000000001\n";
-      assertEquals(expectedOutput, out.toString());
+      try {
+        verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n"));
+        verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n"));
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
     }
 
-
-    @Test(expected=MetricsException.class)
-    public void testCloseAndWrite() throws IOException {
+    @Test
+    public void testFailureAndPutMetrics() throws IOException {
       GraphiteSink sink = new GraphiteSink();
       List<MetricsTag> tags = new ArrayList<MetricsTag>();
       tags.add(new MetricsTag(MsInfo.Context, "all"));
@@ -159,18 +168,38 @@ public void testCloseAndWrite() throws IOException {
       metrics.add(makeMetric("foo2", 2.25));
       MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
 
-      OutputStreamWriter writer = mock(OutputStreamWriter.class);
+      final GraphiteSink.Graphite mockGraphite = makeGraphite();
+      Whitebox.setInternalState(sink, "graphite", mockGraphite);
+
+      // throw exception when first try
+      doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString());
 
-      Whitebox.setInternalState(sink, "writer", writer);
-      sink.close();
       sink.putMetrics(record);
+      verify(mockGraphite).write(anyString());
+      verify(mockGraphite).close();
+
+      // reset mock and try again
+      reset(mockGraphite);
+      when(mockGraphite.isConnected()).thenReturn(false);
+
+      ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
+      sink.putMetrics(record);
+
+      verify(mockGraphite).write(argument.capture());
+      String result = argument.getValue();
+
+      assertEquals(true,
+          result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
+          "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") ||
+          result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" +
+          "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"));
     }
 
     @Test
     public void testClose(){
         GraphiteSink sink = new GraphiteSink();
-        Writer mockWriter = mock(Writer.class);
-        Whitebox.setInternalState(sink, "writer", mockWriter);
+        final GraphiteSink.Graphite mockGraphite = makeGraphite();
+        Whitebox.setInternalState(sink, "graphite", mockGraphite);
         try {
             sink.close();
         } catch (IOException ioe) {
@@ -178,7 +207,7 @@ public void testClose(){
         }
 
         try {
-            verify(mockWriter).close();
+            verify(mockGraphite).close();
         } catch (IOException ioe) {
             ioe.printStackTrace();
         }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index d59e80b58202d..4c2ebc8e0e8f6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -190,7 +190,7 @@ public static class TestSink implements MetricsSink {
       threads[i] = new Thread(new Runnable() {
         private boolean safeAwait(int mySource, CyclicBarrier barrier) {
           try {
-            barrier1.await(2, TimeUnit.SECONDS);
+            barrier.await(2, TimeUnit.SECONDS);
           } catch (InterruptedException e) {
             results[mySource] = "Interrupted";
             return false;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
index 47b496fa57d38..af1ff96b9a575 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMetricsRegistry.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.metrics2.lib;
 
+import org.junit.Ignore;
 import org.junit.Test;
 import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
@@ -56,6 +57,46 @@ public class TestMetricsRegistry {
     });
   }
 
+  /**
+   * Test adding metrics with whitespace in the name
+   */
+  @Test
+  public void testMetricsRegistryIllegalMetricNames() {
+    final MetricsRegistry r = new MetricsRegistry("test");
+    // Fill up with some basics
+    r.newCounter("c1", "c1 desc", 1);
+    r.newGauge("g1", "g1 desc", 1);
+    r.newQuantiles("q1", "q1 desc", "q1 name", "q1 val type", 1);
+    // Add some illegal names
+    expectMetricsException("Metric name 'badcount 2' contains "+
+        "illegal whitespace character", new Runnable() {
+      @Override
+      public void run() { r.newCounter("badcount 2", "c2 desc", 2); }
+    });
+    expectMetricsException("Metric name 'badcount3  ' contains "+
+        "illegal whitespace character", new Runnable() {
+      @Override
+      public void run() { r.newCounter("badcount3  ", "c3 desc", 3); }
+    });
+    expectMetricsException("Metric name '  badcount4' contains "+
+        "illegal whitespace character", new Runnable() {
+      @Override
+      public void run() { r.newCounter("  badcount4", "c4 desc", 4); }
+    });
+    expectMetricsException("Metric name 'withtab5	' contains "+
+        "illegal whitespace character", new Runnable() {
+      @Override
+      public void run() { r.newCounter("withtab5	", "c5 desc", 5); }
+    });
+    expectMetricsException("Metric name 'withnewline6\n' contains "+
+        "illegal whitespace character", new Runnable() {
+      @Override
+      public void run() { r.newCounter("withnewline6\n", "c6 desc", 6); }
+    });
+    // Final validation
+    assertEquals("num metrics in registry", 3, r.metrics().size());
+  }
+
   /**
    * Test the add by name method
    */
@@ -81,6 +122,7 @@ public class TestMetricsRegistry {
     });
   }
 
+  @Ignore
   private void expectMetricsException(String prefix, Runnable fun) {
     try {
       fun.run();
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java
new file mode 100644
index 0000000000000..aa2c2591794ec
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.ganglia;
+
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.hadoop.metrics2.impl.ConfigBuilder;
+import org.junit.Test;
+
+import java.net.DatagramSocket;
+import java.net.MulticastSocket;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestGangliaSink {
+    @Test
+    public void testShouldCreateDatagramSocketByDefault() throws Exception {
+        SubsetConfiguration conf = new ConfigBuilder()
+                .subset("test.sink.ganglia");
+
+        GangliaSink30 gangliaSink = new GangliaSink30();
+        gangliaSink.init(conf);
+        DatagramSocket socket = gangliaSink.getDatagramSocket();
+        assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket);
+    }
+
+    @Test
+    public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception {
+        SubsetConfiguration conf = new ConfigBuilder()
+                .add("test.sink.ganglia.multicast", false)
+                .subset("test.sink.ganglia");
+        GangliaSink30 gangliaSink = new GangliaSink30();
+        gangliaSink.init(conf);
+        DatagramSocket socket = gangliaSink.getDatagramSocket();
+        assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket);
+    }
+
+    @Test
+    public void testShouldCreateMulticastSocket() throws Exception {
+        SubsetConfiguration conf = new ConfigBuilder()
+                .add("test.sink.ganglia.multicast", true)
+                .subset("test.sink.ganglia");
+        GangliaSink30 gangliaSink = new GangliaSink30();
+        gangliaSink.init(conf);
+        DatagramSocket socket = gangliaSink.getDatagramSocket();
+        assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket);
+        int ttl = ((MulticastSocket) socket).getTimeToLive();
+        assertEquals("Did not set default TTL", 1, ttl);
+    }
+
+    @Test
+    public void testShouldSetMulticastSocketTtl() throws Exception {
+        SubsetConfiguration conf = new ConfigBuilder()
+                .add("test.sink.ganglia.multicast", true)
+                .add("test.sink.ganglia.multicast.ttl", 3)
+                .subset("test.sink.ganglia");
+        GangliaSink30 gangliaSink = new GangliaSink30();
+        gangliaSink.init(conf);
+        DatagramSocket socket = gangliaSink.getDatagramSocket();
+        assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket);
+        int ttl = ((MulticastSocket) socket).getTimeToLive();
+        assertEquals("Did not set TTL", 3, ttl);
+    }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java
index f29fcf9ab5e49..649ba1264b74a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSocketIOWithTimeout.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.io.nativeio.NativeIO;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -53,12 +54,14 @@ public class TestSocketIOWithTimeout {
 
   private MultithreadedTestUtil.TestContext ctx = new TestContext();
   
+  private static final int PAGE_SIZE = (int) NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize();
+
   private void doIO(InputStream in, OutputStream out,
       int expectedTimeout) throws IOException {
     /* Keep on writing or reading until we get SocketTimeoutException.
      * It expects this exception to occur within 100 millis of TIMEOUT.
      */
-    byte buf[] = new byte[4192];
+    byte buf[] = new byte[PAGE_SIZE + 19];
     
     while (true) {
       long start = Time.now();
@@ -151,7 +154,7 @@ public void doWork() throws Exception {
       // simulate a partial write scenario.  Attempts were made to switch the
       // test from using a pipe to a network socket and also to use larger and
       // larger buffers in doIO.  Nothing helped the situation though.
-      if (!Shell.WINDOWS && !Shell.PPC_64) {
+      if (!Shell.WINDOWS) {
         try {
           out.write(1);
           fail("Did not throw");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
index 5281694179ce3..a93f9ea5e4e51 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
@@ -34,12 +34,15 @@
 import org.junit.Test;
 
 public class TestTableMapping {
+  private String hostName1 = "1.2.3.4";
+  private String hostName2 = "5.6.7.8";
+
   @Test
   public void testResolve() throws IOException {
     File mapFile = File.createTempFile(getClass().getSimpleName() +
         ".testResolve", ".txt");
-    Files.write("a.b.c /rack1\n" +
-                "1.2.3.4\t/rack2\n", mapFile, Charsets.UTF_8);
+    Files.write(hostName1 + " /rack1\n" +
+                hostName2 + "\t/rack2\n", mapFile, Charsets.UTF_8);
     mapFile.deleteOnExit();
     TableMapping mapping = new TableMapping();
 
@@ -48,8 +51,8 @@ public void testResolve() throws IOException {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
@@ -61,8 +64,8 @@ public void testResolve() throws IOException {
   public void testTableCaching() throws IOException {
     File mapFile = File.createTempFile(getClass().getSimpleName() +
         ".testTableCaching", ".txt");
-    Files.write("a.b.c /rack1\n" +
-                "1.2.3.4\t/rack2\n", mapFile, Charsets.UTF_8);
+    Files.write(hostName1 + " /rack1\n" +
+        hostName2 + "\t/rack2\n", mapFile, Charsets.UTF_8);
     mapFile.deleteOnExit();
     TableMapping mapping = new TableMapping();
 
@@ -71,8 +74,8 @@ public void testTableCaching() throws IOException {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result1 = mapping.resolve(names);
     assertEquals(names.size(), result1.size());
@@ -94,8 +97,8 @@ public void testNoFile() {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
@@ -112,8 +115,8 @@ public void testFileDoesNotExist() {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
@@ -125,8 +128,8 @@ public void testFileDoesNotExist() {
   public void testClearingCachedMappings() throws IOException {
     File mapFile = File.createTempFile(getClass().getSimpleName() +
         ".testClearingCachedMappings", ".txt");
-    Files.write("a.b.c /rack1\n" +
-                "1.2.3.4\t/rack2\n", mapFile, Charsets.UTF_8);
+    Files.write(hostName1 + " /rack1\n" +
+                hostName2 + "\t/rack2\n", mapFile, Charsets.UTF_8);
     mapFile.deleteOnExit();
 
     TableMapping mapping = new TableMapping();
@@ -136,8 +139,8 @@ public void testClearingCachedMappings() throws IOException {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
@@ -149,8 +152,8 @@ public void testClearingCachedMappings() throws IOException {
     mapping.reloadCachedMappings();
 
     names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
@@ -172,8 +175,8 @@ public void testBadFile() throws IOException {
     mapping.setConf(conf);
 
     List<String> names = new ArrayList<String>();
-    names.add("a.b.c");
-    names.add("1.2.3.4");
+    names.add(hostName1);
+    names.add(hostName2);
 
     List<String> result = mapping.resolve(names);
     assertEquals(names.size(), result.size());
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index a814b0d9e6883..36866946eaa72 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -51,6 +51,9 @@ public class TestGroupsCaching {
 
   @Before
   public void setup() {
+    FakeGroupMapping.resetRequestCount();
+    ExceptionalGroupMapping.resetRequestCount();
+
     conf = new Configuration();
     conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
       FakeGroupMapping.class,
@@ -61,16 +64,32 @@ public static class FakeGroupMapping extends ShellBasedUnixGroupsMapping {
     // any to n mapping
     private static Set<String> allGroups = new HashSet<String>();
     private static Set<String> blackList = new HashSet<String>();
+    private static int requestCount = 0;
+    private static long getGroupsDelayMs = 0;
 
     @Override
     public List<String> getGroups(String user) throws IOException {
       LOG.info("Getting groups for " + user);
+      requestCount++;
+
+      delayIfNecessary();
+
       if (blackList.contains(user)) {
         return new LinkedList<String>();
       }
       return new LinkedList<String>(allGroups);
     }
 
+    private void delayIfNecessary() {
+      if (getGroupsDelayMs > 0) {
+        try {
+          Thread.sleep(getGroupsDelayMs);
+        } catch (InterruptedException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+
     @Override
     public void cacheGroupsRefresh() throws IOException {
       LOG.info("Cache is being refreshed.");
@@ -93,6 +112,36 @@ public static void addToBlackList(String user) throws IOException {
       LOG.info("Adding " + user + " to the blacklist");
       blackList.add(user);
     }
+
+    public static int getRequestCount() {
+      return requestCount;
+    }
+
+    public static void resetRequestCount() {
+      requestCount = 0;
+    }
+
+    public static void setGetGroupsDelayMs(long delayMs) {
+      getGroupsDelayMs = delayMs;
+    }
+  }
+
+  public static class ExceptionalGroupMapping extends ShellBasedUnixGroupsMapping {
+    private static int requestCount = 0;
+
+    @Override
+    public List<String> getGroups(String user) throws IOException {
+      requestCount++;
+      throw new IOException("For test");
+    }
+
+    public static int getRequestCount() {
+      return requestCount;
+    }
+
+    public static void resetRequestCount() {
+      requestCount = 0;
+    }
   }
 
   @Test
@@ -219,4 +268,239 @@ public void testNegativeGroupCaching() throws Exception {
     // groups for the user is fetched.
     assertEquals(Arrays.asList(myGroups), groups.getGroups(user));
   }
+
+  @Test
+  public void testCachePreventsImplRequest() throws Exception {
+    // Disable negative cache.
+    conf.setLong(
+      CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 0);
+    Groups groups = new Groups(conf);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    FakeGroupMapping.clearBlackList();
+
+    assertEquals(0, FakeGroupMapping.getRequestCount());
+
+    // First call hits the wire
+    assertTrue(groups.getGroups("me").size() == 2);
+    assertEquals(1, FakeGroupMapping.getRequestCount());
+
+    // Second count hits cache
+    assertTrue(groups.getGroups("me").size() == 2);
+    assertEquals(1, FakeGroupMapping.getRequestCount());
+  }
+
+  @Test
+  public void testExceptionsFromImplNotCachedInNegativeCache() {
+    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+      ExceptionalGroupMapping.class,
+      ShellBasedUnixGroupsMapping.class);
+    conf.setLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 10000);
+    Groups groups = new Groups(conf);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+
+    assertEquals(0, ExceptionalGroupMapping.getRequestCount());
+
+    // First call should hit the wire
+    try {
+      groups.getGroups("anything");
+      fail("Should have thrown");
+    } catch (IOException e) {
+      // okay
+    }
+    assertEquals(1, ExceptionalGroupMapping.getRequestCount());
+
+    // Second call should hit the wire (no negative caching)
+    try {
+      groups.getGroups("anything");
+      fail("Should have thrown");
+    } catch (IOException e) {
+      // okay
+    }
+    assertEquals(2, ExceptionalGroupMapping.getRequestCount());
+  }
+
+  @Test
+  public void testOnlyOneRequestWhenNoEntryIsCached() throws Exception {
+    // Disable negative cache.
+    conf.setLong(
+      CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 0);
+    final Groups groups = new Groups(conf);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    FakeGroupMapping.clearBlackList();
+    FakeGroupMapping.setGetGroupsDelayMs(100);
+
+    ArrayList<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 10; i++) {
+      threads.add(new Thread() {
+        public void run() {
+          try {
+            assertEquals(2, groups.getGroups("me").size());
+          } catch (IOException e) {
+            fail("Should not happen");
+          }
+        }
+      });
+    }
+
+    // We start a bunch of threads who all see no cached value
+    for (Thread t : threads) {
+      t.start();
+    }
+
+    for (Thread t : threads) {
+      t.join();
+    }
+
+    // But only one thread should have made the request
+    assertEquals(1, FakeGroupMapping.getRequestCount());
+  }
+
+  @Test
+  public void testOnlyOneRequestWhenExpiredEntryExists() throws Exception {
+    conf.setLong(
+      CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1);
+    FakeTimer timer = new FakeTimer();
+    final Groups groups = new Groups(conf, timer);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    FakeGroupMapping.clearBlackList();
+    FakeGroupMapping.setGetGroupsDelayMs(100);
+
+    // We make an initial request to populate the cache
+    groups.getGroups("me");
+    int startingRequestCount = FakeGroupMapping.getRequestCount();
+
+    // Then expire that entry
+    timer.advance(400 * 1000);
+    Thread.sleep(100);
+
+    ArrayList<Thread> threads = new ArrayList<Thread>();
+    for (int i = 0; i < 10; i++) {
+      threads.add(new Thread() {
+        public void run() {
+          try {
+            assertEquals(2, groups.getGroups("me").size());
+          } catch (IOException e) {
+            fail("Should not happen");
+          }
+        }
+      });
+    }
+
+    // We start a bunch of threads who all see the cached value
+    for (Thread t : threads) {
+      t.start();
+    }
+
+    for (Thread t : threads) {
+      t.join();
+    }
+
+    // Only one extra request is made
+    assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount());
+  }
+
+  @Test
+  public void testCacheEntriesExpire() throws Exception {
+    conf.setLong(
+      CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1);
+    FakeTimer timer = new FakeTimer();
+    final Groups groups = new Groups(conf, timer);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    FakeGroupMapping.clearBlackList();
+
+    // We make an entry
+    groups.getGroups("me");
+    int startingRequestCount = FakeGroupMapping.getRequestCount();
+
+    timer.advance(20 * 1000);
+
+    // Cache entry has expired so it results in a new fetch
+    groups.getGroups("me");
+    assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount());
+  }
+
+  @Test
+  public void testNegativeCacheClearedOnRefresh() throws Exception {
+    conf.setLong(
+      CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 100);
+    final Groups groups = new Groups(conf);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    FakeGroupMapping.clearBlackList();
+    FakeGroupMapping.addToBlackList("dne");
+
+    try {
+      groups.getGroups("dne");
+      fail("Should have failed to find this group");
+    } catch (IOException e) {
+      // pass
+    }
+
+    int startingRequestCount = FakeGroupMapping.getRequestCount();
+
+    groups.refresh();
+    FakeGroupMapping.addToBlackList("dne");
+
+    try {
+      List<String> g = groups.getGroups("dne");
+      fail("Should have failed to find this group");
+    } catch (IOException e) {
+      // pass
+    }
+
+    assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount());
+  }  
+
+  @Test
+  public void testNegativeCacheEntriesExpire() throws Exception {
+    conf.setLong(
+       CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 2);
+    FakeTimer timer = new FakeTimer();
+    // Ensure that stale entries are removed from negative cache every 2 seconds
+    Groups groups = new Groups(conf, timer);
+    groups.cacheGroupsAdd(Arrays.asList(myGroups));
+    groups.refresh();
+    // Add both these users to blacklist so that they
+    // can be added to negative cache
+    FakeGroupMapping.addToBlackList("user1");
+    FakeGroupMapping.addToBlackList("user2");
+
+    // Put user1 in negative cache.
+    try {
+      groups.getGroups("user1");
+      fail("Did not throw IOException : Failed to obtain groups" +
+            " from FakeGroupMapping.");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains("No groups found for user", e);
+    }
+    // Check if user1 exists in negative cache
+    assertTrue(groups.getNegativeCache().contains("user1"));
+
+    // Advance fake timer
+    timer.advance(1000);
+    // Put user2 in negative cache
+    try {
+      groups.getGroups("user2");
+      fail("Did not throw IOException : Failed to obtain groups" +
+            " from FakeGroupMapping.");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains("No groups found for user", e);
+    }
+    // Check if user2 exists in negative cache
+    assertTrue(groups.getNegativeCache().contains("user2"));
+
+    // Advance timer. Only user2 should be present in negative cache.
+    timer.advance(1100);
+    assertFalse(groups.getNegativeCache().contains("user1"));
+    assertTrue(groups.getNegativeCache().contains("user2"));
+
+    // Advance timer. Even user2 should not be present in negative cache.
+    timer.advance(1000);
+    assertFalse(groups.getNegativeCache().contains("user2"));
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
index ec8ac1d49713e..e6e1d73fb98d9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
@@ -41,6 +41,13 @@ public class TestShellBasedIdMapping {
   private static final Map<Integer, Integer> EMPTY_PASS_THROUGH_MAP =
       new PassThroughMap<Integer>();
   
+  private void createStaticMapFile(final File smapFile, final String smapStr)
+      throws IOException {
+    OutputStream out = new FileOutputStream(smapFile);
+    out.write(smapStr.getBytes());
+    out.close();
+  }
+
   @Test
   public void testStaticMapParsing() throws IOException {
     File tempStaticMapFile = File.createTempFile("nfs-", ".map");
@@ -55,12 +62,13 @@ public void testStaticMapParsing() throws IOException {
         "uid 13 302\n" +
         "gid\t11\t201\n" + // Tabs instead of spaces.
         "\n" + // Entirely empty line.
-        "gid 12 202";
-    OutputStream out = new FileOutputStream(tempStaticMapFile);
-    out.write(staticMapFileContents.getBytes());
-    out.close();
-    StaticMapping parsedMap = ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);
-    
+        "gid 12 202\n" +
+        "uid 4294967294 123\n" +
+        "gid 4294967295 321";
+    createStaticMapFile(tempStaticMapFile, staticMapFileContents);
+
+    StaticMapping parsedMap =
+        ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);    
     assertEquals(10, (int)parsedMap.uidMapping.get(100));
     assertEquals(11, (int)parsedMap.uidMapping.get(201));
     assertEquals(12, (int)parsedMap.uidMapping.get(301));
@@ -71,6 +79,10 @@ public void testStaticMapParsing() throws IOException {
     assertEquals(10000, (int)parsedMap.uidMapping.get(10001));
     // Ensure pass-through of unmapped IDs works.
     assertEquals(1000, (int)parsedMap.uidMapping.get(1000));
+    
+    assertEquals(-2, (int)parsedMap.uidMapping.get(123));
+    assertEquals(-1, (int)parsedMap.gidMapping.get(321));
+    
   }
   
   @Test
@@ -114,6 +126,78 @@ public void testStaticMapping() throws IOException {
     assertEquals(498, (int)gMap.inverse().get("mapred2"));
   }
 
+  // Test staticMap refreshing
+  @Test
+  public void testStaticMapUpdate() throws IOException {
+    File tempStaticMapFile = File.createTempFile("nfs-", ".map");
+    tempStaticMapFile.delete();
+    Configuration conf = new Configuration();
+    conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 1000);    
+    conf.set(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
+        tempStaticMapFile.getPath());
+
+    ShellBasedIdMapping refIdMapping =
+        new ShellBasedIdMapping(conf, true);
+    ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf);
+
+    BiMap<Integer, String> uidNameMap = refIdMapping.getUidNameMap();
+    BiMap<Integer, String> gidNameMap = refIdMapping.getGidNameMap();
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getUid()
+    incrIdMapping.clearNameMaps();
+    uidNameMap = refIdMapping.getUidNameMap();
+    {
+      BiMap.Entry<Integer, String> me = uidNameMap.entrySet().iterator().next();
+      Integer id = me.getKey();
+      String name = me.getValue();
+
+      // The static map is empty, so the id found for "name" would be
+      // the same as "id"
+      Integer nid = incrIdMapping.getUid(name);
+      assertEquals(id, nid);
+      
+      // Clear map and update staticMap file
+      incrIdMapping.clearNameMaps();
+      Integer rid = id + 10000;
+      String smapStr = "uid " + rid + " " + id;
+      createStaticMapFile(tempStaticMapFile, smapStr);
+
+      // Now the id found for "name" should be the id specified by
+      // the staticMap
+      nid = incrIdMapping.getUid(name);
+      assertEquals(rid, nid);
+    }
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getGid()
+    incrIdMapping.clearNameMaps();
+    gidNameMap = refIdMapping.getGidNameMap();
+    {
+      BiMap.Entry<Integer, String> me = gidNameMap.entrySet().iterator().next();
+      Integer id = me.getKey();
+      String name = me.getValue();
+
+      // The static map is empty, so the id found for "name" would be
+      // the same as "id"
+      Integer nid = incrIdMapping.getGid(name);
+      assertEquals(id, nid);
+
+      // Clear map and update staticMap file
+      incrIdMapping.clearNameMaps();
+      Integer rid = id + 10000;
+      String smapStr = "gid " + rid + " " + id;
+      // Sleep a bit to avoid that two changes have the same modification time
+      try {Thread.sleep(1000);} catch (InterruptedException e) {}
+      createStaticMapFile(tempStaticMapFile, smapStr);
+
+      // Now the id found for "name" should be the id specified by
+      // the staticMap
+      nid = incrIdMapping.getGid(name);
+      assertEquals(rid, nid);
+    }
+  }
+
   @Test
   public void testDuplicates() throws IOException {
     assumeTrue(!Shell.WINDOWS);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index c89036272b23b..7ba4bc17e29f2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -47,6 +47,7 @@ public void setup() throws Exception {
     System.setOut(new PrintStream(outContent));
     System.setErr(new PrintStream(errContent));
     final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
+    new File(jksPath.toString()).delete();
     jceksProvider = "jceks://file" + jksPath.toUri();
   }
   
@@ -71,7 +72,7 @@ public void testCredentialSuccessfulLifecycle() throws Exception {
     assertTrue(outContent.toString().contains("credential1"));
 
     outContent.reset();
-    String[] args4 = {"delete", "credential1", "-provider",
+    String[] args4 = {"delete", "credential1", "-f", "-provider",
         jceksProvider};
     rc = cs.run(args4);
     assertEquals(0, rc);
@@ -113,7 +114,7 @@ public void testTransientProviderWarning() throws Exception {
     assertTrue(outContent.toString().contains("WARNING: you are modifying a " +
     		"transient provider."));
 
-    String[] args2 = {"delete", "credential1", "-provider", "user:///"};
+    String[] args2 = {"delete", "credential1", "-f", "-provider", "user:///"};
     rc = cs.run(args2);
     assertEquals(outContent.toString(), 0, rc);
     assertTrue(outContent.toString().contains("credential1 has been successfully " +
@@ -167,7 +168,7 @@ public void testPromptForCredential() throws Exception {
     assertTrue(outContent.toString().contains("credential1 has been successfully " +
         "created."));
     
-    String[] args2 = {"delete", "credential1", "-provider",
+    String[] args2 = {"delete", "credential1", "-f", "-provider",
         jceksProvider};
     rc = shell.run(args2);
     assertEquals(0, rc);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
index 9ef9d7add5364..c473c502da824 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java
@@ -34,6 +34,11 @@ public class TestServiceAuthorization {
   private static final String ACL_CONFIG = "test.protocol.acl";
   private static final String ACL_CONFIG1 = "test.protocol1.acl";
   private static final String ADDRESS =  "0.0.0.0";
+  private static final String HOST_CONFIG = "test.protocol.hosts";
+  private static final String BLOCKED_HOST_CONFIG = "test.protocol.hosts.blocked";
+  private static final String AUTHORIZED_IP = "1.2.3.4";
+  private static final String UNAUTHORIZED_IP = "1.2.3.5";
+  private static final String IP_RANGE = "10.222.0.0/16,10.113.221.221";
 
   public interface TestProtocol1 extends TestProtocol {};
 
@@ -52,7 +57,7 @@ public void testDefaultAcl() {
     ServiceAuthorizationManager serviceAuthorizationManager = 
         new ServiceAuthorizationManager();
     Configuration conf = new Configuration ();
-    //test without setting a default acl
+    // test without setting a default acl
     conf.set(ACL_CONFIG, "user1 group1");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     AccessControlList acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol.class);
@@ -60,7 +65,7 @@ public void testDefaultAcl() {
     acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol1.class);
     assertEquals(AccessControlList.WILDCARD_ACL_VALUE, acl.getAclString());
 
-    //test with a default acl
+    // test with a default acl
     conf.set(
         CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL, 
         "user2 group2");
@@ -81,7 +86,7 @@ public void testBlockedAcl() throws UnknownHostException {
         new ServiceAuthorizationManager();
     Configuration conf = new Configuration ();
 
-    //test without setting a blocked acl
+    // test without setting a blocked acl
     conf.set(ACL_CONFIG, "user1 group1");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -90,7 +95,7 @@ public void testBlockedAcl() throws UnknownHostException {
     } catch (AuthorizationException e) {
       fail();
     }
-    //now set a blocked acl with another user and another group
+    // now set a blocked acl with another user and another group
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group3");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -99,7 +104,7 @@ public void testBlockedAcl() throws UnknownHostException {
     } catch (AuthorizationException e) {
       fail();
     }
-    //now set a blocked acl with the user and another group
+    // now set a blocked acl with the user and another group
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho group3");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -109,7 +114,7 @@ public void testBlockedAcl() throws UnknownHostException {
     } catch (AuthorizationException e) {
 
     }
-    //now set a blocked acl with another user and another group
+    // now set a blocked acl with another user and another group
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group3");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -118,7 +123,7 @@ public void testBlockedAcl() throws UnknownHostException {
     } catch (AuthorizationException e) {
       fail();
     }
-    //now set a blocked acl with another user and group that the user belongs to
+    // now set a blocked acl with another user and group that the user belongs to
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "drwho2 group2");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -126,9 +131,9 @@ public void testBlockedAcl() throws UnknownHostException {
           InetAddress.getByName(ADDRESS));
       fail();
     } catch (AuthorizationException e) {
-      //expects Exception
+      // expects Exception
     }
-    //reset blocked acl so that there is no blocked ACL
+    // reset blocked acl so that there is no blocked ACL
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
@@ -149,7 +154,7 @@ public void testDefaultBlockedAcl() throws UnknownHostException {
         new ServiceAuthorizationManager();
     Configuration conf = new Configuration ();
 
-    //test without setting a default blocked acl
+    // test without setting a default blocked acl
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
     try {
       serviceAuthorizationManager.authorize(drwho, TestProtocol1.class, conf,
@@ -158,27 +163,183 @@ public void testDefaultBlockedAcl() throws UnknownHostException {
       fail();
     }
 
-    //set a restrictive default blocked acl and an non-restricting blocked acl for TestProtocol
+    // set a restrictive default blocked acl and an non-restricting blocked acl for TestProtocol
     conf.set(
         CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL,
         "user2 group2");
     conf.set(ACL_CONFIG + ServiceAuthorizationManager.BLOCKED, "user2");
     serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
-    //drwho is authorized to access TestProtocol
+    // drwho is authorized to access TestProtocol
     try {
       serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
           InetAddress.getByName(ADDRESS));
     } catch (AuthorizationException e) {
       fail();
     }
-    //drwho is not authorized to access TestProtocol1 because it uses the default blocked acl.
+    // drwho is not authorized to access TestProtocol1 because it uses the default blocked acl.
     try {
       serviceAuthorizationManager.authorize(drwho, TestProtocol1.class, conf,
           InetAddress.getByName(ADDRESS));
       fail();
     } catch (AuthorizationException e) {
-      //expects Exception
+      // expects Exception
+    }
+  }
+
+  @Test
+  public void testMachineList() throws UnknownHostException {
+    UserGroupInformation drwho =
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+            new String[] { "group1", "group2" });
+    ServiceAuthorizationManager serviceAuthorizationManager =
+        new ServiceAuthorizationManager();
+    Configuration conf = new Configuration ();
+    conf.set(HOST_CONFIG, "1.2.3.4");
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
+          InetAddress.getByName(AUTHORIZED_IP));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    try {
+      serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
+          InetAddress.getByName(UNAUTHORIZED_IP));
+      fail();
+    } catch (AuthorizationException e) {
+    // expects Exception
+    }
+  }
+
+  @Test
+  public void testDefaultMachineList() throws UnknownHostException {
+    UserGroupInformation drwho =
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+            new String[] { "group1", "group2" });
+    ServiceAuthorizationManager serviceAuthorizationManager =
+        new ServiceAuthorizationManager();
+    Configuration conf = new Configuration ();
+    // test without setting a default MachineList
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
+          InetAddress.getByName(UNAUTHORIZED_IP));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    // test with a default MachineList
+    conf.set(
+        "security.service.authorization.default.hosts",
+        IP_RANGE);
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
+          InetAddress.getByName(UNAUTHORIZED_IP));
+      fail();
+    } catch (AuthorizationException e) {
+    // expects Exception
+    }
+    try {
+      serviceAuthorizationManager.authorize(drwho, TestProtocol.class, conf,
+          InetAddress.getByName("10.222.0.0"));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+  }
+
+  @Test
+  public void testBlockedMachineList() throws UnknownHostException {
+    UserGroupInformation drwho =
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+            new String[] { "group1", "group2" });
+
+    ServiceAuthorizationManager serviceAuthorizationManager =
+        new ServiceAuthorizationManager();
+    Configuration conf = new Configuration ();
+
+    // test without setting a blocked MachineList
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol.class, conf, InetAddress.getByName("10.222.0.0"));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    // now set a blocked MachineList
+    conf.set(BLOCKED_HOST_CONFIG, IP_RANGE);
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol.class, conf, InetAddress.getByName("10.222.0.0"));
+      fail();
+    } catch (AuthorizationException e) {
+    // expects Exception
+    }
+    // reset blocked MachineList
+    conf.set(BLOCKED_HOST_CONFIG, "");
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol.class, conf, InetAddress.getByName("10.222.0.0"));
+    } catch (AuthorizationException e) {
+      fail();
     }
   }
 
+  @Test
+  public void testDefaultBlockedMachineList() throws UnknownHostException {
+    UserGroupInformation drwho =
+        UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
+            new String[] { "group1", "group2" });
+
+    ServiceAuthorizationManager serviceAuthorizationManager =
+        new ServiceAuthorizationManager();
+    Configuration conf = new Configuration ();
+
+    // test without setting a default blocked MachineList
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol1.class, conf, InetAddress.getByName("10.222.0.0"));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    // set a  default blocked MachineList and a blocked MachineList for TestProtocol
+    conf.set(
+        "security.service.authorization.default.hosts.blocked",
+        IP_RANGE);
+    conf.set(BLOCKED_HOST_CONFIG, "1.2.3.4");
+    serviceAuthorizationManager.refresh(conf, new TestPolicyProvider());
+    // TestProtocol can be accessed from "10.222.0.0" because it blocks only "1.2.3.4"
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol.class, conf, InetAddress.getByName("10.222.0.0"));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    // TestProtocol cannot be accessed from  "1.2.3.4"
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+          TestProtocol.class, conf, InetAddress.getByName("1.2.3.4"));
+      fail();
+    } catch (AuthorizationException e) {
+      //expects Exception
+    }
+    // TestProtocol1 can be accessed from "1.2.3.4" because it uses default block list
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+         TestProtocol1.class, conf, InetAddress.getByName("1.2.3.4"));
+    } catch (AuthorizationException e) {
+      fail();
+    }
+    // TestProtocol1 cannot be accessed from "10.222.0.0", 
+    // because "10.222.0.0" is in default block list
+    try {
+      serviceAuthorizationManager.authorize(drwho,
+         TestProtocol1.class, conf, InetAddress.getByName("10.222.0.0"));
+      fail();
+    } catch (AuthorizationException e) {
+      //expects Exception
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 7be71e9ef1752..65c18d13ba791 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -38,6 +38,8 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.log4j.Layout;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.apache.log4j.WriterAppender;
 import org.junit.Assert;
@@ -55,6 +57,46 @@ public abstract class GenericTestUtils {
 
   private static final AtomicInteger sequence = new AtomicInteger();
 
+  @SuppressWarnings("unchecked")
+  public static void disableLog(Log log) {
+    // We expect that commons-logging is a wrapper around Log4j.
+    disableLog((Log4JLogger) log);
+  }
+
+  public static Logger toLog4j(org.slf4j.Logger logger) {
+    return LogManager.getLogger(logger.getName());
+  }
+
+  public static void disableLog(Log4JLogger log) {
+    log.getLogger().setLevel(Level.OFF);
+  }
+
+  public static void disableLog(Logger logger) {
+    logger.setLevel(Level.OFF);
+  }
+
+  public static void disableLog(org.slf4j.Logger logger) {
+    disableLog(toLog4j(logger));
+  }
+
+  @SuppressWarnings("unchecked")
+  public static void setLogLevel(Log log, Level level) {
+    // We expect that commons-logging is a wrapper around Log4j.
+    setLogLevel((Log4JLogger) log, level);
+  }
+
+  public static void setLogLevel(Log4JLogger log, Level level) {
+    log.getLogger().setLevel(level);
+  }
+
+  public static void setLogLevel(Logger logger, Level level) {
+    logger.setLevel(level);
+  }
+
+  public static void setLogLevel(org.slf4j.Logger logger, Level level) {
+    setLogLevel(toLog4j(logger), level);
+  }
+
   /**
    * Extracts the name of the method where the invocation has happened
    * @return String name of the invoking method
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestJUnitSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestJUnitSetup.java
new file mode 100644
index 0000000000000..d6ae04d71bae2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestJUnitSetup.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestJUnitSetup {
+  public static final Log LOG = LogFactory.getLog(TestJUnitSetup.class);
+
+  @Test
+  public void testJavaAssert() {
+    try {
+      assert false : "Good! Java assert is on.";
+    } catch(AssertionError ae) {
+      LOG.info("The AssertionError is expected.", ae);
+      return;
+    }
+    Assert.fail("Java assert does not work.");
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
new file mode 100644
index 0000000000000..9ef3483d25ca8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tracing/TestTraceUtils.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tracing;
+
+import static org.junit.Assert.assertEquals;
+import java.util.LinkedList;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
+import org.apache.htrace.HTraceConfiguration;
+import org.junit.Test;
+
+public class TestTraceUtils {
+  @Test
+  public void testWrappedHadoopConf() {
+    String key = "sampler";
+    String value = "ProbabilitySampler";
+    Configuration conf = new Configuration();
+    conf.set(TraceUtils.HTRACE_CONF_PREFIX + key, value);
+    HTraceConfiguration wrapped = TraceUtils.wrapHadoopConf(conf);
+    assertEquals(value, wrapped.get(key));
+  }
+
+  @Test
+  public void testExtraConfig() {
+    String key = "test.extra.config";
+    String oldValue = "old value";
+    String newValue = "new value";
+    Configuration conf = new Configuration();
+    conf.set(TraceUtils.HTRACE_CONF_PREFIX + key, oldValue);
+    LinkedList<ConfigurationPair> extraConfig =
+        new LinkedList<ConfigurationPair>();
+    extraConfig.add(new ConfigurationPair(key, newValue));
+    HTraceConfiguration wrapped = TraceUtils.wrapHadoopConf(conf, extraConfig);
+    assertEquals(newValue, wrapped.get(key));
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
index cc16493758e23..be8e61ea23ae6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
@@ -87,7 +87,7 @@ public void testConstructUrlsFromClasspath() throws Exception {
     assertEquals(jarFile.toURI().toURL(), urls[2]);
     // nofile should be ignored
   }
-  
+
   @Test
   public void testIsSystemClass() {
     testIsSystemClassInternal("");
@@ -112,8 +112,12 @@ private void testIsSystemClassInternal(String nestedClass) {
         classes("-org.example.Foo,org.example.")));
     assertTrue(isSystemClass("org.example.Bar" + nestedClass,
         classes("-org.example.Foo.,org.example.")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.,-org.example.Foo")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.Foo,-org.example.Foo")));
   }
-  
+
   private List<String> classes(String classes) {
     return Lists.newArrayList(Splitter.on(',').split(classes));
   }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestChunkedArrayList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestChunkedArrayList.java
new file mode 100644
index 0000000000000..a007f85c244c2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestChunkedArrayList.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestChunkedArrayList {
+
+  @Test
+  public void testBasics() {
+    final int N_ELEMS = 100000;
+    ChunkedArrayList<Integer> l = new ChunkedArrayList<Integer>();
+    assertTrue(l.isEmpty());
+    // Insert a bunch of elements.
+    for (int i = 0; i < N_ELEMS; i++) {
+      l.add(i);
+    }
+    assertFalse(l.isEmpty());
+    assertEquals(N_ELEMS, l.size());
+
+    // Check that it got chunked.
+    assertTrue(l.getNumChunks() > 10);
+    assertEquals(8192, l.getMaxChunkSize());
+  }
+  
+  @Test
+  public void testIterator() {
+    ChunkedArrayList<Integer> l = new ChunkedArrayList<Integer>();
+    for (int i = 0; i < 30000; i++) {
+      l.add(i);
+    }
+    
+    int i = 0;
+    for (int fromList : l) {
+      assertEquals(i, fromList);
+      i++;
+    }
+  }
+  
+  @Test
+  public void testPerformance() {
+    String obj = "hello world";
+    
+    final int numElems = 1000000;
+    final int numTrials = 5;
+    
+    for (int trial = 0; trial < numTrials; trial++) {
+      System.gc();
+      {
+        ArrayList<String> arrayList = new ArrayList<String>();
+        StopWatch sw = new StopWatch();
+        sw.start();
+        for (int i = 0; i < numElems; i++) {
+          arrayList.add(obj);
+        }
+        System.out.println("       ArrayList " + sw.now(TimeUnit.MILLISECONDS));
+      }
+      
+      // test ChunkedArrayList
+      System.gc();
+      {
+        ChunkedArrayList<String> chunkedList = new ChunkedArrayList<String>();
+        StopWatch sw = new StopWatch();
+        sw.start();
+        for (int i = 0; i < numElems; i++) {
+          chunkedList.add(obj);
+        }
+        System.out.println("ChunkedArrayList " + sw.now(TimeUnit.MILLISECONDS));
+      }
+    }
+  }
+
+  @Test
+  public void testRemovals() throws Exception {
+    final int NUM_ELEMS = 100000;
+    ChunkedArrayList<Integer> list = new ChunkedArrayList<Integer>();
+    for (int i = 0; i < NUM_ELEMS; i++) {
+      list.add(i);
+    }
+
+    // Iterate through all list elements.
+    Iterator<Integer> iter = list.iterator();
+    for (int i = 0; i < NUM_ELEMS; i++) {
+      Assert.assertTrue(iter.hasNext());
+      Integer val = iter.next();
+      Assert.assertEquals(Integer.valueOf(i), val);
+    }
+    Assert.assertFalse(iter.hasNext());
+    Assert.assertEquals(NUM_ELEMS, list.size());
+
+    // Remove even elements.
+    iter = list.iterator();
+    for (int i = 0; i < NUM_ELEMS; i++) {
+      Assert.assertTrue(iter.hasNext());
+      Integer val = iter.next();
+      Assert.assertEquals(Integer.valueOf(i), val);
+      if (i % 2 == 0) {
+        iter.remove();
+      }
+    }
+    Assert.assertFalse(iter.hasNext());
+    Assert.assertEquals(NUM_ELEMS / 2, list.size());
+
+    // Iterate through all odd list elements.
+    iter = list.iterator();
+    for (int i = 0; i < NUM_ELEMS / 2; i++) {
+      Assert.assertTrue(iter.hasNext());
+      Integer val = iter.next();
+      Assert.assertEquals(Integer.valueOf(1 + (2 * i)), val);
+      iter.remove();
+    }
+    Assert.assertFalse(iter.hasNext());
+
+    // Check that list is now empty.
+    Assert.assertEquals(0, list.size());
+    Assert.assertTrue(list.isEmpty());
+    iter = list.iterator();
+    Assert.assertFalse(iter.hasNext());
+  }
+
+  @Test
+  public void testGet() throws Exception {
+    final int NUM_ELEMS = 100001;
+    ChunkedArrayList<Integer> list = new ChunkedArrayList<Integer>();
+    for (int i = 0; i < NUM_ELEMS; i++) {
+      list.add(i);
+    }
+
+    Assert.assertEquals(Integer.valueOf(100), list.get(100));
+    Assert.assertEquals(Integer.valueOf(1000), list.get(1000));
+    Assert.assertEquals(Integer.valueOf(10000), list.get(10000));
+    Assert.assertEquals(Integer.valueOf(100000), list.get(100000));
+
+    Iterator<Integer> iter = list.iterator();
+    iter.next();
+    iter.remove();
+    Assert.assertEquals(Integer.valueOf(1), list.get(0));
+
+    iter = list.iterator();
+    for (int i = 0; i < 500; i++) {
+      iter.next();
+    }
+    iter.remove();
+
+    Assert.assertEquals(Integer.valueOf(502), list.get(500));
+    Assert.assertEquals(Integer.valueOf(602), list.get(600));
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
index 34fc32aa08f6d..73fd25a7fa241 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java
@@ -21,8 +21,6 @@
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Stopwatch;
-
 import org.apache.hadoop.fs.ChecksumException;
 import org.junit.Test;
 
@@ -147,19 +145,19 @@ public void commonUsagePerfTest() throws Exception {
     Harness h = new Harness(checksum, dataLength, true);
 
     for (int i = 0; i < NUM_RUNS; i++) {
-      Stopwatch s = new Stopwatch().start();
+      StopWatch s = new StopWatch().start();
       // calculate real checksum, make sure it passes
       checksum.calculateChunkedSums(h.dataBuf, h.checksumBuf);
       s.stop();
       System.err.println("Calculate run #" + i + ": " +
-                         s.elapsedTime(TimeUnit.MICROSECONDS) + "us");
+                         s.now(TimeUnit.MICROSECONDS) + "us");
 
-      s = new Stopwatch().start();
+      s = new StopWatch().start();
       // calculate real checksum, make sure it passes
       checksum.verifyChunkedSums(h.dataBuf, h.checksumBuf, "fake file", 0);
       s.stop();
       System.err.println("Verify run #" + i + ": " +
-                         s.elapsedTime(TimeUnit.MICROSECONDS) + "us");
+                         s.now(TimeUnit.MICROSECONDS) + "us");
     }
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFindClass.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFindClass.java
new file mode 100644
index 0000000000000..28389c27d5477
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestFindClass.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.FindClass;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+/**
+ * Test the find class logic
+ */
+public class TestFindClass extends Assert {
+  private static final Log LOG = LogFactory.getLog(TestFindClass.class);
+
+  public static final String LOG4J_PROPERTIES = "log4j.properties";
+
+  /**
+   * Run the tool runner instance
+   * @param expected expected return code
+   * @param args a list of arguments
+   * @throws Exception on any falure that is not handled earlier
+   */
+  private void run(int expected, String... args) throws Exception {
+    int result = ToolRunner.run(new FindClass(), args);
+    assertEquals(expected, result);
+  }
+
+  @Test
+  public void testUsage() throws Throwable {
+    run(FindClass.E_USAGE, "org.apache.hadoop.util.TestFindClass");
+  }
+
+  @Test
+  public void testFindsResource() throws Throwable {
+    run(FindClass.SUCCESS,
+        FindClass.A_RESOURCE, "org/apache/hadoop/util/TestFindClass.class");
+  }
+
+  @Test
+  public void testFailsNoSuchResource() throws Throwable {
+    run(FindClass.E_NOT_FOUND,
+        FindClass.A_RESOURCE,
+        "org/apache/hadoop/util/ThereIsNoSuchClass.class");
+  }
+
+  @Test
+  public void testLoadFindsSelf() throws Throwable {
+    run(FindClass.SUCCESS,
+        FindClass.A_LOAD, "org.apache.hadoop.util.TestFindClass");
+  }
+
+  @Test
+  public void testLoadFailsNoSuchClass() throws Throwable {
+    run(FindClass.E_NOT_FOUND,
+        FindClass.A_LOAD, "org.apache.hadoop.util.ThereIsNoSuchClass");
+  }
+
+  @Test
+  public void testLoadWithErrorInStaticInit() throws Throwable {
+    run(FindClass.E_LOAD_FAILED,
+        FindClass.A_LOAD,
+        "org.apache.hadoop.util.TestFindClass$FailInStaticInit");
+  }
+
+  @Test
+  public void testCreateHandlesBadToString() throws Throwable {
+    run(FindClass.SUCCESS,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$BadToStringClass");
+  }
+
+  @Test
+  public void testCreatesClass() throws Throwable {
+    run(FindClass.SUCCESS,
+        FindClass.A_CREATE, "org.apache.hadoop.util.TestFindClass");
+  }
+
+  @Test
+  public void testCreateFailsInStaticInit() throws Throwable {
+    run(FindClass.E_LOAD_FAILED,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$FailInStaticInit");
+  }
+
+  @Test
+  public void testCreateFailsInConstructor() throws Throwable {
+    run(FindClass.E_CREATE_FAILED,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$FailInConstructor");
+  }
+
+  @Test
+  public void testCreateFailsNoEmptyConstructor() throws Throwable {
+    run(FindClass.E_CREATE_FAILED,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$NoEmptyConstructor");
+  }
+
+  @Test
+  public void testLoadPrivateClass() throws Throwable {
+    run(FindClass.SUCCESS,
+        FindClass.A_LOAD, "org.apache.hadoop.util.TestFindClass$PrivateClass");
+  }
+
+  @Test
+  public void testCreateFailsPrivateClass() throws Throwable {
+    run(FindClass.E_CREATE_FAILED,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$PrivateClass");
+  }
+
+  @Test
+  public void testCreateFailsInPrivateConstructor() throws Throwable {
+    run(FindClass.E_CREATE_FAILED,
+        FindClass.A_CREATE,
+        "org.apache.hadoop.util.TestFindClass$PrivateConstructor");
+  }
+
+  @Test
+  public void testLoadFindsLog4J() throws Throwable {
+    run(FindClass.SUCCESS, FindClass.A_RESOURCE, LOG4J_PROPERTIES);
+  }
+
+  @SuppressWarnings("UseOfSystemOutOrSystemErr")
+  @Test
+  public void testPrintLog4J() throws Throwable {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream out = new PrintStream(baos);
+    FindClass.setOutputStreams(out, System.err);
+    run(FindClass.SUCCESS, FindClass.A_PRINTRESOURCE, LOG4J_PROPERTIES);
+    //here the content should be done
+    out.flush();
+    String body = baos.toString("UTF8");
+    LOG.info(LOG4J_PROPERTIES + " =\n" + body);
+    assertTrue(body.contains("Apache"));
+  }
+  
+  
+  /**
+   * trigger a divide by zero fault in the static init
+   */
+  public static class FailInStaticInit {
+    static {
+      int x = 0;
+      int y = 1 / x;
+    }
+  }
+
+  /**
+   * trigger a divide by zero fault in the constructor
+   */
+  public static class FailInConstructor {
+    public FailInConstructor() {
+      int x = 0;
+      int y = 1 / x;
+    }
+  }
+
+  /**
+   * A class with no parameterless constructor -expect creation to fail
+   */
+  public static class NoEmptyConstructor {
+    public NoEmptyConstructor(String text) {
+    }
+  }
+
+  /**
+   * This has triggers an NPE in the toString() method; checks the logging
+   * code handles this.
+   */
+  public static class BadToStringClass {
+    public BadToStringClass() {
+    }
+
+    @Override
+    public String toString() {
+      throw new NullPointerException("oops");
+    }
+  }
+
+  /**
+   * This has a private constructor
+   * -creating it will trigger an IllegalAccessException
+   */
+  public static class PrivateClass {
+    private PrivateClass() {
+    }
+  }
+
+  /**
+   * This has a private constructor
+   * -creating it will trigger an IllegalAccessException
+   */
+  public static class PrivateConstructor {
+    private PrivateConstructor() {
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStopWatch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStopWatch.java
new file mode 100644
index 0000000000000..6f577b08d241c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStopWatch.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestStopWatch {
+
+  @Test
+  public void testStartAndStop() throws Exception {
+    try (StopWatch sw = new StopWatch()) {
+      Assert.assertFalse(sw.isRunning());
+      sw.start();
+      Assert.assertTrue(sw.isRunning());
+      sw.stop();
+      Assert.assertFalse(sw.isRunning());
+    }
+  }
+
+  @Test
+  public void testStopInTryWithResource() throws Exception {
+    try (StopWatch sw = new StopWatch()) {
+      // make sure that no exception is thrown.
+    }
+  }
+
+  @Test
+  public void testExceptions() throws Exception {
+    StopWatch sw = new StopWatch();
+    try {
+      sw.stop();
+    } catch (Exception e) {
+      Assert.assertTrue("IllegalStateException is expected",
+          e instanceof IllegalStateException);
+    }
+    sw.reset();
+    sw.start();
+    try {
+      sw.start();
+    } catch (Exception e) {
+      Assert.assertTrue("IllegalStateException is expected",
+          e instanceof IllegalStateException);
+    }
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
index 93fa6d5195d03..6ff854d1fb7a0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
@@ -23,6 +23,7 @@
 import static org.junit.Assert.assertTrue;
 
 import java.util.AbstractCollection;
+import java.util.BitSet;
 import java.util.Iterator;
 
 import org.apache.hadoop.util.bloom.BloomFilterCommonTester.BloomFilterTestStrategy;
@@ -237,4 +238,14 @@ public void testFiltersWithMurmurHash() {
                 BloomFilterTestStrategy.FILTER_AND_STRATEGY,
                 BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
   }
+
+  @Test
+  public void testNot() {
+    BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
+    bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
+    BitSet origBitSet = (BitSet) bf.bits.clone();
+    bf.not();
+    assertFalse("BloomFilter#not should have inverted all bits",
+                bf.bits.intersects(origBitSet));
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/hdfs7067.keystore b/hadoop-common-project/hadoop-common/src/test/resources/hdfs7067.keystore
new file mode 100644
index 0000000000000..a0a69b194a31e
Binary files /dev/null and b/hadoop-common-project/hadoop-common/src/test/resources/hdfs7067.keystore differ
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 5196641babe63..f962813b3374c 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -238,7 +238,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] \[-h\] &lt;path&gt; \.\.\. :( )*</expected-output>
+          <expected-output>^-count \[-q\] \[-h\] \[-v\] &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -250,20 +250,28 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output>
+          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output>
+          <expected-output>^( |\t)*or, with the -q option:( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
+          <expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
           <expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
+        </comparator>
       </comparators>
     </test>
 
@@ -786,19 +794,35 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*Print statistics about the file/directory at &lt;path&gt; in the specified format.( )*</expected-output>
+          <expected-output>^( |\t)*Print statistics about the file/directory at &lt;path&gt;( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*in the specified format. Format accepts filesize in( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*blocks \(%b\), type \(%F\), group name of owner \(%g\),( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*name \(%n\), block size \(%o\), replication \(%r\), user name( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*of owner \(%u\), modification date \(%y, %Y\).( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*Format accepts filesize in blocks \(%b\), group name of owner\(%g\), filename \(%n\),( )*</expected-output>
+          <expected-output>^( |\t)*%y shows UTC date as "yyyy-MM-dd HH:mm:ss" and( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*block size \(%o\), replication \(%r\), user name of owner\(%u\), modification date( )*</expected-output>
+          <expected-output>^( |\t)*%Y shows milliseconds since January 1, 1970 UTC.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
+          <expected-output>^( |\t)*If the format is not specified, %y is used by default.( )*</expected-output>
         </comparator>
       </comparators>
     </test>
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
index 1d5b649c83dd4..cba69f4316ec8 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-acls.xml
@@ -41,7 +41,7 @@
     <value>*</value>
     <description>
       ACL for rollover-key operations.
-      If the user does is not in the GET ACL, the key material is not returned
+      If the user is not in the GET ACL, the key material is not returned
       as part of the response.
     </description>
   </property>
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index 88a2b8644ea0d..de9554f8a8c92 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -14,19 +14,13 @@
 #
 
 # Set kms specific environment variables here.
-
-# Settings for the Embedded Tomcat that runs KMS
-# Java System properties for KMS should be specified in this variable
 #
-# export CATALINA_OPTS=
-
-# KMS logs directory
+# hadoop-env.sh is read prior to this file.
 #
-# export KMS_LOG=${KMS_HOME}/logs
 
 # KMS temporary directory
 #
-# export KMS_TEMP=${KMS_HOME}/temp
+# export KMS_TEMP=${HADOOP_PREFIX}/temp
 
 # The HTTP port used by KMS
 #
@@ -34,7 +28,7 @@
 
 # The Admin port used by KMS
 #
-# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+# export KMS_ADMIN_PORT=$((KMS_HTTP_PORT + 1))
 
 # The maximum number of Tomcat handler threads
 #
@@ -44,6 +38,37 @@
 #
 # export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
 
+#
 # The password of the SSL keystore if using SSL
 #
 # export KMS_SSL_KEYSTORE_PASS=password
+
+#
+# The password of the truststore
+#
+# export KMS_SSL_TRUSTSTORE_PASS=
+
+
+##
+## Tomcat specific settings
+##
+#
+# Location of tomcat
+#
+# export KMS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/kms/tomcat
+
+# Java System properties for KMS should be specified in this variable.
+# The java.library.path and hadoop.home.dir properties are automatically
+# configured.  In order to supplement java.library.path,
+# one should add to the JAVA_LIBRARY_PATH env var.
+#
+# export CATALINA_OPTS=
+
+# PID file
+#
+# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid
+
+# Output file
+#
+# export CATALINA_OUT=${KMS_LOG}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out
+
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index c33dd4b60df56..5b67950a30f2e 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -36,6 +36,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Provides access to the <code>AccessControlList</code>s used by KMS,
  * hot-reloading them if the <code>kms-acls.xml</code> file where the ACLs
@@ -70,7 +72,8 @@ public String getBlacklistConfigKey() {
 
   private volatile Map<Type, AccessControlList> acls;
   private volatile Map<Type, AccessControlList> blacklistedAcls;
-  private volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
+  @VisibleForTesting
+  volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
   private final Map<KeyOpType, AccessControlList> defaultKeyAcls =
       new HashMap<KeyOpType, AccessControlList>();
   private final Map<KeyOpType, AccessControlList> whitelistKeyAcls =
@@ -112,7 +115,7 @@ private void setKeyACLs(Configuration conf) {
     Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls =
         new HashMap<String, HashMap<KeyOpType,AccessControlList>>();
     Map<String, String> allKeyACLS =
-        conf.getValByRegex(Pattern.quote(KMSConfiguration.KEY_ACL_PREFIX));
+        conf.getValByRegex(KMSConfiguration.KEY_ACL_PREFIX_REGEX);
     for (Map.Entry<String, String> keyAcl : allKeyACLS.entrySet()) {
       String k = keyAcl.getKey();
       // this should be of type "key.acl.<KEY_NAME>.<OP_TYPE>"
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index a67c68ee0a242..23c983f4e5753 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -38,6 +38,7 @@ public class KMSConfiguration {
   public static final String CONFIG_PREFIX = "hadoop.kms.";
 
   public static final String KEY_ACL_PREFIX = "key.acl.";
+  public static final String KEY_ACL_PREFIX_REGEX = "^key\\.acl\\..+";
   public static final String DEFAULT_KEY_ACL_PREFIX = "default.key.acl.";
   public static final String WHITELIST_KEY_ACL_PREFIX = "whitelist.key.acl.";
 
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
index 3674e7a87aaf3..31fac9f7b2061 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -32,6 +32,7 @@
 import java.io.Writer;
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Type;
+import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
 
@@ -62,7 +63,8 @@ public void writeTo(Object obj, Class<?> aClass, Type type,
       Annotation[] annotations, MediaType mediaType,
       MultivaluedMap<String, Object> stringObjectMultivaluedMap,
       OutputStream outputStream) throws IOException, WebApplicationException {
-    Writer writer = new OutputStreamWriter(outputStream);
+    Writer writer = new OutputStreamWriter(outputStream, Charset
+        .forName("UTF-8"));
     ObjectMapper jsonMapper = new ObjectMapper();
     jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
   }
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
index 3ac929a2fc2d8..c1aa1364af279 100644
--- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
@@ -13,182 +13,64 @@
 #  limitations under the License.
 #
 
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
-  ls=`ls -ld "${PRG}"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '/.*' > /dev/null; then
-    PRG="$link"
-  else
-    PRG=`dirname "${PRG}"`/"$link"
+function hadoop_subproject_init
+{
+  local this
+  local binparent
+  local varlist
+
+  if [[ -z "${HADOOP_KMS_ENV_PROCESSED}" ]]; then
+    if [[ -e "${HADOOP_CONF_DIR}/kms-env.sh" ]]; then
+      . "${HADOOP_CONF_DIR}/kms-env.sh"
+      export HADOOP_KMS_ENV_PROCESSED=true
+    fi
   fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
-
-
-function print() {
-  if [ "${KMS_SILENT}" != "true" ]; then
-    echo "$@"
-  fi
-}
-
-# if KMS_HOME is already set warn it will be ignored
-#
-if [ "${KMS_HOME}" != "" ]; then
-  echo "WARNING: current setting of KMS_HOME ignored"
-fi
 
-print
+  export HADOOP_CATALINA_PREFIX=kms
 
-# setting KMS_HOME to the installation dir, it cannot be changed
-#
-export KMS_HOME=${BASEDIR}
-kms_home=${KMS_HOME}
-print "Setting KMS_HOME:          ${KMS_HOME}"
+  export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_PREFIX}/temp}"
 
-# if the installation has a env file, source it
-# this is for native packages installations
-#
-if [ -e "${KMS_HOME}/bin/kms-env.sh" ]; then
-  print "Sourcing:                    ${KMS_HOME}/bin/kms-env.sh"
-  source ${KMS_HOME}/bin/kms-env.sh
-  grep "^ *export " ${KMS_HOME}/bin/kms-env.sh | sed 's/ *export/  setting/'
-fi
-
-# verify that the sourced env file didn't change KMS_HOME
-# if so, warn and revert
-#
-if [ "${KMS_HOME}" != "${kms_home}" ]; then
-  print "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored"
-  export KMS_HOME=${kms_home}
-  print "  using KMS_HOME:        ${KMS_HOME}"
-fi
-
-if [ "${KMS_CONFIG}" = "" ]; then
-  export KMS_CONFIG=${KMS_HOME}/etc/hadoop
-  print "Setting KMS_CONFIG:        ${KMS_CONFIG}"
-else
-  print "Using   KMS_CONFIG:        ${KMS_CONFIG}"
-fi
-kms_config=${KMS_CONFIG}
-
-# if the configuration dir has a env file, source it
-#
-if [ -e "${KMS_CONFIG}/kms-env.sh" ]; then
-  print "Sourcing:                    ${KMS_CONFIG}/kms-env.sh"
-  source ${KMS_CONFIG}/kms-env.sh
-  grep "^ *export " ${KMS_CONFIG}/kms-env.sh | sed 's/ *export/  setting/'
-fi
-
-# verify that the sourced env file didn't change KMS_HOME
-# if so, warn and revert
-#
-if [ "${KMS_HOME}" != "${kms_home}" ]; then
-  echo "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored"
-  export KMS_HOME=${kms_home}
-fi
-
-# verify that the sourced env file didn't change KMS_CONFIG
-# if so, warn and revert
-#
-if [ "${KMS_CONFIG}" != "${kms_config}" ]; then
-  echo "WARN: KMS_CONFIG resetting to ''${KMS_CONFIG}'' ignored"
-  export KMS_CONFIG=${kms_config}
-fi
+  hadoop_deprecate_envvar KMS_CONFIG HADOOP_CONF_DIR
 
-if [ "${KMS_LOG}" = "" ]; then
-  export KMS_LOG=${KMS_HOME}/logs
-  print "Setting KMS_LOG:           ${KMS_LOG}"
-else
-  print "Using   KMS_LOG:           ${KMS_LOG}"
-fi
+  hadoop_deprecate_envvar KMS_LOG HADOOP_LOG_DIR
 
-if [ ! -f ${KMS_LOG} ]; then
-  mkdir -p ${KMS_LOG}
-fi
+  export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}"
+  export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}"
 
-if [ "${KMS_TEMP}" = "" ]; then
-  export KMS_TEMP=${KMS_HOME}/temp
-  print "Setting KMS_TEMP:           ${KMS_TEMP}"
-else
-  print "Using   KMS_TEMP:           ${KMS_TEMP}"
-fi
+  export HADOOP_CATALINA_HTTP_PORT="${KMS_HTTP_PORT:-16000}"
+  export HADOOP_CATALINA_ADMIN_PORT="${KMS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}"
+  export HADOOP_CATALINA_MAX_THREADS="${KMS_MAX_THREADS:-1000}"
 
-if [ ! -f ${KMS_TEMP} ]; then
-  mkdir -p ${KMS_TEMP}
-fi
+  export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${KMS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
 
-if [ "${KMS_HTTP_PORT}" = "" ]; then
-  export KMS_HTTP_PORT=16000
-  print "Setting KMS_HTTP_PORT:     ${KMS_HTTP_PORT}"
-else
-  print "Using   KMS_HTTP_PORT:     ${KMS_HTTP_PORT}"
-fi
+  # this is undocumented, but older versions would rip the TRUSTSTORE_PASS out of the
+  # CATALINA_OPTS
+  # shellcheck disable=SC2086
+  export KMS_SSL_TRUSTSTORE_PASS=${KMS_SSL_TRUSTSTORE_PASS:-"$(echo ${CATALINA_OPTS} | grep -o 'trustStorePassword=[^ ]*' | cut -f2 -d= )"}
 
-if [ "${KMS_ADMIN_PORT}" = "" ]; then
-  export KMS_ADMIN_PORT=`expr $KMS_HTTP_PORT +  1`
-  print "Setting KMS_ADMIN_PORT:     ${KMS_ADMIN_PORT}"
-else
-  print "Using   KMS_ADMIN_PORT:     ${KMS_ADMIN_PORT}"
-fi
+  export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/kms/tomcat}"
+  export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}"
 
-if [ "${KMS_MAX_THREADS}" = "" ]; then
-  export KMS_MAX_THREADS=1000
-  print "Setting KMS_MAX_THREADS:     ${KMS_MAX_THREADS}"
-else
-  print "Using   KMS_MAX_THREADS:     ${KMS_MAX_THREADS}"
-fi
+  export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}"
 
-if [ "${KMS_SSL_KEYSTORE_FILE}" = "" ]; then
-  export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-  print "Setting KMS_SSL_KEYSTORE_FILE:     ${KMS_SSL_KEYSTORE_FILE}"
-else
-  print "Using   KMS_SSL_KEYSTORE_FILE:     ${KMS_SSL_KEYSTORE_FILE}"
-fi
+  export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid}"
 
-# If KMS_SSL_KEYSTORE_PASS is explicitly set to ""
-# then reset to "password". DO NOT set to "password" if
-# variable is NOT defined.
-if [ "${KMS_SSL_KEYSTORE_PASS}" = "" ]; then
-  if [ -n "${KMS_SSL_KEYSTORE_PASS+1}" ]; then
-    export KMS_SSL_KEYSTORE_PASS=password
-    print "Setting KMS_SSL_KEYSTORE_PASS:     ********"
+  if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
+    varlist=$(env | egrep '(^KMS|^CATALINA)' | cut -f1 -d= | grep -v _PASS)
+    for i in ${varlist}; do
+      hadoop_debug "Setting ${i} to ${!i}"
+    done
   fi
-else
-  KMS_SSL_KEYSTORE_PASS_DISP=`echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'`
-  print "Using   KMS_SSL_KEYSTORE_PASS:     ${KMS_SSL_KEYSTORE_PASS_DISP}"
-fi
-
-if [ "${CATALINA_BASE}" = "" ]; then
-  export CATALINA_BASE=${KMS_HOME}/share/hadoop/kms/tomcat
-  print "Setting CATALINA_BASE:       ${CATALINA_BASE}"
-else
-  print "Using   CATALINA_BASE:       ${CATALINA_BASE}"
-fi
-
-if [ "${KMS_CATALINA_HOME}" = "" ]; then
-  export KMS_CATALINA_HOME=${CATALINA_BASE}
-  print "Setting KMS_CATALINA_HOME:       ${KMS_CATALINA_HOME}"
-else
-  print "Using   KMS_CATALINA_HOME:       ${KMS_CATALINA_HOME}"
-fi
-
-if [ "${CATALINA_OUT}" = "" ]; then
-  export CATALINA_OUT=${KMS_LOG}/kms-catalina.out
-  print "Setting CATALINA_OUT:        ${CATALINA_OUT}"
-else
-  print "Using   CATALINA_OUT:        ${CATALINA_OUT}"
-fi
+}
 
-if [ "${CATALINA_PID}" = "" ]; then
-  export CATALINA_PID=/tmp/kms.pid
-  print "Setting CATALINA_PID:        ${CATALINA_PID}"
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+   [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
+  . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
 else
-  print "Using   CATALINA_PID:        ${CATALINA_PID}"
+  echo "ERROR: Hadoop common not found." 2>&1
+  exit 1
 fi
-
-print
diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
index 24a1f54f964c2..e4d4f93f9ddfe 100644
--- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
@@ -13,62 +13,96 @@
 #  limitations under the License.
 #
 
-# resolve links - $0 may be a softlink
-PRG="${0}"
+function hadoop_usage()
+{
+  echo "Usage: kms.sh [--config confdir] [--debug] --daemon start|status|stop"
+  echo "       kms.sh [--config confdir] [--debug] COMMAND"
+  echo "            where COMMAND is one of:"
+  echo "  run               Start kms in the current window"
+  echo "  run -security     Start in the current window with security manager"
+  echo "  start             Start kms in a separate window"
+  echo "  start -security   Start in a separate window with security manager"
+  echo "  status            Return the LSB compliant status"
+  echo "  stop              Stop kms, waiting up to 5 seconds for the process to end"
+  echo "  stop n            Stop kms, waiting up to n seconds for the process to end"
+  echo "  stop -force       Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
+  echo "  stop n -force     Stop kms, wait up to n seconds and then use kill -KILL if still running"
+}
 
-while [ -h "${PRG}" ]; do
-  ls=`ls -ld "${PRG}"`
-  link=`expr "$ls" : '.*-> \(.*\)$'`
-  if expr "$link" : '/.*' > /dev/null; then
-    PRG="$link"
-  else
-    PRG=`dirname "${PRG}"`/"$link"
-  fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
-
-KMS_SILENT=${KMS_SILENT:-true}
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
 
-source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/kms-config.sh
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/kms-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/kms-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/kms-config.sh." 2>&1
+  exit 1
+fi
 
 # The Java System property 'kms.http.port' it is not used by Kms,
 # it is used in Tomcat's server.xml configuration file
 #
 
 # Mask the trustStorePassword
-KMS_SSL_TRUSTSTORE_PASS=`echo $CATALINA_OPTS | grep -o 'trustStorePassword=[^ ]*' | awk -F'=' '{print $2}'`
-CATALINA_OPTS_DISP=`echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/'`
-print "Using   CATALINA_OPTS:       ${CATALINA_OPTS_DISP}"
+# shellcheck disable=SC2086
+CATALINA_OPTS_DISP="$(echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/')"
+
+hadoop_debug "Using   CATALINA_OPTS:       ${CATALINA_OPTS_DISP}"
 
-catalina_opts="-Dkms.home.dir=${KMS_HOME}";
-catalina_opts="${catalina_opts} -Dkms.config.dir=${KMS_CONFIG}";
-catalina_opts="${catalina_opts} -Dkms.log.dir=${KMS_LOG}";
-catalina_opts="${catalina_opts} -Dkms.temp.dir=${KMS_TEMP}";
-catalina_opts="${catalina_opts} -Dkms.admin.port=${KMS_ADMIN_PORT}";
-catalina_opts="${catalina_opts} -Dkms.http.port=${KMS_HTTP_PORT}";
-catalina_opts="${catalina_opts} -Dkms.max.threads=${KMS_MAX_THREADS}";
-catalina_opts="${catalina_opts} -Dkms.ssl.keystore.file=${KMS_SSL_KEYSTORE_FILE}";
+# We're using hadoop-common, so set up some stuff it might need:
+hadoop_finalize
 
-print "Adding to CATALINA_OPTS:     ${catalina_opts}"
-print "Found KMS_SSL_KEYSTORE_PASS:     `echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'`"
+hadoop_verify_logdir
 
-export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
+if [[ $# = 0 ]]; then
+  case "${HADOOP_DAEMON_MODE}" in
+    status)
+      hadoop_status_daemon "${CATALINA_PID}"
+      exit
+    ;;
+    start)
+      set -- "start"
+    ;;
+    stop)
+      set -- "stop"
+    ;;
+  esac
+fi
+
+hadoop_finalize_catalina_opts
+export CATALINA_OPTS
 
 # A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
 #
-if [ "${1}" = "stop" ]; then
+if [[ "${1}" = "stop" ]]; then
   export JAVA_OPTS=${CATALINA_OPTS}
 fi
 
 # If ssl, the populate the passwords into ssl-server.xml before starting tomcat
-if [ ! "${KMS_SSL_KEYSTORE_PASS}" = "" ] || [ ! "${KMS_SSL_TRUSTSTORE_PASS}" = "" ]; then
-  # Set a KEYSTORE_PASS if not already set
-  KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password}
-  cat ${CATALINA_BASE}/conf/ssl-server.xml.conf \
-    | sed 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \
-    | sed 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' > ${CATALINA_BASE}/conf/ssl-server.xml
-fi 
+#
+# KMS_SSL_KEYSTORE_PASS is a bit odd.
+# if undefined, then the if test will not enable ssl on its own
+# if "", set it to "password".
+# if custom, use provided password
+#
+if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then
+  if [[ -n "${KMS_SSL_KEYSTORE_PASS+x}" ]] || [[ -n "${KMS_SSL_TRUSTSTORE_PASS}" ]]; then
+      export KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password}
+      sed -e 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \
+          -e 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' \
+        "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \
+        > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml"
+      chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1
+  fi
+fi
 
-exec ${KMS_CATALINA_HOME}/bin/catalina.sh "$@"
+exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 11b84d39c786e..a2dcce351e473 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -159,6 +159,15 @@ hadoop-${project.version} $ sbin/kms.sh start
   NOTE: You need to restart the KMS for the configuration changes to take
   effect.
 
+** Loading native libraries
+
+  The following environment variable (which can be set in KMS's
+  <<<etc/hadoop/kms-env.sh>>> script) can be used to specify the location
+  of any required native libraries. For eg. Tomact native Apache Portable
+  Runtime (APR) libraries:
+
+  * JAVA_LIBRARY_PATH
+
 ** KMS Security Configuration
 
 *** Enabling Kerberos HTTP SPNEGO Authentication
@@ -290,7 +299,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
     <value>*</value>
     <description>
       ACL for create-key operations.
-      If the user does is not in the GET ACL, the key material is not returned
+      If the user is not in the GET ACL, the key material is not returned
       as part of the response.
     </description>
   </property>
@@ -300,7 +309,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
     <value>hdfs,foo</value>
     <description>
       Blacklist for create-key operations.
-      If the user does is in the Blacklist, the key material is not returned
+      If the user is in the Blacklist, the key material is not returned
       as part of the response.
     </description>
   </property>
@@ -326,7 +335,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
     <value>*</value>
     <description>
       ACL for rollover-key operations.
-      If the user does is not in the GET ACL, the key material is not returned
+      If the user is not in the GET ACL, the key material is not returned
       as part of the response.
     </description>
   </property>
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 61ce8072be864..70ba95f28d7d0 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -284,6 +284,7 @@ public void testStartStop(final boolean ssl, final boolean kerberos)
       password = null;
     }
 
+    conf.set("hadoop.kms.authentication.token.validity", "1");
     if (kerberos) {
       conf.set("hadoop.kms.authentication.type", "kerberos");
       conf.set("hadoop.kms.authentication.kerberos.keytab",
@@ -303,6 +304,32 @@ public Void call() throws Exception {
             url.getProtocol().equals("https"));
         final URI uri = createKMSUri(getKMSUrl());
 
+        if (ssl) {
+          KeyProvider testKp = new KMSClientProvider(uri, conf);
+          ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
+          while (threadGroup.getParent() != null) {
+            threadGroup = threadGroup.getParent();
+          }
+          Thread[] threads = new Thread[threadGroup.activeCount()];
+          threadGroup.enumerate(threads);
+          Thread reloaderThread = null;
+          for (Thread thread : threads) {
+            if ((thread.getName() != null)
+                && (thread.getName().contains("Truststore reloader thread"))) {
+              reloaderThread = thread;
+            }
+          }
+          Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
+          testKp.close();
+          boolean reloaderStillAlive = true;
+          for (int i = 0; i < 10; i++) {
+            reloaderStillAlive = reloaderThread.isAlive();
+            if (!reloaderStillAlive) break;
+            Thread.sleep(1000);
+          }
+          Assert.assertFalse("Reloader is still alive", reloaderStillAlive);
+        }
+
         if (kerberos) {
           for (String user : new String[]{"client", "client/host"}) {
             doAs(user, new PrivilegedExceptionAction<Void>() {
@@ -311,6 +338,11 @@ public Void run() throws Exception {
                 final KeyProvider kp = new KMSClientProvider(uri, conf);
                 // getKeys() empty
                 Assert.assertTrue(kp.getKeys().isEmpty());
+
+                Thread.sleep(4000);
+                Token<?>[] tokens = ((KMSClientProvider)kp).addDelegationTokens("myuser", new Credentials());
+                Assert.assertEquals(1, tokens.length);
+                Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
                 return null;
               }
             });
@@ -320,6 +352,7 @@ public Void run() throws Exception {
           // getKeys() empty
           Assert.assertTrue(kp.getKeys().isEmpty());
 
+          Thread.sleep(4000);
           Token<?>[] tokens = ((KMSClientProvider)kp).addDelegationTokens("myuser", new Credentials());
           Assert.assertEquals(1, tokens.length);
           Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
index abdf3c21d0225..b4bf50402e22c 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
@@ -26,7 +26,7 @@ public class TestKMSACLs {
 
   @Test
   public void testDefaults() {
-    KMSACLs acls = new KMSACLs(new Configuration(false));
+    final KMSACLs acls = new KMSACLs(new Configuration(false));
     for (KMSACLs.Type type : KMSACLs.Type.values()) {
       Assert.assertTrue(acls.hasAccess(type,
           UserGroupInformation.createRemoteUser("foo")));
@@ -35,11 +35,11 @@ public void testDefaults() {
 
   @Test
   public void testCustom() {
-    Configuration conf = new Configuration(false);
+    final Configuration conf = new Configuration(false);
     for (KMSACLs.Type type : KMSACLs.Type.values()) {
       conf.set(type.getAclConfigKey(), type.toString() + " ");
     }
-    KMSACLs acls = new KMSACLs(conf);
+    final KMSACLs acls = new KMSACLs(conf);
     for (KMSACLs.Type type : KMSACLs.Type.values()) {
       Assert.assertTrue(acls.hasAccess(type,
           UserGroupInformation.createRemoteUser(type.toString())));
@@ -48,4 +48,16 @@ public void testCustom() {
     }
   }
 
+  @Test
+  public void testKeyAclConfigurationLoad() {
+    final Configuration conf = new Configuration(false);
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_1.MANAGEMENT", "CREATE");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_2.ALL", "CREATE");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_3.NONEXISTOPERATION", "CREATE");
+    conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "ROLLOVER");
+    conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT", "DECRYPT_EEK");
+    final KMSACLs acls = new KMSACLs(conf);
+    Assert.assertTrue("expected key ACL size is 2 but got " + acls.keyAcls.size(),
+        acls.keyAcls.size() == 2);
+  }
 }
diff --git a/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml b/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml
index 24a46b86ec49a..6ac4155190356 100644
--- a/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml
+++ b/hadoop-common-project/hadoop-kms/src/test/resources/mini-kms-acls-default.xml
@@ -23,7 +23,7 @@
     <value>*</value>
     <description>
       ACL for create-key operations.
-      If the user does is not in the GET ACL, the key material is not returned
+      If the user is not in the GET ACL, the key material is not returned
       as part of the response.
     </description>
   </property>
@@ -41,7 +41,7 @@
     <value>*</value>
     <description>
       ACL for rollover-key operations.
-      If the user does is not in the GET ACL, the key material is not returned
+      If the user is not in the GET ACL, the key material is not returned
       as part of the response.
     </description>
   </property>
diff --git a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
index 7107b75aaef78..a649bd224e9ca 100644
--- a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
+++ b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java
@@ -79,9 +79,9 @@
 /**
  * Mini KDC based on Apache Directory Server that can be embedded in testcases
  * or used from command line as a standalone KDC.
- * <p/>
+ * <p>
  * <b>From within testcases:</b>
- * <p/>
+ * <p>
  * MiniKdc sets 2 System properties when started and un-sets them when stopped:
  * <ul>
  *   <li>java.security.krb5.conf: set to the MiniKDC real/host/port</li>
@@ -92,7 +92,7 @@
  * For example, running testcases in parallel that start a KDC each. To
  * accomplish this a single MiniKdc should be used for all testcases running
  * in parallel.
- * <p/>
+ * <p>
  * MiniKdc default configuration values are:
  * <ul>
  *   <li>org.name=EXAMPLE (used to create the REALM)</li>
@@ -106,7 +106,6 @@
  *   <li>debug=false</li>
  * </ul>
  * The generated krb5.conf forces TCP connections.
- * <p/>
  */
 public class MiniKdc {
 
@@ -218,7 +217,7 @@ public void run() {
 
   /**
    * Convenience method that returns MiniKdc default configuration.
-   * <p/>
+   * <p>
    * The returned configuration is a copy, it can be customized before using
    * it to create a MiniKdc.
    * @return a MiniKdc default configuration.
@@ -484,7 +483,6 @@ private void initKDCServer() throws Exception {
 
   /**
    * Stops the MiniKdc
-   * @throws Exception
    */
   public synchronized void stop() {
     if (kdc != null) {
diff --git a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
index c052bb1425afa..fac7f0fbd0d7e 100644
--- a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
+++ b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
@@ -37,7 +37,8 @@
 import java.util.Arrays;
 
 public class TestMiniKdc extends KerberosSecurityTestcase {
-
+  private static final boolean IBM_JAVA = System.getProperty("java.vendor")
+      .contains("IBM");
   @Test
   public void testMiniKdcStart() {
     MiniKdc kdc = getKdc();
@@ -94,15 +95,20 @@ private static String getKrb5LoginModuleName() {
     @Override
     public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
       Map<String, String> options = new HashMap<String, String>();
-      options.put("keyTab", keytab);
       options.put("principal", principal);
-      options.put("useKeyTab", "true");
-      options.put("storeKey", "true");
-      options.put("doNotPrompt", "true");
-      options.put("useTicketCache", "true");
-      options.put("renewTGT", "true");
       options.put("refreshKrb5Config", "true");
-      options.put("isInitiator", Boolean.toString(isInitiator));
+      if (IBM_JAVA) {
+        options.put("useKeytab", keytab);
+        options.put("credsType", "both");
+      } else {
+        options.put("keyTab", keytab);
+        options.put("useKeyTab", "true");
+        options.put("storeKey", "true");
+        options.put("doNotPrompt", "true");
+        options.put("useTicketCache", "true");
+        options.put("renewTGT", "true");
+        options.put("isInitiator", Boolean.toString(isInitiator));
+      }
       String ticketCache = System.getenv("KRB5CCNAME");
       if (ticketCache != null) {
         options.put("ticketCache", ticketCache);
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index e30a482edf6c3..409ed757b1043 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -82,11 +82,6 @@
       <artifactId>slf4j-log4j12</artifactId>
       <scope>runtime</scope>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
index 5e101c1de8973..889d45a444f81 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
@@ -19,6 +19,7 @@
 
 import java.util.List;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.NfsExports;
 import org.apache.hadoop.oncrpc.RpcAcceptedReply;
 import org.apache.hadoop.oncrpc.XDR;
@@ -76,7 +77,7 @@ public static XDR writeExportList(XDR xdr, int xid, List<String> exports,
       if (hostGroups.length > 0) {
         for (int j = 0; j < hostGroups.length; j++) {
           xdr.writeBoolean(true); // Value follows - yes
-          xdr.writeVariableOpaque(hostGroups[j].getBytes());
+          xdr.writeVariableOpaque(hostGroups[j].getBytes(Charsets.UTF_8));
         }
       }
       xdr.writeBoolean(false); // Value follows - no more group
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
index 6bc4686157697..415b459f68ffd 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
@@ -22,6 +22,7 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.oncrpc.XDR;
@@ -72,10 +73,8 @@ public FileHandle(String s) {
       return;
     }
 
-    byte[] in = s.getBytes();
-    for (int i = 0; i < in.length; i++) {
-      digest.update(in[i]);
-    }
+    byte[] in = s.getBytes(Charsets.UTF_8);
+    digest.update(in);
 
     byte[] digestbytes = digest.digest();
     for (int i = 0; i < 16; i++) {
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
index 473d527646310..e75ce15b52a39 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
 import org.apache.hadoop.oncrpc.XDR;
@@ -78,7 +79,7 @@ public long getVerf() {
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
     xdr.writeInt(name.length());
-    xdr.writeFixedOpaque(name.getBytes(), name.length());
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8), name.length());
     xdr.writeInt(mode);
     objAttr.serialize(xdr);
   }
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
index 2e959f59f90d3..1dcb85d3d46ef 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -56,6 +57,6 @@ public void serialize(XDR xdr) {
     handle.serialize(xdr);
     fromDirHandle.serialize(xdr);
     xdr.writeInt(fromName.length());
-    xdr.writeFixedOpaque(fromName.getBytes(), fromName.length());
+    xdr.writeFixedOpaque(fromName.getBytes(Charsets.UTF_8), fromName.length());
   }
 }
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
index 4661821a68b4c..0435483e83781 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -53,7 +54,7 @@ public void setName(String name) {
   @VisibleForTesting
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
-    xdr.writeInt(name.getBytes().length);
-    xdr.writeFixedOpaque(name.getBytes());
+    xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
index b3ef828a7ec3f..bba26eeb3015c 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -54,8 +55,8 @@ public SetAttr3 getObjAttr() {
   @Override
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
-    xdr.writeInt(name.getBytes().length);
-    xdr.writeFixedOpaque(name.getBytes());
+    xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
     objAttr.serialize(xdr);
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
index 4a13f879ea3c8..0659dd10c046e 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.NfsFileType;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes.Specdata3;
@@ -79,7 +80,7 @@ public Specdata3 getSpec() {
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
     xdr.writeInt(name.length());
-    xdr.writeFixedOpaque(name.getBytes(), name.length());
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8), name.length());
     objAttr.serialize(xdr);
     if (spec != null) {
       xdr.writeInt(spec.getSpecdata1());
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
index ffd47b0e5dcf0..9ad156d90ea12 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -46,7 +47,7 @@ public String getName() {
   @Override
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
-    xdr.writeInt(name.getBytes().length);
-    xdr.writeFixedOpaque(name.getBytes());
+    xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
index 5144e8a4910c1..c54a8e2da2d84 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -66,10 +67,10 @@ public String getToName() {
   @Override
   public void serialize(XDR xdr) {
     fromDirHandle.serialize(xdr);
-    xdr.writeInt(fromName.getBytes().length);
-    xdr.writeFixedOpaque(fromName.getBytes());
+    xdr.writeInt(fromName.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(fromName.getBytes(Charsets.UTF_8));
     toDirHandle.serialize(xdr);
-    xdr.writeInt(toName.getBytes().length);
-    xdr.writeFixedOpaque(toName.getBytes());
+    xdr.writeInt(toName.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(toName.getBytes(Charsets.UTF_8));
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
index e9977fa5488ff..6ae0de89fbf09 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -46,7 +47,7 @@ public String getName() {
   @Override
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
-    xdr.writeInt(name.getBytes().length);
-    xdr.writeFixedOpaque(name.getBytes());
+    xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
index 288079449dc69..59188e2b50e71 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.oncrpc.XDR;
 
@@ -62,10 +63,10 @@ public String getSymData() {
   @Override
   public void serialize(XDR xdr) {
     handle.serialize(xdr);
-    xdr.writeInt(name.getBytes().length);
-    xdr.writeFixedOpaque(name.getBytes());
+    xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
     symAttr.serialize(xdr);
-    xdr.writeInt(symData.getBytes().length);
-    xdr.writeFixedOpaque(symData.getBytes());
+    xdr.writeInt(symData.getBytes(Charsets.UTF_8).length);
+    xdr.writeFixedOpaque(symData.getBytes(Charsets.UTF_8));
   }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
index 2fdabe2fda789..474559ea92fcb 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 
+import org.apache.commons.io.Charsets;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 
@@ -165,11 +166,11 @@ public void writeVariableOpaque(byte[] src) {
   }
 
   public String readString() {
-    return new String(readVariableOpaque());
+    return new String(readVariableOpaque(), Charsets.UTF_8);
   }
 
   public void writeString(String s) {
-    writeVariableOpaque(s.getBytes());
+    writeVariableOpaque(s.getBytes(Charsets.UTF_8));
   }
 
   private void writePadding() {
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
index 997ce35c60d09..d13c3615b2c98 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
@@ -20,6 +20,7 @@
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.oncrpc.XDR;
 
 /** Credential used by AUTH_SYS */
@@ -93,7 +94,7 @@ public void read(XDR xdr) {
   @Override
   public void write(XDR xdr) {
     // mStamp + mHostName.length + mHostName + mUID + mGID + mAuxGIDs.count
-    mCredentialsLength = 20 + mHostName.getBytes().length;
+    mCredentialsLength = 20 + mHostName.getBytes(Charsets.UTF_8).length;
     // mAuxGIDs
     if (mAuxGIDs != null && mAuxGIDs.length > 0) {
       mCredentialsLength += mAuxGIDs.length * 4;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 0bb6d4b5dd441..4c42ef9952e0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -319,7 +319,7 @@
             <goals>
               <goal>dependencies</goal>
             </goals>
-            <phase>site</phase>
+            <phase>package</phase>
           </execution>
         </executions>
       </plugin>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 367308d421ce0..5b079e940509c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -78,7 +78,7 @@
 
 /**
  * HttpFSServer implementation of the FileSystemAccess FileSystem.
- * <p/>
+ * <p>
  * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
  */
 @InterfaceAudience.Private
@@ -223,7 +223,7 @@ public String getMethod() {
   /**
    * Convenience method that creates a <code>HttpURLConnection</code> for the
    * HttpFSServer file system operations.
-   * <p/>
+   * <p>
    * This methods performs and injects any needed authentication credentials
    * via the {@link #getConnection(URL, String)} method
    *
@@ -289,7 +289,7 @@ public HttpURLConnection run() throws Exception {
 
   /**
    * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
-   * <p/>
+   * <p>
    * This methods performs and injects any needed authentication credentials.
    *
    * @param url url to connect to.
@@ -371,7 +371,7 @@ protected int getDefaultPort() {
 
   /**
    * HttpFSServer subclass of the <code>FSDataInputStream</code>.
-   * <p/>
+   * <p>
    * This implementation does not support the
    * <code>PositionReadable</code> and <code>Seekable</code> methods.
    */
@@ -414,8 +414,8 @@ public boolean seekToNewSource(long targetPos) throws IOException {
 
   /**
    * Opens an FSDataInputStream at the indicated Path.
-   * </p>
-   * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the
+   * <p>
+   * IMPORTANT: the returned <code>FSDataInputStream</code> does not support the
    * <code>PositionReadable</code> and <code>Seekable</code> methods.
    *
    * @param f the file name to open
@@ -434,7 +434,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
 
   /**
    * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
-   * <p/>
+   * <p>
    * This implementation closes the underlying HTTP connection validating the Http connection status
    * at closing time.
    */
@@ -516,7 +516,7 @@ private FSDataOutputStream uploadData(String method, Path f, Map<String, String>
   /**
    * Opens an FSDataOutputStream at the indicated Path with write-progress
    * reporting.
-   * <p/>
+   * <p>
    * IMPORTANT: The <code>Progressable</code> parameter is not used.
    *
    * @param f the file name to open.
@@ -549,7 +549,7 @@ public FSDataOutputStream create(Path f, FsPermission permission,
 
   /**
    * Append to an existing file (optional operation).
-   * <p/>
+   * <p>
    * IMPORTANT: The <code>Progressable</code> parameter is not used.
    *
    * @param f the existing file to be appended.
@@ -838,7 +838,7 @@ public boolean setReplication(Path src, short replication)
    * Modify the ACL entries for a file.
    *
    * @param path Path to modify
-   * @param aclSpec List<AclEntry> describing modifications
+   * @param aclSpec describing modifications
    * @throws IOException
    */
   @Override
@@ -855,7 +855,7 @@ public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
   /**
    * Remove the specified ACL entries from a file
    * @param path Path to modify
-   * @param aclSpec List<AclEntry> describing entries to remove
+   * @param aclSpec describing entries to remove
    * @throws IOException
    */
   @Override
@@ -900,7 +900,7 @@ public void removeAcl(Path path) throws IOException {
   /**
    * Set the ACLs for the given file
    * @param path Path to modify
-   * @param aclSpec List<AclEntry> describing modifications, must include
+   * @param aclSpec describing modifications, must include
    *                entries for user, group, and others for compatibility
    *                with permission bits.
    * @throws IOException
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
index e1cb81b77b712..45d9805d91d03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs.http.client;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.json.simple.parser.JSONParser;
@@ -127,7 +128,7 @@ static URL createURL(Path path, Map<String, String> params, Map<String,
   static Object jsonParse(HttpURLConnection conn) throws IOException {
     try {
       JSONParser parser = new JSONParser();
-      return parser.parse(new InputStreamReader(conn.getInputStream()));
+      return parser.parse(new InputStreamReader(conn.getInputStream(), Charsets.UTF_8));
     } catch (ParseException ex) {
       throw new IOException("JSON parser error, " + ex.getMessage(), ex);
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
index a696cd8c119ea..0a2e08dad0493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
@@ -18,10 +18,10 @@
 package org.apache.hadoop.fs.http.client;
 
 /**
- * HttpFSServer implementation of the FileSystemAccess FileSystem for SSL.
- * <p/>
- * This implementation allows a user to access HDFS over HTTPS via a
- * HttpFSServer server.
+ * <p>HttpFSServer implementation of the FileSystemAccess FileSystem for SSL.
+ * </p>
+ * <p>This implementation allows a user to access HDFS over HTTPS via a
+ * HttpFSServer server.</p>
  */
 public class HttpsFSFileSystem extends HttpFSFileSystem {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
index 67df9a8e642e3..836b4ce9ffd87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
@@ -50,7 +50,7 @@ public class CheckUploadContentTypeFilter implements Filter {
 
   /**
    * Initializes the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    *
    * @param config filter configuration.
@@ -103,7 +103,7 @@ public void doFilter(ServletRequest request, ServletResponse response,
 
   /**
    * Destroys the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    */
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
index 8b332fc6e9abf..004ff388c1ebd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs.http.server;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -24,8 +25,10 @@
 
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
-import java.io.FileReader;
+
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.util.Map;
 import java.util.Properties;
@@ -44,7 +47,7 @@ public class HttpFSAuthenticationFilter
 
   /**
    * Returns the hadoop-auth configuration from HttpFSServer's configuration.
-   * <p/>
+   * <p>
    * It returns all HttpFSServer's configuration properties prefixed with
    * <code>httpfs.authentication</code>. The <code>httpfs.authentication</code>
    * prefix is removed from the returned property names.
@@ -77,7 +80,8 @@ protected Properties getConfiguration(String configPrefix,
 
     try {
       StringBuilder secret = new StringBuilder();
-      Reader reader = new FileReader(signatureSecretFile);
+      Reader reader = new InputStreamReader(new FileInputStream(
+          signatureSecretFile), Charsets.UTF_8);
       int c = reader.read();
       while (c > -1) {
         secret.append((char)c);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
index 3a8d9ada4e688..aed634312349b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
@@ -43,7 +43,6 @@ public class HttpFSExceptionProvider extends ExceptionProvider {
 
   /**
    * Maps different exceptions thrown by HttpFSServer to HTTP status codes.
-   * <p/>
    * <ul>
    * <li>SecurityException : HTTP UNAUTHORIZED</li>
    * <li>FileNotFoundException : HTTP NOT_FOUND</li>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index f9eb454d9a5ff..910371834617e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -85,7 +85,7 @@
 
 /**
  * Main class of HttpFSServer server.
- * <p/>
+ * <p>
  * The <code>HttpFSServer</code> class uses Jersey JAX-RS to binds HTTP requests to the
  * different operations.
  */
@@ -117,7 +117,7 @@ private <T> T fsExecute(UserGroupInformation ugi, FileSystemAccess.FileSystemExe
   /**
    * Returns a filesystem instance. The fileystem instance is wired for release at the completion of
    * the current Servlet request via the {@link FileSystemReleaseFilter}.
-   * <p/>
+   * <p>
    * If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
    * <code>AccessControlException</code> will be thrown.
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
index b7ae3015e3b3f..66438b5f4ab41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
@@ -34,9 +34,9 @@
  * HttpFSServer server, it is a <code>javax.servlet.ServletContextListener
  * </code> implementation that is wired in HttpFSServer's WAR
  * <code>WEB-INF/web.xml</code>.
- * <p/>
+ * <p>
  * It provides acces to the server context via the singleton {@link #get}.
- * <p/>
+ * <p>
  * All the configuration is loaded from configuration properties prefixed
  * with <code>httpfs.</code>.
  */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
index f974159c63867..467ca235653bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
@@ -61,7 +61,7 @@ private XException(ERROR error, String message, Throwable cause) {
 
   /**
    * Creates an XException using another XException as cause.
-   * <p/>
+   * <p>
    * The error code and error message are extracted from the cause.
    *
    * @param cause exception cause.
@@ -95,7 +95,7 @@ public ERROR getError() {
 
   /**
    * Creates a message using a error message template and arguments.
-   * <p/>
+   * <p>
    * The template must be in JDK <code>MessageFormat</code> syntax
    * (using {#} positional parameters).
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
index 088f90058dd24..9d9ce7a1115bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
@@ -44,11 +44,11 @@ public BaseService(String prefix) {
 
   /**
    * Initializes the service.
-   * <p/>
+   * <p>
    * It collects all service properties (properties having the
    * <code>#SERVER#.#SERVICE#.</code> prefix). The property names are then
    * trimmed from the <code>#SERVER#.#SERVICE#.</code> prefix.
-   * <p/>
+   * <p>
    * After collecting  the service properties it delegates to the
    * {@link #init()} method.
    *
@@ -75,7 +75,7 @@ public final void init(Server server) throws ServiceException {
   /**
    * Post initializes the service. This method is called by the
    * {@link Server} after all services of the server have been initialized.
-   * <p/>
+   * <p>
    * This method does a NOP.
    *
    * @throws ServiceException thrown if the service could not be
@@ -88,7 +88,7 @@ public void postInit() throws ServiceException {
   /**
    * Destroy the services.  This method is called once, when the
    * {@link Server} owning the service is being destroyed.
-   * <p/>
+   * <p>
    * This method does a NOP.
    */
   @Override
@@ -98,7 +98,7 @@ public void destroy() {
   /**
    * Returns the service dependencies of this service. The service will be
    * instantiated only if all the service dependencies are already initialized.
-   * <p/>
+   * <p>
    * This method returns an empty array (size 0)
    *
    * @return an empty array (size 0).
@@ -110,7 +110,7 @@ public Class[] getServiceDependencies() {
 
   /**
    * Notification callback when the server changes its status.
-   * <p/>
+   * <p>
    * This method returns an empty array (size 0)
    *
    * @param oldStatus old server status.
@@ -154,7 +154,7 @@ protected String getPrefixedName(String name) {
   /**
    * Returns the service configuration properties. Property
    * names are trimmed off from its prefix.
-   * <p/>
+   * <p>
    * The sevice configuration properties are all properties
    * with names starting with <code>#SERVER#.#SERVICE#.</code>
    * in the server configuration.
@@ -169,7 +169,7 @@ protected Configuration getServiceConfig() {
 
   /**
    * Initializes the server.
-   * <p/>
+   * <p>
    * This method is called by {@link #init(Server)} after all service properties
    * (properties prefixed with
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
index d083831c58a21..5c1bb4f7278bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
@@ -42,40 +42,39 @@
 /**
  * A Server class provides standard configuration, logging and {@link Service}
  * lifecyle management.
- * <p/>
+ * <p>
  * A Server normally has a home directory, a configuration directory, a temp
  * directory and logs directory.
- * <p/>
+ * <p>
  * The Server configuration is loaded from 2 overlapped files,
  * <code>#SERVER#-default.xml</code> and <code>#SERVER#-site.xml</code>. The
  * default file is loaded from the classpath, the site file is laoded from the
  * configuration directory.
- * <p/>
+ * <p>
  * The Server collects all configuration properties prefixed with
  * <code>#SERVER#</code>. The property names are then trimmed from the
  * <code>#SERVER#</code> prefix.
- * <p/>
+ * <p>
  * The Server log configuration is loaded from the
  * <code>#SERVICE#-log4j.properties</code> file in the configuration directory.
- * <p/>
+ * <p>
  * The lifecycle of server is defined in by {@link Server.Status} enum.
  * When a server is create, its status is UNDEF, when being initialized it is
  * BOOTING, once initialization is complete by default transitions to NORMAL.
  * The <code>#SERVER#.startup.status</code> configuration property can be used
  * to specify a different startup status (NORMAL, ADMIN or HALTED).
- * <p/>
+ * <p>
  * Services classes are defined in the <code>#SERVER#.services</code> and
  * <code>#SERVER#.services.ext</code> properties. They are loaded in order
  * (services first, then services.ext).
- * <p/>
+ * <p>
  * Before initializing the services, they are traversed and duplicate service
  * interface are removed from the service list. The last service using a given
  * interface wins (this enables a simple override mechanism).
- * <p/>
+ * <p>
  * After the services have been resoloved by interface de-duplication they are
  * initialized in order. Once all services are initialized they are
  * post-initialized (this enables late/conditional service bindings).
- * <p/>
  */
 @InterfaceAudience.Private
 public class Server {
@@ -152,7 +151,7 @@ public boolean isOperational() {
 
   /**
    * Creates a server instance.
-   * <p/>
+   * <p>
    * The config, log and temp directories are all under the specified home directory.
    *
    * @param name server name.
@@ -177,9 +176,9 @@ public Server(String name, String homeDir, String configDir, String logDir, Stri
 
   /**
    * Creates a server instance.
-   * <p/>
+   * <p>
    * The config, log and temp directories are all under the specified home directory.
-   * <p/>
+   * <p>
    * It uses the provided configuration instead loading it from the config dir.
    *
    * @param name server name.
@@ -192,7 +191,7 @@ public Server(String name, String homeDir, Configuration config) {
 
   /**
    * Creates a server instance.
-   * <p/>
+   * <p>
    * It uses the provided configuration instead loading it from the config dir.
    *
    * @param name server name.
@@ -250,9 +249,9 @@ public Status getStatus() {
 
   /**
    * Sets a new server status.
-   * <p/>
+   * <p>
    * The status must be settable.
-   * <p/>
+   * <p>
    * All services will be notified o the status change via the
    * {@link Service#serverStatusChange(Server.Status, Server.Status)} method. If a service
    * throws an exception during the notification, the server will be destroyed.
@@ -299,7 +298,7 @@ protected void ensureOperational() {
   /**
    * Convenience method that returns a resource as inputstream from the
    * classpath.
-   * <p/>
+   * <p>
    * It first attempts to use the Thread's context classloader and if not
    * set it uses the <code>ClassUtils</code> classloader.
    *
@@ -319,7 +318,7 @@ static InputStream getResource(String name) {
 
   /**
    * Initializes the Server.
-   * <p/>
+   * <p>
    * The initialization steps are:
    * <ul>
    * <li>It verifies the service home and temp directories exist</li>
@@ -335,6 +334,7 @@ static InputStream getResource(String name) {
    * <li>Initializes the services</li>
    * <li>Post-initializes the services</li>
    * <li>Sets the server startup status</li>
+   * </ul>
    *
    * @throws ServerException thrown if the server could not be initialized.
    */
@@ -625,7 +625,7 @@ protected void destroyServices() {
 
   /**
    * Destroys the server.
-   * <p/>
+   * <p>
    * All services are destroyed in reverse order of initialization, then the
    * Log4j framework is shutdown.
    */
@@ -651,7 +651,7 @@ public String getName() {
 
   /**
    * Returns the server prefix for server configuration properties.
-   * <p/>
+   * <p>
    * By default it is the server name.
    *
    * @return the prefix for server configuration properties.
@@ -733,10 +733,10 @@ public <T> T get(Class<T> serviceKlass) {
 
   /**
    * Adds a service programmatically.
-   * <p/>
+   * <p>
    * If a service with the same interface exists, it will be destroyed and
    * removed before the given one is initialized and added.
-   * <p/>
+   * <p>
    * If an exception is thrown the server is destroyed.
    *
    * @param klass service class to add.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java
index 6091415e116aa..ccb15a30bab6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/hadoop/FileSystemAccessService.java
@@ -177,7 +177,7 @@ protected void init() throws ServiceException {
 
     String hadoopConfDirProp = getServiceConfig().get(HADOOP_CONF_DIR, getServer().getConfigDir());
     File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile();
-    if (hadoopConfDir == null) {
+    if (!hadoopConfDir.exists()) {
       hadoopConfDir = new File(getServer().getConfigDir()).getAbsoluteFile();
     }
     if (!hadoopConfDir.exists()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
index 827bcff891657..cf73979f6476d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
@@ -33,7 +33,7 @@
 /**
  * The <code>FileSystemReleaseFilter</code> releases back to the
  * {@link FileSystemAccess} service a <code>FileSystem</code> instance.
- * <p/>
+ * <p>
  * This filter is useful in situations where a servlet request
  * is streaming out HDFS data and the corresponding filesystem
  * instance have to be closed after the streaming completes.
@@ -44,7 +44,7 @@ public abstract class FileSystemReleaseFilter implements Filter {
 
   /**
    * Initializes the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    *
    * @param filterConfig filter configuration.
@@ -83,7 +83,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo
 
   /**
    * Destroys the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    */
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
index dd395f6749549..64f4926f5af1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
@@ -43,7 +43,7 @@ public class HostnameFilter implements Filter {
 
   /**
    * Initializes the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    *
    * @param config filter configuration.
@@ -56,7 +56,7 @@ public void init(FilterConfig config) throws ServletException {
 
   /**
    * Resolves the requester hostname and delegates the request to the chain.
-   * <p/>
+   * <p>
    * The requester hostname is available via the {@link #get} method.
    *
    * @param request servlet request.
@@ -101,7 +101,7 @@ public static String get() {
 
   /**
    * Destroys the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    */
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
index 07b552d7ec510..156cf64ab00cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
@@ -33,7 +33,7 @@
 
 /**
  * Filter that sets request contextual information for the slf4j MDC.
- * <p/>
+ * <p>
  * It sets the following values:
  * <ul>
  * <li>hostname: if the {@link HostnameFilter} is present and configured
@@ -48,7 +48,7 @@ public class MDCFilter implements Filter {
 
   /**
    * Initializes the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    *
    * @param config filter configuration.
@@ -93,7 +93,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha
 
   /**
    * Destroys the filter.
-   * <p/>
+   * <p>
    * This implementation is a NOP.
    */
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
index 9b0ea2a11e1cc..cd1659383bbef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
@@ -75,21 +75,21 @@ protected ServerWebApp(String name, String homeDir, Configuration config) {
   /**
    * Constructor. Subclasses must have a default constructor specifying
    * the server name.
-   * <p/>
+   * <p>
    * The server name is used to resolve the Java System properties that define
    * the server home, config, log and temp directories.
-   * <p/>
+   * <p>
    * The home directory is looked in the Java System property
    * <code>#SERVER_NAME#.home.dir</code>.
-   * <p/>
+   * <p>
    * The config directory is looked in the Java System property
    * <code>#SERVER_NAME#.config.dir</code>, if not defined it resolves to
    * the <code>#SERVER_HOME_DIR#/conf</code> directory.
-   * <p/>
+   * <p>
    * The log directory is looked in the Java System property
    * <code>#SERVER_NAME#.log.dir</code>, if not defined it resolves to
    * the <code>#SERVER_HOME_DIR#/log</code> directory.
-   * <p/>
+   * <p>
    * The temp directory is looked in the Java System property
    * <code>#SERVER_NAME#.temp.dir</code>, if not defined it resolves to
    * the <code>#SERVER_HOME_DIR#/temp</code> directory.
@@ -105,7 +105,7 @@ public ServerWebApp(String name) {
 
   /**
    * Returns the server home directory.
-   * <p/>
+   * <p>
    * It is looked up in the Java System property
    * <code>#SERVER_NAME#.home.dir</code>.
    *
@@ -159,15 +159,15 @@ public void contextInitialized(ServletContextEvent event) {
   }
 
   /**
-   * Resolves the host & port InetSocketAddress the web server is listening to.
-   * <p/>
+   * Resolves the host and port InetSocketAddress the web server is listening to.
+   * <p>
    * This implementation looks for the following 2 properties:
    * <ul>
    *   <li>#SERVER_NAME#.http.hostname</li>
    *   <li>#SERVER_NAME#.http.port</li>
    * </ul>
    *
-   * @return the host & port InetSocketAddress the web server is listening to.
+   * @return the host and port InetSocketAddress the web server is listening to.
    * @throws ServerException thrown if any of the above 2 properties is not defined.
    */
   protected InetSocketAddress resolveAuthority() throws ServerException {
@@ -217,7 +217,7 @@ public InetSocketAddress getAuthority() throws ServerException {
 
   /**
    * Sets an alternate hostname:port InetSocketAddress to use.
-   * <p/>
+   * <p>
    * For testing purposes.
    * 
    * @param authority alterante authority.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
index a398e75845466..31666e83e2711 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
@@ -26,7 +26,7 @@
 
 /**
  * Utility methods to check preconditions.
- * <p/>
+ * <p>
  * Commonly used for method arguments preconditions.
  */
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
index 660eae0835820..6611dd22fefc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
@@ -90,7 +90,7 @@ public static Configuration resolve(Configuration conf) {
 
   /**
    * Create a configuration from an InputStream.
-   * <p/>
+   * <p>
    * ERROR canibalized from <code>Configuration.loadResource()</code>.
    *
    * @param is inputstream to read the configuration from.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONMapProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONMapProvider.java
index 12a2fc680bd6d..0809a85705ca4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONMapProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONMapProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.lib.wsrs;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.json.simple.JSONObject;
 
@@ -55,7 +56,7 @@ public long getSize(Map map, Class<?> aClass, Type type, Annotation[] annotation
   public void writeTo(Map map, Class<?> aClass, Type type, Annotation[] annotations,
                       MediaType mediaType, MultivaluedMap<String, Object> stringObjectMultivaluedMap,
                       OutputStream outputStream) throws IOException, WebApplicationException {
-    Writer writer = new OutputStreamWriter(outputStream);
+    Writer writer = new OutputStreamWriter(outputStream,  Charsets.UTF_8);
     JSONObject.writeJSONString(map, writer);
     writer.write(ENTER);
     writer.flush();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONProvider.java
index 692643eec2f04..15ea8e6bc8bbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/JSONProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.lib.wsrs;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.json.simple.JSONStreamAware;
 
@@ -55,7 +56,7 @@ public long getSize(JSONStreamAware jsonStreamAware, Class<?> aClass, Type type,
   public void writeTo(JSONStreamAware jsonStreamAware, Class<?> aClass, Type type, Annotation[] annotations,
                       MediaType mediaType, MultivaluedMap<String, Object> stringObjectMultivaluedMap,
                       OutputStream outputStream) throws IOException, WebApplicationException {
-    Writer writer = new OutputStreamWriter(outputStream);
+    Writer writer = new OutputStreamWriter(outputStream, Charsets.UTF_8);
     jsonStreamAware.writeJSONString(writer);
     writer.write(ENTER);
     writer.flush();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
index 0f16a9b53e504..e0f62002c70d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
@@ -26,7 +26,7 @@
 
 /**
  * Class that contains all parsed JAX-RS parameters.
- * <p/>
+ * <p>
  * Instances are created by the {@link ParametersProvider} class.
  */
 @InterfaceAudience.Private
@@ -63,7 +63,7 @@ public <V, T extends Param<V>> V get(String name, Class<T> klass) {
    *
    * @param name parameter name.
    * @param klass class of the parameter, used for value casting.
-   * @return List<V> the values of the parameter.
+   * @return the values of the parameter.
    */
   @SuppressWarnings("unchecked")
   public <V, T extends Param<V>> List<V> getValues(String name, Class<T> klass) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
index 2920cd93f8196..f51e74349ed55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
@@ -64,7 +64,7 @@ Hadoop HDFS over HTTP - Documentation Sets ${project.version}
 
   HttpFS was inspired by Hadoop HDFS proxy.
 
-  HttpFS can be seening as a full rewrite of Hadoop HDFS proxy.
+  HttpFS can be seen as a full rewrite of Hadoop HDFS proxy.
 
   Hadoop HDFS proxy provides a subset of file system operations (read only),
   HttpFS provides support for all file system operations.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
index 7566791b06358..9e4aaf538f0ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
@@ -70,4 +70,7 @@ public class NfsConfigKeys {
   public static final int NFS_HTTPS_PORT_DEFAULT = 50579;
   public static final String NFS_HTTPS_ADDRESS_KEY = "nfs.https.address";
   public static final String NFS_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + NFS_HTTPS_PORT_DEFAULT;
+  
+  public static final String  NFS_METRICS_PERCENTILES_INTERVALS_KEY = "nfs.metrics.percentiles.intervals";
+  public static final String  NFS_METRICS_PERCENTILES_INTERVALS_DEFAULT = "";
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java
index 429a457623b0a..ee3f90aa64456 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java
@@ -22,12 +22,11 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 /**
- * This class is a thread pool to easily schedule async data operations.Current
+ * This class is a thread pool to easily schedule async data operations. Current
  * async data operation is write back operation. In the future, we could use it
  * for readahead operations too.
  */
@@ -69,8 +68,8 @@ synchronized void execute(Runnable task) {
     }
     if (LOG.isDebugEnabled()) {
       LOG.debug("Current active thread number: " + executor.getActiveCount()
-          + " queue size:" + executor.getQueue().size()
-          + " scheduled task number:" + executor.getTaskCount());
+          + " queue size: " + executor.getQueue().size()
+          + " scheduled task number: " + executor.getTaskCount());
     }
     executor.execute(task);
   }
@@ -105,10 +104,9 @@ void writeAsync(OpenFileCtx openFileCtx) {
   }
 
   /**
-   * A task for write data back to HDFS for a file. Since only one thread can
-   * write for a file, any time there should be only one task(in queue or
-   * executing) for one file existing, and this should be guaranteed by the
-   * caller.
+   * A task to write data back to HDFS for a file. Since only one thread can
+   * write to a file, there should only be one task at any time for a file
+   * (in queue or executing), and this should be guaranteed by the caller.
    */
   static class WriteBackTask implements Runnable {
 
@@ -135,7 +133,7 @@ public void run() {
       try {
         openFileCtx.executeWriteBack();
       } catch (Throwable t) {
-        LOG.error("Asyn data service got error:", t);
+        LOG.error("Async data service got error: ", t);
       }
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
index 3daf7bb68db9b..ac9abf8b02b00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
@@ -42,7 +42,8 @@ public Nfs3(NfsConfiguration conf) throws IOException {
   
   public Nfs3(NfsConfiguration conf, DatagramSocket registrationSocket,
       boolean allowInsecurePorts) throws IOException {
-    super(new RpcProgramNfs3(conf, registrationSocket, allowInsecurePorts), conf);
+    super(RpcProgramNfs3.createRpcProgramNfs3(conf, registrationSocket,
+        allowInsecurePorts), conf);
     mountd = new Mountd(conf, registrationSocket, allowInsecurePorts);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java
new file mode 100644
index 0000000000000..d36ea732f0f2d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.nfs.nfs3;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+
+/**
+ * This class is for maintaining the various NFS gateway activity statistics and
+ * publishing them through the metrics interfaces.
+ */
+@InterfaceAudience.Private
+@Metrics(about = "Nfs3 metrics", context = "dfs")
+public class Nfs3Metrics {
+  // All mutable rates are in nanoseconds
+  // No metric for nullProcedure;
+  @Metric MutableRate getattr;
+  @Metric MutableRate setattr;
+  @Metric MutableRate lookup;
+  @Metric MutableRate access;
+  @Metric MutableRate readlink;
+  @Metric MutableRate read;
+  final MutableQuantiles[] readNanosQuantiles;
+  @Metric MutableRate write;
+  final MutableQuantiles[] writeNanosQuantiles;
+  @Metric MutableRate create;
+  @Metric MutableRate mkdir;
+  @Metric MutableRate symlink;
+  @Metric MutableRate mknod;
+  @Metric MutableRate remove;
+  @Metric MutableRate rmdir;
+  @Metric MutableRate rename;
+  @Metric MutableRate link;
+  @Metric MutableRate readdir;
+  @Metric MutableRate readdirplus;
+  @Metric MutableRate fsstat;
+  @Metric MutableRate fsinfo;
+  @Metric MutableRate pathconf;
+  @Metric MutableRate commit;
+  final MutableQuantiles[] commitNanosQuantiles;
+
+  @Metric MutableCounterLong bytesWritten;
+  @Metric MutableCounterLong bytesRead;
+
+  final MetricsRegistry registry = new MetricsRegistry("nfs3");
+  final String name;
+  JvmMetrics jvmMetrics = null;
+
+  public Nfs3Metrics(String name, String sessionId, int[] intervals,
+      final JvmMetrics jvmMetrics) {
+    this.name = name;
+    this.jvmMetrics = jvmMetrics;
+    registry.tag(SessionId, sessionId);
+
+    final int len = intervals.length;
+    readNanosQuantiles = new MutableQuantiles[len];
+    writeNanosQuantiles = new MutableQuantiles[len];
+    commitNanosQuantiles = new MutableQuantiles[len];
+
+    for (int i = 0; i < len; i++) {
+      int interval = intervals[i];
+      readNanosQuantiles[i] = registry.newQuantiles("readProcessNanos"
+          + interval + "s", "Read process in ns", "ops", "latency", interval);
+      writeNanosQuantiles[i] = registry.newQuantiles("writeProcessNanos"
+          + interval + "s", " process in ns", "ops", "latency", interval);
+      commitNanosQuantiles[i] = registry.newQuantiles("commitProcessNanos"
+          + interval + "s", "Read process in ns", "ops", "latency", interval);
+    }
+  }
+
+  public static Nfs3Metrics create(Configuration conf, String gatewayName) {
+    String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    JvmMetrics jm = JvmMetrics.create(gatewayName, sessionId, ms);
+
+    // Percentile measurement is [,,,] by default 
+    int[] intervals = conf.getInts(conf.get(
+        NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_KEY,
+        NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_DEFAULT));
+    return ms.register(new Nfs3Metrics(gatewayName, sessionId, intervals, jm));
+  }
+  
+  public String name() {
+    return name;
+  }
+
+  public JvmMetrics getJvmMetrics() {
+    return jvmMetrics;
+  }
+
+  public void incrBytesWritten(long bytes) {
+    bytesWritten.incr(bytes);
+  }
+
+  public void incrBytesRead(long bytes) {
+    bytesRead.incr(bytes);
+  }
+
+  public void addGetattr(long latencyNanos) {
+    getattr.add(latencyNanos);
+  }
+
+  public void addSetattr(long latencyNanos) {
+    setattr.add(latencyNanos);
+  }
+
+  public void addLookup(long latencyNanos) {
+    lookup.add(latencyNanos);
+  }
+
+  public void addAccess(long latencyNanos) {
+    access.add(latencyNanos);
+  }
+
+  public void addReadlink(long latencyNanos) {
+    readlink.add(latencyNanos);
+  }
+
+  public void addRead(long latencyNanos) {
+    read.add(latencyNanos);
+    for (MutableQuantiles q : readNanosQuantiles) {
+      q.add(latencyNanos);
+    }
+  }
+
+  public void addWrite(long latencyNanos) {
+    write.add(latencyNanos);
+    for (MutableQuantiles q : writeNanosQuantiles) {
+      q.add(latencyNanos);
+    }
+  }
+
+  public void addCreate(long latencyNanos) {
+    create.add(latencyNanos);
+  }
+
+  public void addMkdir(long latencyNanos) {
+    mkdir.add(latencyNanos);
+  }
+
+  public void addSymlink(long latencyNanos) {
+    symlink.add(latencyNanos);
+  }
+
+  public void addMknod(long latencyNanos) {
+    mknod.add(latencyNanos);
+  }
+
+  public void addRemove(long latencyNanos) {
+    remove.add(latencyNanos);
+  }
+
+  public void addRmdir(long latencyNanos) {
+    rmdir.add(latencyNanos);
+  }
+
+  public void addRename(long latencyNanos) {
+    rename.add(latencyNanos);
+  }
+
+  public void addLink(long latencyNanos) {
+    link.add(latencyNanos);
+  }
+
+  public void addReaddir(long latencyNanos) {
+    readdir.add(latencyNanos);
+  }
+
+  public void addReaddirplus(long latencyNanos) {
+    readdirplus.add(latencyNanos);
+  }
+
+  public void addFsstat(long latencyNanos) {
+    fsstat.add(latencyNanos);
+  }
+
+  public void addFsinfo(long latencyNanos) {
+    fsinfo.add(latencyNanos);
+  }
+
+  public void addPathconf(long latencyNanos) {
+    pathconf.add(latencyNanos);
+  }
+
+  public void addCommit(long latencyNanos) {
+    commit.add(latencyNanos);
+    for (MutableQuantiles q : commitNanosQuantiles) {
+      q.add(latencyNanos);
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
index 50e83ed4faeb7..cc17394197a5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
@@ -213,4 +213,8 @@ public static byte[] longToByte(long v) {
     data[7] = (byte) (v >>> 0);
     return data;
   }
+  
+  public static long getElapsedTime(long startTimeNano) {
+    return System.nanoTime() - startTimeNano;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
index b31baf58f5d03..9610f48d84e6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
@@ -129,9 +129,8 @@ static class CommitCtx {
     private final Channel channel;
     private final int xid;
     private final Nfs3FileAttributes preOpAttr;
-
-    // Remember time for debug purpose
-    private final long startTime;
+    
+    public final long startTime;
 
     long getOffset() {
       return offset;
@@ -159,7 +158,7 @@ long getStartTime() {
       this.channel = channel;
       this.xid = xid;
       this.preOpAttr = preOpAttr;
-      this.startTime = Time.monotonicNow();
+      this.startTime = System.nanoTime();
     }
 
     @Override
@@ -212,7 +211,7 @@ boolean hasPendingWork() {
   private long updateNonSequentialWriteInMemory(long count) {
     long newValue = nonSequentialWriteInMemory.addAndGet(count);
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Update nonSequentialWriteInMemory by " + count + " new value:"
+      LOG.debug("Update nonSequentialWriteInMemory by " + count + " new value: "
           + newValue);
     }
 
@@ -313,7 +312,7 @@ class Dumper implements Runnable {
     private void dump() {
       // Create dump outputstream for the first time
       if (dumpOut == null) {
-        LOG.info("Create dump file:" + dumpFilePath);
+        LOG.info("Create dump file: " + dumpFilePath);
         File dumpFile = new File(dumpFilePath);
         try {
           synchronized (this) {
@@ -368,8 +367,8 @@ private void dump() {
             updateNonSequentialWriteInMemory(-dumpedDataSize);
           }
         } catch (IOException e) {
-          LOG.error("Dump data failed:" + writeCtx + " with error:" + e
-              + " OpenFileCtx state:" + activeState);
+          LOG.error("Dump data failed: " + writeCtx + " with error: " + e
+              + " OpenFileCtx state: " + activeState);
           // Disable dump
           enabledDump = false;
           return;
@@ -429,8 +428,8 @@ private WriteCtx checkRepeatedWriteRequest(WRITE3Request request,
       return null;
     } else {
       if (xid != writeCtx.getXid()) {
-        LOG.warn("Got a repeated request, same range, with a different xid:"
-            + xid + " xid in old request:" + writeCtx.getXid());
+        LOG.warn("Got a repeated request, same range, with a different xid: "
+            + xid + " xid in old request: " + writeCtx.getXid());
         //TODO: better handling.
       }
       return writeCtx;  
@@ -442,7 +441,7 @@ public void receivedNewWrite(DFSClient dfsClient, WRITE3Request request,
       IdMappingServiceProvider iug) {
     
     if (!activeState) {
-      LOG.info("OpenFileCtx is inactive, fileId:"
+      LOG.info("OpenFileCtx is inactive, fileId: "
           + request.getHandle().getFileId());
       WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
       WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3ERR_IO,
@@ -524,7 +523,7 @@ private synchronized WriteCtx addWritesToCache(WRITE3Request request,
     int originalCount = WriteCtx.INVALID_ORIGINAL_COUNT;
     
     if (LOG.isDebugEnabled()) {
-      LOG.debug("requesed offset=" + offset + " and current offset="
+      LOG.debug("requested offset=" + offset + " and current offset="
           + cachedOffset);
     }
 
@@ -557,7 +556,7 @@ private synchronized WriteCtx addWritesToCache(WRITE3Request request,
     
     // Fail non-append call
     if (offset < cachedOffset) {
-      LOG.warn("(offset,count,nextOffset):" + "(" + offset + "," + count + ","
+      LOG.warn("(offset,count,nextOffset): " + "(" + offset + "," + count + ","
           + nextOffset + ")");
       return null;
     } else {
@@ -569,7 +568,7 @@ private synchronized WriteCtx addWritesToCache(WRITE3Request request,
           dataState);
       if (LOG.isDebugEnabled()) {
         LOG.debug("Add new write to the list with nextOffset " + cachedOffset
-            + " and requesed offset=" + offset);
+            + " and requested offset=" + offset);
       }
       if (writeCtx.getDataState() == WriteCtx.DataState.ALLOW_DUMP) {
         // update the memory size
@@ -585,7 +584,7 @@ private synchronized WriteCtx addWritesToCache(WRITE3Request request,
               + pendingWrites.size());
         }
       } else {
-        LOG.warn("Got a repeated request, same range, with xid:" + xid
+        LOG.warn("Got a repeated request, same range, with xid: " + xid
             + " nextOffset " + +cachedOffset + " req offset=" + offset);
       }
       return writeCtx;
@@ -663,7 +662,7 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
       // offset < nextOffset
       processOverWrite(dfsClient, request, channel, xid, iug);
     } else {
-      // The writes is added to pendingWrites.
+      // The write is added to pendingWrites.
       // Check and start writing back if necessary
       boolean startWriting = checkAndStartWrite(asyncDataService, writeCtx);
       if (!startWriting) {
@@ -675,7 +674,7 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
         // responses of the previous batch. So here send response immediately
         // for unstable non-sequential write
         if (stableHow != WriteStableHow.UNSTABLE) {
-          LOG.info("Have to change stable write to unstable write:"
+          LOG.info("Have to change stable write to unstable write: "
               + request.getStableHow());
           stableHow = WriteStableHow.UNSTABLE;
         }
@@ -687,6 +686,8 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
         WccData fileWcc = new WccData(preOpAttr, latestAttr);
         WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
             fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
+        RpcProgramNfs3.metrics.addWrite(Nfs3Utils
+            .getElapsedTime(writeCtx.startTime));
         Nfs3Utils
             .writeChannel(channel, response.serialize(new XDR(),
                 xid, new VerifierNone()), xid);
@@ -718,7 +719,7 @@ private WRITE3Response processPerfectOverWrite(DFSClient dfsClient,
           + "Continue processing the perfect overwrite.");
     } catch (IOException e) {
       LOG.info("hsync failed when processing possible perfect overwrite, path="
-          + path + " error:" + e);
+          + path + " error: " + e);
       return new WRITE3Response(Nfs3Status.NFS3ERR_IO, wccData, 0, stableHow,
           Nfs3Constant.WRITE_COMMIT_VERF);
     }
@@ -727,7 +728,7 @@ private WRITE3Response processPerfectOverWrite(DFSClient dfsClient,
       fis = dfsClient.createWrappedInputStream(dfsClient.open(path));
       readCount = fis.read(offset, readbuffer, 0, count);
       if (readCount < count) {
-        LOG.error("Can't read back " + count + " bytes, partial read size:"
+        LOG.error("Can't read back " + count + " bytes, partial read size: "
             + readCount);
         return new WRITE3Response(Nfs3Status.NFS3ERR_IO, wccData, 0, stableHow,
             Nfs3Constant.WRITE_COMMIT_VERF);
@@ -756,7 +757,7 @@ private WRITE3Response processPerfectOverWrite(DFSClient dfsClient,
         postOpAttr = Nfs3Utils.getFileAttr(dfsClient, path, iug);
       } catch (IOException e) {
         LOG.info("Got error when processing perfect overwrite, path=" + path
-            + " error:" + e);
+            + " error: " + e);
         return new WRITE3Response(Nfs3Status.NFS3ERR_IO, wccData, 0, stableHow,
             Nfs3Constant.WRITE_COMMIT_VERF);
       }
@@ -807,7 +808,7 @@ public COMMIT_STATUS checkCommit(DFSClient dfsClient, long commitOffset,
           ret = COMMIT_STATUS.COMMIT_ERROR;
         }
       } catch (IOException e) {
-        LOG.error("Got stream error during data sync:" + e);
+        LOG.error("Got stream error during data sync: " + e);
         // Do nothing. Stream will be closed eventually by StreamMonitor.
         // status = Nfs3Status.NFS3ERR_IO;
         ret = COMMIT_STATUS.COMMIT_ERROR;
@@ -971,7 +972,7 @@ public synchronized boolean streamCleanup(long fileId, long streamTimeout) {
     // Check the stream timeout
     if (checkStreamTimeout(streamTimeout)) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("stream can be closed for fileId:" + fileId);
+        LOG.debug("stream can be closed for fileId: " + fileId);
       }
       flag = true;
     }
@@ -987,7 +988,7 @@ public synchronized boolean streamCleanup(long fileId, long streamTimeout) {
   private synchronized WriteCtx offerNextToWrite() {
     if (pendingWrites.isEmpty()) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("The asyn write task has no pending writes, fileId: "
+        LOG.debug("The async write task has no pending writes, fileId: "
             + latestAttr.getFileId());
       }
       // process pending commit again to handle this race: a commit is added
@@ -1020,7 +1021,7 @@ private synchronized WriteCtx offerNextToWrite() {
         this.asyncStatus = false;
       } else if (range.getMin() < offset && range.getMax() > offset) {
         // shouldn't happen since we do sync for overlapped concurrent writers
-        LOG.warn("Got a overlapping write (" + range.getMin() + ","
+        LOG.warn("Got an overlapping write (" + range.getMin() + ", "
             + range.getMax() + "), nextOffset=" + offset
             + ". Silently drop it now");
         pendingWrites.remove(range);
@@ -1043,10 +1044,10 @@ private synchronized WriteCtx offerNextToWrite() {
     return null;
   }
   
-  /** Invoked by AsynDataService to write back to HDFS */
+  /** Invoked by AsyncDataService to write back to HDFS */
   void executeWriteBack() {
     Preconditions.checkState(asyncStatus,
-        "openFileCtx has false asyncStatus, fileId:" + latestAttr.getFileId());
+        "openFileCtx has false asyncStatus, fileId: " + latestAttr.getFileId());
     final long startOffset = asyncWriteBackStartOffset;  
     try {
       while (activeState) {
@@ -1071,10 +1072,10 @@ void executeWriteBack() {
         if (startOffset == asyncWriteBackStartOffset) {
           asyncStatus = false;
         } else {
-          LOG.info("Another asyn task is already started before this one"
-              + " is finalized. fileId:" + latestAttr.getFileId()
-              + " asyncStatus:" + asyncStatus + " original startOffset:"
-              + startOffset + " new startOffset:" + asyncWriteBackStartOffset
+          LOG.info("Another async task is already started before this one"
+              + " is finalized. fileId: " + latestAttr.getFileId()
+              + " asyncStatus: " + asyncStatus + " original startOffset: "
+              + startOffset + " new startOffset: " + asyncWriteBackStartOffset
               + ". Won't change asyncStatus here.");
         }
       }
@@ -1103,7 +1104,7 @@ private void processCommits(long offset) {
       }
       status = Nfs3Status.NFS3ERR_IO;
     } catch (IOException e) {
-      LOG.error("Got stream error during data sync:", e);
+      LOG.error("Got stream error during data sync: ", e);
       // Do nothing. Stream will be closed eventually by StreamMonitor.
       status = Nfs3Status.NFS3ERR_IO;
     }
@@ -1131,14 +1132,16 @@ private void processCommits(long offset) {
 
       COMMIT3Response response = new COMMIT3Response(status, wccData,
           Nfs3Constant.WRITE_COMMIT_VERF);
+      RpcProgramNfs3.metrics.addCommit(Nfs3Utils
+          .getElapsedTime(commit.startTime));
       Nfs3Utils.writeChannelCommit(commit.getChannel(), response
           .serialize(new XDR(), commit.getXid(),
               new VerifierNone()), commit.getXid());
       
       if (LOG.isDebugEnabled()) {
-        LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:"
-            + (Time.monotonicNow() - commit.getStartTime())
-            + "ms. Sent response for commit:" + commit);
+        LOG.debug("FileId: " + latestAttr.getFileId() + " Service time: "
+            + Nfs3Utils.getElapsedTime(commit.startTime)
+            + "ns. Sent response for commit: " + commit);
       }
       entry = pendingCommits.firstEntry();
     }
@@ -1155,13 +1158,14 @@ private void doSingleWrite(final WriteCtx writeCtx) {
     FileHandle handle = writeCtx.getHandle();
     if (LOG.isDebugEnabled()) {
       LOG.debug("do write, fileId: " + handle.getFileId() + " offset: "
-          + offset + " length:" + count + " stableHow:" + stableHow.name());
+          + offset + " length: " + count + " stableHow: " + stableHow.name());
     }
 
     try {
       // The write is not protected by lock. asyncState is used to make sure
       // there is one thread doing write back at any time    
       writeCtx.writeData(fos);
+      RpcProgramNfs3.metrics.incrBytesWritten(writeCtx.getCount());
       
       long flushedOffset = getFlushedOffset();
       if (flushedOffset != (offset + count)) {
@@ -1179,7 +1183,7 @@ private void doSingleWrite(final WriteCtx writeCtx) {
             updateNonSequentialWriteInMemory(-count);
             if (LOG.isDebugEnabled()) {
               LOG.debug("After writing " + handle.getFileId() + " at offset "
-                  + offset + ", updated the memory count, new value:"
+                  + offset + ", updated the memory count, new value: "
                   + nonSequentialWriteInMemory.get());
             }
           }
@@ -1188,18 +1192,18 @@ private void doSingleWrite(final WriteCtx writeCtx) {
       
       if (!writeCtx.getReplied()) {
         if (stableHow != WriteStableHow.UNSTABLE) {
-          LOG.info("Do sync for stable write:" + writeCtx);
+          LOG.info("Do sync for stable write: " + writeCtx);
           try {
             if (stableHow == WriteStableHow.DATA_SYNC) {
               fos.hsync();
             } else {
               Preconditions.checkState(stableHow == WriteStableHow.FILE_SYNC,
-                  "Unknown WriteStableHow:" + stableHow);
+                  "Unknown WriteStableHow: " + stableHow);
               // Sync file data and length
               fos.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
             }
           } catch (IOException e) {
-            LOG.error("hsync failed with writeCtx:" + writeCtx, e);
+            LOG.error("hsync failed with writeCtx: " + writeCtx, e);
             throw e;
           }
         }
@@ -1207,12 +1211,13 @@ private void doSingleWrite(final WriteCtx writeCtx) {
         WccAttr preOpAttr = latestAttr.getWccAttr();
         WccData fileWcc = new WccData(preOpAttr, latestAttr);
         if (writeCtx.getOriginalCount() != WriteCtx.INVALID_ORIGINAL_COUNT) {
-          LOG.warn("Return original count:" + writeCtx.getOriginalCount()
-              + " instead of real data count:" + count);
+          LOG.warn("Return original count: " + writeCtx.getOriginalCount()
+              + " instead of real data count: " + count);
           count = writeCtx.getOriginalCount();
         }
         WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
             fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
+        RpcProgramNfs3.metrics.addWrite(Nfs3Utils.getElapsedTime(writeCtx.startTime));
         Nfs3Utils.writeChannel(channel, response.serialize(
             new XDR(), xid, new VerifierNone()), xid);
       }
@@ -1258,8 +1263,8 @@ synchronized void cleanup() {
         fos.close();
       }
     } catch (IOException e) {
-      LOG.info("Can't close stream for fileId:" + latestAttr.getFileId()
-          + ", error:" + e);
+      LOG.info("Can't close stream for fileId: " + latestAttr.getFileId()
+          + ", error: " + e);
     }
     
     // Reply error for pending writes
@@ -1267,7 +1272,7 @@ synchronized void cleanup() {
     WccAttr preOpAttr = latestAttr.getWccAttr();
     while (!pendingWrites.isEmpty()) {
       OffsetRange key = pendingWrites.firstKey();
-      LOG.info("Fail pending write: (" + key.getMin() + "," + key.getMax()
+      LOG.info("Fail pending write: (" + key.getMin() + ", " + key.getMax()
           + "), nextOffset=" + nextOffset.get());
       
       WriteCtx writeCtx = pendingWrites.remove(key);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
index 98862eda5e7e8..3934d7c494b72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
@@ -26,7 +26,7 @@
 
 /**
  * This class is used to allow the initial registration of the NFS gateway with
- * the system portmap daemon to come from a privileged (< 1024) port. This is
+ * the system portmap daemon to come from a privileged (&lt; 1024) port. This is
  * necessary on certain operating systems to work around this bug in rpcbind:
  * 
  * Red Hat: https://bugzilla.redhat.com/show_bug.cgi?id=731542
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
index c860dd51813d5..05d067410a4d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
@@ -25,6 +25,7 @@
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.EnumSet;
 
 import org.apache.commons.logging.Log;
@@ -47,6 +48,8 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.nfs.AccessPrivilege;
 import org.apache.hadoop.nfs.NfsExports;
 import org.apache.hadoop.nfs.NfsFileType;
@@ -157,12 +160,12 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
   private final long blockSize;
   private final int bufferSize;
   private final boolean aixCompatMode;
-  private Statistics statistics;
   private String writeDumpDir; // The dir save dump files
 
   private final RpcCallCache rpcCallCache;
   private JvmPauseMonitor pauseMonitor;
   private Nfs3HttpServer infoServer = null;
+  static Nfs3Metrics metrics;
 
   public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket,
       boolean allowInsecurePorts) throws IOException {
@@ -208,6 +211,17 @@ public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket
     infoServer = new Nfs3HttpServer(config);
   }
 
+  public static RpcProgramNfs3 createRpcProgramNfs3(NfsConfiguration config,
+      DatagramSocket registrationSocket, boolean allowInsecurePorts)
+      throws IOException {
+    DefaultMetricsSystem.initialize("Nfs3");
+    String displayName = DNS.getDefaultHost("default", "default")
+        + config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
+            NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT);
+    metrics = Nfs3Metrics.create(config, displayName);
+    return new RpcProgramNfs3(config, registrationSocket, allowInsecurePorts);
+  }
+  
   private void clearDirectory(String writeDumpDir) throws IOException {
     File dumpDir = new File(writeDumpDir);
     if (dumpDir.exists()) {
@@ -224,12 +238,13 @@ private void clearDirectory(String writeDumpDir) throws IOException {
   }
   
   @Override
-  public void startDaemons() {    
+  public void startDaemons() {
     if (pauseMonitor == null) {
       pauseMonitor = new JvmPauseMonitor(config);
       pauseMonitor.start();
+      metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
     }
-    writeManager.startAsyncDataSerivce();
+    writeManager.startAsyncDataService();
     try {
       infoServer.start();
     } catch (IOException e) {
@@ -314,8 +329,9 @@ GETATTR3Response getattr(XDR xdr, SecurityHandler securityHandler,
     }
 
     FileHandle handle = request.getHandle();
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("GETATTR for fileId: " + handle.getFileId());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("GETATTR for fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     Nfs3FileAttributes attrs = null;
@@ -339,7 +355,7 @@ GETATTR3Response getattr(XDR xdr, SecurityHandler securityHandler,
       return response;
     }
     if (attrs == null) {
-      LOG.error("Can't get path for fileId:" + handle.getFileId());
+      LOG.error("Can't get path for fileId: " + handle.getFileId());
       response.setStatus(Nfs3Status.NFS3ERR_STALE);
       return response;
     }
@@ -355,7 +371,7 @@ private void setattrInternal(DFSClient dfsClient, String fileIdPath,
 
     if (setMode && updateFields.contains(SetAttrField.MODE)) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("set new mode:" + newAttr.getMode());
+        LOG.debug("set new mode: " + newAttr.getMode());
       }
       dfsClient.setPermission(fileIdPath,
           new FsPermission((short) (newAttr.getMode())));
@@ -375,7 +391,7 @@ private void setattrInternal(DFSClient dfsClient, String fileIdPath,
         .getMilliSeconds() : -1;
     if (atime != -1 || mtime != -1) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("set atime:" + +atime + " mtime:" + mtime);
+        LOG.debug("set atime: " + +atime + " mtime: " + mtime);
       }
       dfsClient.setTimes(fileIdPath, mtime, atime);
     }
@@ -407,7 +423,8 @@ SETATTR3Response setattr(XDR xdr, SecurityHandler securityHandler,
 
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS SETATTR fileId: " + handle.getFileId());
+      LOG.debug("NFS SETATTR fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     if (request.getAttr().getUpdateFields().contains(SetAttrField.SIZE)) {
@@ -422,7 +439,7 @@ SETATTR3Response setattr(XDR xdr, SecurityHandler securityHandler,
     try {
       preOpAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
       if (preOpAttr == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         response.setStatus(Nfs3Status.NFS3ERR_STALE);
         return response;
       }
@@ -493,7 +510,7 @@ LOOKUP3Response lookup(XDR xdr, SecurityHandler securityHandler,
     String fileName = request.getName();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS LOOKUP dir fileId: " + dirHandle.getFileId() + " name: "
-          + fileName);
+          + fileName + " client: " + remoteAddress);
     }
 
     try {
@@ -502,7 +519,7 @@ LOOKUP3Response lookup(XDR xdr, SecurityHandler securityHandler,
           dirHandle, fileName);
       if (postOpObjAttr == null) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("NFS LOOKUP fileId: " + dirHandle.getFileId() + " name:"
+          LOG.debug("NFS LOOKUP fileId: " + dirHandle.getFileId() + " name: "
               + fileName + " does not exist");
         }
         Nfs3FileAttributes postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient,
@@ -514,7 +531,7 @@ LOOKUP3Response lookup(XDR xdr, SecurityHandler securityHandler,
       Nfs3FileAttributes postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient,
           dirFileIdPath, iug);
       if (postOpDirAttr == null) {
-        LOG.info("Can't get path for dir fileId:" + dirHandle.getFileId());
+        LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
         return new LOOKUP3Response(Nfs3Status.NFS3ERR_STALE);
       }
       FileHandle fileHandle = new FileHandle(postOpObjAttr.getFileId());
@@ -561,7 +578,8 @@ ACCESS3Response access(XDR xdr, SecurityHandler securityHandler,
     Nfs3FileAttributes attrs;
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS ACCESS fileId: " + handle.getFileId());
+      LOG.debug("NFS ACCESS fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     try {
@@ -569,7 +587,7 @@ ACCESS3Response access(XDR xdr, SecurityHandler securityHandler,
       attrs = writeManager.getFileAttr(dfsClient, handle, iug);
 
       if (attrs == null) {
-        LOG.error("Can't get path for fileId:" + handle.getFileId());
+        LOG.error("Can't get path for fileId: " + handle.getFileId());
         return new ACCESS3Response(Nfs3Status.NFS3ERR_STALE);
       }
       int access = Nfs3Utils.getAccessRightsForUserGroup(
@@ -627,7 +645,8 @@ READLINK3Response readlink(XDR xdr, SecurityHandler securityHandler,
 
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS READLINK fileId: " + handle.getFileId());
+      LOG.debug("NFS READLINK fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     String fileIdPath = Nfs3Utils.getFileIdPath(handle);
@@ -637,29 +656,30 @@ READLINK3Response readlink(XDR xdr, SecurityHandler securityHandler,
       Nfs3FileAttributes postOpAttr = Nfs3Utils.getFileAttr(dfsClient,
           fileIdPath, iug);
       if (postOpAttr == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new READLINK3Response(Nfs3Status.NFS3ERR_STALE);
       }
       if (postOpAttr.getType() != NfsFileType.NFSLNK.toValue()) {
-        LOG.error("Not a symlink, fileId:" + handle.getFileId());
+        LOG.error("Not a symlink, fileId: " + handle.getFileId());
         return new READLINK3Response(Nfs3Status.NFS3ERR_INVAL);
       }
       if (target == null) {
-        LOG.error("Symlink target should not be null, fileId:"
+        LOG.error("Symlink target should not be null, fileId: "
             + handle.getFileId());
         return new READLINK3Response(Nfs3Status.NFS3ERR_SERVERFAULT);
       }
       int rtmax = config.getInt(NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY,
           NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_DEFAULT);
-      if (rtmax < target.getBytes().length) {
-        LOG.error("Link size: " + target.getBytes().length
+      if (rtmax < target.getBytes(Charset.forName("UTF-8")).length) {
+        LOG.error("Link size: "
+            + target.getBytes(Charset.forName("UTF-8")).length
             + " is larger than max transfer size: " + rtmax);
         return new READLINK3Response(Nfs3Status.NFS3ERR_IO, postOpAttr,
             new byte[0]);
       }
 
       return new READLINK3Response(Nfs3Status.NFS3_OK, postOpAttr,
-          target.getBytes());
+          target.getBytes(Charset.forName("UTF-8")));
 
     } catch (IOException e) {
       LOG.warn("Readlink error: " + e.getClass(), e);
@@ -705,7 +725,7 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler,
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS READ fileId: " + handle.getFileId() + " offset: " + offset
-          + " count: " + count);
+          + " count: " + count + " client: " + remoteAddress);
     }
 
     Nfs3FileAttributes attrs;
@@ -718,13 +738,13 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler,
             Nfs3Utils.getFileIdPath(handle), iug);
       } catch (IOException e) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Get error accessing file, fileId:" + handle.getFileId(), e);
+          LOG.debug("Get error accessing file, fileId: " + handle.getFileId(), e);
         }
         return new READ3Response(Nfs3Status.NFS3ERR_IO);
       }
       if (attrs == null) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Can't get path for fileId:" + handle.getFileId());
+          LOG.debug("Can't get path for fileId: " + handle.getFileId());
         }
         return new READ3Response(Nfs3Status.NFS3ERR_NOENT);
       }
@@ -768,6 +788,7 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler,
 
         try {
           readCount = fis.read(offset, readbuffer, 0, count);
+          metrics.incrBytesRead(readCount);
         } catch (IOException e) {
           // TODO: A cleaner way is to throw a new type of exception
           // which requires incompatible changes.
@@ -784,8 +805,9 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler,
       attrs = Nfs3Utils.getFileAttr(dfsClient, Nfs3Utils.getFileIdPath(handle),
           iug);
       if (readCount < count) {
-        LOG.info("Partical read. Asked offset:" + offset + " count:" + count
-            + " and read back:" + readCount + "file size:" + attrs.getSize());
+        LOG.info("Partical read. Asked offset: " + offset + " count: " + count
+            + " and read back: " + readCount + " file size: "
+            + attrs.getSize());
       }
       // HDFS returns -1 for read beyond file size.
       if (readCount < 0) {
@@ -844,15 +866,15 @@ WRITE3Response write(XDR xdr, Channel channel, int xid,
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS WRITE fileId: " + handle.getFileId() + " offset: "
-          + offset + " length:" + count + " stableHow:" + stableHow.getValue()
-          + " xid:" + xid);
+          + offset + " length: " + count + " stableHow: " + stableHow.getValue()
+          + " xid: " + xid + " client: " + remoteAddress);
     }
 
     Nfs3FileAttributes preOpAttr = null;
     try {
       preOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
       if (preOpAttr == null) {
-        LOG.error("Can't get path for fileId:" + handle.getFileId());
+        LOG.error("Can't get path for fileId: " + handle.getFileId());
         return new WRITE3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -863,7 +885,7 @@ WRITE3Response write(XDR xdr, Channel channel, int xid,
       }
 
       if (LOG.isDebugEnabled()) {
-        LOG.debug("requesed offset=" + offset + " and current filesize="
+        LOG.debug("requested offset=" + offset + " and current filesize="
             + preOpAttr.getSize());
       }
 
@@ -918,7 +940,7 @@ CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
     String fileName = request.getName();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS CREATE dir fileId: " + dirHandle.getFileId()
-          + " filename: " + fileName);
+          + " filename: " + fileName + " client: " + remoteAddress);
     }
 
     int createMode = request.getMode();
@@ -926,7 +948,7 @@ CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
         && request.getObjAttr().getUpdateFields().contains(SetAttrField.SIZE)
         && request.getObjAttr().getSize() != 0) {
       LOG.error("Setting file size is not supported when creating file: "
-          + fileName + " dir fileId:" + dirHandle.getFileId());
+          + fileName + " dir fileId: " + dirHandle.getFileId());
       return new CREATE3Response(Nfs3Status.NFS3ERR_INVAL);
     }
 
@@ -939,7 +961,7 @@ CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
     try {
       preOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (preOpDirAttr == null) {
-        LOG.error("Can't get path for dirHandle:" + dirHandle);
+        LOG.error("Can't get path for dirHandle: " + dirHandle);
         return new CREATE3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -963,7 +985,7 @@ preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
       fos = dfsClient.createWrappedOutputStream(
           dfsClient.create(fileIdPath, permission, flag, false, replication,
               blockSize, null, bufferSize, null),
-          statistics);
+          null);
 
       if ((createMode == Nfs3Constant.CREATE_UNCHECKED)
           || (createMode == Nfs3Constant.CREATE_GUARDED)) {
@@ -991,7 +1013,7 @@ preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
         fos = null;
       } else {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Opened stream for file:" + fileName + ", fileId:"
+          LOG.debug("Opened stream for file: " + fileName + ", fileId: "
               + fileHandle.getFileId());
         }
       }
@@ -1002,7 +1024,7 @@ preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
         try {
           fos.close();
         } catch (IOException e1) {
-          LOG.error("Can't close stream for dirFileId:" + dirHandle.getFileId()
+          LOG.error("Can't close stream for dirFileId: " + dirHandle.getFileId()
               + " filename: " + fileName, e1);
         }
       }
@@ -1011,7 +1033,7 @@ preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
           dirWcc = Nfs3Utils.createWccData(Nfs3Utils.getWccAttr(preOpDirAttr),
               dfsClient, dirFileIdPath, iug);
         } catch (IOException e1) {
-          LOG.error("Can't get postOpDirAttr for dirFileId:"
+          LOG.error("Can't get postOpDirAttr for dirFileId: "
               + dirHandle.getFileId(), e1);
         }
       }
@@ -1049,6 +1071,10 @@ MKDIR3Response mkdir(XDR xdr, SecurityHandler securityHandler,
     }
     FileHandle dirHandle = request.getHandle();
     String fileName = request.getName();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("NFS MKDIR dirId: " + dirHandle.getFileId() + " filename: "
+          + fileName + " client: " + remoteAddress);
+    }
 
     if (request.getObjAttr().getUpdateFields().contains(SetAttrField.SIZE)) {
       LOG.error("Setting file size is not supported when mkdir: " + fileName
@@ -1064,7 +1090,7 @@ MKDIR3Response mkdir(XDR xdr, SecurityHandler securityHandler,
     try {
       preOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (preOpDirAttr == null) {
-        LOG.info("Can't get path for dir fileId:" + dirHandle.getFileId());
+        LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
         return new MKDIR3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1147,7 +1173,7 @@ REMOVE3Response remove(XDR xdr, SecurityHandler securityHandler,
     String fileName = request.getName();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS REMOVE dir fileId: " + dirHandle.getFileId()
-          + " fileName: " + fileName);
+          + " fileName: " + fileName + " client: " + remoteAddress);
     }
 
     String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
@@ -1156,7 +1182,7 @@ REMOVE3Response remove(XDR xdr, SecurityHandler securityHandler,
     try {
       preOpDirAttr =  Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (preOpDirAttr == null) {
-        LOG.info("Can't get path for dir fileId:" + dirHandle.getFileId());
+        LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
         return new REMOVE3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1228,7 +1254,7 @@ RMDIR3Response rmdir(XDR xdr, SecurityHandler securityHandler,
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS RMDIR dir fileId: " + dirHandle.getFileId()
-          + " fileName: " + fileName);
+          + " fileName: " + fileName + " client: " + remoteAddress);
     }
 
     String dirFileIdPath = Nfs3Utils.getFileIdPath(dirHandle);
@@ -1237,7 +1263,7 @@ RMDIR3Response rmdir(XDR xdr, SecurityHandler securityHandler,
     try {
       preOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (preOpDirAttr == null) {
-        LOG.info("Can't get path for dir fileId:" + dirHandle.getFileId());
+        LOG.info("Can't get path for dir fileId: " + dirHandle.getFileId());
         return new RMDIR3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1315,7 +1341,8 @@ RENAME3Response rename(XDR xdr, SecurityHandler securityHandler,
     String toName = request.getToName();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS RENAME from: " + fromHandle.getFileId() + "/" + fromName
-          + " to: " + toHandle.getFileId() + "/" + toName);
+          + " to: " + toHandle.getFileId() + "/" + toName + " client: "
+          + remoteAddress);
     }
 
     String fromDirFileIdPath = Nfs3Utils.getFileIdPath(fromHandle);
@@ -1327,14 +1354,14 @@ RENAME3Response rename(XDR xdr, SecurityHandler securityHandler,
     try {
       fromPreOpAttr = Nfs3Utils.getFileAttr(dfsClient, fromDirFileIdPath, iug);
       if (fromPreOpAttr == null) {
-        LOG.info("Can't get path for fromHandle fileId:"
+        LOG.info("Can't get path for fromHandle fileId: "
             + fromHandle.getFileId());
         return new RENAME3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
       toPreOpAttr = Nfs3Utils.getFileAttr(dfsClient, toDirFileIdPath, iug);
       if (toPreOpAttr == null) {
-        LOG.info("Can't get path for toHandle fileId:" + toHandle.getFileId());
+        LOG.info("Can't get path for toHandle fileId: " + toHandle.getFileId());
         return new RENAME3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1413,7 +1440,8 @@ SYMLINK3Response symlink(XDR xdr, SecurityHandler securityHandler,
     // Don't do any name check to source path, just leave it to HDFS
     String linkIdPath = linkDirIdPath + "/" + name;
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS SYMLINK, target: " + symData + " link: " + linkIdPath);
+      LOG.debug("NFS SYMLINK, target: " + symData + " link: " + linkIdPath
+          + " client: " + remoteAddress);
     }
 
     try {
@@ -1435,7 +1463,7 @@ SYMLINK3Response symlink(XDR xdr, SecurityHandler securityHandler,
           objAttr.getFileId()), objAttr, dirWcc);
 
     } catch (IOException e) {
-      LOG.warn("Exception:" + e);
+      LOG.warn("Exception: " + e);
       int status = mapErrorStatus(e);
       response.setStatus(status);
       return response;
@@ -1462,7 +1490,8 @@ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath,
         throw io;
       }
       // This happens when startAfter was just deleted
-      LOG.info("Cookie couldn't be found: " + new String(startAfter)
+      LOG.info("Cookie couldn't be found: "
+          + new String(startAfter, Charset.forName("UTF-8"))
           + ", do listing from beginning");
       dlisting = dfsClient
           .listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME);
@@ -1500,18 +1529,18 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
     FileHandle handle = request.getHandle();
     long cookie = request.getCookie();
     if (cookie < 0) {
-      LOG.error("Invalid READDIR request, with negitve cookie:" + cookie);
+      LOG.error("Invalid READDIR request, with negative cookie: " + cookie);
       return new READDIR3Response(Nfs3Status.NFS3ERR_INVAL);
     }
     long count = request.getCount();
     if (count <= 0) {
-      LOG.info("Nonpositive count in invalid READDIR request:" + count);
+      LOG.info("Nonpositive count in invalid READDIR request: " + count);
       return new READDIR3Response(Nfs3Status.NFS3_OK);
     }
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS READDIR fileId: " + handle.getFileId() + " cookie: "
-          + cookie + " count: " + count);
+          + cookie + " count: " + count + " client: " + remoteAddress);
     }
 
     HdfsFileStatus dirStatus;
@@ -1522,11 +1551,11 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
       String dirFileIdPath = Nfs3Utils.getFileIdPath(handle);
       dirStatus = dfsClient.getFileInfo(dirFileIdPath);
       if (dirStatus == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new READDIR3Response(Nfs3Status.NFS3ERR_STALE);
       }
       if (!dirStatus.isDir()) {
-        LOG.error("Can't readdir for regular file, fileId:"
+        LOG.error("Can't readdir for regular file, fileId: "
             + handle.getFileId());
         return new READDIR3Response(Nfs3Status.NFS3ERR_NOTDIR);
       }
@@ -1548,7 +1577,9 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
         } else {
           LOG.error("CookieVerf mismatch. request cookieVerf: " + cookieVerf
               + " dir cookieVerf: " + dirStatus.getModificationTime());
-          return new READDIR3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+          return new READDIR3Response(
+              Nfs3Status.NFS3ERR_BAD_COOKIE,
+              Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug));
         }
       }
 
@@ -1559,7 +1590,7 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
 
         if (dotdotStatus == null) {
           // This should not happen
-          throw new IOException("Can't get path for handle path:"
+          throw new IOException("Can't get path for handle path: "
               + dotdotFileIdPath);
         }
         dotdotFileId = dotdotStatus.getFileId();
@@ -1571,13 +1602,13 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
         startAfter = HdfsFileStatus.EMPTY_NAME;
       } else {
         String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
-        startAfter = inodeIdPath.getBytes();
+        startAfter = inodeIdPath.getBytes(Charset.forName("UTF-8"));
       }
 
       dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
       postOpAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (postOpAttr == null) {
-        LOG.error("Can't get path for fileId:" + handle.getFileId());
+        LOG.error("Can't get path for fileId: " + handle.getFileId());
         return new READDIR3Response(Nfs3Status.NFS3ERR_STALE);
       }
     } catch (IOException e) {
@@ -1658,23 +1689,24 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
     FileHandle handle = request.getHandle();
     long cookie = request.getCookie();
     if (cookie < 0) {
-      LOG.error("Invalid READDIRPLUS request, with negitve cookie:" + cookie);
+      LOG.error("Invalid READDIRPLUS request, with negative cookie: " + cookie);
       return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_INVAL);
     }
     long dirCount = request.getDirCount();
     if (dirCount <= 0) {
-      LOG.info("Nonpositive dircount in invalid READDIRPLUS request:" + dirCount);
+      LOG.info("Nonpositive dircount in invalid READDIRPLUS request: " + dirCount);
       return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_INVAL);
     }
     int maxCount = request.getMaxCount();
     if (maxCount <= 0) {
-      LOG.info("Nonpositive maxcount in invalid READDIRPLUS request:" + maxCount);
+      LOG.info("Nonpositive maxcount in invalid READDIRPLUS request: " + maxCount);
       return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_INVAL);
     }
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS READDIRPLUS fileId: " + handle.getFileId() + " cookie: "
-          + cookie + " dirCount: " + dirCount + " maxCount: " + maxCount);
+          + cookie + " dirCount: " + dirCount + " maxCount: " + maxCount
+          + " client: " + remoteAddress);
     }
 
     HdfsFileStatus dirStatus;
@@ -1686,11 +1718,11 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
       String dirFileIdPath = Nfs3Utils.getFileIdPath(handle);
       dirStatus = dfsClient.getFileInfo(dirFileIdPath);
       if (dirStatus == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_STALE);
       }
       if (!dirStatus.isDir()) {
-        LOG.error("Can't readdirplus for regular file, fileId:"
+        LOG.error("Can't readdirplus for regular file, fileId: "
             + handle.getFileId());
         return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_NOTDIR);
       }
@@ -1710,7 +1742,10 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
         } else {
           LOG.error("cookieverf mismatch. request cookieverf: " + cookieVerf
               + " dir cookieverf: " + dirStatus.getModificationTime());
-          return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+          return new READDIRPLUS3Response(
+              Nfs3Status.NFS3ERR_BAD_COOKIE,
+              Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug),
+              0, null);
         }
       }
 
@@ -1721,7 +1756,7 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
 
         if (dotdotStatus == null) {
           // This should not happen
-          throw new IOException("Can't get path for handle path:"
+          throw new IOException("Can't get path for handle path: "
               + dotdotFileIdPath);
         }
         dotdotFileId = dotdotStatus.getFileId();
@@ -1733,13 +1768,13 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
         startAfter = HdfsFileStatus.EMPTY_NAME;
       } else {
         String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
-        startAfter = inodeIdPath.getBytes();
+        startAfter = inodeIdPath.getBytes(Charset.forName("UTF-8"));
       }
 
       dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
       postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
       if (postOpDirAttr == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_STALE);
       }
     } catch (IOException e) {
@@ -1771,7 +1806,7 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
         try {
           attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
         } catch (IOException e) {
-          LOG.error("Can't get file attributes for fileId:" + fileId, e);
+          LOG.error("Can't get file attributes for fileId: " + fileId, e);
           continue;
         }
         entries[i] = new READDIRPLUS3Response.EntryPlus3(fileId,
@@ -1788,7 +1823,7 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
         try {
           attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
         } catch (IOException e) {
-          LOG.error("Can't get file attributes for fileId:" + fileId, e);
+          LOG.error("Can't get file attributes for fileId: " + fileId, e);
           continue;
         }
         entries[i] = new READDIRPLUS3Response.EntryPlus3(fileId,
@@ -1833,7 +1868,8 @@ FSSTAT3Response fsstat(XDR xdr, SecurityHandler securityHandler,
 
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS FSSTAT fileId: " + handle.getFileId());
+      LOG.debug("NFS FSSTAT fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     try {
@@ -1844,7 +1880,7 @@ FSSTAT3Response fsstat(XDR xdr, SecurityHandler securityHandler,
       Nfs3FileAttributes attrs = writeManager.getFileAttr(dfsClient, handle,
           iug);
       if (attrs == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new FSSTAT3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1907,7 +1943,8 @@ FSINFO3Response fsinfo(XDR xdr, SecurityHandler securityHandler,
 
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS FSINFO fileId: " + handle.getFileId());
+      LOG.debug("NFS FSINFO fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     try {
@@ -1924,7 +1961,7 @@ FSINFO3Response fsinfo(XDR xdr, SecurityHandler securityHandler,
       Nfs3FileAttributes attrs = Nfs3Utils.getFileAttr(dfsClient,
           Nfs3Utils.getFileIdPath(handle), iug);
       if (attrs == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new FSINFO3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -1973,14 +2010,15 @@ PATHCONF3Response pathconf(XDR xdr, SecurityHandler securityHandler,
     Nfs3FileAttributes attrs;
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("NFS PATHCONF fileId: " + handle.getFileId());
+      LOG.debug("NFS PATHCONF fileId: " + handle.getFileId() + " client: "
+          + remoteAddress);
     }
 
     try {
       attrs = Nfs3Utils.getFileAttr(dfsClient, Nfs3Utils.getFileIdPath(handle),
           iug);
       if (attrs == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new PATHCONF3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -2024,7 +2062,8 @@ COMMIT3Response commit(XDR xdr, Channel channel, int xid,
     FileHandle handle = request.getHandle();
     if (LOG.isDebugEnabled()) {
       LOG.debug("NFS COMMIT fileId: " + handle.getFileId() + " offset="
-          + request.getOffset() + " count=" + request.getCount());
+          + request.getOffset() + " count=" + request.getCount() + " client: "
+          + remoteAddress);
     }
 
     String fileIdPath = Nfs3Utils.getFileIdPath(handle);
@@ -2032,7 +2071,7 @@ COMMIT3Response commit(XDR xdr, Channel channel, int xid,
     try {
       preOpAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
       if (preOpAttr == null) {
-        LOG.info("Can't get path for fileId:" + handle.getFileId());
+        LOG.info("Can't get path for fileId: " + handle.getFileId());
         return new COMMIT3Response(Nfs3Status.NFS3ERR_STALE);
       }
 
@@ -2046,8 +2085,8 @@ COMMIT3Response commit(XDR xdr, Channel channel, int xid,
           : (request.getOffset() + request.getCount());
 
       // Insert commit as an async request
-      writeManager.handleCommit(dfsClient, handle, commitOffset,
-          channel, xid, preOpAttr);
+      writeManager.handleCommit(dfsClient, handle, commitOffset, channel, xid,
+          preOpAttr);
       return null;
     } catch (IOException e) {
       LOG.warn("Exception ", e);
@@ -2129,20 +2168,29 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
         }
       }
     }
-
+    
+    // Since write and commit could be async, they use their own startTime and
+    // only record success requests.
+    final long startTime = System.nanoTime();
+    
     NFS3Response response = null;
     if (nfsproc3 == NFSPROC3.NULL) {
       response = nullProcedure();
     } else if (nfsproc3 == NFSPROC3.GETATTR) {
       response = getattr(xdr, info);
+      metrics.addGetattr(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.SETATTR) {
       response = setattr(xdr, info);
+      metrics.addSetattr(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.LOOKUP) {
       response = lookup(xdr, info);
+      metrics.addLookup(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.ACCESS) {
       response = access(xdr, info);
+      metrics.addAccess(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.READLINK) {
       response = readlink(xdr, info);
+      metrics.addReadlink(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.READ) {
       if (LOG.isDebugEnabled()) {
           LOG.debug(Nfs3Utils.READ_RPC_START + xid);
@@ -2151,6 +2199,7 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
       if (LOG.isDebugEnabled() && (nfsproc3 == NFSPROC3.READ)) {
         LOG.debug(Nfs3Utils.READ_RPC_END + xid);
       }
+      metrics.addRead(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.WRITE) {
       if (LOG.isDebugEnabled()) {
           LOG.debug(Nfs3Utils.WRITE_RPC_START + xid);
@@ -2159,30 +2208,43 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
       // Write end debug trace is in Nfs3Utils.writeChannel
     } else if (nfsproc3 == NFSPROC3.CREATE) {
       response = create(xdr, info);
+      metrics.addCreate(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.MKDIR) {
       response = mkdir(xdr, info);
+      metrics.addMkdir(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.SYMLINK) {
       response = symlink(xdr, info);
+      metrics.addSymlink(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.MKNOD) {
       response = mknod(xdr, info);
+      metrics.addMknod(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.REMOVE) {
       response = remove(xdr, info);
+      metrics.addRemove(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.RMDIR) {
       response = rmdir(xdr, info);
+      metrics.addRmdir(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.RENAME) {
       response = rename(xdr, info);
+      metrics.addRename(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.LINK) {
       response = link(xdr, info);
+      metrics.addLink(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.READDIR) {
       response = readdir(xdr, info);
+      metrics.addReaddir(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.READDIRPLUS) {
       response = readdirplus(xdr, info);
+      metrics.addReaddirplus(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.FSSTAT) {
       response = fsstat(xdr, info);
+      metrics.addFsstat(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.FSINFO) {
       response = fsinfo(xdr, info);
+      metrics.addFsinfo(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.PATHCONF) {
-      response = pathconf(xdr,info);
+      response = pathconf(xdr, info);
+      metrics.addPathconf(Nfs3Utils.getElapsedTime(startTime));
     } else if (nfsproc3 == NFSPROC3.COMMIT) {
       response = commit(xdr, info);
     } else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
index 758fd3998b868..82c826fda1eef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
@@ -84,7 +84,8 @@ public int getOriginalCount() {
   private long dumpFileOffset;
   
   private volatile DataState dataState;
-
+  public final long startTime;
+  
   public DataState getDataState() {
     return dataState;
   }
@@ -235,6 +236,7 @@ void setReplied(boolean replied) {
     this.replied = replied;
     this.dataState = dataState;
     raf = null;
+    this.startTime = System.nanoTime();
   }
   
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
index e71eaa51488d4..7810ce2d1a877 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
@@ -18,10 +18,12 @@
 package org.apache.hadoop.hdfs.nfs.nfs3;
 
 import java.io.IOException;
+import java.util.EnumSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
@@ -99,7 +101,7 @@ boolean addOpenFileStream(FileHandle h, OpenFileCtx ctx) {
     this.fileContextCache = new OpenFileCtxCache(config, streamTimeout);
   }
 
-  void startAsyncDataSerivce() {
+  void startAsyncDataService() {
     if (asyncDataServiceStarted) {
       return;
     }
@@ -137,7 +139,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
     FileHandle fileHandle = request.getHandle();
     OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
     if (openFileCtx == null) {
-      LOG.info("No opened stream for fileId:" + fileHandle.getFileId());
+      LOG.info("No opened stream for fileId: " + fileHandle.getFileId());
 
       String fileIdPath = Nfs3Utils.getFileIdPath(fileHandle.getFileId());
       HdfsDataOutputStream fos = null;
@@ -147,20 +149,21 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
             CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
             CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
         
-        fos = dfsClient.append(fileIdPath, bufferSize, null, null);
+        fos = dfsClient.append(fileIdPath, bufferSize,
+            EnumSet.of(CreateFlag.APPEND), null, null);
 
         latestAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
       } catch (RemoteException e) {
         IOException io = e.unwrapRemoteException();
         if (io instanceof AlreadyBeingCreatedException) {
-          LOG.warn("Can't append file:" + fileIdPath
-              + ". Possibly the file is being closed. Drop the request:"
+          LOG.warn("Can't append file: " + fileIdPath
+              + ". Possibly the file is being closed. Drop the request: "
               + request + ", wait for the client to retry...");
           return;
         }
         throw e;
       } catch (IOException e) {
-        LOG.error("Can't apapend to file:" + fileIdPath, e);
+        LOG.error("Can't append to file: " + fileIdPath, e);
         if (fos != null) {
           fos.close();
         }
@@ -185,7 +188,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
         try {
           fos.close();
         } catch (IOException e) {
-          LOG.error("Can't close stream for fileId:" + handle.getFileId(), e);
+          LOG.error("Can't close stream for fileId: " + handle.getFileId(), e);
         }
         // Notify client to retry
         WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
@@ -198,7 +201,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel,
       }
 
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Opened stream for appending file:" + fileHandle.getFileId());
+        LOG.debug("Opened stream for appending file: " + fileHandle.getFileId());
       }
     }
 
@@ -217,13 +220,14 @@ int commitBeforeRead(DFSClient dfsClient, FileHandle fileHandle,
 
     if (openFileCtx == null) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("No opened stream for fileId:" + fileHandle.getFileId()
+        LOG.debug("No opened stream for fileId: " + fileHandle.getFileId()
             + " commitOffset=" + commitOffset
             + ". Return success in this case.");
       }
       status = Nfs3Status.NFS3_OK;
 
     } else {
+      // commit request triggered by read won't create pending comment obj
       COMMIT_STATUS ret = openFileCtx.checkCommit(dfsClient, commitOffset,
           null, 0, null, true);
       switch (ret) {
@@ -250,8 +254,8 @@ int commitBeforeRead(DFSClient dfsClient, FileHandle fileHandle,
         status = Nfs3Status.NFS3_OK;
         break;
       default:
-        LOG.error("Should not get commit return code:" + ret.name());
-        throw new RuntimeException("Should not get commit return code:"
+        LOG.error("Should not get commit return code: " + ret.name());
+        throw new RuntimeException("Should not get commit return code: "
             + ret.name());
       }
     }
@@ -260,11 +264,12 @@ int commitBeforeRead(DFSClient dfsClient, FileHandle fileHandle,
   
   void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
       long commitOffset, Channel channel, int xid, Nfs3FileAttributes preOpAttr) {
+    long startTime = System.nanoTime();
     int status;
     OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
 
     if (openFileCtx == null) {
-      LOG.info("No opened stream for fileId:" + fileHandle.getFileId()
+      LOG.info("No opened stream for fileId: " + fileHandle.getFileId()
           + " commitOffset=" + commitOffset + ". Return success in this case.");
       status = Nfs3Status.NFS3_OK;
       
@@ -290,8 +295,8 @@ void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
         status = Nfs3Status.NFS3_OK;
         break;
       default:
-        LOG.error("Should not get commit return code:" + ret.name());
-        throw new RuntimeException("Should not get commit return code:"
+        LOG.error("Should not get commit return code: " + ret.name());
+        throw new RuntimeException("Should not get commit return code: "
             + ret.name());
       }
     }
@@ -306,9 +311,9 @@ void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
     WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr), postOpAttr);
     COMMIT3Response response = new COMMIT3Response(status, fileWcc,
         Nfs3Constant.WRITE_COMMIT_VERF);
+    RpcProgramNfs3.metrics.addCommit(Nfs3Utils.getElapsedTime(startTime));
     Nfs3Utils.writeChannelCommit(channel,
-        response.serialize(new XDR(), xid, new VerifierNone()),
-        xid);
+        response.serialize(new XDR(), xid, new VerifierNone()), xid);
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
index 6536cd80f4f8e..54b7ee7fd18fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestDFSClientCache.java
@@ -39,7 +39,7 @@ public void testEviction() throws IOException {
     conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://localhost");
 
     // Only one entry will be in the cache
-    final int MAX_CACHE_SIZE = 2;
+    final int MAX_CACHE_SIZE = 1;
 
     DFSClientCache cache = new DFSClientCache(conf, MAX_CACHE_SIZE);
 
@@ -50,7 +50,8 @@ public void testEviction() throws IOException {
 
     cache.getDfsClient("test2");
     assertTrue(isDfsClientClose(c1));
-    assertEquals(MAX_CACHE_SIZE - 1, cache.clientCache.size());
+    assertTrue("cache size should be the max size or less",
+        cache.clientCache.size() <= MAX_CACHE_SIZE);
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java
index d44e9abe68022..46dbd42f4c97b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java
@@ -48,6 +48,10 @@ public static void setUp() throws Exception {
         HttpConfig.Policy.HTTP_AND_HTTPS.name());
     conf.set(NfsConfigKeys.NFS_HTTP_ADDRESS_KEY, "localhost:0");
     conf.set(NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY, "localhost:0");
+    // Use emphral port in case tests are running in parallel
+    conf.setInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0);
+    conf.setInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0);
+    
     File base = new File(BASEDIR);
     FileUtil.fullyDelete(base);
     base.mkdirs();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
new file mode 100644
index 0000000000000..2ef8527334aae
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -0,0 +1,4 @@
+  BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
+
+    HDFS-7347. Configurable erasure coding policy for individual files and
+    directories ( Zhe Zhang via vinayakumarb )
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 769be433084a8..eda3744e6febe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -18,6 +18,11 @@ Trunk (Unreleased)
 
     HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
 
+    HDFS-3689. Add support for variable length block. (jing9)
+
+    HDFS-7720. Quota by Storage Type API, tools and ClientNameNode Protocol
+    changes. (Xiaoyu Yao via Arpit Agarwal)
+
   IMPROVEMENTS
 
     HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
@@ -125,6 +130,12 @@ Trunk (Unreleased)
     HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
     directory. (Jing Zhao via wheat9)
 
+    HDFS-7591. hdfs classpath command should support same options as hadoop
+    classpath (Varun Saxena via Arpit Agarwal)
+
+    HDFS-7430. Rewrite the BlockScanner to use O(1) memory and use multiple
+    threads (cmccabe)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -257,6 +268,25 @@ Trunk (Unreleased)
 
     HDFS-7407. Minor typo in privileged pid/out/log names (aw)
 
+    HDFS-7581. HDFS documentation needs updating post-shell rewrite (aw)
+
+    HADOOP-11484. hadoop-mapreduce-client-nativetask fails to build on ARM
+    AARCH64 due to x86 asm statements (Edward Nevill via Colin P. McCabe)
+
+    HDFS-7667. Various typos and improvements to HDFS Federation doc
+    (Charles Lamb via aw)
+
+    HDFS-3750. API docs don't include HDFS (Jolly Chen via aw)
+
+    HDFS-7320. The appearance of hadoop-hdfs-httpfs site docs is inconsistent 
+    (Masatake Iwasaki via aw)
+
+    HDFS-7721. The HDFS BlockScanner may run fast during the first hour
+    (cmccabe)
+
+    HDFS-7751. Fix TestHDFSCLI for quota with storage type.  (Xiaoyu Yao
+    via szetszwo)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -273,6 +303,16 @@ Release 2.7.0 - UNRELEASED
     (Maysam Yabandeh via wang)
 
     HDFS-7424. Add web UI for NFS gateway (brandonli)
+    
+    HDFS-7449. Add metrics to NFS gateway (brandonli)
+
+    HDFS-3107. Introduce truncate. (Plamen Jeliazkov via shv)
+
+    HDFS-7056. Snapshot support for truncate. (Plamen Jeliazkov and shv)
+
+    HDFS-6673. Add delimited format support to PB OIV tool. (Eddy Xu via wang)
+
+    HDFS-7655. Expose truncate API for Web HDFS. (yliu)
 
   IMPROVEMENTS
 
@@ -441,11 +481,126 @@ Release 2.7.0 - UNRELEASED
     HDFS-7476. Consolidate ACL-related operations to a single class.
     (wheat9 via cnauroth)
 
+    HDFS-7384. 'getfacl' command and 'getAclStatus' output should be in sync.
+    (Vinayakumar B via cnauroth)
+
+    HDFS-7486. Consolidate XAttr-related implementation into a single class.
+    (wheat9)
+
+    HDFS-7498. Simplify the logic in INodesInPath. (jing9)
+
+    HDFS-7463. Simplify FSNamesystem#getBlockLocationsUpdateTimes. (wheat9)
+
+    HDFS-7509. Avoid resolving path multiple times. (jing9)
+
+    HDFS-7426. Change nntop JMX format to be a JSON blob. (wang)
+
+    HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)
+
+    HDFS-7536. Remove unused CryptoCodec in org.apache.hadoop.fs.Hdfs.
+    (Yi Liu via wheat9)
+
+    HDFS-7528. Consolidate symlink-related implementation into a single class.
+    (wheat9)
+
+    HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin
+    P. McCabe)
+
+    HDFS-7373. Clean up temporary files after fsimage transfer failures.
+    (kihwal)
+
+    HDFS-7543. Avoid path resolution when getting FileStatus for audit logs.
+    (wheat9)
+
+    HDFS-7530. Allow renaming of encryption zone roots. (Charles Lamb via wang)
+
+    HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter.
+    (jing9)
+
+    HADOOP-11032. Replace use of Guava's Stopwatch with Hadoop's StopWatch
+    (ozawa)
+
+    HADOOP-11470. Remove some uses of obsolete guava APIs from the hadoop
+    codebase. (Sangjin Lee via Colin P. McCabe)
+
+    HDFS-7182. JMX metrics aren't accessible when NN is busy. (Ming Ma via jing9)
+
+    HDFS-7323. Move the get/setStoragePolicy commands out from dfsadmin.
+    (jing9 via yliu)
+
+    HDFS-7326: Add documentation for hdfs debug commands (Vijay Bhat via Colin
+    P. McCabe)
+
+    HDFS-7598. Remove dependency on old version of guava in
+    TestDFSClientCache#testEviction. (Sangjin Lee via Colin P. McCabe)
+
+    HDFS-7600. Refine hdfs admin classes to reuse common code. (jing9)
+
+    HDFS-2219. Change fsck to support fully qualified paths so that a 
+    particular namenode in a federated cluster with multiple namenodes
+    can be specified in the path parameter.  (szetszwo)
+
+    HDFS-7457. DatanodeID generates excessive garbage. (daryn via kihwal)
+
+    HDFS-7189. Add trace spans for DFSClient metadata operations. (Colin P.
+    McCabe via yliu)
+
+    HDFS-7573. Consolidate the implementation of delete() into a single class.
+    (wheat9)
+
+    HDFS-7640. Print NFS Client in the NFS log. (Brandon Li via wheat9)
+
+    HDFS-7623. Add htrace configuration properties to core-default.xml and
+    update user doc about how to enable htrace. (yliu)
+
+    HDFS-7224. Allow reuse of NN connections via webhdfs (Eric Payne via
+    kihwal)
+
+    HDFS-7683. Combine usages and percent stats in NameNode UI.
+    (Vinayakumar B via wheat9)
+
+    HDFS-7675. Remove unused member DFSClient#spanReceiverHost (cmccabe)
+
+    HDFS-7603. The background replication queue initialization may not let
+    others run (kihwal)
+
+    HDFS-7706. Switch BlockManager logging to use slf4j. (wang)
+
+    HDFS-5631. Change BlockMetadataHeader.readHeader(..), ChunkChecksum
+    class and constructor to public; and fix FsDatasetSpi to use generic type
+    instead of FsVolumeImpl.  (David Powell and Joe Pallas via szetszwo)
+
+    HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter
+    type instead of concrete classes Block and ReplicaInfo.  (David Powell
+    and Joe Pallas via szetszwo)
+
+    HDFS-7681. Change ReplicaInputStreams constructor to take InputStream(s)
+    instead of FileDescriptor(s).  (Joe Pallas via szetszwo)
+
+    HDFS-7712. Switch blockStateChangeLog to use slf4j. (wang)
+
+    HDFS-7270. Add congestion signaling capability to DataNode write protocol.
+    (wheat9)
+
+    HDFS-7732. Fix the order of the parameters in DFSConfigKeys.
+    (Brahma Reddy Battula via aajisaka)
+
+    HDFS-7710. Remove dead code in BackupImage.java. (Xiaoyu Yao via aajisaka)
+
+    HDFS-7738. Revise the exception message for recover lease; add more truncate
+    tests such as truncate with HA setup, negative tests, truncate with other
+    operations and multiple truncates.  (szetszwo)
+
+    HDFS-7743. Code cleanup of BlockInfo and rename BlockInfo to
+    BlockInfoContiguous. (jing9)
+
   OPTIMIZATIONS
 
     HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
     (Vinayakumar B via wheat9)
 
+    HDFS-7615. Remove longReadLock (kihwal)
+
   BUG FIXES
 
     HDFS-6741. Improve permission denied message when
@@ -547,6 +702,176 @@ Release 2.7.0 - UNRELEASED
     HDFS-7472. Fix typo in message of ReplicaNotFoundException.
     (Masatake Iwasaki via wheat9)
 
+    HDFS-7473. Document setting dfs.namenode.fs-limits.max-directory-items to 0
+    is invalid. (Akira AJISAKA via cnauroth)
+
+    HDFS-7481. Add ACL indicator to the "Permission Denied" exception.
+    (vinayakumarb)
+
+    HDFS-7502. Fix findbugs warning in hdfs-nfs project.
+    (Brandon Li via wheat9)
+
+    HDFS-5578. [JDK8] Fix Javadoc errors caused by incorrect or illegal tags
+    in doc comments. (Andrew Purtell via wheat9)
+
+    HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved
+    deterministic. (Xiaoyu Yao via Arpit Agarwal)
+
+    HDFS-7515. Fix new findbugs warnings in hadoop-hdfs. (wheat9)
+
+    HDFS-7497. Inconsistent report of decommissioning DataNodes between
+    dfsadmin and NameNode webui. (Yongjun Zhang via wang)
+
+    HDFS-7517. Remove redundant non-null checks in FSNamesystem#
+    getBlockLocations. (wheat9)
+
+    HDFS-7514. TestTextCommand fails on Windows. (Arpit Agarwal)
+
+    HDFS-7506. Consolidate implementation of setting inode attributes into a
+    single class. (wheat9)
+    
+    HDFS-7516. Fix findbugs warnings in hdfs-nfs project. (brandonli)
+
+    HDFS-6425. Large postponedMisreplicatedBlocks has impact on blockReport
+    latency. (Ming Ma via kihwal)
+
+    HDFS-7494. Checking of closed in DFSInputStream#pread() should be protected
+    by synchronization (Ted Yu via Colin P. McCabe)
+
+    HDFS-7431. log message for InvalidMagicNumberException may be incorrect.
+    (Yi Liu via cnauroth)
+
+    HDFS-7552. Change FsVolumeList toString() to fix
+    TestDataNodeVolumeFailureToleration (Liang Xie via Colin P. McCabe)
+
+    HDFS-7557. Fix spacing for a few keys in DFSConfigKeys.java 
+    (Colin P.McCabe)
+
+    HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
+    restart/failover. (Vinayakumar B via cnauroth)
+
+    HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
+    memory footprint of NameNode (vinayakumarb)
+
+    HDFS-7563. NFS gateway parseStaticMap NumberFormatException 
+    (Yongjun Zhang via brandonli)
+
+    HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
+    Windows. (Arpit Agarwal via cnauroth)
+
+    HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+
+    HDFS-7564. NFS gateway dynamically reload UID/GID mapping file /etc/nfs.map
+    (Yongjun Zhang via brandonli)
+
+    HDFS-7561. TestFetchImage should write fetched-image-dir under target.
+    (Liang Xie via shv)
+
+    HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs.
+    (Zhanwei Wang via cnauroth)
+
+    HDFS-7579. Improve log reporting during block report rpc failure.
+    (Charles Lamb via cnauroth)
+
+    HDFS-7596. NameNode should prune dead storages from storageMap.
+    (Arpit Agarwal via cnauroth)
+
+    HDFS-7533. Datanode sometimes does not shutdown on receiving upgrade
+    shutdown command (Eric Payne via kihwal)
+
+    HDFS-5445. PacketReceiver populates the packetLen field in PacketHeader
+    incorrectly (Jonathan Mace via Colin P. McCabe)
+
+    HDFS-7470. SecondaryNameNode need twice memory when calling
+    reloadFromImageFile. (zhaoyunjiong via cnauroth)
+
+    HDFS-7585. Get TestEnhancedByteBufferAccess working on CPU architectures
+    with page sizes other than 4096 (Sam Liu via Colin P. McCabe)
+
+    HDFS-7635. Remove TestCorruptFilesJsp from branch-2. (cnauroth)
+
+    HDFS-7632. MiniDFSCluster configures DataNode data directories incorrectly if
+    using more than 1 DataNode and more than 2 storage locations per DataNode.
+    (cnauroth)
+
+    HDFS-7637. Fix the check condition for reserved path. (Yi Liu via jing9)
+
+    HDFS-7641. Update archival storage user doc for list/set/get block storage
+    policies. (yliu)
+
+    HDFS-7496. Fix FsVolume removal race conditions on the DataNode by
+    reference-counting the volume instances (lei via cmccabe)
+
+    HDFS-7610. Fix removal of dynamically added DN volumes (Lei (Eddy) Xu via
+    Colin P. McCabe)
+
+    HDFS-7548. Corrupt block reporting delayed until datablock scanner thread
+    detects it (Rushabh Shah via kihwal)
+
+    HDFS-7575. Upgrade should generate a unique storage ID for each
+    volume. (Arpit Agarwal)
+
+    HDFS-3519. Checkpoint upload may interfere with a concurrent saveNamespace.
+    (Ming Ma via cnauroth)
+
+    HDFS-7660. BlockReceiver#close() might be called multiple times, which
+    causes the fsvolume reference being released incorrectly. (Lei Xu via
+    yliu)
+
+    HDFS-7644. minor typo in HttpFS doc (Charles Lamb via aw)
+
+    HDFS-7606. Fix potential NPE in INodeFile.getBlocks(). (Byron Wong via shv)
+
+    HDFS-7638: Small fix and few refinements for FSN#truncate. (yliu)
+
+    HDFS-7634. Disallow truncation of Lazy persist files. (Yi Liu via
+    Arpit Agarwal)
+
+    HDFS-7643. Test case to ensure lazy persist files cannot be truncated.
+    (Yi Liu via Arpit Agarwal)
+
+    HDFS-7659. truncate should check negative value of the new length.
+    (Yi Liu via shv)
+
+    HDFS-7676. Fix TestFileTruncate to avoid bug of HDFS-7611. (shv)
+
+    HDFS-49. MiniDFSCluster.stopDataNode will always shut down a node in
+    the cluster if a matching name is not found. (stevel)
+
+    HDFS-7566. Remove obsolete entries from hdfs-default.xml (Ray Chiang
+    via aw)
+
+    HDFS-7677. DistributedFileSystem#truncate should resolve symlinks. (yliu)
+
+    HDFS-7611. deleteSnapshot and delete of a file can leave orphaned blocks
+    in the blocksMap on NameNode restart. (jing9 and Byron Wong)
+
+    HDFS-7423. various typos and message formatting fixes in nfs daemon and
+    doc. (Charles Lamb via yliu)
+
+    HDFS-7696. In FsDatasetImpl, the getBlockInputStream(..) and
+    getTmpInputStreams(..) methods may leak file descriptors.  (szetszwo)
+
+    HDFS-6651. Deletion failure can leak inodes permanently.
+    (Jing Zhao via wheat9)
+
+    HDFS-7707. Edit log corruption due to delayed block removal again.
+    (Yongjun Zhang via kihwal)
+
+    HDFS-7734. Class cast exception in NameNode#main. (yliu via wang)
+
+    HDFS-7719. BlockPoolSliceStorage#removeVolumes fails to remove some
+    in-memory state associated with volumes. (Lei (Eddy) Xu via Colin P.
+    McCabe)
+
+    HDFS-7709. Fix findbug warnings in httpfs. (Rakesh R via ozawa)
+
+    HDFS-7698. Fix locking on HDFS read statistics and add a method for
+    clearing them. (Colin P. McCabe via yliu)
+
+    HDFS-7741. Remove unnecessary synchronized in FSDataInputStream and
+    HdfsDataInputStream. (yliu)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -565,6 +890,21 @@ Release 2.6.1 - UNRELEASED
     HDFS-4882. Prevent the Namenode's LeaseManager from looping forever in
     checkLeases (Ravi Prakash via Colin P. McCabe)
 
+    HDFS-7489. Incorrect locking in FsVolumeList#checkDirs can hang datanodes
+    (Noah Lorang via Colin P. McCabe)
+
+    HDFS-7503. Namenode restart after large deletions can cause slow
+    processReport. (Arpit Agarwal)
+
+    HDFS-7443. Datanode upgrade to BLOCKID_BASED_LAYOUT fails if duplicate
+    block files are present in the same volume (cmccabe)
+
+    HDFS-3443. Fix NPE when namenode transition to active during startup by
+    adding checkNNStartup() in NameNodeRpcServer.  (Vinayakumar B via szetszwo)
+
+    HDFS-7733. NFS: readdir/readdirplus return null directory
+    attribute on failure. (Arpit Agarwal)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/checkstyle.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/checkstyle.xml
index 5b25fe2cb5aca..eda4a4716365e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/checkstyle.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/checkstyle.xml
@@ -134,7 +134,6 @@
         <!-- Checks for common coding problems               -->
         <!-- See http://checkstyle.sf.net/config_coding.html -->
         <!-- module name="AvoidInlineConditionals"/-->
-        <module name="DoubleCheckedLocking"/>
         <module name="EmptyStatement"/>
         <module name="EqualsHashCode"/>
         <module name="HiddenField">
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index bad1792546716..5efce5c037f22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -182,7 +182,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>org.htrace</groupId>
+      <groupId>org.apache.htrace</groupId>
       <artifactId>htrace-core</artifactId>
     </dependency>
     <dependency>
@@ -198,6 +198,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.fusesource.leveldbjni</groupId>
+      <artifactId>leveldbjni-all</artifactId>
+      <version>1.8</version>
+    </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
       <groupId>org.bouncycastle</groupId>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 227be45da5e34..8da49c011633a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -172,9 +172,15 @@ target_link_libraries(test_libhdfs_write
 
 add_library(native_mini_dfs
     main/native/libhdfs/native_mini_dfs.c
+    main/native/libhdfs/common/htable.c
+    main/native/libhdfs/exception.c
+    main/native/libhdfs/jni_helper.c
+    ${OS_DIR}/mutexes.c
+    ${OS_DIR}/thread_local_storage.c
 )
 target_link_libraries(native_mini_dfs
-    hdfs
+    ${JAVA_JVM_LIBRARY}
+    ${OS_LINK_LIBRARIES}
 )
 
 add_executable(test_native_mini_dfs
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c
index e41f950828d07..dc58318b02569 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c
@@ -21,21 +21,14 @@
 #include <curl/curl.h>
 
 #include "hdfs_http_client.h"
+#include "exception.h"
 
 static pthread_mutex_t curlInitMutex = PTHREAD_MUTEX_INITIALIZER;
 static volatile int curlGlobalInited = 0;
 
 const char *hdfs_strerror(int errnoval)
 {
-    const char *msg = NULL;
-    if (errnoval < 0 || errnoval >= sys_nerr) {
-        msg = "Invalid Error Code";
-    } else if (sys_errlist == NULL) {
-        msg = "Unknown Error";
-    } else {
-        msg = sys_errlist[errnoval];
-    }
-    return msg;
+  return terror(errnoval);
 }
 
 int initResponseBuffer(struct ResponseBuffer **buffer)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index ee666f3a2512a..5f8ff1843b620 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -49,7 +49,7 @@ function hadoop_usage
   echo "  secondarynamenode    run the DFS secondary namenode"
   echo "  snapshotDiff         diff two snapshots of a directory or diff the"
   echo "                       current directory contents with a snapshot"
-  echo "  storagepolicies      get all the existing block storage policies"
+  echo "  storagepolicies      list/get/set block storage policies"
   echo "  version              print the version"
   echo "  zkfc                 run the ZK Failover Controller daemon"
   echo ""
@@ -94,9 +94,7 @@ case ${COMMAND} in
     CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
   ;;
   classpath)
-    hadoop_finalize
-    echo "${CLASSPATH}"
-    exit
+    hadoop_do_classpath_subcommand "$@"
   ;;
   crypto)
     CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
@@ -228,7 +226,7 @@ case ${COMMAND} in
     CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
   ;;
   storagepolicies)
-    CLASS=org.apache.hadoop.hdfs.tools.GetStoragePolicies
+    CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
   ;;
   version)
     CLASS=org.apache.hadoop.util.VersionInfo
@@ -247,6 +245,8 @@ case ${COMMAND} in
   ;;
 esac
 
+hadoop_verify_user "${COMMAND}"
+
 if [[ -n "${secure_service}" ]]; then
   HADOOP_SECURE_USER="${secure_user}"
   hadoop_verify_secure_prereq
@@ -273,7 +273,6 @@ if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
   fi
 fi
 
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
 hadoop_finalize
 
 if [[ -n "${supportdaemonization}" ]]; then
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
index ed7b0420ae6cb..244e5a9da5fa8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -34,30 +34,23 @@ function hadoop_subproject_init
   # used interchangeable from here on out
   # ...
   # this should get deprecated at some point.
-  HADOOP_LOG_DIR="${HADOOP_HDFS_LOG_DIR:-$HADOOP_LOG_DIR}"
-  HADOOP_HDFS_LOG_DIR="${HADOOP_LOG_DIR}"
-  
-  HADOOP_LOGFILE="${HADOOP_HDFS_LOGFILE:-$HADOOP_LOGFILE}"
-  HADOOP_HDFS_LOGFILE="${HADOOP_LOGFILE}"
-  
-  HADOOP_NICENESS=${HADOOP_HDFS_NICENESS:-$HADOOP_NICENESS}
-  HADOOP_HDFS_NICENESS="${HADOOP_NICENESS}"
-  
-  HADOOP_STOP_TIMEOUT=${HADOOP_HDFS_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}
-  HADOOP_HDFS_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
-  
-  HADOOP_PID_DIR="${HADOOP_HDFS_PID_DIR:-$HADOOP_PID_DIR}"
-  HADOOP_HDFS_PID_DIR="${HADOOP_PID_DIR}"
+
+  hadoop_deprecate_envvar HADOOP_HDFS_LOG_DIR HADOOP_LOG_DIR
+
+  hadoop_deprecate_envvar HADOOP_HDFS_LOGFILE HADOOP_LOGFILE
+
+  hadoop_deprecate_envvar HADOOP_HDFS_NICENESS HADOOP_NICENESS
+
+  hadoop_deprecate_envvar HADOOP_HDFS_STOP_TIMEOUT HADOOP_STOP_TIMEOUT
   
-  HADOOP_ROOT_LOGGER=${HADOOP_HDFS_ROOT_LOGGER:-$HADOOP_ROOT_LOGGER}
-  HADOOP_HDFS_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
+  hadoop_deprecate_envvar HADOOP_HDFS_PID_DIR HADOOP_PID_DIR
+
+  hadoop_deprecate_envvar HADOOP_HDFS_ROOT_LOGGER HADOOP_ROOT_LOGGER
+
+  hadoop_deprecate_envvar HADOOP_HDFS_IDENT_STRING HADOOP_IDENT_STRING
   
   HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_PREFIX}"
   
-  HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}"
-  HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}"
-
-  
   # turn on the defaults
   export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}
   export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
@@ -66,8 +59,6 @@ function hadoop_subproject_init
   export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"}
   export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"}
   export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"}
-  
-  
 }
 
 if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 00ecfe2795908..78832524c9e40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -51,7 +51,15 @@ if "%1" == "--loglevel" (
       goto print_usage
   )
 
-  set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies
+  if %hdfs-command% == classpath (
+    if not defined hdfs-command-arguments (
+      @rem No need to bother starting up a JVM for this simple case.
+      @echo %CLASSPATH%
+      exit /b
+    )
+  )
+
+  set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath
   for %%i in ( %hdfscommands% ) do (
     if %hdfs-command% == %%i set hdfscommand=true
   )
@@ -122,6 +130,10 @@ goto :eof
   set CLASS=org.apache.hadoop.hdfs.tools.JMXGet
   goto :eof
 
+:classpath
+  set CLASS=org.apache.hadoop.util.Classpath
+  goto :eof
+
 :oiv
   set CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
   goto :eof
@@ -160,7 +172,7 @@ goto :eof
   goto :eof
 
 :storagepolicies
-  set CLASS=org.apache.hadoop.hdfs.tools.GetStoragePolicies
+  set CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
   goto :eof
 
 @rem This changes %1, %2 etc. Hence those cannot be used after calling this.
@@ -216,7 +228,7 @@ goto :eof
   @echo 						Use -help to see options
   @echo   cacheadmin           configure the HDFS cache
   @echo   mover                run a utility to move block replicas across storage types
-  @echo   storagepolicies      get all the existing block storage policies
+  @echo   storagepolicies      list/get/set block storage policies
   @echo.
   @echo Most commands print help when invoked w/o parameters.
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 111630c5f59dd..1b9f515deb8f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -30,7 +30,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.CryptoCodec;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
@@ -61,7 +60,6 @@
 public class Hdfs extends AbstractFileSystem {
 
   DFSClient dfs;
-  final CryptoCodec factory;
   private boolean verifyChecksum = true;
 
   static {
@@ -88,7 +86,6 @@ public class Hdfs extends AbstractFileSystem {
     }
 
     this.dfs = new DFSClient(theUri, conf, getStatistics());
-    this.factory = CryptoCodec.getInstance(conf);
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
index 13e0a5226854a..7e40917fff009 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
@@ -668,7 +668,6 @@ private BlockReader getRemoteBlockReaderFromTcp() throws IOException {
       Peer peer = null;
       try {
         curPeer = nextTcpPeer();
-        if (curPeer == null) break;
         if (curPeer.fromCache) remainingCacheTries--;
         peer = curPeer.peer;
         blockReader = getRemoteBlockReader(peer);
@@ -699,7 +698,6 @@ private BlockReader getRemoteBlockReaderFromTcp() throws IOException {
         }
       }
     }
-    return null;
   }
 
   public static class BlockReaderPeer {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
index 2a9ce964d0438..5b697e075d939 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
@@ -33,12 +33,12 @@
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica;
 import org.apache.hadoop.util.DirectBufferPool;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
 
 /**
  * BlockReaderLocal enables local short circuited reads. If the DFS client is on
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
index f7ff94ac01045..3582f6781257c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
@@ -46,9 +46,9 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 /**
  * BlockReaderLocalLegacy enables local short circuited reads. If the DFS client is on
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
index ba749783f5f54..c809017645e67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
@@ -48,6 +48,10 @@
 import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.token.Token;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -71,7 +75,7 @@ class BlockStorageLocationUtil {
    */
   private static List<VolumeBlockLocationCallable> createVolumeBlockLocationCallables(
       Configuration conf, Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks,
-      int timeout, boolean connectToDnViaHostname) {
+      int timeout, boolean connectToDnViaHostname, Span parent) {
     
     if (datanodeBlocks.isEmpty()) {
       return Lists.newArrayList();
@@ -111,7 +115,7 @@ private static List<VolumeBlockLocationCallable> createVolumeBlockLocationCallab
       }
       VolumeBlockLocationCallable callable = new VolumeBlockLocationCallable(
           conf, datanode, poolId, blockIds, dnTokens, timeout, 
-          connectToDnViaHostname);
+          connectToDnViaHostname, parent);
       callables.add(callable);
     }
     return callables;
@@ -131,11 +135,11 @@ private static List<VolumeBlockLocationCallable> createVolumeBlockLocationCallab
   static Map<DatanodeInfo, HdfsBlocksMetadata> queryDatanodesForHdfsBlocksMetadata(
       Configuration conf, Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks,
       int poolsize, int timeoutMs, boolean connectToDnViaHostname)
-      throws InvalidBlockTokenException {
+        throws InvalidBlockTokenException {
 
     List<VolumeBlockLocationCallable> callables = 
         createVolumeBlockLocationCallables(conf, datanodeBlocks, timeoutMs, 
-            connectToDnViaHostname);
+            connectToDnViaHostname, Trace.currentSpan());
     
     // Use a thread pool to execute the Callables in parallel
     List<Future<HdfsBlocksMetadata>> futures = 
@@ -319,11 +323,12 @@ private static class VolumeBlockLocationCallable implements
     private final long[] blockIds;
     private final List<Token<BlockTokenIdentifier>> dnTokens;
     private final boolean connectToDnViaHostname;
+    private final Span parentSpan;
     
     VolumeBlockLocationCallable(Configuration configuration,
         DatanodeInfo datanode, String poolId, long []blockIds,
         List<Token<BlockTokenIdentifier>> dnTokens, int timeout, 
-        boolean connectToDnViaHostname) {
+        boolean connectToDnViaHostname, Span parentSpan) {
       this.configuration = configuration;
       this.timeout = timeout;
       this.datanode = datanode;
@@ -331,6 +336,7 @@ private static class VolumeBlockLocationCallable implements
       this.blockIds = blockIds;
       this.dnTokens = dnTokens;
       this.connectToDnViaHostname = connectToDnViaHostname;
+      this.parentSpan = parentSpan;
     }
     
     public DatanodeInfo getDatanodeInfo() {
@@ -342,6 +348,8 @@ public HdfsBlocksMetadata call() throws Exception {
       HdfsBlocksMetadata metadata = null;
       // Create the RPC proxy and make the RPC
       ClientDatanodeProtocol cdp = null;
+      TraceScope scope =
+          Trace.startSpan("getHdfsBlocksMetadata", parentSpan);
       try {
         cdp = DFSUtil.createClientDatanodeProtocolProxy(datanode, configuration,
             timeout, connectToDnViaHostname);
@@ -350,6 +358,7 @@ public HdfsBlocksMetadata call() throws Exception {
         // Bubble this up to the caller, handle with the Future
         throw e;
       } finally {
+        scope.close();
         if (cdp != null) {
           RPC.stopProxy(cdp);
         }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 62db1fac8ccc8..e5bf98d727112 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -167,6 +167,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -211,16 +212,17 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.tracing.SpanReceiverHost;
-import org.apache.hadoop.tracing.TraceSamplerFactory;
+import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.Daemon;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.DataChecksum.Type;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Time;
-import org.htrace.Sampler;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.SamplerBuilder;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
@@ -275,7 +277,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
   private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
   @VisibleForTesting
   KeyProvider provider;
-  private final SpanReceiverHost spanReceiverHost;
   private final Sampler<?> traceSampler;
 
   /**
@@ -620,8 +621,8 @@ public DFSClient(URI nameNodeUri, Configuration conf,
   public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
       Configuration conf, FileSystem.Statistics stats)
     throws IOException {
-    spanReceiverHost = SpanReceiverHost.getInstance(conf);
-    traceSampler = TraceSamplerFactory.createSampler(conf);
+    SpanReceiverHost.getInstance(conf);
+    traceSampler = new SamplerBuilder(TraceUtils.wrapHadoopConf(conf)).build();
     // Copy only the required DFSClient configuration
     this.dfsClientConf = new Conf(conf);
     if (this.dfsClientConf.useLegacyBlockReaderLocal) {
@@ -998,11 +999,14 @@ public long getDefaultBlockSize() {
    * @see ClientProtocol#getPreferredBlockSize(String)
    */
   public long getBlockSize(String f) throws IOException {
+    TraceScope scope = getPathTraceScope("getBlockSize", f);
     try {
       return namenode.getPreferredBlockSize(f);
     } catch (IOException ie) {
       LOG.warn("Problem getting block size", ie);
       throw ie;
+    } finally {
+      scope.close();
     }
   }
 
@@ -1035,17 +1039,20 @@ public String getCanonicalServiceName() {
   public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
       throws IOException {
     assert dtService != null;
-    Token<DelegationTokenIdentifier> token =
-      namenode.getDelegationToken(renewer);
-
-    if (token != null) {
-      token.setService(this.dtService);
-      LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
-    } else {
-      LOG.info("Cannot get delegation token from " + renewer);
+    TraceScope scope = Trace.startSpan("getDelegationToken", traceSampler);
+    try {
+      Token<DelegationTokenIdentifier> token =
+        namenode.getDelegationToken(renewer);
+      if (token != null) {
+        token.setService(this.dtService);
+        LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
+      } else {
+        LOG.info("Cannot get delegation token from " + renewer);
+      }
+      return token;
+    } finally {
+      scope.close();
     }
-    return token;
-
   }
 
   /**
@@ -1216,7 +1223,12 @@ public LocatedBlocks getLocatedBlocks(String src, long start)
   @VisibleForTesting
   public LocatedBlocks getLocatedBlocks(String src, long start, long length)
       throws IOException {
-    return callGetBlockLocations(namenode, src, start, length);
+    TraceScope scope = getPathTraceScope("getBlockLocations", src);
+    try {
+      return callGetBlockLocations(namenode, src, start, length);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -1243,12 +1255,15 @@ static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
   boolean recoverLease(String src) throws IOException {
     checkOpen();
 
+    TraceScope scope = getPathTraceScope("recoverLease", src);
     try {
       return namenode.recoverLease(src, clientName);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(FileNotFoundException.class,
                                      AccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -1265,14 +1280,19 @@ boolean recoverLease(String src) throws IOException {
    * as the data-block the task processes. 
    */
   public BlockLocation[] getBlockLocations(String src, long start, 
-    long length) throws IOException, UnresolvedLinkException {
-    LocatedBlocks blocks = getLocatedBlocks(src, start, length);
-    BlockLocation[] locations =  DFSUtil.locatedBlocks2Locations(blocks);
-    HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
-    for (int i = 0; i < locations.length; i++) {
-      hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
+        long length) throws IOException, UnresolvedLinkException {
+    TraceScope scope = getPathTraceScope("getBlockLocations", src);
+    try {
+      LocatedBlocks blocks = getLocatedBlocks(src, start, length);
+      BlockLocation[] locations =  DFSUtil.locatedBlocks2Locations(blocks);
+      HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
+      for (int i = 0; i < locations.length; i++) {
+        hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
+      }
+      return hdfsLocations;
+    } finally {
+      scope.close();
     }
-    return hdfsLocations;
   }
   
   /**
@@ -1326,15 +1346,21 @@ public BlockStorageLocation[] getBlockStorageLocations(
     }
         
     // Make RPCs to the datanodes to get volume locations for its replicas
-    Map<DatanodeInfo, HdfsBlocksMetadata> metadatas = BlockStorageLocationUtil
-        .queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
-            getConf().getFileBlockStorageLocationsNumThreads,
-            getConf().getFileBlockStorageLocationsTimeoutMs,
-            getConf().connectToDnViaHostname);
-    
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("metadata returned: "
-          + Joiner.on("\n").withKeyValueSeparator("=").join(metadatas));
+    TraceScope scope =
+      Trace.startSpan("getBlockStorageLocations", traceSampler);
+    Map<DatanodeInfo, HdfsBlocksMetadata> metadatas;
+    try {
+      metadatas = BlockStorageLocationUtil.
+          queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
+              getConf().getFileBlockStorageLocationsNumThreads,
+              getConf().getFileBlockStorageLocationsTimeoutMs,
+              getConf().connectToDnViaHostname);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("metadata returned: "
+            + Joiner.on("\n").withKeyValueSeparator("=").join(metadatas));
+      }
+    } finally {
+      scope.close();
     }
     
     // Regroup the returned VolumeId metadata to again be grouped by
@@ -1354,19 +1380,24 @@ public BlockStorageLocation[] getBlockStorageLocations(
    */
   private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo
       feInfo) throws IOException {
-    if (provider == null) {
-      throw new IOException("No KeyProvider is configured, cannot access" +
-          " an encrypted file");
-    }
-    EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
-        feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
-        feInfo.getEncryptedDataEncryptionKey());
+    TraceScope scope = Trace.startSpan("decryptEDEK", traceSampler);
     try {
-      KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
-          .createKeyProviderCryptoExtension(provider);
-      return cryptoProvider.decryptEncryptedKey(ekv);
-    } catch (GeneralSecurityException e) {
-      throw new IOException(e);
+      if (provider == null) {
+        throw new IOException("No KeyProvider is configured, cannot access" +
+            " an encrypted file");
+      }
+      EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption(
+          feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
+          feInfo.getEncryptedDataEncryptionKey());
+      try {
+        KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
+            .createKeyProviderCryptoExtension(provider);
+        return cryptoProvider.decryptEncryptedKey(ekv);
+      } catch (GeneralSecurityException e) {
+        throw new IOException(e);
+      }
+    } finally {
+      scope.close();
     }
   }
 
@@ -1504,7 +1535,12 @@ public DFSInputStream open(String src, int buffersize, boolean verifyChecksum)
       throws IOException, UnresolvedLinkException {
     checkOpen();
     //    Get block info from namenode
-    return new DFSInputStream(this, src, verifyChecksum);
+    TraceScope scope = getPathTraceScope("newDFSInputStream", src);
+    try {
+      return new DFSInputStream(this, src, verifyChecksum);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -1621,9 +1657,8 @@ public DFSOutputStream create(String src,
    * @param checksumOpt checksum options
    * 
    * @return output stream
-   * 
-   * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
-   * boolean, short, long) for detailed description of exceptions thrown
+   *
+   * @see ClientProtocol#create for detailed description of exceptions thrown
    */
   public DFSOutputStream create(String src, 
                              FsPermission permission,
@@ -1697,7 +1732,7 @@ private DFSOutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
         }
         return null;
       }
-      return callAppend(src, buffersize, progress);
+      return callAppend(src, buffersize, flag, progress);
     }
     return null;
   }
@@ -1737,6 +1772,7 @@ public DFSOutputStream primitiveCreate(String src,
    */
   public void createSymlink(String target, String link, boolean createParent)
       throws IOException {
+    TraceScope scope = getPathTraceScope("createSymlink", target);
     try {
       FsPermission dirPerm = 
           FsPermission.getDefault().applyUMask(dfsClientConf.uMask); 
@@ -1750,6 +1786,8 @@ public void createSymlink(String target, String link, boolean createParent)
                                      DSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -1760,20 +1798,28 @@ public void createSymlink(String target, String link, boolean createParent)
    */
   public String getLinkTarget(String path) throws IOException { 
     checkOpen();
+    TraceScope scope = getPathTraceScope("getLinkTarget", path);
     try {
       return namenode.getLinkTarget(path);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class);
+    } finally {
+      scope.close();
     }
   }
 
   /** Method to get stream returned by append call */
-  private DFSOutputStream callAppend(String src,
-      int buffersize, Progressable progress) throws IOException {
-    LastBlockWithStatus lastBlockWithStatus = null;
+  private DFSOutputStream callAppend(String src, int buffersize,
+      EnumSet<CreateFlag> flag, Progressable progress) throws IOException {
+    CreateFlag.validateForAppend(flag);
     try {
-      lastBlockWithStatus = namenode.append(src, clientName);
+      LastBlockWithStatus blkWithStatus = namenode.append(src, clientName,
+          new EnumSetWritable<>(flag, CreateFlag.class));
+      return DFSOutputStream.newStreamForAppend(this, src,
+          flag.contains(CreateFlag.NEW_BLOCK),
+          buffersize, progress, blkWithStatus.getLastBlock(),
+          blkWithStatus.getFileStatus(), dfsClientConf.createChecksum());
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
@@ -1783,10 +1829,6 @@ private DFSOutputStream callAppend(String src,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
     }
-    HdfsFileStatus newStat = lastBlockWithStatus.getFileStatus();
-    return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
-        lastBlockWithStatus.getLastBlock(), newStat,
-        dfsClientConf.createChecksum());
   }
   
   /**
@@ -1794,23 +1836,25 @@ private DFSOutputStream callAppend(String src,
    * 
    * @param src file name
    * @param buffersize buffer size
+   * @param flag indicates whether to append data to a new block instead of
+   *             the last block
    * @param progress for reporting write-progress; null is acceptable.
    * @param statistics file system statistics; null is acceptable.
    * @return an output stream for writing into the file
    * 
-   * @see ClientProtocol#append(String, String) 
+   * @see ClientProtocol#append(String, String, EnumSetWritable)
    */
   public HdfsDataOutputStream append(final String src, final int buffersize,
-      final Progressable progress, final FileSystem.Statistics statistics
-      ) throws IOException {
-    final DFSOutputStream out = append(src, buffersize, progress);
+      EnumSet<CreateFlag> flag, final Progressable progress,
+      final FileSystem.Statistics statistics) throws IOException {
+    final DFSOutputStream out = append(src, buffersize, flag, progress);
     return createWrappedOutputStream(out, statistics, out.getInitialLen());
   }
 
-  private DFSOutputStream append(String src, int buffersize, Progressable progress) 
-      throws IOException {
+  private DFSOutputStream append(String src, int buffersize,
+      EnumSet<CreateFlag> flag, Progressable progress) throws IOException {
     checkOpen();
-    final DFSOutputStream result = callAppend(src, buffersize, progress);
+    final DFSOutputStream result = callAppend(src, buffersize, flag, progress);
     beginFileLease(result.getFileId(), result);
     return result;
   }
@@ -1824,6 +1868,7 @@ private DFSOutputStream append(String src, int buffersize, Progressable progress
    */
   public boolean setReplication(String src, short replication)
       throws IOException {
+    TraceScope scope = getPathTraceScope("setReplication", src);
     try {
       return namenode.setReplication(src, replication);
     } catch(RemoteException re) {
@@ -1833,6 +1878,8 @@ public boolean setReplication(String src, short replication)
                                      DSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -1843,6 +1890,7 @@ public boolean setReplication(String src, short replication)
    */
   public void setStoragePolicy(String src, String policyName)
       throws IOException {
+    TraceScope scope = getPathTraceScope("setStoragePolicy", src);
     try {
       namenode.setStoragePolicy(src, policyName);
     } catch (RemoteException e) {
@@ -1852,6 +1900,8 @@ public void setStoragePolicy(String src, String policyName)
                                     NSQuotaExceededException.class,
                                     UnresolvedPathException.class,
                                     SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -1859,7 +1909,12 @@ public void setStoragePolicy(String src, String policyName)
    * @return All the existing storage policies
    */
   public BlockStoragePolicy[] getStoragePolicies() throws IOException {
-    return namenode.getStoragePolicies();
+    TraceScope scope = Trace.startSpan("getStoragePolicies", traceSampler);
+    try {
+      return namenode.getStoragePolicies();
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -1870,6 +1925,7 @@ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
   @Deprecated
   public boolean rename(String src, String dst) throws IOException {
     checkOpen();
+    TraceScope scope = getSrcDstTraceScope("rename", src, dst);
     try {
       return namenode.rename(src, dst);
     } catch(RemoteException re) {
@@ -1878,21 +1934,26 @@ public boolean rename(String src, String dst) throws IOException {
                                      DSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
   /**
    * Move blocks from src to trg and delete src
-   * See {@link ClientProtocol#concat(String, String [])}. 
+   * See {@link ClientProtocol#concat}.
    */
   public void concat(String trg, String [] srcs) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("concat", traceSampler);
     try {
       namenode.concat(trg, srcs);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
   /**
@@ -1902,6 +1963,7 @@ public void concat(String trg, String [] srcs) throws IOException {
   public void rename(String src, String dst, Options.Rename... options)
       throws IOException {
     checkOpen();
+    TraceScope scope = getSrcDstTraceScope("rename2", src, dst);
     try {
       namenode.rename2(src, dst, options);
     } catch(RemoteException re) {
@@ -1914,8 +1976,32 @@ public void rename(String src, String dst, Options.Rename... options)
                                      NSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
+
+  /**
+   * Truncate a file to an indicated size
+   * See {@link ClientProtocol#truncate}.
+   */
+  public boolean truncate(String src, long newLength) throws IOException {
+    checkOpen();
+    if (newLength < 0) {
+      throw new HadoopIllegalArgumentException(
+          "Cannot truncate to a negative file size: " + newLength + ".");
+    }
+    TraceScope scope = getPathTraceScope("truncate", src);
+    try {
+      return namenode.truncate(src, newLength, clientName);
+    } catch (RemoteException re) {
+      throw re.unwrapRemoteException(AccessControlException.class,
+          UnresolvedPathException.class);
+    } finally {
+      scope.close();
+    }
+  }
+
   /**
    * Delete file or directory.
    * See {@link ClientProtocol#delete(String, boolean)}. 
@@ -1923,7 +2009,7 @@ public void rename(String src, String dst, Options.Rename... options)
   @Deprecated
   public boolean delete(String src) throws IOException {
     checkOpen();
-    return namenode.delete(src, true);
+    return delete(src, true);
   }
 
   /**
@@ -1935,6 +2021,7 @@ public boolean delete(String src) throws IOException {
    */
   public boolean delete(String src, boolean recursive) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("delete", src);
     try {
       return namenode.delete(src, recursive);
     } catch(RemoteException re) {
@@ -1943,6 +2030,8 @@ public boolean delete(String src, boolean recursive) throws IOException {
                                      SafeModeException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
   
@@ -1972,15 +2061,17 @@ public DirectoryListing listPaths(String src,  byte[] startAfter)
    * @see ClientProtocol#getListing(String, byte[], boolean)
    */
   public DirectoryListing listPaths(String src,  byte[] startAfter,
-      boolean needLocation) 
-    throws IOException {
+      boolean needLocation) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("listPaths", src);
     try {
       return namenode.getListing(src, startAfter, needLocation);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -1994,12 +2085,15 @@ public DirectoryListing listPaths(String src,  byte[] startAfter,
    */
   public HdfsFileStatus getFileInfo(String src) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getFileInfo", src);
     try {
       return namenode.getFileInfo(src);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
@@ -2009,12 +2103,15 @@ public HdfsFileStatus getFileInfo(String src) throws IOException {
    */
   public boolean isFileClosed(String src) throws IOException{
     checkOpen();
+    TraceScope scope = getPathTraceScope("isFileClosed", src);
     try {
       return namenode.isFileClosed(src);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
@@ -2028,12 +2125,15 @@ public boolean isFileClosed(String src) throws IOException{
    */
   public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getFileLinkInfo", src);
     try {
       return namenode.getFileLinkInfo(src);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      UnresolvedPathException.class);
-     }
+    } finally {
+      scope.close();
+    }
    }
   
   @InterfaceAudience.Private
@@ -2330,6 +2430,7 @@ private Type inferChecksumTypeByReading(LocatedBlock lb, DatanodeInfo dn)
   public void setPermission(String src, FsPermission permission)
       throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("setPermission", src);
     try {
       namenode.setPermission(src, permission);
     } catch(RemoteException re) {
@@ -2338,6 +2439,8 @@ public void setPermission(String src, FsPermission permission)
                                      SafeModeException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -2352,6 +2455,7 @@ public void setPermission(String src, FsPermission permission)
   public void setOwner(String src, String username, String groupname)
       throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("setOwner", src);
     try {
       namenode.setOwner(src, username, groupname);
     } catch(RemoteException re) {
@@ -2360,6 +2464,18 @@ public void setOwner(String src, String username, String groupname)
                                      SafeModeException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);                                   
+    } finally {
+      scope.close();
+    }
+  }
+
+  private long[] callGetStats() throws IOException {
+    checkOpen();
+    TraceScope scope = Trace.startSpan("getStats", traceSampler);
+    try {
+      return namenode.getStats();
+    } finally {
+      scope.close();
     }
   }
 
@@ -2367,7 +2483,7 @@ public void setOwner(String src, String username, String groupname)
    * @see ClientProtocol#getStats()
    */
   public FsStatus getDiskStatus() throws IOException {
-    long rawNums[] = namenode.getStats();
+    long rawNums[] = callGetStats();
     return new FsStatus(rawNums[0], rawNums[1], rawNums[2]);
   }
 
@@ -2377,7 +2493,7 @@ public FsStatus getDiskStatus() throws IOException {
    * @throws IOException
    */ 
   public long getMissingBlocksCount() throws IOException {
-    return namenode.getStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
+    return callGetStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
   }
   
   /**
@@ -2386,7 +2502,7 @@ public long getMissingBlocksCount() throws IOException {
    * @throws IOException
    */
   public long getMissingReplOneBlocksCount() throws IOException {
-    return namenode.getStats()[ClientProtocol.
+    return callGetStats()[ClientProtocol.
         GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX];
   }
 
@@ -2395,7 +2511,7 @@ public long getMissingReplOneBlocksCount() throws IOException {
    * @throws IOException
    */ 
   public long getUnderReplicatedBlocksCount() throws IOException {
-    return namenode.getStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
+    return callGetStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
   }
   
   /**
@@ -2403,7 +2519,7 @@ public long getUnderReplicatedBlocksCount() throws IOException {
    * @throws IOException
    */ 
   public long getCorruptBlocksCount() throws IOException {
-    return namenode.getStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
+    return callGetStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
   }
   
   /**
@@ -2412,18 +2528,37 @@ public long getCorruptBlocksCount() throws IOException {
    */
   public CorruptFileBlocks listCorruptFileBlocks(String path,
                                                  String cookie)
-    throws IOException {
-    return namenode.listCorruptFileBlocks(path, cookie);
+        throws IOException {
+    checkOpen();
+    TraceScope scope = getPathTraceScope("listCorruptFileBlocks", path);
+    try {
+      return namenode.listCorruptFileBlocks(path, cookie);
+    } finally {
+      scope.close();
+    }
   }
 
   public DatanodeInfo[] datanodeReport(DatanodeReportType type)
-  throws IOException {
-    return namenode.getDatanodeReport(type);
+      throws IOException {
+    checkOpen();
+    TraceScope scope = Trace.startSpan("datanodeReport", traceSampler);
+    try {
+      return namenode.getDatanodeReport(type);
+    } finally {
+      scope.close();
+    }
   }
     
   public DatanodeStorageReport[] getDatanodeStorageReport(
       DatanodeReportType type) throws IOException {
-    return namenode.getDatanodeStorageReport(type);
+    checkOpen();
+    TraceScope scope =
+        Trace.startSpan("datanodeStorageReport", traceSampler);
+    try {
+      return namenode.getDatanodeStorageReport(type);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -2447,7 +2582,13 @@ public boolean setSafeMode(SafeModeAction action) throws IOException {
    * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction, boolean)
    */
   public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException{
-    return namenode.setSafeMode(action, isChecked);    
+    TraceScope scope =
+        Trace.startSpan("setSafeMode", traceSampler);
+    try {
+      return namenode.setSafeMode(action, isChecked);
+    } finally {
+      scope.close();
+    }
   }
  
   /**
@@ -2461,10 +2602,13 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOEx
   public String createSnapshot(String snapshotRoot, String snapshotName)
       throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("createSnapshot", traceSampler);
     try {
       return namenode.createSnapshot(snapshotRoot, snapshotName);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
@@ -2479,10 +2623,14 @@ public String createSnapshot(String snapshotRoot, String snapshotName)
    */
   public void deleteSnapshot(String snapshotRoot, String snapshotName)
       throws IOException {
+    checkOpen();
+    TraceScope scope = Trace.startSpan("deleteSnapshot", traceSampler);
     try {
       namenode.deleteSnapshot(snapshotRoot, snapshotName);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
@@ -2497,10 +2645,13 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName)
   public void renameSnapshot(String snapshotDir, String snapshotOldName,
       String snapshotNewName) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("renameSnapshot", traceSampler);
     try {
       namenode.renameSnapshot(snapshotDir, snapshotOldName, snapshotNewName);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
@@ -2513,10 +2664,14 @@ public void renameSnapshot(String snapshotDir, String snapshotOldName,
   public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
       throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("getSnapshottableDirListing",
+        traceSampler);
     try {
       return namenode.getSnapshottableDirListing();
     } catch(RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
@@ -2527,10 +2682,13 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
    */
   public void allowSnapshot(String snapshotRoot) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("allowSnapshot", traceSampler);
     try {
       namenode.allowSnapshot(snapshotRoot);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
@@ -2541,10 +2699,13 @@ public void allowSnapshot(String snapshotRoot) throws IOException {
    */
   public void disallowSnapshot(String snapshotRoot) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("disallowSnapshot", traceSampler);
     try {
       namenode.disallowSnapshot(snapshotRoot);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
@@ -2556,78 +2717,99 @@ public void disallowSnapshot(String snapshotRoot) throws IOException {
   public SnapshotDiffReport getSnapshotDiffReport(String snapshotDir,
       String fromSnapshot, String toSnapshot) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("getSnapshotDiffReport", traceSampler);
     try {
       return namenode.getSnapshotDiffReport(snapshotDir,
           fromSnapshot, toSnapshot);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
   public long addCacheDirective(
       CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("addCacheDirective", traceSampler);
     try {
       return namenode.addCacheDirective(info, flags);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
   public void modifyCacheDirective(
       CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("modifyCacheDirective", traceSampler);
     try {
       namenode.modifyCacheDirective(info, flags);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
   public void removeCacheDirective(long id)
       throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("removeCacheDirective", traceSampler);
     try {
       namenode.removeCacheDirective(id);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
   
   public RemoteIterator<CacheDirectiveEntry> listCacheDirectives(
       CacheDirectiveInfo filter) throws IOException {
-    return new CacheDirectiveIterator(namenode, filter);
+    return new CacheDirectiveIterator(namenode, filter, traceSampler);
   }
 
   public void addCachePool(CachePoolInfo info) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("addCachePool", traceSampler);
     try {
       namenode.addCachePool(info);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
   public void modifyCachePool(CachePoolInfo info) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("modifyCachePool", traceSampler);
     try {
       namenode.modifyCachePool(info);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
   public void removeCachePool(String poolName) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("removeCachePool", traceSampler);
     try {
       namenode.removeCachePool(poolName);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException();
+    } finally {
+      scope.close();
     }
   }
 
   public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
-    return new CachePoolIterator(namenode);
+    return new CachePoolIterator(namenode, traceSampler);
   }
 
   /**
@@ -2636,10 +2818,13 @@ public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
    * @see ClientProtocol#saveNamespace()
    */
   void saveNamespace() throws AccessControlException, IOException {
+    TraceScope scope = Trace.startSpan("saveNamespace", traceSampler);
     try {
       namenode.saveNamespace();
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -2650,10 +2835,13 @@ void saveNamespace() throws AccessControlException, IOException {
    * @see ClientProtocol#rollEdits()
    */
   long rollEdits() throws AccessControlException, IOException {
+    TraceScope scope = Trace.startSpan("rollEdits", traceSampler);
     try {
       return namenode.rollEdits();
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -2669,7 +2857,12 @@ ExtendedBlock getPreviousBlock(long fileId) {
    */
   boolean restoreFailedStorage(String arg)
       throws AccessControlException, IOException{
-    return namenode.restoreFailedStorage(arg);
+    TraceScope scope = Trace.startSpan("restoreFailedStorage", traceSampler);
+    try {
+      return namenode.restoreFailedStorage(arg);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -2680,7 +2873,12 @@ boolean restoreFailedStorage(String arg)
    * @see ClientProtocol#refreshNodes()
    */
   public void refreshNodes() throws IOException {
-    namenode.refreshNodes();
+    TraceScope scope = Trace.startSpan("refreshNodes", traceSampler);
+    try {
+      namenode.refreshNodes();
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -2689,7 +2887,12 @@ public void refreshNodes() throws IOException {
    * @see ClientProtocol#metaSave(String)
    */
   public void metaSave(String pathname) throws IOException {
-    namenode.metaSave(pathname);
+    TraceScope scope = Trace.startSpan("metaSave", traceSampler);
+    try {
+      namenode.metaSave(pathname);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -2701,18 +2904,33 @@ public void metaSave(String pathname) throws IOException {
    * @see ClientProtocol#setBalancerBandwidth(long)
    */
   public void setBalancerBandwidth(long bandwidth) throws IOException {
-    namenode.setBalancerBandwidth(bandwidth);
+    TraceScope scope = Trace.startSpan("setBalancerBandwidth", traceSampler);
+    try {
+      namenode.setBalancerBandwidth(bandwidth);
+    } finally {
+      scope.close();
+    }
   }
     
   /**
    * @see ClientProtocol#finalizeUpgrade()
    */
   public void finalizeUpgrade() throws IOException {
-    namenode.finalizeUpgrade();
+    TraceScope scope = Trace.startSpan("finalizeUpgrade", traceSampler);
+    try {
+      namenode.finalizeUpgrade();
+    } finally {
+      scope.close();
+    }
   }
 
   RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException {
-    return namenode.rollingUpgrade(action);
+    TraceScope scope = Trace.startSpan("rollingUpgrade", traceSampler);
+    try {
+      return namenode.rollingUpgrade(action);
+    } finally {
+      scope.close();
+    }
   }
 
   /**
@@ -2769,6 +2987,7 @@ public boolean primitiveMkdir(String src, FsPermission absPermission,
     if(LOG.isDebugEnabled()) {
       LOG.debug(src + ": masked=" + absPermission);
     }
+    TraceScope scope = Trace.startSpan("mkdir", traceSampler);
     try {
       return namenode.mkdirs(src, absPermission, createParent);
     } catch(RemoteException re) {
@@ -2782,28 +3001,33 @@ public boolean primitiveMkdir(String src, FsPermission absPermission,
                                      DSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
   
   /**
    * Get {@link ContentSummary} rooted at the specified directory.
-   * @param path The string representation of the path
+   * @param src The string representation of the path
    * 
    * @see ClientProtocol#getContentSummary(String)
    */
   ContentSummary getContentSummary(String src) throws IOException {
+    TraceScope scope = getPathTraceScope("getContentSummary", src);
     try {
       return namenode.getContentSummary(src);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   /**
    * Sets or resets quotas for a directory.
-   * @see ClientProtocol#setQuota(String, long, long)
+   * @see ClientProtocol#setQuota(String, long, long, StorageType)
    */
   void setQuota(String src, long namespaceQuota, long diskspaceQuota) 
       throws IOException {
@@ -2817,8 +3041,10 @@ void setQuota(String src, long namespaceQuota, long diskspaceQuota)
                                          diskspaceQuota);
                                          
     }
+    TraceScope scope = getPathTraceScope("setQuota", src);
     try {
-      namenode.setQuota(src, namespaceQuota, diskspaceQuota);
+      // Pass null as storage type for traditional space/namespace quota.
+      namenode.setQuota(src, namespaceQuota, diskspaceQuota, null);
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
@@ -2826,9 +3052,39 @@ void setQuota(String src, long namespaceQuota, long diskspaceQuota)
                                      DSQuotaExceededException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
+  /**
+   * Sets or resets quotas by storage type for a directory.
+   * @see ClientProtocol#setQuota(String, long, long, StorageType)
+   */
+  void setQuotaByStorageType(String src, StorageType type, long spaceQuota)
+      throws IOException {
+    if (spaceQuota <= 0 && spaceQuota != HdfsConstants.QUOTA_DONT_SET &&
+        spaceQuota != HdfsConstants.QUOTA_RESET) {
+      throw new IllegalArgumentException("Invalid values for quota :" +
+        spaceQuota);
+    }
+    if (type == null) {
+      throw new IllegalArgumentException("Invalid storage type(null)");
+    }
+    if (!type.supportTypeQuota()) {
+      throw new IllegalArgumentException("Don't support Quota for storage type : "
+        + type.toString());
+    }
+    try {
+      namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, spaceQuota, type);
+    } catch (RemoteException re) {
+      throw re.unwrapRemoteException(AccessControlException.class,
+        FileNotFoundException.class,
+        QuotaByStorageTypeExceededException.class,
+        UnresolvedPathException.class,
+        SnapshotAccessControlException.class);
+    }
+  }
   /**
    * set the modification and access time of a file
    * 
@@ -2836,6 +3092,7 @@ void setQuota(String src, long namespaceQuota, long diskspaceQuota)
    */
   public void setTimes(String src, long mtime, long atime) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("setTimes", src);
     try {
       namenode.setTimes(src, mtime, atime);
     } catch(RemoteException re) {
@@ -2843,6 +3100,8 @@ public void setTimes(String src, long mtime, long atime) throws IOException {
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class,
                                      SnapshotAccessControlException.class);
+    } finally {
+      scope.close();
     }
   }
 
@@ -2894,6 +3153,7 @@ public ClientContext getClientContext() {
   public void modifyAclEntries(String src, List<AclEntry> aclSpec)
       throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("modifyAclEntries", src);
     try {
       namenode.modifyAclEntries(src, aclSpec);
     } catch(RemoteException re) {
@@ -2904,12 +3164,15 @@ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void removeAclEntries(String src, List<AclEntry> aclSpec)
       throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("removeAclEntries", traceSampler);
     try {
       namenode.removeAclEntries(src, aclSpec);
     } catch(RemoteException re) {
@@ -2920,11 +3183,14 @@ public void removeAclEntries(String src, List<AclEntry> aclSpec)
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void removeDefaultAcl(String src) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("removeDefaultAcl", traceSampler);
     try {
       namenode.removeDefaultAcl(src);
     } catch(RemoteException re) {
@@ -2935,11 +3201,14 @@ public void removeDefaultAcl(String src) throws IOException {
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void removeAcl(String src) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("removeAcl", traceSampler);
     try {
       namenode.removeAcl(src);
     } catch(RemoteException re) {
@@ -2950,11 +3219,14 @@ public void removeAcl(String src) throws IOException {
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
     checkOpen();
+    TraceScope scope = Trace.startSpan("setAcl", traceSampler);
     try {
       namenode.setAcl(src, aclSpec);
     } catch(RemoteException re) {
@@ -2965,11 +3237,14 @@ public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public AclStatus getAclStatus(String src) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getAclStatus", src);
     try {
       return namenode.getAclStatus(src);
     } catch(RemoteException re) {
@@ -2977,41 +3252,50 @@ public AclStatus getAclStatus(String src) throws IOException {
                                      AclException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
   public void createEncryptionZone(String src, String keyName)
     throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("createEncryptionZone", src);
     try {
       namenode.createEncryptionZone(src, keyName);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      SafeModeException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public EncryptionZone getEZForPath(String src)
           throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getEZForPath", src);
     try {
       return namenode.getEZForPath(src);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public RemoteIterator<EncryptionZone> listEncryptionZones()
       throws IOException {
     checkOpen();
-    return new EncryptionZoneIterator(namenode);
+    return new EncryptionZoneIterator(namenode, traceSampler);
   }
 
   public void setXAttr(String src, String name, byte[] value, 
       EnumSet<XAttrSetFlag> flag) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("setXAttr", src);
     try {
       namenode.setXAttr(src, XAttrHelper.buildXAttr(name, value), flag);
     } catch (RemoteException re) {
@@ -3021,11 +3305,14 @@ public void setXAttr(String src, String name, byte[] value,
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
   public byte[] getXAttr(String src, String name) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getXAttr", src);
     try {
       final List<XAttr> xAttrs = XAttrHelper.buildXAttrAsList(name);
       final List<XAttr> result = namenode.getXAttrs(src, xAttrs);
@@ -3034,23 +3321,29 @@ public byte[] getXAttr(String src, String name) throws IOException {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
   public Map<String, byte[]> getXAttrs(String src) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getXAttrs", src);
     try {
       return XAttrHelper.buildXAttrMap(namenode.getXAttrs(src, null));
     } catch(RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
   public Map<String, byte[]> getXAttrs(String src, List<String> names) 
       throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("getXAttrs", src);
     try {
       return XAttrHelper.buildXAttrMap(namenode.getXAttrs(
           src, XAttrHelper.buildXAttrs(names)));
@@ -3058,12 +3351,15 @@ public Map<String, byte[]> getXAttrs(String src, List<String> names)
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
   
   public List<String> listXAttrs(String src)
           throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("listXAttrs", src);
     try {
       final Map<String, byte[]> xattrs =
         XAttrHelper.buildXAttrMap(namenode.listXAttrs(src));
@@ -3072,11 +3368,14 @@ public List<String> listXAttrs(String src)
       throw re.unwrapRemoteException(AccessControlException.class,
                                      FileNotFoundException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void removeXAttr(String src, String name) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("removeXAttr", src);
     try {
       namenode.removeXAttr(src, XAttrHelper.buildXAttr(name));
     } catch(RemoteException re) {
@@ -3086,27 +3385,32 @@ public void removeXAttr(String src, String name) throws IOException {
                                      SafeModeException.class,
                                      SnapshotAccessControlException.class,
                                      UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public void checkAccess(String src, FsAction mode) throws IOException {
     checkOpen();
+    TraceScope scope = getPathTraceScope("checkAccess", src);
     try {
       namenode.checkAccess(src, mode);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
           FileNotFoundException.class,
           UnresolvedPathException.class);
+    } finally {
+      scope.close();
     }
   }
 
   public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
-    return new DFSInotifyEventInputStream(namenode);
+    return new DFSInotifyEventInputStream(traceSampler, namenode);
   }
 
   public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid)
       throws IOException {
-    return new DFSInotifyEventInputStream(namenode, lastReadTxid);
+    return new DFSInotifyEventInputStream(traceSampler, namenode, lastReadTxid);
   }
 
   @Override // RemotePeerFactory
@@ -3225,4 +3529,24 @@ TraceScope getPathTraceScope(String description, String path) {
     }
     return scope;
   }
+
+  private static final byte[] SRC = "src".getBytes(Charset.forName("UTF-8"));
+
+  private static final byte[] DST = "dst".getBytes(Charset.forName("UTF-8"));
+
+  TraceScope getSrcDstTraceScope(String description, String src, String dst) {
+    TraceScope scope = Trace.startSpan(description, traceSampler);
+    Span span = scope.getSpan();
+    if (span != null) {
+      if (src != null) {
+        span.addKVAnnotation(SRC,
+            src.getBytes(Charset.forName("UTF-8")));
+      }
+      if (dst != null) {
+        span.addKVAnnotation(DST,
+            dst.getBytes(Charset.forName("UTF-8")));
+      }
+    }
+    return scope;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 78cae9cb258ba..04a631ffef8ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -217,6 +217,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3;
   public static final String  DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min";
   public static final int     DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1;
+  public static final String  DFS_NAMENODE_STRIPE_MIN_KEY = "dfs.namenode.stripe.min";
+  public static final int     DFS_NAMENODE_STRIPE_MIN_DEFAULT = 1;
   public static final String  DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY = "dfs.namenode.replication.pending.timeout-sec";
   public static final int     DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT = -1;
   public static final String  DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = "dfs.namenode.replication.max-streams";
@@ -325,6 +327,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY = "dfs.namenode.write.stale.datanode.ratio";
   public static final float DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT = 0.5f;
 
+  // Number of blocks to rescan for each iteration of postponedMisreplicatedBlocks.
+  public static final String DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY = "dfs.namenode.blocks.per.postponedblocks.rescan";
+  public static final long DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT = 10000;
+
   // Replication monitoring related keys
   public static final String DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION =
       "dfs.namenode.invalidate.work.pct.per.iteration";
@@ -437,6 +443,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final int     DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096;
   public static final String  DFS_DATANODE_SCAN_PERIOD_HOURS_KEY = "dfs.datanode.scan.period.hours";
   public static final int     DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT = 0;
+  public static final String  DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND = "dfs.block.scanner.volume.bytes.per.second";
+  public static final long    DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT = 1048576L;
   public static final String  DFS_DATANODE_TRANSFERTO_ALLOWED_KEY = "dfs.datanode.transferTo.allowed";
   public static final boolean DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT = true;
   public static final String  DFS_HEARTBEAT_INTERVAL_KEY = "dfs.heartbeat.interval";
@@ -499,11 +507,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY = "dfs.client.read.shortcircuit.skip.checksum";
   public static final boolean DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_DEFAULT = false;
   public static final String DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY = "dfs.client.read.shortcircuit.buffer.size";
+  public static final int DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_DEFAULT = 1024 * 1024;
   public static final String DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_KEY = "dfs.client.read.shortcircuit.streams.cache.size";
   public static final int DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_DEFAULT = 256;
   public static final String DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY = "dfs.client.read.shortcircuit.streams.cache.expiry.ms";
   public static final long DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_DEFAULT = 5 * 60 * 1000;
-  public static final int DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_DEFAULT = 1024 * 1024;
   public static final String DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC = "dfs.client.domain.socket.data.traffic";
   public static final boolean DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT = false;
   public static final String DFS_CLIENT_MMAP_ENABLED= "dfs.client.mmap.enabled";
@@ -599,6 +607,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_STORAGE_POLICY_ENABLED_KEY = "dfs.storage.policy.enabled";
   public static final boolean DFS_STORAGE_POLICY_ENABLED_DEFAULT = true;
 
+  public static final String  DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY = "dfs.quota.by.storage.type.enabled";
+  public static final boolean DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT = true;
+
   // HA related configuration
   public static final String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes";
   public static final String DFS_HA_NAMENODE_ID_KEY = "dfs.ha.namenode.id";
@@ -723,17 +734,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.client.hedged.read.threadpool.size";
   public static final int     DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE = 0;
 
-   // Slow io warning log threshold settings for dfsclient and datanode.
-   public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
-     "dfs.client.slow.io.warning.threshold.ms";
-   public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
-   public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
-     "dfs.datanode.slow.io.warning.threshold.ms";
-   public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300;
+  // Slow io warning log threshold settings for dfsclient and datanode.
+  public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
+    "dfs.client.slow.io.warning.threshold.ms";
+  public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
+  public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
+    "dfs.datanode.slow.io.warning.threshold.ms";
+  public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300;
 
-   public static final String DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY =
-       "dfs.datanode.block.id.layout.upgrade.threads";
-   public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 12;
+  public static final String DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY =
+      "dfs.datanode.block.id.layout.upgrade.threads";
+  public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 12;
 
   public static final String DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY =
       "dfs.namenode.inotify.max.events.per.rpc";
@@ -758,4 +769,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String NNTOP_WINDOWS_MINUTES_KEY =
       "dfs.namenode.top.windows.minutes";
   public static final String[] NNTOP_WINDOWS_MINUTES_DEFAULT = {"1","5","25"};
+  public static final String DFS_PIPELINE_ECN_ENABLED = "dfs.pipeline.ecn";
+  public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
index 83b92b95387f5..1f9e3e992e3a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
@@ -26,6 +26,9 @@
 import org.apache.hadoop.hdfs.inotify.MissingEventsException;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.util.Time;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,6 +47,11 @@ public class DFSInotifyEventInputStream {
   public static Logger LOG = LoggerFactory.getLogger(DFSInotifyEventInputStream
       .class);
 
+  /**
+   * The trace sampler to use when making RPCs to the NameNode.
+   */
+  private final Sampler<?> traceSampler;
+
   private final ClientProtocol namenode;
   private Iterator<EventBatch> it;
   private long lastReadTxid;
@@ -59,12 +67,15 @@ public class DFSInotifyEventInputStream {
 
   private static final int INITIAL_WAIT_MS = 10;
 
-  DFSInotifyEventInputStream(ClientProtocol namenode) throws IOException {
-    this(namenode, namenode.getCurrentEditLogTxid()); // only consider new txn's
+  DFSInotifyEventInputStream(Sampler<?> traceSampler, ClientProtocol namenode)
+        throws IOException {
+    // Only consider new transaction IDs.
+    this(traceSampler, namenode, namenode.getCurrentEditLogTxid());
   }
 
-  DFSInotifyEventInputStream(ClientProtocol namenode, long lastReadTxid)
-      throws IOException {
+  DFSInotifyEventInputStream(Sampler traceSampler, ClientProtocol namenode,
+        long lastReadTxid) throws IOException {
+    this.traceSampler = traceSampler;
     this.namenode = namenode;
     this.it = Iterators.emptyIterator();
     this.lastReadTxid = lastReadTxid;
@@ -87,39 +98,45 @@ public class DFSInotifyEventInputStream {
    * The next available batch of events will be returned.
    */
   public EventBatch poll() throws IOException, MissingEventsException {
-    // need to keep retrying until the NN sends us the latest committed txid
-    if (lastReadTxid == -1) {
-      LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
-      lastReadTxid = namenode.getCurrentEditLogTxid();
-      return null;
-    }
-    if (!it.hasNext()) {
-      EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
-      if (el.getLastTxid() != -1) {
-        // we only want to set syncTxid when we were actually able to read some
-        // edits on the NN -- otherwise it will seem like edits are being
-        // generated faster than we can read them when the problem is really
-        // that we are temporarily unable to read edits
-        syncTxid = el.getSyncTxid();
-        it = el.getBatches().iterator();
-        long formerLastReadTxid = lastReadTxid;
-        lastReadTxid = el.getLastTxid();
-        if (el.getFirstTxid() != formerLastReadTxid + 1) {
-          throw new MissingEventsException(formerLastReadTxid + 1,
-              el.getFirstTxid());
+    TraceScope scope =
+        Trace.startSpan("inotifyPoll", traceSampler);
+    try {
+      // need to keep retrying until the NN sends us the latest committed txid
+      if (lastReadTxid == -1) {
+        LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
+        lastReadTxid = namenode.getCurrentEditLogTxid();
+        return null;
+      }
+      if (!it.hasNext()) {
+        EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
+        if (el.getLastTxid() != -1) {
+          // we only want to set syncTxid when we were actually able to read some
+          // edits on the NN -- otherwise it will seem like edits are being
+          // generated faster than we can read them when the problem is really
+          // that we are temporarily unable to read edits
+          syncTxid = el.getSyncTxid();
+          it = el.getBatches().iterator();
+          long formerLastReadTxid = lastReadTxid;
+          lastReadTxid = el.getLastTxid();
+          if (el.getFirstTxid() != formerLastReadTxid + 1) {
+            throw new MissingEventsException(formerLastReadTxid + 1,
+                el.getFirstTxid());
+          }
+        } else {
+          LOG.debug("poll(): read no edits from the NN when requesting edits " +
+            "after txid {}", lastReadTxid);
+          return null;
         }
+      }
+
+      if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
+        // newly seen edit log ops actually got converted to events
+        return it.next();
       } else {
-        LOG.debug("poll(): read no edits from the NN when requesting edits " +
-          "after txid {}", lastReadTxid);
         return null;
       }
-    }
-
-    if (it.hasNext()) { // can be empty if el.getLastTxid != -1 but none of the
-      // newly seen edit log ops actually got converted to events
-      return it.next();
-    } else {
-      return null;
+    } finally {
+      scope.close();
     }
   }
 
@@ -163,25 +180,29 @@ public long getTxidsBehindEstimate() {
    */
   public EventBatch poll(long time, TimeUnit tu) throws IOException,
       InterruptedException, MissingEventsException {
-    long initialTime = Time.monotonicNow();
-    long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
-    long nextWait = INITIAL_WAIT_MS;
+    TraceScope scope = Trace.startSpan("inotifyPollWithTimeout", traceSampler);
     EventBatch next = null;
-    while ((next = poll()) == null) {
-      long timeLeft = totalWait - (Time.monotonicNow() - initialTime);
-      if (timeLeft <= 0) {
-        LOG.debug("timed poll(): timed out");
-        break;
-      } else if (timeLeft < nextWait * 2) {
-        nextWait = timeLeft;
-      } else {
-        nextWait *= 2;
+    try {
+      long initialTime = Time.monotonicNow();
+      long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
+      long nextWait = INITIAL_WAIT_MS;
+      while ((next = poll()) == null) {
+        long timeLeft = totalWait - (Time.monotonicNow() - initialTime);
+        if (timeLeft <= 0) {
+          LOG.debug("timed poll(): timed out");
+          break;
+        } else if (timeLeft < nextWait * 2) {
+          nextWait = timeLeft;
+        } else {
+          nextWait *= 2;
+        }
+        LOG.debug("timed poll(): poll() returned null, sleeping for {} ms",
+            nextWait);
+        Thread.sleep(nextWait);
       }
-      LOG.debug("timed poll(): poll() returned null, sleeping for {} ms",
-          nextWait);
-      Thread.sleep(nextWait);
+    } finally {
+      scope.close();
     }
-
     return next;
   }
 
@@ -196,18 +217,23 @@ public EventBatch poll(long time, TimeUnit tu) throws IOException,
    */
   public EventBatch take() throws IOException, InterruptedException,
       MissingEventsException {
+    TraceScope scope = Trace.startSpan("inotifyTake", traceSampler);
     EventBatch next = null;
-    int nextWaitMin = INITIAL_WAIT_MS;
-    while ((next = poll()) == null) {
-      // sleep for a random period between nextWaitMin and nextWaitMin * 2
-      // to avoid stampedes at the NN if there are multiple clients
-      int sleepTime = nextWaitMin + rng.nextInt(nextWaitMin);
-      LOG.debug("take(): poll() returned null, sleeping for {} ms", sleepTime);
-      Thread.sleep(sleepTime);
-      // the maximum sleep is 2 minutes
-      nextWaitMin = Math.min(60000, nextWaitMin * 2);
+    try {
+      int nextWaitMin = INITIAL_WAIT_MS;
+      while ((next = poll()) == null) {
+        // sleep for a random period between nextWaitMin and nextWaitMin * 2
+        // to avoid stampedes at the NN if there are multiple clients
+        int sleepTime = nextWaitMin + rng.nextInt(nextWaitMin);
+        LOG.debug("take(): poll() returned null, sleeping for {} ms", sleepTime);
+        Thread.sleep(sleepTime);
+        // the maximum sleep is 2 minutes
+        nextWaitMin = Math.min(60000, nextWaitMin * 2);
+      }
+    } finally {
+      scope.close();
     }
 
     return next;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index b8b1d905e590d..9e7533348d1c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -72,11 +73,11 @@
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.IdentityHashStore;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
 
 /****************************************************************
  * DFSInputStream provides bytes from a named file.  It handles 
@@ -90,7 +91,7 @@ public class DFSInputStream extends FSInputStream
   public static boolean tcpReadsDisabledForTesting = false;
   private long hedgedReadOpsLoopNumForTesting = 0;
   private final DFSClient dfsClient;
-  private boolean closed = false;
+  private AtomicBoolean closed = new AtomicBoolean(false);
   private final String src;
   private final boolean verifyChecksum;
 
@@ -130,10 +131,7 @@ public class DFSInputStream extends FSInputStream
 
   public static class ReadStatistics {
     public ReadStatistics() {
-      this.totalBytesRead = 0;
-      this.totalLocalBytesRead = 0;
-      this.totalShortCircuitBytesRead = 0;
-      this.totalZeroCopyBytesRead = 0;
+      clear();
     }
 
     public ReadStatistics(ReadStatistics rhs) {
@@ -202,6 +200,13 @@ void addZeroCopyBytes(long amt) {
       this.totalShortCircuitBytesRead += amt;
       this.totalZeroCopyBytesRead += amt;
     }
+
+    void clear() {
+      this.totalBytesRead = 0;
+      this.totalLocalBytesRead = 0;
+      this.totalShortCircuitBytesRead = 0;
+      this.totalZeroCopyBytesRead = 0;
+    }
     
     private long totalBytesRead;
 
@@ -411,7 +416,7 @@ synchronized public ExtendedBlock getCurrentBlock() {
   /**
    * Return collection of blocks that has already been located.
    */
-  public synchronized List<LocatedBlock> getAllBlocks() throws IOException {
+  public List<LocatedBlock> getAllBlocks() throws IOException {
     return getBlockRange(0, getFileLength());
   }
 
@@ -661,7 +666,8 @@ private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
    */
   @Override
   public synchronized void close() throws IOException {
-    if (closed) {
+    if (!closed.compareAndSet(false, true)) {
+      DFSClient.LOG.warn("DFSInputStream has been closed already");
       return;
     }
     dfsClient.checkOpen();
@@ -685,7 +691,6 @@ public void accept(ByteBuffer k, Object v) {
       blockReader = null;
     }
     super.close();
-    closed = true;
   }
 
   @Override
@@ -699,26 +704,28 @@ public synchronized int read() throws IOException {
    * strategy-agnostic.
    */
   private interface ReaderStrategy {
-    public int doRead(BlockReader blockReader, int off, int len,
-        ReadStatistics readStatistics) throws ChecksumException, IOException;
+    public int doRead(BlockReader blockReader, int off, int len)
+        throws ChecksumException, IOException;
   }
 
-  private static void updateReadStatistics(ReadStatistics readStatistics, 
+  private void updateReadStatistics(ReadStatistics readStatistics, 
         int nRead, BlockReader blockReader) {
     if (nRead <= 0) return;
-    if (blockReader.isShortCircuit()) {
-      readStatistics.addShortCircuitBytes(nRead);
-    } else if (blockReader.isLocal()) {
-      readStatistics.addLocalBytes(nRead);
-    } else {
-      readStatistics.addRemoteBytes(nRead);
+    synchronized(infoLock) {
+      if (blockReader.isShortCircuit()) {
+        readStatistics.addShortCircuitBytes(nRead);
+      } else if (blockReader.isLocal()) {
+        readStatistics.addLocalBytes(nRead);
+      } else {
+        readStatistics.addRemoteBytes(nRead);
+      }
     }
   }
   
   /**
    * Used to read bytes into a byte[]
    */
-  private static class ByteArrayStrategy implements ReaderStrategy {
+  private class ByteArrayStrategy implements ReaderStrategy {
     final byte[] buf;
 
     public ByteArrayStrategy(byte[] buf) {
@@ -726,26 +733,26 @@ public ByteArrayStrategy(byte[] buf) {
     }
 
     @Override
-    public int doRead(BlockReader blockReader, int off, int len,
-            ReadStatistics readStatistics) throws ChecksumException, IOException {
-        int nRead = blockReader.read(buf, off, len);
-        updateReadStatistics(readStatistics, nRead, blockReader);
-        return nRead;
+    public int doRead(BlockReader blockReader, int off, int len)
+          throws ChecksumException, IOException {
+      int nRead = blockReader.read(buf, off, len);
+      updateReadStatistics(readStatistics, nRead, blockReader);
+      return nRead;
     }
   }
 
   /**
    * Used to read bytes into a user-supplied ByteBuffer
    */
-  private static class ByteBufferStrategy implements ReaderStrategy {
+  private class ByteBufferStrategy implements ReaderStrategy {
     final ByteBuffer buf;
     ByteBufferStrategy(ByteBuffer buf) {
       this.buf = buf;
     }
 
     @Override
-    public int doRead(BlockReader blockReader, int off, int len,
-        ReadStatistics readStatistics) throws ChecksumException, IOException {
+    public int doRead(BlockReader blockReader, int off, int len)
+        throws ChecksumException, IOException {
       int oldpos = buf.position();
       int oldlimit = buf.limit();
       boolean success = false;
@@ -784,7 +791,7 @@ private synchronized int readBuffer(ReaderStrategy reader, int off, int len,
     while (true) {
       // retry as many times as seekToNewSource allows.
       try {
-        return reader.doRead(blockReader, off, len, readStatistics);
+        return reader.doRead(blockReader, off, len);
       } catch ( ChecksumException ce ) {
         DFSClient.LOG.warn("Found Checksum error for "
             + getCurrentBlock() + " from " + currentNode
@@ -822,7 +829,7 @@ private synchronized int readBuffer(ReaderStrategy reader, int off, int len,
 
   private synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
     dfsClient.checkOpen();
-    if (closed) {
+    if (closed.get()) {
       throw new IOException("Stream closed");
     }
     Map<ExtendedBlock,Set<DatanodeInfo>> corruptedBlockMap 
@@ -1375,7 +1382,7 @@ private int pread(long position, byte[] buffer, int offset, int length)
       throws IOException {
     // sanity checks
     dfsClient.checkOpen();
-    if (closed) {
+    if (closed.get()) {
       throw new IOException("Stream closed");
     }
     failures = 0;
@@ -1484,7 +1491,7 @@ public synchronized void seek(long targetPos) throws IOException {
     if (targetPos < 0) {
       throw new EOFException("Cannot seek to negative offset");
     }
-    if (closed) {
+    if (closed.get()) {
       throw new IOException("Stream is closed!");
     }
     boolean done = false;
@@ -1571,7 +1578,7 @@ public synchronized long getPos() throws IOException {
    */
   @Override
   public synchronized int available() throws IOException {
-    if (closed) {
+    if (closed.get()) {
       throw new IOException("Stream closed");
     }
 
@@ -1611,8 +1618,19 @@ private static final class DNAddrPair {
   /**
    * Get statistics about the reads which this DFSInputStream has done.
    */
-  public synchronized ReadStatistics getReadStatistics() {
-    return new ReadStatistics(readStatistics);
+  public ReadStatistics getReadStatistics() {
+    synchronized(infoLock) {
+      return new ReadStatistics(readStatistics);
+    }
+  }
+
+  /**
+   * Clear statistics about the reads which this DFSInputStream has done.
+   */
+  public void clearReadStatistics() {
+    synchronized(infoLock) {
+      readStatistics.clear();
+    }
   }
 
   public FileEncryptionInfo getFileEncryptionInfo() {
@@ -1774,7 +1792,9 @@ private synchronized ByteBuffer tryReadZeroCopy(int maxLength,
       buffer.position((int)blockPos);
       buffer.limit((int)(blockPos + length));
       extendedReadBuffers.put(buffer, clientMmap);
-      readStatistics.addZeroCopyBytes(length);
+      synchronized (infoLock) {
+        readStatistics.addZeroCopyBytes(length);
+      }
       if (DFSClient.LOG.isDebugEnabled()) {
         DFSClient.LOG.debug("readZeroCopy read " + length + 
             " bytes from offset " + curPos + " via the zero-copy read " +
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index e574d1d080d8f..9560e0154258b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -39,6 +39,7 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -93,9 +94,9 @@
 import org.apache.hadoop.util.DataChecksum.Type;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Time;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -241,8 +242,6 @@ private static class Packet {
     /**
      * Create a new packet.
      * 
-     * @param pktSize maximum size of the packet, 
-     *                including checksum data and actual data.
      * @param chunksPerPkt maximum number of chunks per packet.
      * @param offsetInBlock offset in bytes into the HDFS block.
      */
@@ -358,9 +357,9 @@ private boolean isHeartbeatPacket() {
     
     @Override
     public String toString() {
-      return "packet seqno:" + this.seqno +
-      " offsetInBlock:" + this.offsetInBlock + 
-      " lastPacketInBlock:" + this.lastPacketInBlock +
+      return "packet seqno: " + this.seqno +
+      " offsetInBlock: " + this.offsetInBlock +
+      " lastPacketInBlock: " + this.lastPacketInBlock +
       " lastByteOffsetInBlock: " + this.getLastByteOffsetBlock();
     }
   }
@@ -405,7 +404,8 @@ public DatanodeInfo load(DatanodeInfo key) throws Exception {
     private String[] favoredNodes;
     volatile boolean hasError = false;
     volatile int errorIndex = -1;
-    volatile int restartingNodeIndex = -1; // Restarting node index
+    // Restarting node index
+    AtomicInteger restartingNodeIndex = new AtomicInteger(-1);
     private long restartDeadline = 0; // Deadline of DN restart
     private BlockConstructionStage stage;  // block construction stage
     private long bytesSent = 0; // number of bytes that've been sent
@@ -426,15 +426,16 @@ public DatanodeInfo load(DatanodeInfo key) throws Exception {
     /**
      * construction with tracing info
      */
-    private DataStreamer(HdfsFileStatus stat, Span span) {
+    private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, Span span) {
       isAppend = false;
       isLazyPersistFile = isLazyPersist(stat);
+      this.block = block;
       stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
       traceSpan = span;
     }
     
     /**
-     * Construct a data streamer for append
+     * Construct a data streamer for appending to the last partial block
      * @param lastBlock last block of the file to be appended
      * @param stat status of the file to be appended
      * @param bytesPerChecksum number of bytes per checksum
@@ -556,7 +557,7 @@ public void run() {
         try {
           // process datanode IO errors if any
           boolean doSleep = false;
-          if (hasError && (errorIndex >= 0 || restartingNodeIndex >= 0)) {
+          if (hasError && (errorIndex >= 0 || restartingNodeIndex.get() >= 0)) {
             doSleep = processDatanodeError();
           }
 
@@ -699,7 +700,7 @@ public void run() {
           }
         } catch (Throwable e) {
           // Log warning if there was a real error.
-          if (restartingNodeIndex == -1) {
+          if (restartingNodeIndex.get() == -1) {
             DFSClient.LOG.warn("DataStreamer Exception", e);
           }
           if (e instanceof IOException) {
@@ -708,7 +709,7 @@ public void run() {
             setLastException(new IOException("DataStreamer Exception: ",e));
           }
           hasError = true;
-          if (errorIndex == -1 && restartingNodeIndex == -1) {
+          if (errorIndex == -1 && restartingNodeIndex.get() == -1) {
             // Not a datanode issue
             streamerClosed = true;
           }
@@ -806,7 +807,7 @@ synchronized void setErrorIndex(int idx) {
 
     /** Set the restarting node index. Called by responder */
     synchronized void setRestartingNodeIndex(int idx) {
-      restartingNodeIndex = idx;
+      restartingNodeIndex.set(idx);
       // If the data streamer has already set the primary node
       // bad, clear it. It is likely that the write failed due to
       // the DN shutdown. Even if it was a real failure, the pipeline
@@ -821,7 +822,7 @@ synchronized void setRestartingNodeIndex(int idx) {
      */
     synchronized void tryMarkPrimaryDatanodeFailed() {
       // There should be no existing error and no ongoing restart.
-      if ((errorIndex == -1) && (restartingNodeIndex == -1)) {
+      if ((errorIndex == -1) && (restartingNodeIndex.get() == -1)) {
         errorIndex = 0;
       }
     }
@@ -891,7 +892,8 @@ public void run() {
             long seqno = ack.getSeqno();
             // processes response status from datanodes.
             for (int i = ack.getNumOfReplies()-1; i >=0  && dfsClient.clientRunning; i--) {
-              final Status reply = ack.getReply(i);
+              final Status reply = PipelineAck.getStatusFromHeader(ack
+                .getReply(i));
               // Restart will not be treated differently unless it is
               // the local node or the only one in the pipeline.
               if (PipelineAck.isRestartOOBStatus(reply) &&
@@ -962,7 +964,7 @@ public void run() {
               synchronized (dataQueue) {
                 dataQueue.notifyAll();
               }
-              if (restartingNodeIndex == -1) {
+              if (restartingNodeIndex.get() == -1) {
                 DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception "
                      + " for block " + block, e);
               }
@@ -1186,7 +1188,7 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
         // Sleep before reconnect if a dn is restarting.
         // This process will be repeated until the deadline or the datanode
         // starts back up.
-        if (restartingNodeIndex >= 0) {
+        if (restartingNodeIndex.get() >= 0) {
           // 4 seconds or the configured deadline period, whichever is shorter.
           // This is the retry interval and recovery will be retried in this
           // interval until timeout or success.
@@ -1196,7 +1198,7 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
             Thread.sleep(delay);
           } catch (InterruptedException ie) {
             lastException.set(new IOException("Interrupted while waiting for " +
-                "datanode to restart. " + nodes[restartingNodeIndex]));
+                "datanode to restart. " + nodes[restartingNodeIndex.get()]));
             streamerClosed = true;
             return false;
           }
@@ -1237,21 +1239,21 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
           setPipeline(newnodes, newStorageTypes, newStorageIDs);
 
           // Just took care of a node error while waiting for a node restart
-          if (restartingNodeIndex >= 0) {
+          if (restartingNodeIndex.get() >= 0) {
             // If the error came from a node further away than the restarting
             // node, the restart must have been complete.
-            if (errorIndex > restartingNodeIndex) {
-              restartingNodeIndex = -1;
-            } else if (errorIndex < restartingNodeIndex) {
+            if (errorIndex > restartingNodeIndex.get()) {
+              restartingNodeIndex.set(-1);
+            } else if (errorIndex < restartingNodeIndex.get()) {
               // the node index has shifted.
-              restartingNodeIndex--;
+              restartingNodeIndex.decrementAndGet();
             } else {
               // this shouldn't happen...
               assert false;
             }
           }
 
-          if (restartingNodeIndex == -1) {
+          if (restartingNodeIndex.get() == -1) {
             hasError = false;
           }
           lastException.set(null);
@@ -1293,10 +1295,10 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
           success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
         }
 
-        if (restartingNodeIndex >= 0) {
+        if (restartingNodeIndex.get() >= 0) {
           assert hasError == true;
           // check errorIndex set above
-          if (errorIndex == restartingNodeIndex) {
+          if (errorIndex == restartingNodeIndex.get()) {
             // ignore, if came from the restarting node
             errorIndex = -1;
           }
@@ -1306,8 +1308,8 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
           }
           // expired. declare the restarting node dead
           restartDeadline = 0;
-          int expiredNodeIndex = restartingNodeIndex;
-          restartingNodeIndex = -1;
+          int expiredNodeIndex = restartingNodeIndex.get();
+          restartingNodeIndex.set(-1);
           DFSClient.LOG.warn("Datanode did not restart in time: " +
               nodes[expiredNodeIndex]);
           // Mark the restarting node as failed. If there is any other failed
@@ -1459,7 +1461,7 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
           // from the local datanode. Thus it is safe to treat this as a
           // regular node error.
           if (PipelineAck.isRestartOOBStatus(pipelineStatus) &&
-            restartingNodeIndex == -1) {
+            restartingNodeIndex.get() == -1) {
             checkRestart = true;
             throw new IOException("A datanode is restarting.");
           }
@@ -1476,10 +1478,10 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
           assert null == blockStream : "Previous blockStream unclosed";
           blockStream = out;
           result =  true; // success
-          restartingNodeIndex = -1;
+          restartingNodeIndex.set(-1);
           hasError = false;
         } catch (IOException ie) {
-          if (restartingNodeIndex == -1) {
+          if (restartingNodeIndex.get() == -1) {
             DFSClient.LOG.info("Exception in createBlockOutputStream", ie);
           }
           if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
@@ -1511,10 +1513,10 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
           if (checkRestart && shouldWaitForRestart(errorIndex)) {
             restartDeadline = dfsClient.getConf().datanodeRestartTimeout +
                 Time.now();
-            restartingNodeIndex = errorIndex;
+            restartingNodeIndex.set(errorIndex);
             errorIndex = -1;
             DFSClient.LOG.info("Waiting for the datanode to be restarted: " +
-                nodes[restartingNodeIndex]);
+                nodes[restartingNodeIndex.get()]);
           }
           hasError = true;
           setLastException(ie);
@@ -1716,7 +1718,7 @@ private DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,
     if (Trace.isTracing()) {
       traceSpan = Trace.startSpan(this.getClass().getSimpleName()).detach();
     }
-    streamer = new DataStreamer(stat, traceSpan);
+    streamer = new DataStreamer(stat, null, traceSpan);
     if (favoredNodes != null && favoredNodes.length != 0) {
       streamer.setFavoredNodes(favoredNodes);
     }
@@ -1773,7 +1775,7 @@ static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src,
   }
 
   /** Construct a new output stream for append. */
-  private DFSOutputStream(DFSClient dfsClient, String src,
+  private DFSOutputStream(DFSClient dfsClient, String src, boolean toNewBlock,
       Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat,
       DataChecksum checksum) throws IOException {
     this(dfsClient, src, progress, stat, checksum);
@@ -1785,21 +1787,24 @@ private DFSOutputStream(DFSClient dfsClient, String src,
     }
 
     // The last partial block of the file has to be filled.
-    if (lastBlock != null) {
+    if (!toNewBlock && lastBlock != null) {
       // indicate that we are appending to an existing block
       bytesCurBlock = lastBlock.getBlockSize();
       streamer = new DataStreamer(lastBlock, stat, bytesPerChecksum, traceSpan);
     } else {
-      computePacketChunkSize(dfsClient.getConf().writePacketSize, bytesPerChecksum);
-      streamer = new DataStreamer(stat, traceSpan);
+      computePacketChunkSize(dfsClient.getConf().writePacketSize,
+          bytesPerChecksum);
+      streamer = new DataStreamer(stat,
+          lastBlock != null ? lastBlock.getBlock() : null, traceSpan);
     }
     this.fileEncryptionInfo = stat.getFileEncryptionInfo();
   }
 
   static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src,
-      int buffersize, Progressable progress, LocatedBlock lastBlock,
-      HdfsFileStatus stat, DataChecksum checksum) throws IOException {
-    final DFSOutputStream out = new DFSOutputStream(dfsClient, src,
+      boolean toNewBlock, int bufferSize, Progressable progress,
+      LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum)
+      throws IOException {
+    final DFSOutputStream out = new DFSOutputStream(dfsClient, src, toNewBlock,
         progress, lastBlock, stat, checksum);
     out.start();
     return out;
@@ -1995,35 +2000,37 @@ private void flushOrSync(boolean isSync, EnumSet<SyncFlag> syncFlags)
       long toWaitFor;
       long lastBlockLength = -1L;
       boolean updateLength = syncFlags.contains(SyncFlag.UPDATE_LENGTH);
+      boolean endBlock = syncFlags.contains(SyncFlag.END_BLOCK);
       synchronized (this) {
-        // flush checksum buffer, but keep checksum buffer intact
-        int numKept = flushBuffer(true, true);
+        // flush checksum buffer, but keep checksum buffer intact if we do not
+        // need to end the current block
+        int numKept = flushBuffer(!endBlock, true);
         // bytesCurBlock potentially incremented if there was buffered data
 
         if (DFSClient.LOG.isDebugEnabled()) {
-          DFSClient.LOG.debug(
-            "DFSClient flush() :" +
-            " bytesCurBlock " + bytesCurBlock +
-            " lastFlushOffset " + lastFlushOffset);
+          DFSClient.LOG.debug("DFSClient flush(): "
+              + " bytesCurBlock=" + bytesCurBlock
+              + " lastFlushOffset=" + lastFlushOffset
+              + " createNewBlock=" + endBlock);
         }
         // Flush only if we haven't already flushed till this offset.
         if (lastFlushOffset != bytesCurBlock) {
           assert bytesCurBlock > lastFlushOffset;
           // record the valid offset of this flush
           lastFlushOffset = bytesCurBlock;
-          if (isSync && currentPacket == null) {
+          if (isSync && currentPacket == null && !endBlock) {
             // Nothing to send right now,
             // but sync was requested.
-            // Send an empty packet
+            // Send an empty packet if we do not end the block right now
             currentPacket = createPacket(packetSize, chunksPerPacket,
                 bytesCurBlock, currentSeqno++);
           }
         } else {
-          if (isSync && bytesCurBlock > 0) {
+          if (isSync && bytesCurBlock > 0 && !endBlock) {
             // Nothing to send right now,
             // and the block was partially written,
             // and sync was requested.
-            // So send an empty sync packet.
+            // So send an empty sync packet if we do not end the block right now
             currentPacket = createPacket(packetSize, chunksPerPacket,
                 bytesCurBlock, currentSeqno++);
           } else if (currentPacket != null) {
@@ -2036,10 +2043,21 @@ private void flushOrSync(boolean isSync, EnumSet<SyncFlag> syncFlags)
           currentPacket.syncBlock = isSync;
           waitAndQueueCurrentPacket();          
         }
-        // Restore state of stream. Record the last flush offset 
-        // of the last full chunk that was flushed.
-        //
-        bytesCurBlock -= numKept;
+        if (endBlock && bytesCurBlock > 0) {
+          // Need to end the current block, thus send an empty packet to
+          // indicate this is the end of the block and reset bytesCurBlock
+          currentPacket = createPacket(0, 0, bytesCurBlock, currentSeqno++);
+          currentPacket.lastPacketInBlock = true;
+          currentPacket.syncBlock = shouldSyncBlock || isSync;
+          waitAndQueueCurrentPacket();
+          bytesCurBlock = 0;
+          lastFlushOffset = 0;
+        } else {
+          // Restore state of stream. Record the last flush offset
+          // of the last full chunk that was flushed.
+          bytesCurBlock -= numKept;
+        }
+
         toWaitFor = lastQueuedSeqno;
       } // end synchronized
 
@@ -2058,8 +2076,8 @@ private void flushOrSync(boolean isSync, EnumSet<SyncFlag> syncFlags)
       // namenode.
       if (persistBlocks.getAndSet(false) || updateLength) {
         try {
-          dfsClient.namenode.fsync(src, fileId,
-              dfsClient.clientName, lastBlockLength);
+          dfsClient.namenode.fsync(src, fileId, dfsClient.clientName,
+              lastBlockLength);
         } catch (IOException ioe) {
           DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe);
           // If we got an error here, it might be because some other thread called
@@ -2086,7 +2104,7 @@ private void flushOrSync(boolean isSync, EnumSet<SyncFlag> syncFlags)
       DFSClient.LOG.warn("Error while syncing", e);
       synchronized (this) {
         if (!isClosed()) {
-          lastException.set(new IOException("IOException flush:" + e));
+          lastException.set(new IOException("IOException flush: " + e));
           closeThreads(true);
         }
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index f1bfcb4fafa75..8b3f5121bf037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -341,15 +341,20 @@ public static byte[] string2Bytes(String str) {
   /**
    * Given a list of path components returns a path as a UTF8 String
    */
-  public static String byteArray2PathString(byte[][] pathComponents) {
+  public static String byteArray2PathString(byte[][] pathComponents,
+      int offset, int length) {
     if (pathComponents.length == 0) {
       return "";
-    } else if (pathComponents.length == 1
+    }
+    Preconditions.checkArgument(offset >= 0 && offset < pathComponents.length);
+    Preconditions.checkArgument(length >= 0 && offset + length <=
+        pathComponents.length);
+    if (pathComponents.length == 1
         && (pathComponents[0] == null || pathComponents[0].length == 0)) {
       return Path.SEPARATOR;
     }
     StringBuilder result = new StringBuilder();
-    for (int i = 0; i < pathComponents.length; i++) {
+    for (int i = offset; i < offset + length; i++) {
       result.append(new String(pathComponents[i], Charsets.UTF_8));
       if (i < pathComponents.length - 1) {
         result.append(Path.SEPARATOR_CHAR);
@@ -358,6 +363,10 @@ public static String byteArray2PathString(byte[][] pathComponents) {
     return result.toString();
   }
 
+  public static String byteArray2PathString(byte[][] pathComponents) {
+    return byteArray2PathString(pathComponents, 0, pathComponents.length);
+  }
+
   /**
    * Converts a list of path components into a path using Path.SEPARATOR.
    * 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index d4653acf66e30..63d48bcdead8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -314,13 +314,19 @@ public FSDataInputStream next(final FileSystem fs, final Path p)
   @Override
   public FSDataOutputStream append(Path f, final int bufferSize,
       final Progressable progress) throws IOException {
+    return append(f, EnumSet.of(CreateFlag.APPEND), bufferSize, progress);
+  }
+
+  public FSDataOutputStream append(Path f, final EnumSet<CreateFlag> flag,
+      final int bufferSize, final Progressable progress) throws IOException {
     statistics.incrementWriteOps(1);
     Path absF = fixRelativePart(f);
     return new FileSystemLinkResolver<FSDataOutputStream>() {
       @Override
       public FSDataOutputStream doCall(final Path p)
-          throws IOException, UnresolvedLinkException {
-        return dfs.append(getPathName(p), bufferSize, progress, statistics);
+          throws IOException {
+        return dfs.append(getPathName(p), bufferSize, flag, progress,
+            statistics);
       }
       @Override
       public FSDataOutputStream next(final FileSystem fs, final Path p)
@@ -626,7 +632,25 @@ public Void next(final FileSystem fs, final Path p)
       }.resolve(this, absDst);
     }
   }
-  
+
+  @Override
+  public boolean truncate(Path f, final long newLength) throws IOException {
+    statistics.incrementWriteOps(1);
+    Path absF = fixRelativePart(f);
+    return new FileSystemLinkResolver<Boolean>() {
+      @Override
+      public Boolean doCall(final Path p)
+          throws IOException, UnresolvedLinkException {
+        return dfs.truncate(getPathName(p), newLength);
+      }
+      @Override
+      public Boolean next(final FileSystem fs, final Path p)
+          throws IOException {
+        return fs.truncate(p, newLength);
+      }
+    }.resolve(this, absF);
+  }
+
   @Override
   public boolean delete(Path f, final boolean recursive) throws IOException {
     statistics.incrementWriteOps(1);
@@ -664,7 +688,7 @@ public ContentSummary next(final FileSystem fs, final Path p)
   }
 
   /** Set a directory's quotas
-   * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long) 
+   * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long, StorageType)
    */
   public void setQuota(Path src, final long namespaceQuota,
       final long diskspaceQuota) throws IOException {
@@ -686,6 +710,35 @@ public Void next(final FileSystem fs, final Path p)
     }.resolve(this, absF);
   }
 
+  /**
+   * Set the per type storage quota of a directory.
+   *
+   * @param src target directory whose quota is to be modified.
+   * @param type storage type of the specific storage type quota to be modified.
+   * @param spaceQuota value of the specific storage type quota to be modified.
+   * Maybe {@link HdfsConstants#QUOTA_RESET} to clear quota by storage type.
+   */
+  public void setQuotaByStorageType(
+    Path src, final StorageType type, final long spaceQuota)
+    throws IOException {
+    Path absF = fixRelativePart(src);
+    new FileSystemLinkResolver<Void>() {
+      @Override
+      public Void doCall(final Path p)
+        throws IOException, UnresolvedLinkException {
+        dfs.setQuotaByStorageType(getPathName(p), type, spaceQuota);
+        return null;
+      }
+      @Override
+      public Void next(final FileSystem fs, final Path p)
+        throws IOException {
+        // setQuotaByStorageType is not defined in FileSystem, so we only can resolve
+        // within this DFS
+        return doCall(p);
+      }
+    }.resolve(this, absF);
+  }
+
   private FileStatus[] listStatusInternal(Path p) throws IOException {
     String src = getPathName(p);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
index f2d3395dc8c44..628c61030f1fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
@@ -46,9 +46,9 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
index bc0db568bf022..3f133b69293ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
@@ -51,11 +51,11 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
 
 /**
  * This is a wrapper around connection to datanode
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
index 88cc7d6b7e24c..a26ed91d1177c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java
@@ -45,12 +45,18 @@ public enum StorageType {
 
   private static final StorageType[] VALUES = values();
 
-  StorageType(boolean isTransient) { this.isTransient = isTransient; }
+  StorageType(boolean isTransient) {
+    this.isTransient = isTransient;
+  }
 
   public boolean isTransient() {
     return isTransient;
   }
 
+  public boolean supportTypeQuota() {
+    return !isTransient;
+  }
+
   public boolean isMovable() {
     return !isTransient;
   }
@@ -60,12 +66,28 @@ public static List<StorageType> asList() {
   }
 
   public static List<StorageType> getMovableTypes() {
-    List<StorageType> movableTypes = new ArrayList<StorageType>();
+    return getNonTransientTypes();
+  }
+
+  public static List<StorageType> getTypesSupportingQuota() {
+    return getNonTransientTypes();
+  }
+
+  public static StorageType parseStorageType(int i) {
+    return VALUES[i];
+  }
+
+  public static StorageType parseStorageType(String s) {
+    return StorageType.valueOf(s.toUpperCase());
+  }
+
+  private static List<StorageType> getNonTransientTypes() {
+    List<StorageType> nonTransientTypes = new ArrayList<>();
     for (StorageType t : VALUES) {
       if ( t.isTransient == false ) {
-        movableTypes.add(t);
+        nonTransientTypes.add(t);
       }
     }
-    return movableTypes;
+    return nonTransientTypes;
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index 6280d675d0132..ca80ec46f0033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
@@ -116,6 +117,32 @@ public void setSpaceQuota(Path src, long spaceQuota) throws IOException {
   public void clearSpaceQuota(Path src) throws IOException {
     dfs.setQuota(src, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_RESET);
   }
+
+  /**
+   * Set the quota by storage type for a directory. Note that
+   * directories and sym links do not occupy disk space.
+   *
+   * @param src the target directory to set the quota by storage type
+   * @param type the storage type to set for quota by storage type
+   * @param spaceQuota the value to set for quota by storage type
+   * @throws IOException in the event of error
+   */
+  public void setQuotaByStorageType(Path src, StorageType type, long spaceQuota)
+      throws IOException {
+    dfs.setQuotaByStorageType(src, type, spaceQuota);
+  }
+
+  /**
+   * Clear the space quota by storage type for a directory. Note that
+   * directories and sym links do not occupy disk space.
+   *
+   * @param src the target directory to clear the quota by storage type
+   * @param type the storage type to clear for quota by storage type
+   * @throws IOException in the event of error
+   */
+  public void clearQuotaByStorageType(Path src, StorageType type) throws IOException {
+    dfs.setQuotaByStorageType(src, type, HdfsConstants.QUOTA_RESET);
+  }
   
   /**
    * Allow snapshot on a directory.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
index e1269c49a3bd0..e8ac6865e4346 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
@@ -83,7 +83,7 @@ public ExtendedBlock getCurrentBlock() {
   /**
    * Get the collection of blocks that has already been located.
    */
-  public synchronized List<LocatedBlock> getAllBlocks() throws IOException {
+  public List<LocatedBlock> getAllBlocks() throws IOException {
     return getDFSInputStream().getAllBlocks();
   }
 
@@ -103,7 +103,11 @@ public long getVisibleLength() throws IOException {
    * be higher than you would expect just by adding up the number of
    * bytes read through HdfsDataInputStream.
    */
-  public synchronized DFSInputStream.ReadStatistics getReadStatistics() {
+  public DFSInputStream.ReadStatistics getReadStatistics() {
     return getDFSInputStream().getReadStatistics();
   }
+
+  public void clearReadStatistics() {
+    getDFSInputStream().clearReadStatistics();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
index 214967863e77a..745ca7e7d8996 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
@@ -101,6 +101,12 @@ public static enum SyncFlag {
      * When doing sync to DataNodes, also update the metadata (block length) in
      * the NameNode.
      */
-    UPDATE_LENGTH;
+    UPDATE_LENGTH,
+
+    /**
+     * Sync the data to DataNode, close the current block, and allocate a new
+     * block
+     */
+    END_BLOCK;
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index e8a34e7c21d0c..a6de289b6799f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -101,6 +101,7 @@ public static enum INodeType {
     private FsPermission perms;
     private String symlinkTarget;
     private boolean overwrite;
+    private long defaultBlockSize;
 
     public static class Builder {
       private INodeType iNodeType;
@@ -112,6 +113,7 @@ public static class Builder {
       private FsPermission perms;
       private String symlinkTarget;
       private boolean overwrite;
+      private long defaultBlockSize = 0;
 
       public Builder iNodeType(INodeType type) {
         this.iNodeType = type;
@@ -158,6 +160,11 @@ public Builder overwrite(boolean overwrite) {
         return this;
       }
 
+      public Builder defaultBlockSize(long defaultBlockSize) {
+        this.defaultBlockSize = defaultBlockSize;
+        return this;
+      }
+
       public CreateEvent build() {
         return new CreateEvent(this);
       }
@@ -174,6 +181,7 @@ private CreateEvent(Builder b) {
       this.perms = b.perms;
       this.symlinkTarget = b.symlinkTarget;
       this.overwrite = b.overwrite;
+      this.defaultBlockSize = b.defaultBlockSize;
     }
 
     public INodeType getiNodeType() {
@@ -220,6 +228,10 @@ public String getSymlinkTarget() {
     public boolean getOverwrite() {
       return overwrite;
     }
+
+    public long getDefaultBlockSize() {
+      return defaultBlockSize;
+    }
   }
 
   /**
@@ -398,11 +410,36 @@ public static class RenameEvent extends Event {
     private String dstPath;
     private long timestamp;
 
-    public RenameEvent(String srcPath, String dstPath, long timestamp) {
+    public static class Builder {
+      private String srcPath;
+      private String dstPath;
+      private long timestamp;
+
+      public Builder srcPath(String srcPath) {
+        this.srcPath = srcPath;
+        return this;
+      }
+
+      public Builder dstPath(String dstPath) {
+        this.dstPath = dstPath;
+        return this;
+      }
+
+      public Builder timestamp(long timestamp) {
+        this.timestamp = timestamp;
+        return this;
+      }
+
+      public RenameEvent build() {
+        return new RenameEvent(this);
+      }
+    }
+
+    private RenameEvent(Builder builder) {
       super(EventType.RENAME);
-      this.srcPath = srcPath;
-      this.dstPath = dstPath;
-      this.timestamp = timestamp;
+      this.srcPath = builder.srcPath;
+      this.dstPath = builder.dstPath;
+      this.timestamp = builder.timestamp;
     }
 
     public String getSrcPath() {
@@ -426,15 +463,40 @@ public long getTimestamp() {
    */
   public static class AppendEvent extends Event {
     private String path;
+    private boolean newBlock;
+
+    public static class Builder {
+      private String path;
+      private boolean newBlock;
+
+      public Builder path(String path) {
+        this.path = path;
+        return this;
+      }
+
+      public Builder newBlock(boolean newBlock) {
+        this.newBlock = newBlock;
+        return this;
+      }
+
+      public AppendEvent build() {
+        return new AppendEvent(this);
+      }
+    }
 
-    public AppendEvent(String path) {
+    private AppendEvent(Builder b) {
       super(EventType.APPEND);
-      this.path = path;
+      this.path = b.path;
+      this.newBlock = b.newBlock;
     }
 
     public String getPath() {
       return path;
     }
+
+    public boolean toNewBlock() {
+      return newBlock;
+    }
   }
 
   /**
@@ -444,10 +506,29 @@ public static class UnlinkEvent extends Event {
     private String path;
     private long timestamp;
 
-    public UnlinkEvent(String path, long timestamp) {
+    public static class Builder {
+      private String path;
+      private long timestamp;
+
+      public Builder path(String path) {
+        this.path = path;
+        return this;
+      }
+
+      public Builder timestamp(long timestamp) {
+        this.timestamp = timestamp;
+        return this;
+      }
+
+      public UnlinkEvent build() {
+        return new UnlinkEvent(this);
+      }
+    }
+
+    private UnlinkEvent(Builder builder) {
       super(EventType.UNLINK);
-      this.path = path;
-      this.timestamp = timestamp;
+      this.path = builder.path;
+      this.timestamp = builder.timestamp;
     }
 
     public String getPath() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index 8a0b7316c8319..4389714986f45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.datanode.Replica;
 
 /**
  * This class provides an interface for accessing list of blocks that
@@ -85,8 +85,8 @@ private int index2BlockId(int blockIndex) {
    * @param finalized - list of finalized blocks
    * @param uc - list of under construction blocks
    */
-  public BlockListAsLongs(final List<? extends Block> finalized,
-                          final List<ReplicaInfo> uc) {
+  public BlockListAsLongs(final List<? extends Replica> finalized,
+                          final List<? extends Replica> uc) {
     int finalizedSize = finalized == null ? 0 : finalized.size();
     int ucSize = uc == null ? 0 : uc.size();
     int len = HEADER_SIZE
@@ -113,8 +113,34 @@ public BlockListAsLongs(final List<? extends Block> finalized,
     }
   }
 
+  /**
+   * Create block report from a list of finalized blocks.  Used by
+   * NNThroughputBenchmark.
+   *
+   * @param blocks - list of finalized blocks
+   */
+  public BlockListAsLongs(final List<? extends Block> blocks) {
+    int finalizedSize = blocks == null ? 0 : blocks.size();
+    int len = HEADER_SIZE
+              + (finalizedSize + 1) * LONGS_PER_FINALIZED_BLOCK;
+
+    blockList = new long[len];
+
+    // set the header
+    blockList[0] = finalizedSize;
+    blockList[1] = 0;
+
+    // set finalized blocks
+    for (int i = 0; i < finalizedSize; i++) {
+      setBlock(i, blocks.get(i));
+    }
+
+    // set invalid delimiting block
+    setDelimitingBlock(finalizedSize);
+  }
+
   public BlockListAsLongs() {
-    this(null);
+    this((long[])null);
   }
 
   /**
@@ -279,18 +305,30 @@ public long corruptBlockLengthForTesting(final int blockIndex, Random rand) {
   /**
    * Set the indexTh block
    * @param index - the index of the block to set
-   * @param b - the block is set to the value of the this block
+   * @param r - the block is set to the value of the this Replica
    */
-  private <T extends Block> void setBlock(final int index, final T b) {
+  private void setBlock(final int index, final Replica r) {
     int pos = index2BlockId(index);
-    blockList[pos] = b.getBlockId();
-    blockList[pos + 1] = b.getNumBytes();
-    blockList[pos + 2] = b.getGenerationStamp();
+    blockList[pos] = r.getBlockId();
+    blockList[pos + 1] = r.getNumBytes();
+    blockList[pos + 2] = r.getGenerationStamp();
     if(index < getNumberOfFinalizedReplicas())
       return;
-    assert ((ReplicaInfo)b).getState() != ReplicaState.FINALIZED :
+    assert r.getState() != ReplicaState.FINALIZED :
       "Must be under-construction replica.";
-    blockList[pos + 3] = ((ReplicaInfo)b).getState().getValue();
+    blockList[pos + 3] = r.getState().getValue();
+  }
+
+  /**
+   * Set the indexTh block
+   * @param index - the index of the block to set
+   * @param b - the block is set to the value of the this Block
+   */
+  private void setBlock(final int index, final Block b) {
+    int pos = index2BlockId(index);
+    blockList[pos] = b.getBlockId();
+    blockList[pos + 1] = b.getNumBytes();
+    blockList[pos + 2] = b.getGenerationStamp();
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
index 676106de10f79..923cdb4a29d8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
@@ -25,6 +25,9 @@
 import org.apache.hadoop.fs.BatchedRemoteIterator;
 import org.apache.hadoop.fs.InvalidRequestException;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.base.Preconditions;
 
@@ -39,12 +42,14 @@ public class CacheDirectiveIterator
 
   private CacheDirectiveInfo filter;
   private final ClientProtocol namenode;
+  private final Sampler<?> traceSampler;
 
   public CacheDirectiveIterator(ClientProtocol namenode,
-      CacheDirectiveInfo filter) {
+      CacheDirectiveInfo filter, Sampler<?> traceSampler) {
     super(0L);
     this.namenode = namenode;
     this.filter = filter;
+    this.traceSampler = traceSampler;
   }
 
   private static CacheDirectiveInfo removeIdFromFilter(CacheDirectiveInfo filter) {
@@ -89,6 +94,7 @@ public boolean hasMore() {
   public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
       throws IOException {
     BatchedEntries<CacheDirectiveEntry> entries = null;
+    TraceScope scope = Trace.startSpan("listCacheDirectives", traceSampler);
     try {
       entries = namenode.listCacheDirectives(prevKey, filter);
     } catch (IOException e) {
@@ -110,6 +116,8 @@ public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
             "Did not find requested id " + id);
       }
       throw e;
+    } finally {
+      scope.close();
     }
     Preconditions.checkNotNull(entries);
     return entries;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
index 44d6b45174208..e9481f7284124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
@@ -23,6 +23,9 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 /**
  * CachePoolIterator is a remote iterator that iterates cache pools.
@@ -34,16 +37,23 @@ public class CachePoolIterator
     extends BatchedRemoteIterator<String, CachePoolEntry> {
 
   private final ClientProtocol namenode;
+  private final Sampler traceSampler;
 
-  public CachePoolIterator(ClientProtocol namenode) {
+  public CachePoolIterator(ClientProtocol namenode, Sampler traceSampler) {
     super("");
     this.namenode = namenode;
+    this.traceSampler = traceSampler;
   }
 
   @Override
   public BatchedEntries<CachePoolEntry> makeRequest(String prevKey)
       throws IOException {
-    return namenode.listCachePools(prevKey);
+    TraceScope scope = Trace.startSpan("listCachePools", traceSampler);
+    try {
+      return namenode.listCachePools(prevKey);
+    } finally {
+      scope.close();
+    }
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 2301575f0e3fd..f5fc9378d201c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.retry.AtMostOnce;
@@ -203,6 +204,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
    * Append to the end of the file. 
    * @param src path of the file being created.
    * @param clientName name of the current client.
+   * @param flag indicates whether the data is appended to a new block.
    * @return wrapper with information about the last partial block and file
    *    status if any
    * @throws AccessControlException if permission to append file is 
@@ -225,10 +227,10 @@ public HdfsFileStatus create(String src, FsPermission masked,
    * @throws UnsupportedOperationException if append is not supported
    */
   @AtMostOnce
-  public LastBlockWithStatus append(String src, String clientName)
-      throws AccessControlException, DSQuotaExceededException,
-      FileNotFoundException, SafeModeException, UnresolvedLinkException,
-      SnapshotAccessControlException, IOException;
+  public LastBlockWithStatus append(String src, String clientName,
+      EnumSetWritable<CreateFlag> flag) throws AccessControlException,
+      DSQuotaExceededException, FileNotFoundException, SafeModeException,
+      UnresolvedLinkException, SnapshotAccessControlException, IOException;
 
   /**
    * Set replication for an existing file.
@@ -521,7 +523,37 @@ public void rename2(String src, String dst, Options.Rename... options)
       FileAlreadyExistsException, FileNotFoundException,
       NSQuotaExceededException, ParentNotDirectoryException, SafeModeException,
       UnresolvedLinkException, SnapshotAccessControlException, IOException;
-  
+
+  /**
+   * Truncate file src to new size.
+   * <ul>
+   * <li>Fails if src is a directory.
+   * <li>Fails if src does not exist.
+   * <li>Fails if src is not closed.
+   * <li>Fails if new size is greater than current size.
+   * </ul>
+   * <p>
+   * This implementation of truncate is purely a namespace operation if truncate
+   * occurs at a block boundary. Requires DataNode block recovery otherwise.
+   * <p>
+   * @param src  existing file
+   * @param newLength  the target size
+   *
+   * @return true if client does not need to wait for block recovery,
+   * false if client needs to wait for block recovery.
+   *
+   * @throws AccessControlException If access is denied
+   * @throws FileNotFoundException If file <code>src</code> is not found
+   * @throws SafeModeException truncate not allowed in safemode
+   * @throws UnresolvedLinkException If <code>src</code> contains a symlink
+   * @throws SnapshotAccessControlException if path is in RO snapshot
+   * @throws IOException If an I/O error occurred
+   */
+  @Idempotent
+  public boolean truncate(String src, long newLength, String clientName)
+      throws AccessControlException, FileNotFoundException, SafeModeException,
+      UnresolvedLinkException, SnapshotAccessControlException, IOException;
+
   /**
    * Delete the given file or directory from the file system.
    * <p>
@@ -920,7 +952,9 @@ public ContentSummary getContentSummary(String path)
    * @param namespaceQuota Limit on the number of names in the tree rooted 
    *                       at the directory
    * @param diskspaceQuota Limit on disk space occupied all the files under
-   *                       this directory. 
+   *                       this directory.
+   * @param type StorageType that the space quota is intended to be set on.
+   *             It may be null when called by traditional space/namespace quota.
    * <br><br>
    *                       
    * The quota can have three types of values : (1) 0 or more will set 
@@ -937,8 +971,8 @@ public ContentSummary getContentSummary(String path)
    * @throws IOException If an I/O error occurred
    */
   @Idempotent
-  public void setQuota(String path, long namespaceQuota, long diskspaceQuota)
-      throws AccessControlException, FileNotFoundException,
+  public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
+      StorageType type) throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, SnapshotAccessControlException, IOException;
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
index c781e5b9eceff..779e3b905f1a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
@@ -46,6 +46,8 @@ public class DatanodeID implements Comparable<DatanodeID> {
   private int infoPort;      // info server port
   private int infoSecurePort; // info server port
   private int ipcPort;       // IPC server port
+  private String xferAddr;
+  private int hashCode = -1;
 
   /**
    * UUID identifying a given datanode. For upgraded Datanodes this is the
@@ -86,10 +88,12 @@ public DatanodeID(String ipAddr, String hostName, String datanodeUuid,
     this.infoPort = infoPort;
     this.infoSecurePort = infoSecurePort;
     this.ipcPort = ipcPort;
+    updateXferAddrAndInvalidateHashCode();
   }
   
   public void setIpAddr(String ipAddr) {
     this.ipAddr = ipAddr;
+    updateXferAddrAndInvalidateHashCode();
   }
 
   public void setPeerHostName(String peerHostName) {
@@ -106,6 +110,7 @@ public String getDatanodeUuid() {
   @VisibleForTesting
   public void setDatanodeUuidForTesting(String datanodeUuid) {
     this.datanodeUuid = datanodeUuid;
+    updateXferAddrAndInvalidateHashCode();
   }
 
   private String checkDatanodeUuid(String uuid) {
@@ -141,7 +146,7 @@ public String getPeerHostName() {
    * @return IP:xferPort string
    */
   public String getXferAddr() {
-    return ipAddr + ":" + xferPort;
+    return xferAddr;
   }
 
   /**
@@ -237,7 +242,11 @@ public boolean equals(Object to) {
   
   @Override
   public int hashCode() {
-    return getXferAddr().hashCode()^ datanodeUuid.hashCode();
+    if (hashCode == -1) {
+      int newHashCode = xferAddr.hashCode() ^ datanodeUuid.hashCode();
+      hashCode = newHashCode & Integer.MAX_VALUE;
+    }
+    return hashCode;
   }
   
   @Override
@@ -257,6 +266,7 @@ public void updateRegInfo(DatanodeID nodeReg) {
     infoPort = nodeReg.getInfoPort();
     infoSecurePort = nodeReg.getInfoSecurePort();
     ipcPort = nodeReg.getIpcPort();
+    updateXferAddrAndInvalidateHashCode();
   }
     
   /**
@@ -269,4 +279,13 @@ public void updateRegInfo(DatanodeID nodeReg) {
   public int compareTo(DatanodeID that) {
     return getXferAddr().compareTo(that.getXferAddr());
   }
+
+  // NOTE: mutable hash codes are dangerous, however this class chooses to
+  // use them.  this method must be called when a value mutates that is used
+  // to compute the hash, equality, or comparison of instances.
+  private void updateXferAddrAndInvalidateHashCode() {
+    xferAddr = ipAddr + ":" + xferPort;
+    // can't compute new hash yet because uuid might still null...
+    hashCode = -1;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
index b8c21b048ac0e..0141215da1ab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
@@ -23,6 +23,9 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 /**
  * EncryptionZoneIterator is a remote iterator that iterates over encryption
@@ -34,16 +37,24 @@ public class EncryptionZoneIterator
     extends BatchedRemoteIterator<Long, EncryptionZone> {
 
   private final ClientProtocol namenode;
+  private final Sampler<?> traceSampler;
 
-  public EncryptionZoneIterator(ClientProtocol namenode) {
+  public EncryptionZoneIterator(ClientProtocol namenode,
+                                Sampler<?> traceSampler) {
     super(Long.valueOf(0));
     this.namenode = namenode;
+    this.traceSampler = traceSampler;
   }
 
   @Override
   public BatchedEntries<EncryptionZone> makeRequest(Long prevId)
       throws IOException {
-    return namenode.listEncryptionZones(prevId);
+    TraceScope scope = Trace.startSpan("listEncryptionZones", traceSampler);
+    try {
+      return namenode.listEncryptionZones(prevId);
+    } finally {
+      scope.close();
+    }
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 54da8ebe8d4a9..7d50360e7780c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -170,6 +170,7 @@ public static enum DatanodeReportType {
   public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
   public static final String HOT_STORAGE_POLICY_NAME = "HOT";
   public static final String WARM_STORAGE_POLICY_NAME = "WARM";
+  public static final String EC_STORAGE_POLICY_NAME = "EC";
   public static final String COLD_STORAGE_POLICY_NAME = "COLD";
 
   public static final byte MEMORY_STORAGE_POLICY_ID = 15;
@@ -177,5 +178,10 @@ public static enum DatanodeReportType {
   public static final byte ONESSD_STORAGE_POLICY_ID = 10;
   public static final byte HOT_STORAGE_POLICY_ID = 7;
   public static final byte WARM_STORAGE_POLICY_ID = 5;
+  public static final byte EC_STORAGE_POLICY_ID = 4;
   public static final byte COLD_STORAGE_POLICY_ID = 2;
+
+  public static final byte NUM_DATA_BLOCKS = 3;
+  public static final byte NUM_PARITY_BLOCKS = 2;
+  public static final byte MAX_BLOCKS_IN_GROUP = 16;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java
new file mode 100644
index 0000000000000..1de0a307a65e2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.StorageType;
+
+import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class QuotaByStorageTypeExceededException extends QuotaExceededException {
+  protected static final long serialVersionUID = 1L;
+  protected StorageType type;
+
+  public QuotaByStorageTypeExceededException() {}
+
+  public QuotaByStorageTypeExceededException(String msg) {
+    super(msg);
+  }
+
+  public QuotaByStorageTypeExceededException(long quota, long count, StorageType type) {
+    super(quota, count);
+    this.type = type;
+  }
+
+  @Override
+  public String getMessage() {
+    String msg = super.getMessage();
+    if (msg == null) {
+      return "Quota by storage type : " + type.toString() +
+          " on path : " + (pathName==null ? "": pathName) +
+          " is exceeded. quota = "  + long2String(quota, "B", 2) +
+          " but space consumed = " + long2String(count, "B", 2);
+    } else {
+      return msg;
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
index b91e17a3b658c..2ef3c3f8d5eba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
@@ -31,10 +31,10 @@
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceInfo;
-import org.htrace.TraceScope;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceInfo;
+import org.apache.htrace.TraceScope;
 
 /**
  * Static utilities for dealing with the protocol buffers used by the
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
index 0de445c222de9..3045a13b2009c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
@@ -178,7 +178,7 @@ private void doRead(ReadableByteChannel ch, InputStream in)
     if (curHeader == null) {
       curHeader = new PacketHeader();
     }
-    curHeader.setFieldsFromData(dataPlusChecksumLen, headerBuf);
+    curHeader.setFieldsFromData(payloadLen, headerBuf);
     
     // Compute the sub-slices of the packet
     int checksumLen = dataPlusChecksumLen - curHeader.getDataLen();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
index 6d4065386bfb7..35e5bb84dc7b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
@@ -22,17 +22,21 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_DEFAULT;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos;
+
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
 import com.google.protobuf.TextFormat;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
 
 /** Pipeline Acknowledgment **/
 @InterfaceAudience.Private
@@ -46,6 +50,55 @@ public class PipelineAck {
   // place holder for timeout value of each OOB type
   final static long[] OOB_TIMEOUT;
 
+  public enum ECN {
+    DISABLED(0),
+    SUPPORTED(1),
+    SUPPORTED2(2),
+    CONGESTED(3);
+
+    private final int value;
+    private static final ECN[] VALUES = values();
+    static ECN valueOf(int value) {
+      return VALUES[value];
+    }
+
+    ECN(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+  }
+
+  private enum StatusFormat {
+    STATUS(null, 4),
+    RESERVED(STATUS.BITS, 1),
+    ECN_BITS(RESERVED.BITS, 2);
+
+    private final LongBitFormat BITS;
+
+    StatusFormat(LongBitFormat prev, int bits) {
+      BITS = new LongBitFormat(name(), prev, bits, 0);
+    }
+
+    static Status getStatus(int header) {
+      return Status.valueOf((int) STATUS.BITS.retrieve(header));
+    }
+
+    static ECN getECN(int header) {
+      return ECN.valueOf((int) ECN_BITS.BITS.retrieve(header));
+    }
+
+    public static int setStatus(int old, Status status) {
+      return (int) STATUS.BITS.combine(status.getNumber(), old);
+    }
+
+    public static int setECN(int old, ECN ecn) {
+      return (int) ECN_BITS.BITS.combine(ecn.getValue(), old);
+    }
+  }
+
   static {
     OOB_TIMEOUT = new long[NUM_OOB_TYPES];
     HdfsConfiguration conf = new HdfsConfiguration();
@@ -65,7 +118,7 @@ public PipelineAck() {
    * @param seqno sequence number
    * @param replies an array of replies
    */
-  public PipelineAck(long seqno, Status[] replies) {
+  public PipelineAck(long seqno, int[] replies) {
     this(seqno, replies, 0L);
   }
 
@@ -75,10 +128,15 @@ public PipelineAck(long seqno, Status[] replies) {
    * @param replies an array of replies
    * @param downstreamAckTimeNanos ack RTT in nanoseconds, 0 if no next DN in pipeline
    */
-  public PipelineAck(long seqno, Status[] replies, long downstreamAckTimeNanos) {
+  public PipelineAck(long seqno, int[] replies,
+                     long downstreamAckTimeNanos) {
+    ArrayList<Integer> replyList = Lists.newArrayList();
+    for (int r : replies) {
+      replyList.add(r);
+    }
     proto = PipelineAckProto.newBuilder()
       .setSeqno(seqno)
-      .addAllStatus(Arrays.asList(replies))
+      .addAllReply(replyList)
       .setDownstreamAckTimeNanos(downstreamAckTimeNanos)
       .build();
   }
@@ -96,15 +154,15 @@ public long getSeqno() {
    * @return the number of replies
    */
   public short getNumOfReplies() {
-    return (short)proto.getStatusCount();
+    return (short)proto.getReplyCount();
   }
   
   /**
    * get the ith reply
    * @return the the ith reply
    */
-  public Status getReply(int i) {
-    return proto.getStatus(i);
+  public int getReply(int i) {
+    return proto.getReply(i);
   }
 
   /**
@@ -120,8 +178,8 @@ public long getDownstreamAckTimeNanos() {
    * @return true if all statuses are SUCCESS
    */
   public boolean isSuccess() {
-    for (Status reply : proto.getStatusList()) {
-      if (reply != Status.SUCCESS) {
+    for (int reply : proto.getReplyList()) {
+      if (StatusFormat.getStatus(reply) != Status.SUCCESS) {
         return false;
       }
     }
@@ -138,11 +196,12 @@ public Status getOOBStatus() {
     if (getSeqno() != UNKOWN_SEQNO) {
       return null;
     }
-    for (Status reply : proto.getStatusList()) {
+    for (int reply : proto.getReplyList()) {
       // The following check is valid because protobuf guarantees to
       // preserve the ordering of enum elements.
-      if (reply.getNumber() >= OOB_START && reply.getNumber() <= OOB_END) {
-        return reply;
+      Status s = StatusFormat.getStatus(reply);
+      if (s.getNumber() >= OOB_START && s.getNumber() <= OOB_END) {
+        return s;
       }
     }
     return null;
@@ -184,4 +243,19 @@ public void write(OutputStream out) throws IOException {
   public String toString() {
     return TextFormat.shortDebugString(proto);
   }
+
+  public static Status getStatusFromHeader(int header) {
+    return StatusFormat.getStatus(header);
+  }
+
+  public static int setStatusForHeader(int old, Status status) {
+    return StatusFormat.setStatus(old, status);
+  }
+
+  public static int combineHeader(ECN ecn, Status status) {
+    int header = 0;
+    header = StatusFormat.setStatus(header, status);
+    header = StatusFormat.setECN(header, ecn);
+    return header;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 8192925166fc2..2ea17871356a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -40,7 +40,7 @@
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
 import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
-import org.htrace.TraceScope;
+import org.apache.htrace.TraceScope;
 
 /** Receiver */
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index 1ae9da53ca039..844c27081445b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -48,8 +48,8 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DataChecksum;
 
-import org.htrace.Trace;
-import org.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.Span;
 
 import com.google.protobuf.Message;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
index eb494a87e171e..cc9b0c69807c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
@@ -31,14 +31,26 @@
 public class InvalidMagicNumberException extends IOException {
 
   private static final long serialVersionUID = 1L;
+  private final boolean handshake4Encryption;
 
   /**
    * Creates a new InvalidMagicNumberException.
    *
    * @param magicNumber expected value
    */
-  public InvalidMagicNumberException(int magicNumber) {
+  public InvalidMagicNumberException(final int magicNumber, 
+      final boolean handshake4Encryption) {
     super(String.format("Received %x instead of %x from client.",
         magicNumber, SASL_TRANSFER_MAGIC_NUMBER));
+    this.handshake4Encryption = handshake4Encryption;
+  }
+  
+  /**
+   * Return true if it's handshake for encryption
+   * 
+   * @return boolean true if it's handshake for encryption
+   */
+  public boolean isHandshake4Encryption() {
+    return handshake4Encryption;
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
index 1d2b30b88bba8..f060beb3475c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
@@ -357,7 +357,8 @@ private IOStreamPair doSaslHandshake(OutputStream underlyingOut,
 
     int magicNumber = in.readInt();
     if (magicNumber != SASL_TRANSFER_MAGIC_NUMBER) {
-      throw new InvalidMagicNumberException(magicNumber);
+      throw new InvalidMagicNumberException(magicNumber, 
+          dnConf.getEncryptDataTransfer());
     }
     try {
       // step 1
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 5b6609bfbbee8..30a46df29b21b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -18,12 +18,14 @@
 package org.apache.hadoop.hdfs.protocolPB;
 
 import java.io.IOException;
+import java.util.EnumSet;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
 import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -65,6 +67,8 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto;
@@ -181,12 +185,12 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathResponseProto;
@@ -207,6 +211,7 @@
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
@@ -410,8 +415,11 @@ public CreateResponseProto create(RpcController controller,
   public AppendResponseProto append(RpcController controller,
       AppendRequestProto req) throws ServiceException {
     try {
+      EnumSetWritable<CreateFlag> flags = req.hasFlag() ?
+          PBHelper.convertCreateFlag(req.getFlag()) :
+          new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND));
       LastBlockWithStatus result = server.append(req.getSrc(),
-          req.getClientName());
+          req.getClientName(), flags);
       AppendResponseProto.Builder builder = AppendResponseProto.newBuilder();
       if (result.getLastBlock() != null) {
         builder.setBlock(PBHelper.convert(result.getLastBlock()));
@@ -520,7 +528,7 @@ public GetAdditionalDatanodeResponseProto getAdditionalDatanode(
       throw new ServiceException(e);
     }
   }
-  
+
   @Override
   public CompleteResponseProto complete(RpcController controller,
       CompleteRequestProto req) throws ServiceException {
@@ -584,6 +592,18 @@ public Rename2ResponseProto rename2(RpcController controller,
     return VOID_RENAME2_RESPONSE;
   }
 
+  @Override
+  public TruncateResponseProto truncate(RpcController controller,
+      TruncateRequestProto req) throws ServiceException {
+    try {
+      boolean result = server.truncate(req.getSrc(), req.getNewLength(),
+          req.getClientName());
+      return TruncateResponseProto.newBuilder().setResult(result).build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
   @Override
   public DeleteResponseProto delete(RpcController controller,
     DeleteRequestProto req) throws ServiceException {
@@ -867,7 +887,9 @@ public SetQuotaResponseProto setQuota(RpcController controller,
       SetQuotaRequestProto req) throws ServiceException {
     try {
       server.setQuota(req.getPath(), req.getNamespaceQuota(),
-          req.getDiskspaceQuota());
+          req.getDiskspaceQuota(),
+          req.hasStorageType() ?
+          PBHelper.convertStorageType(req.getStorageType()): null);
       return VOID_SETQUOTA_RESPONSE;
     } catch (IOException e) {
       throw new ServiceException(e);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 58049204fc4ab..9ab380b1760e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -28,7 +28,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.crypto.CipherSuite;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
 import org.apache.hadoop.fs.CacheFlag;
@@ -85,6 +84,7 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto;
@@ -155,15 +155,14 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetReplicationRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetSafeModeRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.TruncateRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
@@ -173,6 +172,7 @@
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.ProtobufHelper;
@@ -302,14 +302,28 @@ public HdfsFileStatus create(String src, FsPermission masked,
   }
 
   @Override
-  public LastBlockWithStatus append(String src, String clientName)
-      throws AccessControlException, DSQuotaExceededException,
-      FileNotFoundException, SafeModeException, UnresolvedLinkException,
-      IOException {
-    AppendRequestProto req = AppendRequestProto.newBuilder()
+  public boolean truncate(String src, long newLength, String clientName)
+      throws IOException, UnresolvedLinkException {
+    TruncateRequestProto req = TruncateRequestProto.newBuilder()
         .setSrc(src)
+        .setNewLength(newLength)
         .setClientName(clientName)
         .build();
+    try {
+      return rpcProxy.truncate(null, req).getResult();
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
+  public LastBlockWithStatus append(String src, String clientName,
+      EnumSetWritable<CreateFlag> flag) throws AccessControlException,
+      DSQuotaExceededException, FileNotFoundException, SafeModeException,
+      UnresolvedLinkException, IOException {
+    AppendRequestProto req = AppendRequestProto.newBuilder().setSrc(src)
+        .setClientName(clientName).setFlag(PBHelper.convertCreateFlag(flag))
+        .build();
     try {
       AppendResponseProto res = rpcProxy.append(null, req);
       LocatedBlock lastBlock = res.hasBlock() ? PBHelper
@@ -792,14 +806,19 @@ public ContentSummary getContentSummary(String path)
   }
 
   @Override
-  public void setQuota(String path, long namespaceQuota, long diskspaceQuota)
+  public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
+                       StorageType type)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    SetQuotaRequestProto req = SetQuotaRequestProto.newBuilder()
+    final SetQuotaRequestProto.Builder builder
+        = SetQuotaRequestProto.newBuilder()
         .setPath(path)
         .setNamespaceQuota(namespaceQuota)
-        .setDiskspaceQuota(diskspaceQuota)
-        .build();
+        .setDiskspaceQuota(diskspaceQuota);
+    if (type != null) {
+      builder.setStorageType(PBHelper.convertStorageType(type));
+    }
+    final SetQuotaRequestProto req = builder.build();
     try {
       rpcProxy.setQuota(null, req);
     } catch (ServiceException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
index 087c697c5871c..ba0a8fc24ac37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
@@ -76,12 +76,12 @@ public UpdateReplicaUnderRecoveryResponseProto updateReplicaUnderRecovery(
     final String storageID;
     try {
       storageID = impl.updateReplicaUnderRecovery(
-          PBHelper.convert(request.getBlock()),
-          request.getRecoveryId(), request.getNewLength());
+          PBHelper.convert(request.getBlock()), request.getRecoveryId(),
+          request.getNewBlockId(), request.getNewLength());
     } catch (IOException e) {
       throw new ServiceException(e);
     }
     return UpdateReplicaUnderRecoveryResponseProto.newBuilder()
         .setStorageUuid(storageID).build();
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
index 5174d86188292..fee62a4e99462 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
@@ -102,11 +102,12 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
 
   @Override
   public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
-      long recoveryId, long newLength) throws IOException {
+      long recoveryId, long newBlockId, long newLength) throws IOException {
     UpdateReplicaUnderRecoveryRequestProto req = 
         UpdateReplicaUnderRecoveryRequestProto.newBuilder()
         .setBlock(PBHelper.convert(oldBlock))
-        .setNewLength(newLength).setRecoveryId(recoveryId).build();
+        .setNewLength(newLength).setNewBlockId(newBlockId)
+        .setRecoveryId(recoveryId).build();
     try {
       return rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req
           ).getStorageUuid();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 2a5edc8dc9443..e4746cc400bb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -607,14 +607,19 @@ public static RecoveringBlockProto convert(RecoveringBlock b) {
       return null;
     }
     LocatedBlockProto lb = PBHelper.convert((LocatedBlock)b);
-    return RecoveringBlockProto.newBuilder().setBlock(lb)
-        .setNewGenStamp(b.getNewGenerationStamp()).build();
+    RecoveringBlockProto.Builder builder = RecoveringBlockProto.newBuilder();
+    builder.setBlock(lb).setNewGenStamp(b.getNewGenerationStamp());
+    if(b.getNewBlock() != null)
+      builder.setTruncateBlock(PBHelper.convert(b.getNewBlock()));
+    return builder.build();
   }
 
   public static RecoveringBlock convert(RecoveringBlockProto b) {
     ExtendedBlock block = convert(b.getBlock().getB());
     DatanodeInfo[] locs = convert(b.getBlock().getLocsList());
-    return new RecoveringBlock(block, locs, b.getNewGenStamp());
+    return (b.hasTruncateBlock()) ?
+        new RecoveringBlock(block, locs, PBHelper.convert(b.getTruncateBlock())) :
+        new RecoveringBlock(block, locs, b.getNewGenStamp());
   }
   
   public static DatanodeInfoProto.AdminState convert(
@@ -1368,6 +1373,9 @@ public static int convertCreateFlag(EnumSetWritable<CreateFlag> flag) {
     if (flag.contains(CreateFlag.LAZY_PERSIST)) {
       value |= CreateFlagProto.LAZY_PERSIST.getNumber();
     }
+    if (flag.contains(CreateFlag.NEW_BLOCK)) {
+      value |= CreateFlagProto.NEW_BLOCK.getNumber();
+    }
     return value;
   }
   
@@ -1388,7 +1396,11 @@ public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
         == CreateFlagProto.LAZY_PERSIST_VALUE) {
       result.add(CreateFlag.LAZY_PERSIST);
     }
-    return new EnumSetWritable<CreateFlag>(result);
+    if ((flag & CreateFlagProto.NEW_BLOCK_VALUE)
+        == CreateFlagProto.NEW_BLOCK_VALUE) {
+      result.add(CreateFlag.NEW_BLOCK);
+    }
+    return new EnumSetWritable<CreateFlag>(result, CreateFlag.class);
   }
 
   public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
@@ -2278,15 +2290,24 @@ public static List<AclEntry> convertAclEntry(List<AclEntryProto> aclSpec) {
 
   public static AclStatus convert(GetAclStatusResponseProto e) {
     AclStatusProto r = e.getResult();
-    return new AclStatus.Builder().owner(r.getOwner()).group(r.getGroup())
-        .stickyBit(r.getSticky())
-        .addEntries(convertAclEntry(r.getEntriesList())).build();
+    AclStatus.Builder builder = new AclStatus.Builder();
+    builder.owner(r.getOwner()).group(r.getGroup()).stickyBit(r.getSticky())
+        .addEntries(convertAclEntry(r.getEntriesList()));
+    if (r.hasPermission()) {
+      builder.setPermission(convert(r.getPermission()));
+    }
+    return builder.build();
   }
 
   public static GetAclStatusResponseProto convert(AclStatus e) {
-    AclStatusProto r = AclStatusProto.newBuilder().setOwner(e.getOwner())
+    AclStatusProto.Builder builder = AclStatusProto.newBuilder();
+    builder.setOwner(e.getOwner())
         .setGroup(e.getGroup()).setSticky(e.isStickyBit())
-        .addAllEntries(convertAclEntryProto(e.getEntries())).build();
+        .addAllEntries(convertAclEntryProto(e.getEntries()));
+    if (e.getPermission() != null) {
+      builder.setPermission(convert(e.getPermission()));
+    }
+    AclStatusProto r = builder.build();
     return GetAclStatusResponseProto.newBuilder().setResult(r).build();
   }
   
@@ -2557,6 +2578,7 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
                 .replication(create.getReplication())
                 .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
                     create.getSymlinkTarget())
+                .defaultBlockSize(create.getDefaultBlockSize())
                 .overwrite(create.getOverwrite()).build());
             break;
           case EVENT_METADATA:
@@ -2583,19 +2605,26 @@ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
           case EVENT_RENAME:
             InotifyProtos.RenameEventProto rename =
                 InotifyProtos.RenameEventProto.parseFrom(p.getContents());
-            events.add(new Event.RenameEvent(rename.getSrcPath(),
-                rename.getDestPath(), rename.getTimestamp()));
+            events.add(new Event.RenameEvent.Builder()
+                  .srcPath(rename.getSrcPath())
+                  .dstPath(rename.getDestPath())
+                  .timestamp(rename.getTimestamp())
+                  .build());
             break;
           case EVENT_APPEND:
-            InotifyProtos.AppendEventProto reopen =
+            InotifyProtos.AppendEventProto append =
                 InotifyProtos.AppendEventProto.parseFrom(p.getContents());
-            events.add(new Event.AppendEvent(reopen.getPath()));
+            events.add(new Event.AppendEvent.Builder().path(append.getPath())
+                .newBlock(append.hasNewBlock() && append.getNewBlock())
+                .build());
             break;
           case EVENT_UNLINK:
             InotifyProtos.UnlinkEventProto unlink =
                 InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
-            events.add(new Event.UnlinkEvent(unlink.getPath(),
-                unlink.getTimestamp()));
+            events.add(new Event.UnlinkEvent.Builder()
+                  .path(unlink.getPath())
+                  .timestamp(unlink.getTimestamp())
+                  .build());
             break;
           default:
             throw new RuntimeException("Unexpected inotify event type: " +
@@ -2641,6 +2670,7 @@ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList
                         .setReplication(ce2.getReplication())
                         .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
                             "" : ce2.getSymlinkTarget())
+                        .setDefaultBlockSize(ce2.getDefaultBlockSize())
                         .setOverwrite(ce2.getOverwrite()).build().toByteString()
                 ).build());
             break;
@@ -2687,10 +2717,10 @@ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList
             Event.AppendEvent re2 = (Event.AppendEvent) e;
             events.add(InotifyProtos.EventProto.newBuilder()
                 .setType(InotifyProtos.EventType.EVENT_APPEND)
-                .setContents(
-                    InotifyProtos.AppendEventProto.newBuilder()
-                        .setPath(re2.getPath()).build().toByteString()
-                ).build());
+                .setContents(InotifyProtos.AppendEventProto.newBuilder()
+                    .setPath(re2.getPath())
+                    .setNewBlock(re2.toNewBlock()).build().toByteString())
+                .build());
             break;
           case UNLINK:
             Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index e37869c797551..6938f571e5c3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -52,10 +52,10 @@
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.StopWatch;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
 import com.google.common.net.InetAddresses;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
@@ -143,7 +143,7 @@ public class IPCLoggerChannel implements AsyncLogger {
   /**
    * Stopwatch which starts counting on each heartbeat that is sent
    */
-  private final Stopwatch lastHeartbeatStopwatch = new Stopwatch();
+  private final StopWatch lastHeartbeatStopwatch = new StopWatch();
   
   private static final long HEARTBEAT_INTERVAL_MILLIS = 1000;
 
@@ -463,8 +463,8 @@ private void throwIfOutOfSync()
    * written.
    */
   private void heartbeatIfNecessary() throws IOException {
-    if (lastHeartbeatStopwatch.elapsedMillis() > HEARTBEAT_INTERVAL_MILLIS ||
-        !lastHeartbeatStopwatch.isRunning()) {
+    if (lastHeartbeatStopwatch.now(TimeUnit.MILLISECONDS)
+        > HEARTBEAT_INTERVAL_MILLIS || !lastHeartbeatStopwatch.isRunning()) {
       try {
         getProxy().heartbeat(createReqInfo());
       } finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index 9be56b822de53..9d11ca59e9bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.lang.math.LongRange;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -61,14 +62,12 @@
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StopWatch;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
 import com.google.protobuf.TextFormat;
 
 /**
@@ -374,15 +373,20 @@ synchronized void journal(RequestInfo reqInfo,
     
     curSegment.writeRaw(records, 0, records.length);
     curSegment.setReadyToFlush();
-    Stopwatch sw = new Stopwatch();
+    StopWatch sw = new StopWatch();
     sw.start();
     curSegment.flush(shouldFsync);
     sw.stop();
-    
-    metrics.addSync(sw.elapsedTime(TimeUnit.MICROSECONDS));
-    if (sw.elapsedTime(TimeUnit.MILLISECONDS) > WARN_SYNC_MILLIS_THRESHOLD) {
+
+    long nanoSeconds = sw.now();
+    metrics.addSync(
+        TimeUnit.MICROSECONDS.convert(nanoSeconds, TimeUnit.NANOSECONDS));
+    long milliSeconds = TimeUnit.MILLISECONDS.convert(
+        nanoSeconds, TimeUnit.NANOSECONDS);
+
+    if (milliSeconds > WARN_SYNC_MILLIS_THRESHOLD) {
       LOG.warn("Sync of transaction range " + firstTxnId + "-" + lastTxnId +
-               " took " + sw.elapsedTime(TimeUnit.MILLISECONDS) + "ms");
+               " took " + milliSeconds + "ms");
     }
 
     if (isLagging) {
@@ -788,8 +792,8 @@ public synchronized void acceptRecovery(RequestInfo reqInfo,
         // Paranoid sanity check: if the new log is shorter than the log we
         // currently have, we should not end up discarding any transactions
         // which are already Committed.
-        if (txnRange(currentSegment).contains(committedTxnId.get()) &&
-            !txnRange(segment).contains(committedTxnId.get())) {
+        if (txnRange(currentSegment).containsLong(committedTxnId.get()) &&
+            !txnRange(segment).containsLong(committedTxnId.get())) {
           throw new AssertionError(
               "Cannot replace segment " +
               TextFormat.shortDebugString(currentSegment) +
@@ -807,7 +811,7 @@ public synchronized void acceptRecovery(RequestInfo reqInfo,
         
         // If we're shortening the log, update our highest txid
         // used for lag metrics.
-        if (txnRange(currentSegment).contains(highestWrittenTxId)) {
+        if (txnRange(currentSegment).containsLong(highestWrittenTxId)) {
           highestWrittenTxId = segment.getEndTxId();
         }
       }
@@ -851,10 +855,10 @@ public synchronized void acceptRecovery(RequestInfo reqInfo,
         TextFormat.shortDebugString(newData));
   }
 
-  private Range<Long> txnRange(SegmentStateProto seg) {
+  private LongRange txnRange(SegmentStateProto seg) {
     Preconditions.checkArgument(seg.hasEndTxId(),
         "invalid segment: %s", seg);
-    return Ranges.closed(seg.getStartTxId(), seg.getEndTxId());
+    return new LongRange(seg.getStartTxId(), seg.getEndTxId());
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
index 588bc580ce54c..a5a40f1158fea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
@@ -233,6 +233,7 @@ private File getLogDir(String jid) {
     Preconditions.checkArgument(jid != null &&
         !jid.isEmpty(),
         "bad journal identifier: %s", jid);
+    assert jid != null;
     return new File(new File(dir), jid);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index 9ef227400c41f..15476118b9eec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -31,7 +31,7 @@ public interface BlockCollection {
   /**
    * Get the last block of the collection.
    */
-  public BlockInfo getLastBlock();
+  public BlockInfoContiguous getLastBlock();
 
   /** 
    * Get content summary.
@@ -46,7 +46,7 @@ public interface BlockCollection {
   /**
    * Get the blocks.
    */
-  public BlockInfo[] getBlocks();
+  public BlockInfoContiguous[] getBlocks();
 
   /**
    * Get preferred block size for the collection 
@@ -73,13 +73,13 @@ public interface BlockCollection {
   /**
    * Set the block at the given index.
    */
-  public void setBlock(int index, BlockInfo blk);
+  public void setBlock(int index, BlockInfoContiguous blk);
 
   /**
    * Convert the last block of the collection to an under-construction block
    * and set the locations.
    */
-  public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock,
+  public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous lastBlock,
       DatanodeStorageInfo[] targets) throws IOException;
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index 1c69203afccf8..e7f8a05f34d06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -53,10 +53,12 @@ public class BlockIdManager {
    * The global block ID space for this file system.
    */
   private final SequentialBlockIdGenerator blockIdGenerator;
+  private final SequentialBlockGroupIdGenerator blockGroupIdGenerator;
 
   public BlockIdManager(BlockManager blockManager) {
     this.generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
     this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager);
+    this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager);
   }
 
   /**
@@ -190,6 +192,10 @@ public long nextBlockId() {
     return blockIdGenerator.nextValue();
   }
 
+  public long nextBlockGroupId() {
+    return blockGroupIdGenerator.nextValue();
+  }
+
   public boolean isGenStampInFuture(Block block) {
     if (isLegacyBlock(block)) {
       return block.getGenerationStamp() > getGenerationStampV1();
@@ -205,4 +211,12 @@ public void clear() {
       .LAST_RESERVED_BLOCK_ID);
     generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
   }
-}
\ No newline at end of file
+
+  public static boolean isStripedBlockID(long id) {
+    return id < 0;
+  }
+
+  public static long convertToGroupID(long id) {
+    return id & (~(HdfsConstants.MAX_BLOCKS_IN_GROUP - 1));
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
similarity index 82%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index 5d0b473f40d46..48069c1799c58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -30,8 +30,9 @@
  * the block are stored.
  */
 @InterfaceAudience.Private
-public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
-  public static final BlockInfo[] EMPTY_ARRAY = {}; 
+public class BlockInfoContiguous extends Block
+    implements LightWeightGSet.LinkedElement {
+  public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
 
   private BlockCollection bc;
 
@@ -56,12 +57,12 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement {
    * Construct an entry for blocksmap
    * @param replication the block's replication factor
    */
-  public BlockInfo(short replication) {
+  public BlockInfoContiguous(short replication) {
     this.triplets = new Object[3*replication];
     this.bc = null;
   }
   
-  public BlockInfo(Block blk, short replication) {
+  public BlockInfoContiguous(Block blk, short replication) {
     super(blk);
     this.triplets = new Object[3*replication];
     this.bc = null;
@@ -72,7 +73,7 @@ public BlockInfo(Block blk, short replication) {
    * This is used to convert BlockInfoUnderConstruction
    * @param from BlockInfo to copy from.
    */
-  protected BlockInfo(BlockInfo from) {
+  protected BlockInfoContiguous(BlockInfoContiguous from) {
     this(from, from.bc.getBlockReplication());
     this.bc = from.bc;
   }
@@ -96,23 +97,23 @@ DatanodeStorageInfo getStorageInfo(int index) {
     return (DatanodeStorageInfo)triplets[index*3];
   }
 
-  private BlockInfo getPrevious(int index) {
+  private BlockInfoContiguous getPrevious(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+1];
+    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
     assert info == null || 
-        info.getClass().getName().startsWith(BlockInfo.class.getName()) : 
+        info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) :
               "BlockInfo is expected at " + index*3;
     return info;
   }
 
-  BlockInfo getNext(int index) {
+  BlockInfoContiguous getNext(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+2];
-    assert info == null || 
-        info.getClass().getName().startsWith(BlockInfo.class.getName()) : 
-              "BlockInfo is expected at " + index*3;
+    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
+    assert info == null || info.getClass().getName().startsWith(
+        BlockInfoContiguous.class.getName()) :
+        "BlockInfo is expected at " + index*3;
     return info;
   }
 
@@ -130,10 +131,10 @@ private void setStorageInfo(int index, DatanodeStorageInfo storage) {
    * @param to - block to be set to previous on the list of blocks
    * @return current previous block on the list of blocks
    */
-  private BlockInfo setPrevious(int index, BlockInfo to) {
-	assert this.triplets != null : "BlockInfo is not initialized";
-	assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+1];
+  private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
     triplets[index*3+1] = to;
     return info;
   }
@@ -146,10 +147,10 @@ private BlockInfo setPrevious(int index, BlockInfo to) {
    * @param to - block to be set to next on the list of blocks
    *    * @return current next block on the list of blocks
    */
-  private BlockInfo setNext(int index, BlockInfo to) {
-	assert this.triplets != null : "BlockInfo is not initialized";
-	assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+2];
+  private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
     triplets[index*3+2] = to;
     return info;
   }
@@ -241,6 +242,7 @@ boolean findDatanode(DatanodeDescriptor dn) {
     }
     return false;
   }
+
   /**
    * Find specified DatanodeStorageInfo.
    * @return DatanodeStorageInfo or null if not found.
@@ -265,10 +267,12 @@ int findStorageInfo(DatanodeStorageInfo storageInfo) {
     int len = getCapacity();
     for(int idx = 0; idx < len; idx++) {
       DatanodeStorageInfo cur = getStorageInfo(idx);
-      if(cur == storageInfo)
+      if (cur == storageInfo) {
         return idx;
-      if(cur == null)
+      }
+      if (cur == null) {
         break;
+      }
     }
     return -1;
   }
@@ -279,7 +283,8 @@ int findStorageInfo(DatanodeStorageInfo storageInfo) {
    * If the head is null then form a new list.
    * @return current block as the new head of the list.
    */
-  BlockInfo listInsert(BlockInfo head, DatanodeStorageInfo storage) {
+  BlockInfoContiguous listInsert(BlockInfoContiguous head,
+      DatanodeStorageInfo storage) {
     int dnIndex = this.findStorageInfo(storage);
     assert dnIndex >= 0 : "Data node is not found: current";
     assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : 
@@ -299,15 +304,16 @@ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
    * @return the new head of the list or null if the list becomes
    * empy after deletion.
    */
-  BlockInfo listRemove(BlockInfo head, DatanodeStorageInfo storage) {
+  BlockInfoContiguous listRemove(BlockInfoContiguous head,
+      DatanodeStorageInfo storage) {
     if(head == null)
       return null;
     int dnIndex = this.findStorageInfo(storage);
     if(dnIndex < 0) // this block is not on the data-node list
       return head;
 
-    BlockInfo next = this.getNext(dnIndex);
-    BlockInfo prev = this.getPrevious(dnIndex);
+    BlockInfoContiguous next = this.getNext(dnIndex);
+    BlockInfoContiguous prev = this.getPrevious(dnIndex);
     this.setNext(dnIndex, null);
     this.setPrevious(dnIndex, null);
     if(prev != null)
@@ -325,25 +331,26 @@ BlockInfo listRemove(BlockInfo head, DatanodeStorageInfo storage) {
    *
    * @return the new head of the list.
    */
-  public BlockInfo moveBlockToHead(BlockInfo head, DatanodeStorageInfo storage,
-      int curIndex, int headIndex) {
+  public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head,
+      DatanodeStorageInfo storage, int curIndex, int headIndex) {
     if (head == this) {
       return this;
     }
-    BlockInfo next = this.setNext(curIndex, head);
-    BlockInfo prev = this.setPrevious(curIndex, null);
+    BlockInfoContiguous next = this.setNext(curIndex, head);
+    BlockInfoContiguous prev = this.setPrevious(curIndex, null);
 
     head.setPrevious(headIndex, this);
     prev.setNext(prev.findStorageInfo(storage), next);
-    if (next != null)
+    if (next != null) {
       next.setPrevious(next.findStorageInfo(storage), prev);
+    }
     return this;
   }
 
   /**
    * BlockInfo represents a block that is not being constructed.
    * In order to start modifying the block, the BlockInfo should be converted
-   * to {@link BlockInfoUnderConstruction}.
+   * to {@link BlockInfoContiguousUnderConstruction}.
    * @return {@link BlockUCState#COMPLETE}
    */
   public BlockUCState getBlockUCState() {
@@ -363,16 +370,18 @@ public boolean isComplete() {
    * Convert a complete block to an under construction block.
    * @return BlockInfoUnderConstruction -  an under construction block.
    */
-  public BlockInfoUnderConstruction convertToBlockUnderConstruction(
+  public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
       BlockUCState s, DatanodeStorageInfo[] targets) {
     if(isComplete()) {
-      BlockInfoUnderConstruction ucBlock = new BlockInfoUnderConstruction(this,
+      BlockInfoContiguousUnderConstruction ucBlock =
+          new BlockInfoContiguousUnderConstruction(this,
           getBlockCollection().getBlockReplication(), s, targets);
       ucBlock.setBlockCollection(getBlockCollection());
       return ucBlock;
     }
     // the block is already under construction
-    BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this;
+    BlockInfoContiguousUnderConstruction ucBlock =
+        (BlockInfoContiguousUnderConstruction)this;
     ucBlock.setBlockUCState(s);
     ucBlock.setExpectedLocations(targets);
     ucBlock.setBlockCollection(getBlockCollection());
@@ -384,7 +393,7 @@ public int hashCode() {
     // Super implementation is sufficient
     return super.hashCode();
   }
-  
+
   @Override
   public boolean equals(Object obj) {
     // Sufficient to rely on super's implementation
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
similarity index 93%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index f19ad1c511bad..91b76ccb0e266 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@
  * Represents a block that is currently being constructed.<br>
  * This is usually the last block of a file opened for write or append.
  */
-public class BlockInfoUnderConstruction extends BlockInfo {
+public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
   /** Block state. See {@link BlockUCState} */
   private BlockUCState blockUCState;
 
@@ -54,6 +54,11 @@ public class BlockInfoUnderConstruction extends BlockInfo {
    */
   private long blockRecoveryId = 0;
 
+  /**
+   * The block source to use in the event of copy-on-write truncate.
+   */
+  private Block truncateBlock;
+
   /**
    * ReplicaUnderConstruction contains information about replicas while
    * they are under construction.
@@ -153,16 +158,14 @@ public void appendStringTo(StringBuilder sb) {
    * Create block and set its state to
    * {@link BlockUCState#UNDER_CONSTRUCTION}.
    */
-  public BlockInfoUnderConstruction(Block blk, short replication) {
+  public BlockInfoContiguousUnderConstruction(Block blk, short replication) {
     this(blk, replication, BlockUCState.UNDER_CONSTRUCTION, null);
   }
 
   /**
    * Create a block that is currently being constructed.
    */
-  public BlockInfoUnderConstruction(Block blk, short replication,
-                             BlockUCState state,
-                             DatanodeStorageInfo[] targets) {
+  public BlockInfoContiguousUnderConstruction(Block blk, short replication, BlockUCState state, DatanodeStorageInfo[] targets) {
     super(blk, replication);
     assert getBlockUCState() != BlockUCState.COMPLETE :
       "BlockInfoUnderConstruction cannot be in COMPLETE state";
@@ -179,10 +182,10 @@ assert getBlockUCState() != BlockUCState.COMPLETE :
    * the client or it does not have at least a minimal number of replicas 
    * reported from data-nodes. 
    */
-  BlockInfo convertToCompleteBlock() throws IOException {
+  BlockInfoContiguous convertToCompleteBlock() throws IOException {
     assert getBlockUCState() != BlockUCState.COMPLETE :
       "Trying to convert a COMPLETE block";
-    return new BlockInfo(this);
+    return new BlockInfoContiguous(this);
   }
 
   /** Set expected locations */
@@ -229,6 +232,15 @@ public long getBlockRecoveryId() {
     return blockRecoveryId;
   }
 
+  /** Get recover block */
+  public Block getTruncateBlock() {
+    return truncateBlock;
+  }
+
+  public void setTruncateBlock(Block recoveryBlock) {
+    this.truncateBlock = recoveryBlock;
+  }
+
   /**
    * Process the recorded replicas. When about to commit or finish the
    * pipeline recovery sort out bad replicas.
@@ -246,7 +258,7 @@ public void setGenerationStampAndVerifyReplicas(long genStamp) {
       if (genStamp != r.getGenerationStamp()) {
         r.getExpectedStorageLocation().removeBlock(this);
         NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
-            + "from location: " + r.getExpectedStorageLocation());
+            + "from location: {}", r.getExpectedStorageLocation());
       }
     }
   }
@@ -313,8 +325,8 @@ public void initializeBlockRecovery(long recoveryId) {
     if (primary != null) {
       primary.getExpectedStorageLocation().getDatanodeDescriptor().addBlockToBeRecovered(this);
       primary.setChosenAsPrimary(true);
-      NameNode.blockStateChangeLog.info("BLOCK* " + this
-        + " recovery started, primary=" + primary);
+      NameNode.blockStateChangeLog.info(
+          "BLOCK* {} recovery started, primary={}", this, primary);
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 26766961423f4..8610b79b380d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -37,8 +37,6 @@
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -89,6 +87,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Keeps information related to the blocks stored in the Hadoop cluster.
@@ -96,8 +96,8 @@
 @InterfaceAudience.Private
 public class BlockManager {
 
-  static final Log LOG = LogFactory.getLog(BlockManager.class);
-  public static final Log blockLog = NameNode.blockStateChangeLog;
+  public static final Logger LOG = LoggerFactory.getLogger(BlockManager.class);
+  public static final Logger blockLog = NameNode.blockStateChangeLog;
 
   private static final String QUEUE_REASON_CORRUPT_STATE =
     "it has the wrong state or generation stamp";
@@ -535,8 +535,8 @@ private void dumpBlockMeta(Block block, PrintWriter out) {
     int usableReplicas = numReplicas.liveReplicas() +
                          numReplicas.decommissionedReplicas();
     
-    if (block instanceof BlockInfo) {
-      BlockCollection bc = ((BlockInfo) block).getBlockCollection();
+    if (block instanceof BlockInfoContiguous) {
+      BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection();
       String fileName = (bc == null) ? "[orphaned]" : bc.getName();
       out.print(fileName + ": ");
     }
@@ -590,8 +590,9 @@ public boolean checkMinReplication(Block block) {
    * @throws IOException if the block does not have at least a minimal number
    * of replicas reported from data-nodes.
    */
-  private static boolean commitBlock(final BlockInfoUnderConstruction block,
-      final Block commitBlock) throws IOException {
+  private static boolean commitBlock(
+      final BlockInfoContiguousUnderConstruction block, final Block commitBlock)
+      throws IOException {
     if (block.getBlockUCState() == BlockUCState.COMMITTED)
       return false;
     assert block.getNumBytes() <= commitBlock.getNumBytes() :
@@ -615,13 +616,14 @@ public boolean commitOrCompleteLastBlock(BlockCollection bc,
       Block commitBlock) throws IOException {
     if(commitBlock == null)
       return false; // not committing, this is a block allocation retry
-    BlockInfo lastBlock = bc.getLastBlock();
+    BlockInfoContiguous lastBlock = bc.getLastBlock();
     if(lastBlock == null)
       return false; // no blocks in file yet
     if(lastBlock.isComplete())
       return false; // already completed (e.g. by syncBlock)
     
-    final boolean b = commitBlock((BlockInfoUnderConstruction)lastBlock, commitBlock);
+    final boolean b = commitBlock(
+        (BlockInfoContiguousUnderConstruction) lastBlock, commitBlock);
     if(countNodes(lastBlock).liveReplicas() >= minReplication)
       completeBlock(bc, bc.numBlocks()-1, false);
     return b;
@@ -634,14 +636,15 @@ public boolean commitOrCompleteLastBlock(BlockCollection bc,
    * @throws IOException if the block does not have at least a minimal number
    * of replicas reported from data-nodes.
    */
-  private BlockInfo completeBlock(final BlockCollection bc,
+  private BlockInfoContiguous completeBlock(final BlockCollection bc,
       final int blkIndex, boolean force) throws IOException {
     if(blkIndex < 0)
       return null;
-    BlockInfo curBlock = bc.getBlocks()[blkIndex];
+    BlockInfoContiguous curBlock = bc.getBlocks()[blkIndex];
     if(curBlock.isComplete())
       return curBlock;
-    BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)curBlock;
+    BlockInfoContiguousUnderConstruction ucBlock =
+        (BlockInfoContiguousUnderConstruction) curBlock;
     int numNodes = ucBlock.numNodes();
     if (!force && numNodes < minReplication)
       throw new IOException("Cannot complete block: " +
@@ -649,7 +652,7 @@ private BlockInfo completeBlock(final BlockCollection bc,
     if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED)
       throw new IOException(
           "Cannot complete block: block has not been COMMITTED by the client");
-    BlockInfo completeBlock = ucBlock.convertToCompleteBlock();
+    BlockInfoContiguous completeBlock = ucBlock.convertToCompleteBlock();
     // replace penultimate block in file
     bc.setBlock(blkIndex, completeBlock);
     
@@ -667,9 +670,9 @@ private BlockInfo completeBlock(final BlockCollection bc,
     return blocksMap.replaceBlock(completeBlock);
   }
 
-  private BlockInfo completeBlock(final BlockCollection bc,
-      final BlockInfo block, boolean force) throws IOException {
-    BlockInfo[] fileBlocks = bc.getBlocks();
+  private BlockInfoContiguous completeBlock(final BlockCollection bc,
+      final BlockInfoContiguous block, boolean force) throws IOException {
+    BlockInfoContiguous[] fileBlocks = bc.getBlocks();
     for(int idx = 0; idx < fileBlocks.length; idx++)
       if(fileBlocks[idx] == block) {
         return completeBlock(bc, idx, force);
@@ -682,8 +685,8 @@ private BlockInfo completeBlock(final BlockCollection bc,
    * regardless of whether enough replicas are present. This is necessary
    * when tailing edit logs as a Standby.
    */
-  public BlockInfo forceCompleteBlock(final BlockCollection bc,
-      final BlockInfoUnderConstruction block) throws IOException {
+  public BlockInfoContiguous forceCompleteBlock(final BlockCollection bc,
+      final BlockInfoContiguousUnderConstruction block) throws IOException {
     block.commitBlock(block);
     return completeBlock(bc, block, true);
   }
@@ -700,20 +703,22 @@ public BlockInfo forceCompleteBlock(final BlockCollection bc,
    * The client is supposed to allocate a new block with the next call.
    *
    * @param bc file
+   * @param bytesToRemove num of bytes to remove from block
    * @return the last block locations if the block is partial or null otherwise
    */
   public LocatedBlock convertLastBlockToUnderConstruction(
-      BlockCollection bc) throws IOException {
-    BlockInfo oldBlock = bc.getLastBlock();
+      BlockCollection bc, long bytesToRemove) throws IOException {
+    BlockInfoContiguous oldBlock = bc.getLastBlock();
     if(oldBlock == null ||
-        bc.getPreferredBlockSize() == oldBlock.getNumBytes())
+       bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove)
       return null;
     assert oldBlock == getStoredBlock(oldBlock) :
       "last block of the file is not in blocksMap";
 
     DatanodeStorageInfo[] targets = getStorages(oldBlock);
 
-    BlockInfoUnderConstruction ucBlock = bc.setLastBlock(oldBlock, targets);
+    BlockInfoContiguousUnderConstruction ucBlock =
+        bc.setLastBlock(oldBlock, targets);
     blocksMap.replaceBlock(ucBlock);
 
     // Remove block from replication queue.
@@ -755,7 +760,8 @@ private List<DatanodeStorageInfo> getValidLocations(Block block) {
     return locations;
   }
   
-  private List<LocatedBlock> createLocatedBlockList(final BlockInfo[] blocks,
+  private List<LocatedBlock> createLocatedBlockList(
+      final BlockInfoContiguous[] blocks,
       final long offset, final long length, final int nrBlocksToReturn,
       final AccessMode mode) throws IOException {
     int curBlk = 0;
@@ -785,7 +791,7 @@ private List<LocatedBlock> createLocatedBlockList(final BlockInfo[] blocks,
     return results;
   }
 
-  private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
+  private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks,
       final long endPos, final AccessMode mode) throws IOException {
     int curBlk = 0;
     long curPos = 0;
@@ -801,7 +807,7 @@ private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
     return createLocatedBlock(blocks[curBlk], curPos, mode);
   }
   
-  private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
+  private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos,
     final BlockTokenSecretManager.AccessMode mode) throws IOException {
     final LocatedBlock lb = createLocatedBlock(blk, pos);
     if (mode != null) {
@@ -811,15 +817,16 @@ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
   }
 
   /** @return a LocatedBlock for the given block */
-  private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
+  private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos
       ) throws IOException {
-    if (blk instanceof BlockInfoUnderConstruction) {
+    if (blk instanceof BlockInfoContiguousUnderConstruction) {
       if (blk.isComplete()) {
         throw new IOException(
             "blk instanceof BlockInfoUnderConstruction && blk.isComplete()"
             + ", blk=" + blk);
       }
-      final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk;
+      final BlockInfoContiguousUnderConstruction uc =
+          (BlockInfoContiguousUnderConstruction) blk;
       final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations();
       final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk);
       return new LocatedBlock(eb, storages, pos, false);
@@ -858,7 +865,7 @@ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
   }
 
   /** Create a LocatedBlocks. */
-  public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
+  public LocatedBlocks createLocatedBlocks(final BlockInfoContiguous[] blocks,
       final long fileSizeExcludeBlocksUnderConstruction,
       final boolean isFileUnderConstruction, final long offset,
       final long length, final boolean needBlockToken,
@@ -881,7 +888,7 @@ public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
       final LocatedBlock lastlb;
       final boolean isComplete;
       if (!inSnapshot) {
-        final BlockInfo last = blocks[blocks.length - 1];
+        final BlockInfoContiguous last = blocks[blocks.length - 1];
         final long lastPos = last.isComplete()?
             fileSizeExcludeBlocksUnderConstruction - last.getNumBytes()
             : fileSizeExcludeBlocksUnderConstruction;
@@ -970,7 +977,7 @@ public void verifyReplication(String src,
   /**
    * Check if a block is replicated to at least the minimum replication.
    */
-  public boolean isSufficientlyReplicated(BlockInfo b) {
+  public boolean isSufficientlyReplicated(BlockInfoContiguous b) {
     // Compare against the lesser of the minReplication and number of live DNs.
     final int replication =
         Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes());
@@ -1001,8 +1008,8 @@ private BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
       final long size) throws UnregisteredNodeException {
     final DatanodeDescriptor node = getDatanodeManager().getDatanode(datanode);
     if (node == null) {
-      blockLog.warn("BLOCK* getBlocks: "
-          + "Asking for blocks from an unrecorded node " + datanode);
+      blockLog.warn("BLOCK* getBlocks: Asking for blocks from an" +
+          " unrecorded node {}", datanode);
       throw new HadoopIllegalArgumentException(
           "Datanode " + datanode + " not found.");
     }
@@ -1011,7 +1018,7 @@ private BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
     if(numBlocks == 0) {
       return new BlocksWithLocations(new BlockWithLocations[0]);
     }
-    Iterator<BlockInfo> iter = node.getBlockIterator();
+    Iterator<BlockInfoContiguous> iter = node.getBlockIterator();
     int startBlock = DFSUtil.getRandom().nextInt(numBlocks); // starting from a random block
     // skip blocks
     for(int i=0; i<startBlock; i++) {
@@ -1019,7 +1026,7 @@ private BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
     }
     List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
     long totalSize = 0;
-    BlockInfo curBlock;
+    BlockInfoContiguous curBlock;
     while(totalSize<size && iter.hasNext()) {
       curBlock = iter.next();
       if(!curBlock.isComplete())  continue;
@@ -1050,21 +1057,6 @@ void removeBlocksAssociatedTo(final DatanodeDescriptor node) {
 
     node.resetBlocks();
     invalidateBlocks.remove(node);
-    
-    // If the DN hasn't block-reported since the most recent
-    // failover, then we may have been holding up on processing
-    // over-replicated blocks because of it. But we can now
-    // process those blocks.
-    boolean stale = false;
-    for(DatanodeStorageInfo storage : node.getStorageInfos()) {
-      if (storage.areBlockContentsStale()) {
-        stale = true;
-        break;
-      }
-    }
-    if (stale) {
-      rescanPostponedMisreplicatedBlocks();
-    }
   }
 
   /** Remove the blocks associated to the given DatanodeStorageInfo. */
@@ -1106,8 +1098,7 @@ private void addToInvalidates(Block b) {
       datanodes.append(node).append(" ");
     }
     if (datanodes.length() != 0) {
-      blockLog.info("BLOCK* addToInvalidates: " + b + " "
-          + datanodes);
+      blockLog.info("BLOCK* addToInvalidates: {} {}", b, datanodes.toString());
     }
   }
 
@@ -1134,14 +1125,13 @@ void removeFromInvalidates(final DatanodeInfo datanode) {
   public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
       final DatanodeInfo dn, String storageID, String reason) throws IOException {
     assert namesystem.hasWriteLock();
-    final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock());
+    final BlockInfoContiguous storedBlock = getStoredBlock(blk.getLocalBlock());
     if (storedBlock == null) {
       // Check if the replica is in the blockMap, if not
       // ignore the request for now. This could happen when BlockScanner
       // thread of Datanode reports bad block before Block reports are sent
       // by the Datanode on startup
-      blockLog.info("BLOCK* findAndMarkBlockAsCorrupt: "
-          + blk + " not found");
+      blockLog.info("BLOCK* findAndMarkBlockAsCorrupt: {} not found", blk);
       return;
     }
 
@@ -1170,8 +1160,8 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
 
     BlockCollection bc = b.corrupted.getBlockCollection();
     if (bc == null) {
-      blockLog.info("BLOCK markBlockAsCorrupt: " + b
-          + " cannot be marked as corrupt as it does not belong to any file");
+      blockLog.info("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
+          " corrupt as it does not belong to any file", b);
       addToInvalidates(b.corrupted, node);
       return;
     } 
@@ -1218,7 +1208,7 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
    */
   private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
       ) throws IOException {
-    blockLog.info("BLOCK* invalidateBlock: " + b + " on " + dn);
+    blockLog.info("BLOCK* invalidateBlock: {} on {}", b, dn);
     DatanodeDescriptor node = getDatanodeManager().getDatanode(dn);
     if (node == null) {
       throw new IOException("Cannot invalidate " + b
@@ -1232,20 +1222,22 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
           "invalidation of " + b + " on " + dn + " because " +
           nr.replicasOnStaleNodes() + " replica(s) are located on nodes " +
           "with potentially out-of-date block reports");
+      blockLog.info("BLOCK* invalidateBlocks: postponing " +
+          "invalidation of {} on {} because {} replica(s) are located on " +
+          "nodes with potentially out-of-date block reports", b, dn,
+          nr.replicasOnStaleNodes());
       postponeBlock(b.corrupted);
       return false;
     } else if (nr.liveReplicas() >= 1) {
       // If we have at least one copy on a live node, then we can delete it.
       addToInvalidates(b.corrupted, dn);
       removeStoredBlock(b.stored, node);
-      if(blockLog.isDebugEnabled()) {
-        blockLog.debug("BLOCK* invalidateBlocks: "
-            + b + " on " + dn + " listed for deletion.");
-      }
+      blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.",
+          b, dn);
       return true;
     } else {
-      blockLog.info("BLOCK* invalidateBlocks: " + b
-          + " on " + dn + " is the only copy and was not deleted");
+      blockLog.info("BLOCK* invalidateBlocks: {} on {} is the only copy and" +
+          " was not deleted", b, dn);
       return false;
     }
   }
@@ -1377,8 +1369,8 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
               if ( (pendingReplications.getNumReplicas(block) > 0) ||
                    (blockHasEnoughRacks(block)) ) {
                 neededReplications.remove(block, priority); // remove from neededReplications
-                blockLog.info("BLOCK* Removing " + block
-                    + " from neededReplications as it has enough replicas");
+                blockLog.info("BLOCK* Removing {} from neededReplications as" +
+                        " it has enough replicas", block);
                 continue;
               }
             }
@@ -1447,8 +1439,8 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
                  (blockHasEnoughRacks(block)) ) {
               neededReplications.remove(block, priority); // remove from neededReplications
               rw.targets = null;
-              blockLog.info("BLOCK* Removing " + block
-                  + " from neededReplications as it has enough replicas");
+              blockLog.info("BLOCK* Removing {} from neededReplications as" +
+                      " it has enough replicas", block);
               continue;
             }
           }
@@ -1472,11 +1464,8 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
           // replications that fail after an appropriate amount of time.
           pendingReplications.increment(block,
               DatanodeStorageInfo.toDatanodeDescriptors(targets));
-          if(blockLog.isDebugEnabled()) {
-            blockLog.debug(
-                "BLOCK* block " + block
-                + " is moved from neededReplications to pendingReplications");
-          }
+          blockLog.debug("BLOCK* block {} is moved from neededReplications to "
+                  + "pendingReplications", block);
 
           // remove from neededReplications
           if(numEffectiveReplicas + targets.length >= requiredReplication) {
@@ -1498,16 +1487,13 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
             targetList.append(' ');
             targetList.append(targets[k].getDatanodeDescriptor());
           }
-          blockLog.info("BLOCK* ask " + rw.srcNode
-              + " to replicate " + rw.block + " to " + targetList);
+          blockLog.info("BLOCK* ask {} to replicate {} to {}", rw.srcNode,
+              rw.block, targetList);
         }
       }
     }
-    if(blockLog.isDebugEnabled()) {
-        blockLog.debug(
-          "BLOCK* neededReplications = " + neededReplications.size()
-          + " pendingReplications = " + pendingReplications.size());
-    }
+    blockLog.debug("BLOCK* neededReplications = {} pendingReplications = {}",
+        neededReplications.size(), pendingReplications.size());
 
     return scheduledWork;
   }
@@ -1716,11 +1702,11 @@ private void processPendingReplications() {
    * reported by the datanode in the block report. 
    */
   static class StatefulBlockInfo {
-    final BlockInfoUnderConstruction storedBlock;
+    final BlockInfoContiguousUnderConstruction storedBlock;
     final Block reportedBlock;
     final ReplicaState reportedState;
     
-    StatefulBlockInfo(BlockInfoUnderConstruction storedBlock, 
+    StatefulBlockInfo(BlockInfoContiguousUnderConstruction storedBlock,
         Block reportedBlock, ReplicaState reportedState) {
       this.storedBlock = storedBlock;
       this.reportedBlock = reportedBlock;
@@ -1734,15 +1720,16 @@ static class StatefulBlockInfo {
    */
   private static class BlockToMarkCorrupt {
     /** The corrupted block in a datanode. */
-    final BlockInfo corrupted;
+    final BlockInfoContiguous corrupted;
     /** The corresponding block stored in the BlockManager. */
-    final BlockInfo stored;
+    final BlockInfoContiguous stored;
     /** The reason to mark corrupt. */
     final String reason;
     /** The reason code to be stored */
     final Reason reasonCode;
 
-    BlockToMarkCorrupt(BlockInfo corrupted, BlockInfo stored, String reason,
+    BlockToMarkCorrupt(BlockInfoContiguous corrupted,
+        BlockInfoContiguous stored, String reason,
         Reason reasonCode) {
       Preconditions.checkNotNull(corrupted, "corrupted is null");
       Preconditions.checkNotNull(stored, "stored is null");
@@ -1753,13 +1740,14 @@ private static class BlockToMarkCorrupt {
       this.reasonCode = reasonCode;
     }
 
-    BlockToMarkCorrupt(BlockInfo stored, String reason, Reason reasonCode) {
+    BlockToMarkCorrupt(BlockInfoContiguous stored, String reason,
+        Reason reasonCode) {
       this(stored, stored, reason, reasonCode);
     }
 
-    BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
+    BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason,
         Reason reasonCode) {
-      this(new BlockInfo(stored), stored, reason, reasonCode);
+      this(new BlockInfoContiguous(stored), stored, reason, reasonCode);
       //the corrupted block in datanode has a different generation stamp
       corrupted.setGenerationStamp(gs);
     }
@@ -1785,6 +1773,8 @@ public boolean processReport(final DatanodeID nodeID,
     final long startTime = Time.now(); //after acquiring write lock
     final long endTime;
     DatanodeDescriptor node;
+    Collection<Block> invalidatedBlocks = null;
+
     try {
       node = datanodeManager.getDatanode(nodeID);
       if (node == null || !node.isAlive) {
@@ -1803,8 +1793,8 @@ public boolean processReport(final DatanodeID nodeID,
       if (namesystem.isInStartupSafeMode()
           && storageInfo.getBlockReportCount() > 0) {
         blockLog.info("BLOCK* processReport: "
-            + "discarded non-initial block report from " + nodeID
-            + " because namenode still in startup phase");
+            + "discarded non-initial block report from {}"
+            + " because namenode still in startup phase", nodeID);
         return !node.hasStaleStorages();
       }
 
@@ -1813,75 +1803,116 @@ public boolean processReport(final DatanodeID nodeID,
         // ordinary block reports.  This shortens restart times.
         processFirstBlockReport(storageInfo, newReport);
       } else {
-        processReport(storageInfo, newReport);
+        invalidatedBlocks = processReport(storageInfo, newReport);
       }
       
-      // Now that we have an up-to-date block report, we know that any
-      // deletions from a previous NN iteration have been accounted for.
-      boolean staleBefore = storageInfo.areBlockContentsStale();
       storageInfo.receivedBlockReport();
-      if (staleBefore && !storageInfo.areBlockContentsStale()) {
-        LOG.info("BLOCK* processReport: Received first block report from "
-            + storage + " after starting up or becoming active. Its block "
-            + "contents are no longer considered stale");
-        rescanPostponedMisreplicatedBlocks();
-      }
-      
     } finally {
       endTime = Time.now();
       namesystem.writeUnlock();
     }
 
+    if (invalidatedBlocks != null) {
+      for (Block b : invalidatedBlocks) {
+        blockLog.info("BLOCK* processReport: {} on node {} size {} does not " +
+            "belong to any file", b, node, b.getNumBytes());
+      }
+    }
+
     // Log the block report processing stats from Namenode perspective
     final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
     if (metrics != null) {
       metrics.addBlockReport((int) (endTime - startTime));
     }
-    blockLog.info("BLOCK* processReport: from storage " + storage.getStorageID()
-        + " node " + nodeID + ", blocks: " + newReport.getNumberOfBlocks()
-        + ", hasStaleStorages: " + node.hasStaleStorages()
-        + ", processing time: " + (endTime - startTime) + " msecs");
+    blockLog.info("BLOCK* processReport: from storage {} node {}, " +
+        "blocks: {}, hasStaleStorage: {}, processing time: {} msecs", storage
+        .getStorageID(), nodeID, newReport.getNumberOfBlocks(),
+        node.hasStaleStorages(), (endTime - startTime));
     return !node.hasStaleStorages();
   }
 
   /**
    * Rescan the list of blocks which were previously postponed.
    */
-  private void rescanPostponedMisreplicatedBlocks() {
-    for (Iterator<Block> it = postponedMisreplicatedBlocks.iterator();
-         it.hasNext();) {
-      Block b = it.next();
-      
-      BlockInfo bi = blocksMap.getStoredBlock(b);
-      if (bi == null) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
-              "Postponed mis-replicated block " + b + " no longer found " +
-              "in block map.");
+  void rescanPostponedMisreplicatedBlocks() {
+    if (getPostponedMisreplicatedBlocksCount() == 0) {
+      return;
+    }
+    long startTimeRescanPostponedMisReplicatedBlocks = Time.now();
+    long startPostponedMisReplicatedBlocksCount =
+        getPostponedMisreplicatedBlocksCount();
+    namesystem.writeLock();
+    try {
+      // blocksPerRescan is the configured number of blocks per rescan.
+      // Randomly select blocksPerRescan consecutive blocks from the HashSet
+      // when the number of blocks remaining is larger than blocksPerRescan.
+      // The reason we don't always pick the first blocksPerRescan blocks is to
+      // handle the case if for some reason some datanodes remain in
+      // content stale state for a long time and only impact the first
+      // blocksPerRescan blocks.
+      int i = 0;
+      long startIndex = 0;
+      long blocksPerRescan =
+          datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan();
+      long base = getPostponedMisreplicatedBlocksCount() - blocksPerRescan;
+      if (base > 0) {
+        startIndex = DFSUtil.getRandom().nextLong() % (base+1);
+        if (startIndex < 0) {
+          startIndex += (base+1);
         }
-        it.remove();
-        postponedMisreplicatedBlocksCount.decrementAndGet();
-        continue;
       }
-      MisReplicationResult res = processMisReplicatedBlock(bi);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
-            "Re-scanned block " + b + ", result is " + res);
+      Iterator<Block> it = postponedMisreplicatedBlocks.iterator();
+      for (int tmp = 0; tmp < startIndex; tmp++) {
+        it.next();
       }
-      if (res != MisReplicationResult.POSTPONE) {
-        it.remove();
-        postponedMisreplicatedBlocksCount.decrementAndGet();
+
+      for (;it.hasNext(); i++) {
+        Block b = it.next();
+        if (i >= blocksPerRescan) {
+          break;
+        }
+
+        BlockInfoContiguous bi = getStoredBlock(b);
+        if (bi == null) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
+                "Postponed mis-replicated block " + b + " no longer found " +
+                "in block map.");
+          }
+          it.remove();
+          postponedMisreplicatedBlocksCount.decrementAndGet();
+          continue;
+        }
+        MisReplicationResult res = processMisReplicatedBlock(bi);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
+              "Re-scanned block " + b + ", result is " + res);
+        }
+        if (res != MisReplicationResult.POSTPONE) {
+          it.remove();
+          postponedMisreplicatedBlocksCount.decrementAndGet();
+        }
       }
+    } finally {
+      namesystem.writeUnlock();
+      long endPostponedMisReplicatedBlocksCount =
+          getPostponedMisreplicatedBlocksCount();
+      LOG.info("Rescan of postponedMisreplicatedBlocks completed in " +
+          (Time.now() - startTimeRescanPostponedMisReplicatedBlocks) +
+          " msecs. " + endPostponedMisReplicatedBlocksCount +
+          " blocks are left. " + (startPostponedMisReplicatedBlocksCount -
+          endPostponedMisReplicatedBlocksCount) + " blocks are removed.");
     }
   }
   
-  private void processReport(final DatanodeStorageInfo storageInfo,
-                             final BlockListAsLongs report) throws IOException {
+  private Collection<Block> processReport(
+      final DatanodeStorageInfo storageInfo,
+      final BlockListAsLongs report) throws IOException {
     // Normal case:
     // Modify the (block-->datanode) map, according to the difference
     // between the old and new block report.
     //
-    Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
+    Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
     Collection<Block> toRemove = new TreeSet<Block>();
     Collection<Block> toInvalidate = new LinkedList<Block>();
     Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
@@ -1898,23 +1929,22 @@ private void processReport(final DatanodeStorageInfo storageInfo,
       removeStoredBlock(b, node);
     }
     int numBlocksLogged = 0;
-    for (BlockInfo b : toAdd) {
+    for (BlockInfoContiguous b : toAdd) {
       addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog);
       numBlocksLogged++;
     }
     if (numBlocksLogged > maxNumBlocksToLog) {
-      blockLog.info("BLOCK* processReport: logged info for " + maxNumBlocksToLog
-          + " of " + numBlocksLogged + " reported.");
+      blockLog.info("BLOCK* processReport: logged info for {} of {} " +
+          "reported.", maxNumBlocksToLog, numBlocksLogged);
     }
     for (Block b : toInvalidate) {
-      blockLog.info("BLOCK* processReport: "
-          + b + " on " + node + " size " + b.getNumBytes()
-          + " does not belong to any file");
       addToInvalidates(b, node);
     }
     for (BlockToMarkCorrupt b : toCorrupt) {
       markBlockAsCorrupt(b, storageInfo, node);
     }
+
+    return toInvalidate;
   }
 
   /**
@@ -1947,7 +1977,7 @@ private void processFirstBlockReport(
         continue;
       }
       
-      BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
+      BlockInfoContiguous storedBlock = getStoredBlock(iblk);
       // If block does not belong to any file, we are done.
       if (storedBlock == null) continue;
       
@@ -1970,12 +2000,13 @@ private void processFirstBlockReport(
       
       // If block is under construction, add this replica to its list
       if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
-        ((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent(
-            storageInfo, iblk, reportedState);
+        ((BlockInfoContiguousUnderConstruction)storedBlock)
+            .addReplicaIfNotPresent(storageInfo, iblk, reportedState);
         // OpenFileBlocks only inside snapshots also will be added to safemode
         // threshold. So we need to update such blocks to safemode
         // refer HDFS-5283
-        BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock;
+        BlockInfoContiguousUnderConstruction blockUC =
+            (BlockInfoContiguousUnderConstruction) storedBlock;
         if (namesystem.isInSnapshot(blockUC)) {
           int numOfReplicas = blockUC.getNumExpectedLocations();
           namesystem.incrementSafeBlockCount(numOfReplicas);
@@ -1991,7 +2022,7 @@ private void processFirstBlockReport(
 
   private void reportDiff(DatanodeStorageInfo storageInfo, 
       BlockListAsLongs newReport, 
-      Collection<BlockInfo> toAdd,              // add to DatanodeDescriptor
+      Collection<BlockInfoContiguous> toAdd,              // add to DatanodeDescriptor
       Collection<Block> toRemove,           // remove from DatanodeDescriptor
       Collection<Block> toInvalidate,       // should be removed from DN
       Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
@@ -1999,7 +2030,7 @@ private void reportDiff(DatanodeStorageInfo storageInfo,
 
     // place a delimiter in the list which separates blocks 
     // that have been reported from those that have not
-    BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
+    BlockInfoContiguous delimiter = new BlockInfoContiguous(new Block(), (short) 1);
     AddBlockResult result = storageInfo.addBlock(delimiter);
     assert result == AddBlockResult.ADDED 
         : "Delimiting block cannot be present in the node";
@@ -2014,7 +2045,7 @@ private void reportDiff(DatanodeStorageInfo storageInfo,
     while(itBR.hasNext()) {
       Block iblk = itBR.next();
       ReplicaState iState = itBR.getCurrentReplicaState();
-      BlockInfo storedBlock = processReportedBlock(storageInfo,
+      BlockInfoContiguous storedBlock = processReportedBlock(storageInfo,
           iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
 
       // move block to the head of the list
@@ -2026,7 +2057,8 @@ private void reportDiff(DatanodeStorageInfo storageInfo,
 
     // collect blocks that have not been reported
     // all of them are next to the delimiter
-    Iterator<BlockInfo> it = storageInfo.new BlockIterator(delimiter.getNext(0));
+    Iterator<BlockInfoContiguous> it =
+        storageInfo.new BlockIterator(delimiter.getNext(0));
     while(it.hasNext())
       toRemove.add(it.next());
     storageInfo.removeBlock(delimiter);
@@ -2063,10 +2095,10 @@ private void reportDiff(DatanodeStorageInfo storageInfo,
    * @return the up-to-date stored block, if it should be kept.
    *         Otherwise, null.
    */
-  private BlockInfo processReportedBlock(
+  private BlockInfoContiguous processReportedBlock(
       final DatanodeStorageInfo storageInfo,
       final Block block, final ReplicaState reportedState, 
-      final Collection<BlockInfo> toAdd, 
+      final Collection<BlockInfoContiguous> toAdd,
       final Collection<Block> toInvalidate, 
       final Collection<BlockToMarkCorrupt> toCorrupt,
       final Collection<StatefulBlockInfo> toUC) {
@@ -2087,7 +2119,7 @@ private BlockInfo processReportedBlock(
     }
     
     // find block by blockId
-    BlockInfo storedBlock = blocksMap.getStoredBlock(block);
+    BlockInfoContiguous storedBlock = getStoredBlock(block);
     if(storedBlock == null) {
       // If blocksMap does not contain reported block id,
       // the replica should be removed from the data-node.
@@ -2130,7 +2162,8 @@ private BlockInfo processReportedBlock(
     }
 
     if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
-      toUC.add(new StatefulBlockInfo((BlockInfoUnderConstruction) storedBlock,
+      toUC.add(new StatefulBlockInfo(
+          (BlockInfoContiguousUnderConstruction) storedBlock,
           new Block(block), reportedState));
       return storedBlock;
     }
@@ -2218,7 +2251,7 @@ public void processAllPendingDNMessages() throws IOException {
    */
   private BlockToMarkCorrupt checkReplicaCorrupt(
       Block reported, ReplicaState reportedState, 
-      BlockInfo storedBlock, BlockUCState ucState, 
+      BlockInfoContiguous storedBlock, BlockUCState ucState,
       DatanodeDescriptor dn) {
     switch(reportedState) {
     case FINALIZED:
@@ -2291,7 +2324,7 @@ private BlockToMarkCorrupt checkReplicaCorrupt(
     }
   }
 
-  private boolean isBlockUnderConstruction(BlockInfo storedBlock, 
+  private boolean isBlockUnderConstruction(BlockInfoContiguous storedBlock,
       BlockUCState ucState, ReplicaState reportedState) {
     switch(reportedState) {
     case FINALIZED:
@@ -2314,7 +2347,7 @@ private boolean isBlockUnderConstruction(BlockInfo storedBlock,
 
   void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
       DatanodeStorageInfo storageInfo) throws IOException {
-    BlockInfoUnderConstruction block = ucBlock.storedBlock;
+    BlockInfoContiguousUnderConstruction block = ucBlock.storedBlock;
     block.addReplicaIfNotPresent(
         storageInfo, ucBlock.reportedBlock, ucBlock.reportedState);
 
@@ -2325,9 +2358,8 @@ void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
   } 
 
   /**
-   * Faster version of
-   * {@link #addStoredBlock(BlockInfo, DatanodeStorageInfo, DatanodeDescriptor, boolean)}
-   * , intended for use with initial block report at startup. If not in startup
+   * Faster version of {@link #addStoredBlock},
+   * intended for use with initial block report at startup. If not in startup
    * safe mode, will call standard addStoredBlock(). Assumes this method is
    * called "immediately" so there is no need to refresh the storedBlock from
    * blocksMap. Doesn't handle underReplication/overReplication, or worry about
@@ -2336,7 +2368,7 @@ void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
    * 
    * @throws IOException
    */
-  private void addStoredBlockImmediate(BlockInfo storedBlock,
+  private void addStoredBlockImmediate(BlockInfoContiguous storedBlock,
       DatanodeStorageInfo storageInfo)
   throws IOException {
     assert (storedBlock != null && namesystem.hasWriteLock());
@@ -2368,25 +2400,25 @@ private void addStoredBlockImmediate(BlockInfo storedBlock,
    * needed replications if this takes care of the problem.
    * @return the block that is stored in blockMap.
    */
-  private Block addStoredBlock(final BlockInfo block,
+  private Block addStoredBlock(final BlockInfoContiguous block,
                                DatanodeStorageInfo storageInfo,
                                DatanodeDescriptor delNodeHint,
                                boolean logEveryBlock)
   throws IOException {
     assert block != null && namesystem.hasWriteLock();
-    BlockInfo storedBlock;
+    BlockInfoContiguous storedBlock;
     DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
-    if (block instanceof BlockInfoUnderConstruction) {
+    if (block instanceof BlockInfoContiguousUnderConstruction) {
       //refresh our copy in case the block got completed in another thread
-      storedBlock = blocksMap.getStoredBlock(block);
+      storedBlock = getStoredBlock(block);
     } else {
       storedBlock = block;
     }
     if (storedBlock == null || storedBlock.getBlockCollection() == null) {
       // If this block does not belong to anyfile, then we are done.
-      blockLog.info("BLOCK* addStoredBlock: " + block + " on "
-          + node + " size " + block.getNumBytes()
-          + " but it does not belong to any file");
+      blockLog.info("BLOCK* addStoredBlock: {} on {} size {} but it does not" +
+          " belong to any file", block, node, block.getNumBytes());
+
       // we could add this block to invalidate set of this datanode.
       // it will happen in next block report otherwise.
       return block;
@@ -2408,6 +2440,8 @@ private Block addStoredBlock(final BlockInfo block,
       blockLog.warn("BLOCK* addStoredBlock: " + "block " + storedBlock
           + " moved to storageType " + storageInfo.getStorageType()
           + " on node " + node);
+      blockLog.warn("BLOCK* addStoredBlock: block {} moved to storageType " +
+          "{} on node {}", storedBlock, storageInfo.getStorageType(), node);
     } else {
       // if the same block is added again and the replica was corrupt
       // previously because of a wrong gen stamp, remove it from the
@@ -2415,9 +2449,9 @@ private Block addStoredBlock(final BlockInfo block,
       corruptReplicas.removeFromCorruptReplicasMap(block, node,
           Reason.GENSTAMP_MISMATCH);
       curReplicaDelta = 0;
-      blockLog.warn("BLOCK* addStoredBlock: "
-          + "Redundant addStoredBlock request received for " + storedBlock
-          + " on " + node + " size " + storedBlock.getNumBytes());
+      blockLog.warn("BLOCK* addStoredBlock: Redundant addStoredBlock request"
+              + " received for {} on node {} size {}", storedBlock, node,
+          storedBlock.getNumBytes());
     }
 
     // Now check for completion of blocks and safe block count
@@ -2473,7 +2507,8 @@ private Block addStoredBlock(final BlockInfo block,
     return storedBlock;
   }
 
-  private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
+  private void logAddStoredBlock(BlockInfoContiguous storedBlock,
+      DatanodeDescriptor node) {
     if (!blockLog.isInfoEnabled()) {
       return;
     }
@@ -2485,7 +2520,7 @@ private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
     storedBlock.appendStringTo(sb);
     sb.append(" size " )
       .append(storedBlock.getNumBytes());
-    blockLog.info(sb);
+    blockLog.info(sb.toString());
   }
   /**
    * Invalidate corrupt replicas.
@@ -2500,7 +2535,7 @@ private void logAddStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
    *
    * @param blk Block whose corrupt replicas need to be invalidated
    */
-  private void invalidateCorruptReplicas(BlockInfo blk) {
+  private void invalidateCorruptReplicas(BlockInfoContiguous blk) {
     Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk);
     boolean removedFromBlocksMap = true;
     if (nodes == null)
@@ -2515,8 +2550,8 @@ private void invalidateCorruptReplicas(BlockInfo blk) {
           removedFromBlocksMap = false;
         }
       } catch (IOException e) {
-        blockLog.info("invalidateCorruptReplicas "
-            + "error in deleting bad block " + blk + " on " + node, e);
+        blockLog.info("invalidateCorruptReplicas error in deleting bad block"
+            + " {} on {}", blk, node, e);
         removedFromBlocksMap = false;
       }
     }
@@ -2579,16 +2614,19 @@ private void processMisReplicatesAsync() throws InterruptedException {
     long nrInvalid = 0, nrOverReplicated = 0;
     long nrUnderReplicated = 0, nrPostponed = 0, nrUnderConstruction = 0;
     long startTimeMisReplicatedScan = Time.now();
-    Iterator<BlockInfo> blocksItr = blocksMap.getBlocks().iterator();
+    Iterator<BlockInfoContiguous> blocksItr = blocksMap.getBlocks().iterator();
     long totalBlocks = blocksMap.size();
     replicationQueuesInitProgress = 0;
     long totalProcessed = 0;
+    long sleepDuration =
+        Math.max(1, Math.min(numBlocksPerIteration/1000, 10000));
+
     while (namesystem.isRunning() && !Thread.currentThread().isInterrupted()) {
       int processed = 0;
       namesystem.writeLockInterruptibly();
       try {
         while (processed < numBlocksPerIteration && blocksItr.hasNext()) {
-          BlockInfo block = blocksItr.next();
+          BlockInfoContiguous block = blocksItr.next();
           MisReplicationResult res = processMisReplicatedBlock(block);
           if (LOG.isTraceEnabled()) {
             LOG.trace("block " + block + ": " + res);
@@ -2639,6 +2677,8 @@ private void processMisReplicatesAsync() throws InterruptedException {
         }
       } finally {
         namesystem.writeUnlock();
+        // Make sure it is out of the write lock for sufficiently long time.
+        Thread.sleep(sleepDuration);
       }
     }
     if (Thread.currentThread().isInterrupted()) {
@@ -2660,7 +2700,7 @@ public double getReplicationQueuesInitProgress() {
    * appropriate queues if necessary, and returns a result code indicating
    * what happened with it.
    */
-  private MisReplicationResult processMisReplicatedBlock(BlockInfo block) {
+  private MisReplicationResult processMisReplicatedBlock(BlockInfoContiguous block) {
     BlockCollection bc = block.getBlockCollection();
     if (bc == null) {
       // block does not belong to any file
@@ -2843,7 +2883,7 @@ private void chooseExcessReplicates(final Collection<DatanodeStorageInfo> nonExc
       //
       addToInvalidates(b, cur.getDatanodeDescriptor());
       blockLog.info("BLOCK* chooseExcessReplicates: "
-                +"("+cur+", "+b+") is added to invalidated blocks set");
+                +"({}, {}) is added to invalidated blocks set", cur, b);
     }
   }
 
@@ -2877,11 +2917,8 @@ private void addToExcessReplicate(DatanodeInfo dn, Block block) {
     }
     if (excessBlocks.add(block)) {
       excessBlocksCount.incrementAndGet();
-      if(blockLog.isDebugEnabled()) {
-        blockLog.debug("BLOCK* addToExcessReplicate:"
-            + " (" + dn + ", " + block
-            + ") is added to excessReplicateMap");
-      }
+      blockLog.debug("BLOCK* addToExcessReplicate: ({}, {}) is added to"
+          + " excessReplicateMap", dn, block);
     }
   }
 
@@ -2890,17 +2927,12 @@ private void addToExcessReplicate(DatanodeInfo dn, Block block) {
    * removed block is still valid.
    */
   public void removeStoredBlock(Block block, DatanodeDescriptor node) {
-    if(blockLog.isDebugEnabled()) {
-      blockLog.debug("BLOCK* removeStoredBlock: "
-          + block + " from " + node);
-    }
+    blockLog.debug("BLOCK* removeStoredBlock: {} from {}", block, node);
     assert (namesystem.hasWriteLock());
     {
       if (!blocksMap.removeNode(block, node)) {
-        if(blockLog.isDebugEnabled()) {
-          blockLog.debug("BLOCK* removeStoredBlock: "
-              + block + " has already been removed from node " + node);
-        }
+        blockLog.debug("BLOCK* removeStoredBlock: {} has already been" +
+            " removed from node {}", block, node);
         return;
       }
 
@@ -2925,10 +2957,8 @@ public void removeStoredBlock(Block block, DatanodeDescriptor node) {
       if (excessBlocks != null) {
         if (excessBlocks.remove(block)) {
           excessBlocksCount.decrementAndGet();
-          if(blockLog.isDebugEnabled()) {
-            blockLog.debug("BLOCK* removeStoredBlock: "
-                + block + " is removed from excessBlocks");
-          }
+          blockLog.debug("BLOCK* removeStoredBlock: {} is removed from " +
+              "excessBlocks", block);
           if (excessBlocks.size() == 0) {
             excessReplicateMap.remove(node.getDatanodeUuid());
           }
@@ -2981,8 +3011,8 @@ void addBlock(DatanodeStorageInfo storageInfo, Block block, String delHint)
     if (delHint != null && delHint.length() != 0) {
       delHintNode = datanodeManager.getDatanode(delHint);
       if (delHintNode == null) {
-        blockLog.warn("BLOCK* blockReceived: " + block
-            + " is expected to be removed from an unrecorded node " + delHint);
+        blockLog.warn("BLOCK* blockReceived: {} is expected to be removed " +
+            "from an unrecorded node {}", block, delHint);
       }
     }
 
@@ -2999,7 +3029,7 @@ private void processAndHandleReportedBlock(
       ReplicaState reportedState, DatanodeDescriptor delHintNode)
       throws IOException {
     // blockReceived reports a finalized block
-    Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
+    Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
     Collection<Block> toInvalidate = new LinkedList<Block>();
     Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
     Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
@@ -3016,18 +3046,17 @@ private void processAndHandleReportedBlock(
       addStoredBlockUnderConstruction(b, storageInfo);
     }
     long numBlocksLogged = 0;
-    for (BlockInfo b : toAdd) {
+    for (BlockInfoContiguous b : toAdd) {
       addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog);
       numBlocksLogged++;
     }
     if (numBlocksLogged > maxNumBlocksToLog) {
-      blockLog.info("BLOCK* addBlock: logged info for " + maxNumBlocksToLog
-          + " of " + numBlocksLogged + " reported.");
+      blockLog.info("BLOCK* addBlock: logged info for {} of {} reported.",
+          maxNumBlocksToLog, numBlocksLogged);
     }
     for (Block b : toInvalidate) {
-      blockLog.info("BLOCK* addBlock: block "
-          + b + " on " + node + " size " + b.getNumBytes()
-          + " does not belong to any file");
+      blockLog.info("BLOCK* addBlock: block {} on node {} size {} does not " +
+          "belong to any file", b, node, b.getNumBytes());
       addToInvalidates(b, node);
     }
     for (BlockToMarkCorrupt b : toCorrupt) {
@@ -3050,10 +3079,8 @@ public void processIncrementalBlockReport(final DatanodeID nodeID,
     int receiving = 0;
     final DatanodeDescriptor node = datanodeManager.getDatanode(nodeID);
     if (node == null || !node.isAlive) {
-      blockLog
-          .warn("BLOCK* processIncrementalBlockReport"
-              + " is received from dead or unregistered node "
-              + nodeID);
+      blockLog.warn("BLOCK* processIncrementalBlockReport"
+              + " is received from dead or unregistered node {}", nodeID);
       throw new IOException(
           "Got incremental block report from unregistered or dead node");
     }
@@ -3092,17 +3119,12 @@ public void processIncrementalBlockReport(final DatanodeID nodeID,
         assert false : msg; // if assertions are enabled, throw.
         break;
       }
-      if (blockLog.isDebugEnabled()) {
-        blockLog.debug("BLOCK* block "
-            + (rdbi.getStatus()) + ": " + rdbi.getBlock()
-            + " is received from " + nodeID);
-      }
-    }
-    if (blockLog.isDebugEnabled()) {
-      blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: " + "from "
-        + nodeID + " receiving: " + receiving + ", " + " received: " + received
-        + ", " + " deleted: " + deleted);
+      blockLog.debug("BLOCK* block {}: {} is received from {}",
+          rdbi.getStatus(), rdbi.getBlock(), nodeID);
     }
+    blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from "
+            + "{} receiving: {}, received: {}, deleted: {}", nodeID, receiving,
+        received, deleted);
   }
 
   /**
@@ -3148,7 +3170,7 @@ public NumberReplicas countNodes(Block b) {
    * @param b - the block being tested
    * @return count of live nodes for this block
    */
-  int countLiveNodes(BlockInfo b) {
+  int countLiveNodes(BlockInfoContiguous b) {
     if (!namesystem.isInStartupSafeMode()) {
       return countNodes(b).liveReplicas();
     }
@@ -3303,7 +3325,7 @@ public int getActiveBlockCount() {
     return blocksMap.size();
   }
 
-  public DatanodeStorageInfo[] getStorages(BlockInfo block) {
+  public DatanodeStorageInfo[] getStorages(BlockInfoContiguous block) {
     final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()];
     int i = 0;
     for(DatanodeStorageInfo s : blocksMap.getStorages(block)) {
@@ -3333,8 +3355,16 @@ public void removeBlock(Block block) {
     }
   }
 
-  public BlockInfo getStoredBlock(Block block) {
-    return blocksMap.getStoredBlock(block);
+  public BlockInfoContiguous getStoredBlock(Block block) {
+    BlockInfoContiguous info = null;
+    if (BlockIdManager.isStripedBlockID(block.getBlockId())) {
+      info = blocksMap.getStoredBlock(
+          new Block(BlockIdManager.convertToGroupID(block.getBlockId())));
+    }
+    if (info == null) {
+      info = blocksMap.getStoredBlock(block);
+    }
+    return info;
   }
 
   /** updates a block in under replication queue */
@@ -3426,10 +3456,8 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) {
     } finally {
       namesystem.writeUnlock();
     }
-    if (blockLog.isInfoEnabled()) {
-      blockLog.info("BLOCK* " + getClass().getSimpleName()
-          + ": ask " + dn + " to delete " + toInvalidate);
-    }
+    blockLog.info("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(),
+        dn, toInvalidate);
     return toInvalidate.size();
   }
 
@@ -3483,7 +3511,8 @@ public long getMissingReplOneBlocksCount() {
     return this.neededReplications.getCorruptReplOneBlockSize();
   }
 
-  public BlockInfo addBlockCollection(BlockInfo block, BlockCollection bc) {
+  public BlockInfoContiguous addBlockCollection(BlockInfoContiguous block,
+      BlockCollection bc) {
     return blocksMap.addBlockCollection(block, bc);
   }
 
@@ -3570,6 +3599,7 @@ public void run() {
           if (namesystem.isPopulatingReplQueues()) {
             computeDatanodeWork();
             processPendingReplications();
+            rescanPostponedMisreplicatedBlocks();
           }
           Thread.sleep(replicationRecheckInterval);
         } catch (Throwable t) {
@@ -3584,7 +3614,8 @@ public void run() {
             LOG.info("Stopping ReplicationMonitor for testing.");
             break;
           }
-          LOG.fatal("ReplicationMonitor thread received Runtime exception. ", t);
+          LOG.error("ReplicationMonitor thread received Runtime exception. ",
+              t);
           terminate(1, t);
         }
       }
@@ -3638,6 +3669,8 @@ public void clearQueues() {
     excessReplicateMap.clear();
     invalidateBlocks.clear();
     datanodeManager.clearPendingQueues();
+    postponedMisreplicatedBlocks.clear();
+    postponedMisreplicatedBlocksCount.set(0);
   };
   
 
@@ -3688,7 +3721,7 @@ private void chooseTargets(BlockPlacementPolicy blockplacement,
 
   /**
    * A simple result enum for the result of
-   * {@link BlockManager#processMisReplicatedBlock(BlockInfo)}.
+   * {@link BlockManager#processMisReplicatedBlock(BlockInfoContiguous)}.
    */
   enum MisReplicationResult {
     /** The block should be invalidated since it belongs to a deleted file. */
@@ -3709,4 +3742,9 @@ public void shutdown() {
     stopReplicationInitializer();
     blocksMap.close();
   }
+  
+  public void clear() {
+    clearQueues();
+    blocksMap.clear();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index ce87b06fc7298..c81dc5b135369 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -77,6 +77,11 @@ public static BlockStoragePolicySuite createDefaultSuite() {
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
         new StorageType[]{StorageType.DISK, StorageType.ARCHIVE});
+    final byte ecId = HdfsConstants.EC_STORAGE_POLICY_ID;
+    policies[ecId] = new BlockStoragePolicy(ecId,
+        HdfsConstants.EC_STORAGE_POLICY_NAME,
+        new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
+        new StorageType[]{StorageType.ARCHIVE});
     final byte coldId = HdfsConstants.COLD_STORAGE_POLICY_ID;
     policies[coldId] = new BlockStoragePolicy(coldId,
         HdfsConstants.COLD_STORAGE_POLICY_NAME,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 6664034abe37a..806a4cbb92bc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -36,10 +36,10 @@
  */
 class BlocksMap {
   private static class StorageIterator implements Iterator<DatanodeStorageInfo> {
-    private final BlockInfo blockInfo;
+    private final BlockInfoContiguous blockInfo;
     private int nextIdx = 0;
       
-    StorageIterator(BlockInfo blkInfo) {
+    StorageIterator(BlockInfoContiguous blkInfo) {
       this.blockInfo = blkInfo;
     }
 
@@ -63,14 +63,14 @@ public void remove()  {
   /** Constant {@link LightWeightGSet} capacity. */
   private final int capacity;
   
-  private GSet<Block, BlockInfo> blocks;
+  private GSet<Block, BlockInfoContiguous> blocks;
 
   BlocksMap(int capacity) {
     // Use 2% of total memory to size the GSet capacity
     this.capacity = capacity;
-    this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity) {
+    this.blocks = new LightWeightGSet<Block, BlockInfoContiguous>(capacity) {
       @Override
-      public Iterator<BlockInfo> iterator() {
+      public Iterator<BlockInfoContiguous> iterator() {
         SetIterator iterator = new SetIterator();
         /*
          * Not tracking any modifications to set. As this set will be used
@@ -86,22 +86,26 @@ public Iterator<BlockInfo> iterator() {
 
 
   void close() {
+    clear();
+    blocks = null;
+  }
+  
+  void clear() {
     if (blocks != null) {
       blocks.clear();
-      blocks = null;
     }
   }
 
   BlockCollection getBlockCollection(Block b) {
-    BlockInfo info = blocks.get(b);
+    BlockInfoContiguous info = blocks.get(b);
     return (info != null) ? info.getBlockCollection() : null;
   }
 
   /**
    * Add block b belonging to the specified block collection to the map.
    */
-  BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) {
-    BlockInfo info = blocks.get(b);
+  BlockInfoContiguous addBlockCollection(BlockInfoContiguous b, BlockCollection bc) {
+    BlockInfoContiguous info = blocks.get(b);
     if (info != b) {
       info = b;
       blocks.put(info);
@@ -116,7 +120,7 @@ BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) {
    * and remove all data-node locations associated with the block.
    */
   void removeBlock(Block block) {
-    BlockInfo blockInfo = blocks.remove(block);
+    BlockInfoContiguous blockInfo = blocks.remove(block);
     if (blockInfo == null)
       return;
 
@@ -128,7 +132,7 @@ void removeBlock(Block block) {
   }
   
   /** Returns the block object it it exists in the map. */
-  BlockInfo getStoredBlock(Block b) {
+  BlockInfoContiguous getStoredBlock(Block b) {
     return blocks.get(b);
   }
 
@@ -160,7 +164,7 @@ public boolean apply(DatanodeStorageInfo storage) {
    * For a block that has already been retrieved from the BlocksMap
    * returns {@link Iterable} of the storages the block belongs to.
    */
-  Iterable<DatanodeStorageInfo> getStorages(final BlockInfo storedBlock) {
+  Iterable<DatanodeStorageInfo> getStorages(final BlockInfoContiguous storedBlock) {
     return new Iterable<DatanodeStorageInfo>() {
       @Override
       public Iterator<DatanodeStorageInfo> iterator() {
@@ -171,7 +175,7 @@ public Iterator<DatanodeStorageInfo> iterator() {
 
   /** counts number of containing nodes. Better than using iterator. */
   int numNodes(Block b) {
-    BlockInfo info = blocks.get(b);
+    BlockInfoContiguous info = blocks.get(b);
     return info == null ? 0 : info.numNodes();
   }
 
@@ -181,7 +185,7 @@ int numNodes(Block b) {
    * only if it does not belong to any file and data-nodes.
    */
   boolean removeNode(Block b, DatanodeDescriptor node) {
-    BlockInfo info = blocks.get(b);
+    BlockInfoContiguous info = blocks.get(b);
     if (info == null)
       return false;
 
@@ -199,7 +203,7 @@ int size() {
     return blocks.size();
   }
 
-  Iterable<BlockInfo> getBlocks() {
+  Iterable<BlockInfoContiguous> getBlocks() {
     return blocks;
   }
   
@@ -214,8 +218,8 @@ int getCapacity() {
    * @param newBlock - block for replacement
    * @return new block
    */
-  BlockInfo replaceBlock(BlockInfo newBlock) {
-    BlockInfo currentBlock = blocks.get(newBlock);
+  BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) {
+    BlockInfoContiguous currentBlock = blocks.get(newBlock);
     assert currentBlock != null : "the block if not in blocksMap";
     // replace block in data-node lists
     for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index a0f35039e47d4..bf5ece9bc1618 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -369,7 +369,7 @@ private void rescanCacheDirectives() {
    * @param file The file.
    */
   private void rescanFile(CacheDirective directive, INodeFile file) {
-    BlockInfo[] blockInfos = file.getBlocks();
+    BlockInfoContiguous[] blockInfos = file.getBlocks();
 
     // Increment the "needed" statistics
     directive.addFilesNeeded(1);
@@ -394,7 +394,7 @@ private void rescanFile(CacheDirective directive, INodeFile file) {
     }
 
     long cachedTotal = 0;
-    for (BlockInfo blockInfo : blockInfos) {
+    for (BlockInfoContiguous blockInfo : blockInfos) {
       if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) {
         // We don't try to cache blocks that are under construction.
         LOG.trace("Directive {}: can't cache block {} because it is in state "
@@ -453,7 +453,7 @@ private void rescanFile(CacheDirective directive, INodeFile file) {
   }
 
   private String findReasonForNotCaching(CachedBlock cblock, 
-          BlockInfo blockInfo) {
+          BlockInfoContiguous blockInfo) {
     if (blockInfo == null) {
       // Somehow, a cache report with the block arrived, but the block
       // reports from the DataNode haven't (yet?) described such a block.
@@ -513,7 +513,7 @@ private void rescanCachedBlockMap() {
           iter.remove();
         }
       }
-      BlockInfo blockInfo = blockManager.
+      BlockInfoContiguous blockInfo = blockManager.
             getStoredBlock(new Block(cblock.getBlockId()));
       String reason = findReasonForNotCaching(cblock, blockInfo);
       int neededCached = 0;
@@ -628,7 +628,7 @@ private void addNewPendingCached(final int neededCached,
       List<DatanodeDescriptor> pendingCached) {
     // To figure out which replicas can be cached, we consult the
     // blocksMap.  We don't want to try to cache a corrupt replica, though.
-    BlockInfo blockInfo = blockManager.
+    BlockInfoContiguous blockInfo = blockManager.
           getStoredBlock(new Block(cachedBlock.getBlockId()));
     if (blockInfo == null) {
       LOG.debug("Block {}: can't add new cached replicas," +
@@ -667,7 +667,7 @@ private void addNewPendingCached(final int neededCached,
       Iterator<CachedBlock> it = datanode.getPendingCached().iterator();
       while (it.hasNext()) {
         CachedBlock cBlock = it.next();
-        BlockInfo info =
+        BlockInfoContiguous info =
             blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
         if (info != null) {
           pendingBytes -= info.getNumBytes();
@@ -677,7 +677,7 @@ private void addNewPendingCached(final int neededCached,
       // Add pending uncached blocks from effective capacity
       while (it.hasNext()) {
         CachedBlock cBlock = it.next();
-        BlockInfo info =
+        BlockInfoContiguous info =
             blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
         if (info != null) {
           pendingBytes += info.getNumBytes();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
index 764f25dddadb5..fc2e23408f249 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
@@ -73,18 +73,15 @@ void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
     }
     
     if (!nodes.keySet().contains(dn)) {
-      NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
-                                   blk.getBlockName() +
-                                   " added as corrupt on " + dn +
-                                   " by " + Server.getRemoteIp() +
-                                   reasonText);
+      NameNode.blockStateChangeLog.info(
+          "BLOCK NameSystem.addToCorruptReplicasMap: {} added as corrupt on "
+              + "{} by {} {}", blk.getBlockName(), dn, Server.getRemoteIp(),
+          reasonText);
     } else {
-      NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
-                                   "duplicate requested for " + 
-                                   blk.getBlockName() + " to add as corrupt " +
-                                   "on " + dn +
-                                   " by " + Server.getRemoteIp() +
-                                   reasonText);
+      NameNode.blockStateChangeLog.info(
+          "BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for" +
+              " {} to add as corrupt on {} by {} {}", blk.getBlockName(), dn,
+              Server.getRemoteIp(), reasonText);
     }
     // Add the node or update the reason.
     nodes.put(dn, reasonCode);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index c808482a17e7c..833ab560f220e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -197,8 +197,8 @@ public CachedBlocksList getPendingUncached() {
   /** A queue of blocks to be replicated by this datanode */
   private final BlockQueue<BlockTargetPair> replicateBlocks = new BlockQueue<BlockTargetPair>();
   /** A queue of blocks to be recovered by this datanode */
-  private final BlockQueue<BlockInfoUnderConstruction> recoverBlocks =
-                                new BlockQueue<BlockInfoUnderConstruction>();
+  private final BlockQueue<BlockInfoContiguousUnderConstruction> recoverBlocks =
+                                new BlockQueue<BlockInfoContiguousUnderConstruction>();
   /** A set of blocks to be invalidated by this datanode */
   private final LightWeightHashSet<Block> invalidateBlocks = new LightWeightHashSet<Block>();
 
@@ -284,7 +284,7 @@ boolean hasStaleStorages() {
    * Remove block from the list of blocks belonging to the data-node. Remove
    * data-node from the block.
    */
-  boolean removeBlock(BlockInfo b) {
+  boolean removeBlock(BlockInfoContiguous b) {
     final DatanodeStorageInfo s = b.findStorageInfo(this);
     // if block exists on this datanode
     if (s != null) {
@@ -297,7 +297,7 @@ boolean removeBlock(BlockInfo b) {
    * Remove block from the list of blocks belonging to the data-node. Remove
    * data-node from the block.
    */
-  boolean removeBlock(String storageID, BlockInfo b) {
+  boolean removeBlock(String storageID, BlockInfoContiguous b) {
     DatanodeStorageInfo s = getStorageInfo(storageID);
     if (s != null) {
       return s.removeBlock(b);
@@ -416,6 +416,46 @@ public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity,
     if (checkFailedStorages) {
       updateFailedStorage(failedStorageInfos);
     }
+
+    if (storageMap.size() != reports.length) {
+      pruneStorageMap(reports);
+    }
+  }
+
+  /**
+   * Remove stale storages from storageMap. We must not remove any storages
+   * as long as they have associated block replicas.
+   */
+  private void pruneStorageMap(final StorageReport[] reports) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Number of storages reported in heartbeat=" + reports.length +
+                    "; Number of storages in storageMap=" + storageMap.size());
+    }
+
+    HashMap<String, DatanodeStorageInfo> excessStorages;
+
+    synchronized (storageMap) {
+      // Init excessStorages with all known storages.
+      excessStorages = new HashMap<String, DatanodeStorageInfo>(storageMap);
+
+      // Remove storages that the DN reported in the heartbeat.
+      for (final StorageReport report : reports) {
+        excessStorages.remove(report.getStorage().getStorageID());
+      }
+
+      // For each remaining storage, remove it if there are no associated
+      // blocks.
+      for (final DatanodeStorageInfo storageInfo : excessStorages.values()) {
+        if (storageInfo.numBlocks() == 0) {
+          storageMap.remove(storageInfo.getStorageID());
+          LOG.info("Removed storage " + storageInfo + " from DataNode" + this);
+        } else if (LOG.isDebugEnabled()) {
+          // This can occur until all block reports are received.
+          LOG.debug("Deferring removal of stale storage " + storageInfo +
+                        " with " + storageInfo.numBlocks() + " blocks");
+        }
+      }
+    }
   }
 
   private void updateFailedStorage(
@@ -428,12 +468,12 @@ private void updateFailedStorage(
     }
   }
 
-  private static class BlockIterator implements Iterator<BlockInfo> {
+  private static class BlockIterator implements Iterator<BlockInfoContiguous> {
     private int index = 0;
-    private final List<Iterator<BlockInfo>> iterators;
+    private final List<Iterator<BlockInfoContiguous>> iterators;
     
     private BlockIterator(final DatanodeStorageInfo... storages) {
-      List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
+      List<Iterator<BlockInfoContiguous>> iterators = new ArrayList<Iterator<BlockInfoContiguous>>();
       for (DatanodeStorageInfo e : storages) {
         iterators.add(e.getBlockIterator());
       }
@@ -447,7 +487,7 @@ public boolean hasNext() {
     }
 
     @Override
-    public BlockInfo next() {
+    public BlockInfoContiguous next() {
       update();
       return iterators.get(index).next();
     }
@@ -464,10 +504,10 @@ private void update() {
     }
   }
 
-  Iterator<BlockInfo> getBlockIterator() {
+  Iterator<BlockInfoContiguous> getBlockIterator() {
     return new BlockIterator(getStorageInfos());
   }
-  Iterator<BlockInfo> getBlockIterator(final String storageID) {
+  Iterator<BlockInfoContiguous> getBlockIterator(final String storageID) {
     return new BlockIterator(getStorageInfo(storageID));
   }
 
@@ -490,7 +530,7 @@ void addBlockToBeReplicated(Block block, DatanodeStorageInfo[] targets) {
   /**
    * Store block recovery work.
    */
-  void addBlockToBeRecovered(BlockInfoUnderConstruction block) {
+  void addBlockToBeRecovered(BlockInfoContiguousUnderConstruction block) {
     if(recoverBlocks.contains(block)) {
       // this prevents adding the same block twice to the recovery queue
       BlockManager.LOG.info(block + " is already in the recovery queue");
@@ -532,11 +572,11 @@ public List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
     return replicateBlocks.poll(maxTransfers);
   }
 
-  public BlockInfoUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
-    List<BlockInfoUnderConstruction> blocks = recoverBlocks.poll(maxTransfers);
+  public BlockInfoContiguousUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
+    List<BlockInfoContiguousUnderConstruction> blocks = recoverBlocks.poll(maxTransfers);
     if(blocks == null)
       return null;
-    return blocks.toArray(new BlockInfoUnderConstruction[blocks.size()]);
+    return blocks.toArray(new BlockInfoContiguousUnderConstruction[blocks.size()]);
   }
 
   /**
@@ -747,8 +787,6 @@ DatanodeStorageInfo updateStorage(DatanodeStorage s) {
         // For backwards compatibility, make sure that the type and
         // state are updated. Some reports from older datanodes do
         // not include these fields so we may have assumed defaults.
-        // This check can be removed in the next major release after
-        // 2.4.
         storage.updateFromStorage(s);
         storageMap.put(storage.getStorageID(), storage);
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 356a4a3cf0dac..a33d9907417b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -133,13 +133,18 @@ public class DatanodeManager {
    * writing to stale datanodes, i.e., continue using stale nodes for writing.
    */
   private final float ratioUseStaleDataNodesForWrite;
-  
+
   /** The number of stale DataNodes */
   private volatile int numStaleNodes;
 
   /** The number of stale storages */
   private volatile int numStaleStorages;
 
+  /**
+   * Number of blocks to check for each postponedMisreplicatedBlocks iteration
+   */
+  private final long blocksPerPostponedMisreplicatedBlocksRescan;
+
   /**
    * Whether or not this cluster has ever consisted of more than 1 rack,
    * according to the NetworkTopology.
@@ -259,6 +264,9 @@ public class DatanodeManager {
     this.timeBetweenResendingCachingDirectivesMs = conf.getLong(
         DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS,
         DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT);
+    this.blocksPerPostponedMisreplicatedBlocksRescan = conf.getLong(
+        DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY,
+        DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT);
   }
 
   private static long getStaleIntervalFromConf(Configuration conf,
@@ -483,7 +491,7 @@ public DatanodeDescriptor getDatanode(DatanodeID nodeID
     if (!node.getXferAddr().equals(nodeID.getXferAddr())) {
       final UnregisteredNodeException e = new UnregisteredNodeException(
           nodeID, node);
-      NameNode.stateChangeLog.fatal("BLOCK* NameSystem.getDatanode: "
+      NameNode.stateChangeLog.error("BLOCK* NameSystem.getDatanode: "
                                     + e.getLocalizedMessage());
       throw e;
     }
@@ -1112,16 +1120,8 @@ public int getNumDeadDataNodes() {
   public List<DatanodeDescriptor> getDecommissioningNodes() {
     // There is no need to take namesystem reader lock as
     // getDatanodeListForReport will synchronize on datanodeMap
-    final List<DatanodeDescriptor> decommissioningNodes
-        = new ArrayList<DatanodeDescriptor>();
-    final List<DatanodeDescriptor> results = getDatanodeListForReport(
-        DatanodeReportType.LIVE);
-    for(DatanodeDescriptor node : results) {
-      if (node.isDecommissionInProgress()) {
-        decommissioningNodes.add(node);
-      }
-    }
-    return decommissioningNodes;
+    // A decommissioning DN may be "alive" or "dead".
+    return getDatanodeListForReport(DatanodeReportType.DECOMMISSIONING);
   }
   
   /* Getter and Setter for stale DataNodes related attributes */
@@ -1141,6 +1141,10 @@ public boolean shouldAvoidStaleDataNodesForWrite() {
             * ratioUseStaleDataNodesForWrite);
   }
 
+  public long getBlocksPerPostponedMisreplicatedBlocksRescan() {
+    return blocksPerPostponedMisreplicatedBlocksRescan;
+  }
+
   /**
    * @return The time interval used to mark DataNodes as stale.
    */
@@ -1413,12 +1417,12 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
         }
 
         //check lease recovery
-        BlockInfoUnderConstruction[] blocks = nodeinfo
+        BlockInfoContiguousUnderConstruction[] blocks = nodeinfo
             .getLeaseRecoveryCommand(Integer.MAX_VALUE);
         if (blocks != null) {
           BlockRecoveryCommand brCommand = new BlockRecoveryCommand(
               blocks.length);
-          for (BlockInfoUnderConstruction b : blocks) {
+          for (BlockInfoContiguousUnderConstruction b : blocks) {
             final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations();
             // Skip stale nodes during recovery - not heart beated for some time (30s by default).
             final List<DatanodeStorageInfo> recoveryLocations =
@@ -1428,24 +1432,37 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
                 recoveryLocations.add(storages[i]);
               }
             }
+            // If we are performing a truncate recovery than set recovery fields
+            // to old block.
+            boolean truncateRecovery = b.getTruncateBlock() != null;
+            boolean copyOnTruncateRecovery = truncateRecovery &&
+                b.getTruncateBlock().getBlockId() != b.getBlockId();
+            ExtendedBlock primaryBlock = (copyOnTruncateRecovery) ?
+                new ExtendedBlock(blockPoolId, b.getTruncateBlock()) :
+                new ExtendedBlock(blockPoolId, b);
             // If we only get 1 replica after eliminating stale nodes, then choose all
             // replicas for recovery and let the primary data node handle failures.
+            DatanodeInfo[] recoveryInfos;
             if (recoveryLocations.size() > 1) {
               if (recoveryLocations.size() != storages.length) {
                 LOG.info("Skipped stale nodes for recovery : " +
                     (storages.length - recoveryLocations.size()));
               }
-              brCommand.add(new RecoveringBlock(
-                  new ExtendedBlock(blockPoolId, b),
-                  DatanodeStorageInfo.toDatanodeInfos(recoveryLocations),
-                  b.getBlockRecoveryId()));
+              recoveryInfos =
+                  DatanodeStorageInfo.toDatanodeInfos(recoveryLocations);
             } else {
               // If too many replicas are stale, then choose all replicas to participate
               // in block recovery.
-              brCommand.add(new RecoveringBlock(
-                  new ExtendedBlock(blockPoolId, b),
-                  DatanodeStorageInfo.toDatanodeInfos(storages),
-                  b.getBlockRecoveryId()));
+              recoveryInfos = DatanodeStorageInfo.toDatanodeInfos(storages);
+            }
+            if(truncateRecovery) {
+              Block recoveryBlock = (copyOnTruncateRecovery) ? b :
+                  b.getTruncateBlock();
+              brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos,
+                                                recoveryBlock));
+            } else {
+              brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos,
+                                                b.getBlockRecoveryId()));
             }
           }
           return new DatanodeCommand[] { brCommand };
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index a3198e20ef35a..d5ad5fe31afcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -80,10 +80,10 @@ public void updateFromStorage(DatanodeStorage storage) {
   /**
    * Iterates over the list of blocks belonging to the data-node.
    */
-  class BlockIterator implements Iterator<BlockInfo> {
-    private BlockInfo current;
+  class BlockIterator implements Iterator<BlockInfoContiguous> {
+    private BlockInfoContiguous current;
 
-    BlockIterator(BlockInfo head) {
+    BlockIterator(BlockInfoContiguous head) {
       this.current = head;
     }
 
@@ -91,8 +91,8 @@ public boolean hasNext() {
       return current != null;
     }
 
-    public BlockInfo next() {
-      BlockInfo res = current;
+    public BlockInfoContiguous next() {
+      BlockInfoContiguous res = current;
       current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this));
       return res;
     }
@@ -112,7 +112,7 @@ public void remove() {
   private volatile long remaining;
   private long blockPoolUsed;
 
-  private volatile BlockInfo blockList = null;
+  private volatile BlockInfoContiguous blockList = null;
   private int numBlocks = 0;
 
   /** The number of block reports received */
@@ -215,7 +215,7 @@ long getBlockPoolUsed() {
     return blockPoolUsed;
   }
 
-  public AddBlockResult addBlock(BlockInfo b) {
+  public AddBlockResult addBlock(BlockInfoContiguous b) {
     // First check whether the block belongs to a different storage
     // on the same DN.
     AddBlockResult result = AddBlockResult.ADDED;
@@ -240,7 +240,7 @@ public AddBlockResult addBlock(BlockInfo b) {
     return result;
   }
 
-  boolean removeBlock(BlockInfo b) {
+  public boolean removeBlock(BlockInfoContiguous b) {
     blockList = b.listRemove(blockList, this);
     if (b.removeStorage(this)) {
       numBlocks--;
@@ -254,7 +254,7 @@ int numBlocks() {
     return numBlocks;
   }
   
-  Iterator<BlockInfo> getBlockIterator() {
+  Iterator<BlockInfoContiguous> getBlockIterator() {
     return new BlockIterator(blockList);
 
   }
@@ -263,7 +263,7 @@ Iterator<BlockInfo> getBlockIterator() {
    * Move block to the head of the list of blocks belonging to the data-node.
    * @return the index of the head of the blockList
    */
-  int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
+  int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) {
     blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex);
     return curIndex;
   }
@@ -273,7 +273,7 @@ int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
    * @return the head of the blockList
    */
   @VisibleForTesting
-  BlockInfo getBlockListHeadForTesting(){
+  BlockInfoContiguous getBlockListHeadForTesting(){
     return blockList;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
index 66649bb4e379c..a465f85d60479 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
@@ -26,7 +26,6 @@
 import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.commons.logging.Log;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -37,6 +36,7 @@
 import org.apache.hadoop.hdfs.DFSUtil;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
 
 /**
  * Keeps a Collection for every named machine containing blocks
@@ -67,7 +67,7 @@ class InvalidateBlocks {
     printBlockDeletionTime(BlockManager.LOG);
   }
 
-  private void printBlockDeletionTime(final Log log) {
+  private void printBlockDeletionTime(final Logger log) {
     log.info(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY
         + " is set to " + DFSUtil.durationToString(pendingPeriodInMs));
     SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss");
@@ -112,8 +112,8 @@ synchronized void add(final Block block, final DatanodeInfo datanode,
     if (set.add(block)) {
       numBlocks++;
       if (log) {
-        NameNode.blockStateChangeLog.info("BLOCK* " + getClass().getSimpleName()
-            + ": add " + block + " to " + datanode);
+        NameNode.blockStateChangeLog.info("BLOCK* {}: add {} to {}",
+            getClass().getSimpleName(), block, datanode);
       }
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
index 2b507e74acf1a..57c296262514a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
@@ -28,9 +28,9 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.util.Daemon;
+import org.slf4j.Logger;
 
 /***************************************************
  * PendingReplicationBlocks does the bookkeeping of all
@@ -44,7 +44,7 @@
  *
  ***************************************************/
 class PendingReplicationBlocks {
-  private static final Log LOG = BlockManager.LOG;
+  private static final Logger LOG = BlockManager.LOG;
 
   private final Map<Block, PendingBlockInfo> pendingReplications;
   private final ArrayList<Block> timedOutItems;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
new file mode 100644
index 0000000000000..e9e22ee46d539
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.util.SequentialNumber;
+
+/**
+ * Generate the next valid block group ID by incrementing the maximum block
+ * group ID allocated so far, with the first 2^10 block group IDs reserved.
+ * HDFS-EC introduces a hierarchical protocol to name blocks and groups:
+ * Contiguous: {reserved block IDs | flag | block ID}
+ * Striped: {reserved block IDs | flag | block group ID | index in group}
+ *
+ * Following n bits of reserved block IDs, The (n+1)th bit in an ID
+ * distinguishes contiguous (0) and striped (1) blocks. For a striped block,
+ * bits (n+2) to (64-m) represent the ID of its block group, while the last m
+ * bits represent its index of the group. The value m is determined by the
+ * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP).
+ */
+@InterfaceAudience.Private
+public class SequentialBlockGroupIdGenerator extends SequentialNumber {
+
+  private final BlockManager blockManager;
+
+  SequentialBlockGroupIdGenerator(BlockManager blockManagerRef) {
+    super(Long.MIN_VALUE);
+    this.blockManager = blockManagerRef;
+  }
+
+  @Override // NumberGenerator
+  public long nextValue() {
+    // Skip to next legitimate block group ID based on the naming protocol
+    while (super.getCurrentValue() % HdfsConstants.MAX_BLOCKS_IN_GROUP > 0) {
+      super.nextValue();
+    }
+    // Make sure there's no conflict with existing random block IDs
+    while (hasValidBlockInRange(super.getCurrentValue())) {
+      super.skipTo(super.getCurrentValue() +
+          HdfsConstants.MAX_BLOCKS_IN_GROUP);
+    }
+    if (super.getCurrentValue() >= 0) {
+      BlockManager.LOG.warn("All negative block group IDs are used, " +
+          "growing into positive IDs, " +
+          "which might conflict with non-erasure coded blocks.");
+    }
+    return super.getCurrentValue();
+  }
+
+  /**
+   *
+   * @param id The starting ID of the range
+   * @return true if any ID in the range
+   *      {id, id+HdfsConstants.MAX_BLOCKS_IN_GROUP} is pointed-to by a file
+   */
+  private boolean hasValidBlockInRange(long id) {
+    for (int i = 0; i < HdfsConstants.MAX_BLOCKS_IN_GROUP; i++) {
+      Block b = new Block(id + i);
+      if (blockManager.getBlockCollection(b) != null) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
index eef8857b042f4..c97de4b806aaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
@@ -19,7 +19,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.util.SequentialNumber;
 
 /**
@@ -54,6 +53,11 @@ public long nextValue() {
     while(isValidBlock(b)) {
       b.setBlockId(super.nextValue());
     }
+    if (b.getBlockId() < 0) {
+      BlockManager.LOG.warn("All positive block IDs are used, " +
+          "wrapping to negative IDs, " +
+          "which might conflict with erasure coded block groups.");
+    }
     return b.getBlockId();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index 7d0eea4c070b4..1daa0ee2bc72f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
@@ -195,15 +195,12 @@ synchronized boolean add(Block block,
           expectedReplicas == 1) {
         corruptReplOneBlocks++;
       }
-      if(NameNode.blockStateChangeLog.isDebugEnabled()) {
-        NameNode.blockStateChangeLog.debug(
-          "BLOCK* NameSystem.UnderReplicationBlock.add:"
-          + block
-          + " has only " + curReplicas
-          + " replicas and need " + expectedReplicas
-          + " replicas so is added to neededReplications"
-          + " at priority level " + priLevel);
-      }
+      NameNode.blockStateChangeLog.debug(
+          "BLOCK* NameSystem.UnderReplicationBlock.add: {}"
+              + " has only {} replicas and need {} replicas so is added to" +
+              " neededReplications at priority level {}", block, curReplicas,
+          expectedReplicas, priLevel);
+
       return true;
     }
     return false;
@@ -247,24 +244,18 @@ synchronized boolean remove(Block block,
   boolean remove(Block block, int priLevel) {
     if(priLevel >= 0 && priLevel < LEVEL 
         && priorityQueues.get(priLevel).remove(block)) {
-      if(NameNode.blockStateChangeLog.isDebugEnabled()) {
-        NameNode.blockStateChangeLog.debug(
-          "BLOCK* NameSystem.UnderReplicationBlock.remove: "
-          + "Removing block " + block
-          + " from priority queue "+ priLevel);
-      }
+      NameNode.blockStateChangeLog.debug(
+        "BLOCK* NameSystem.UnderReplicationBlock.remove: Removing block {}" +
+            " from priority queue {}", block, priLevel);
       return true;
     } else {
       // Try to remove the block from all queues if the block was
       // not found in the queue for the given priority level.
       for (int i = 0; i < LEVEL; i++) {
         if (priorityQueues.get(i).remove(block)) {
-          if(NameNode.blockStateChangeLog.isDebugEnabled()) {
-            NameNode.blockStateChangeLog.debug(
-              "BLOCK* NameSystem.UnderReplicationBlock.remove: "
-              + "Removing block " + block
-              + " from priority queue "+ i);
-          }
+          NameNode.blockStateChangeLog.debug(
+              "BLOCK* NameSystem.UnderReplicationBlock.remove: Removing block" +
+                  " {} from priority queue {}", block, priLevel);
           return true;
         }
       }
@@ -310,15 +301,12 @@ synchronized void update(Block block, int curReplicas,
       remove(block, oldPri);
     }
     if(priorityQueues.get(curPri).add(block)) {
-      if(NameNode.blockStateChangeLog.isDebugEnabled()) {
-        NameNode.blockStateChangeLog.debug(
-          "BLOCK* NameSystem.UnderReplicationBlock.update:"
-          + block
-          + " has only "+ curReplicas
-          + " replicas and needs " + curExpectedReplicas
-          + " replicas so is added to neededReplications"
-          + " at priority level " + curPri);
-      }
+      NameNode.blockStateChangeLog.debug(
+          "BLOCK* NameSystem.UnderReplicationBlock.update: {} has only {} " +
+              "replicas and needs {} replicas so is added to " +
+              "neededReplications at priority level {}", block, curReplicas,
+          curExpectedReplicas, curPri);
+
     }
     if (oldPri != curPri || expectedReplicasDelta != 0) {
       // corruptReplOneBlocks could possibly change
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 31fdb84385683..e6bd5b289332d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -727,7 +727,7 @@ FileLock tryLock() throws IOException {
         file.close();
         throw e;
       }
-      if (res != null && !deletionHookAdded) {
+      if (!deletionHookAdded) {
         // If the file existed prior to our startup, we didn't
         // call deleteOnExit above. But since we successfully locked
         // the dir, we can take care of cleaning it up.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 4a54bed404b5e..dfeacdef1808a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -656,9 +656,6 @@ private boolean processCommandFromActive(DatanodeCommand cmd,
       //
       Block toDelete[] = bcmd.getBlocks();
       try {
-        if (dn.blockScanner != null) {
-          dn.blockScanner.deleteBlocks(bcmd.getBlockPoolId(), toDelete);
-        }
         // using global fsdataset
         dn.getFSDataset().invalidate(bcmd.getBlockPoolId(), toDelete);
       } catch(IOException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index d94d0561e55bf..e3967270f8b80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -22,7 +22,10 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.SocketTimeoutException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 
 import com.google.common.base.Joiner;
 import org.apache.commons.logging.Log;
@@ -458,7 +461,7 @@ List<DatanodeCommand> blockReport() throws IOException {
       return null;
     }
 
-    ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
+    final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
 
     // Flush any block information that precedes the block report. Otherwise
     // we have a chance that we will miss the delHint information
@@ -485,40 +488,54 @@ List<DatanodeCommand> blockReport() throws IOException {
     }
 
     // Send the reports to the NN.
-    int numReportsSent;
+    int numReportsSent = 0;
+    int numRPCs = 0;
+    boolean success = false;
     long brSendStartTime = now();
-    if (totalBlockCount < dnConf.blockReportSplitThreshold) {
-      // Below split threshold, send all reports in a single message.
-      numReportsSent = 1;
-      DatanodeCommand cmd =
-          bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports);
-      if (cmd != null) {
-        cmds.add(cmd);
-      }
-    } else {
-      // Send one block report per message.
-      numReportsSent = i;
-      for (StorageBlockReport report : reports) {
-        StorageBlockReport singleReport[] = { report };
+    try {
+      if (totalBlockCount < dnConf.blockReportSplitThreshold) {
+        // Below split threshold, send all reports in a single message.
         DatanodeCommand cmd = bpNamenode.blockReport(
-            bpRegistration, bpos.getBlockPoolId(), singleReport);
+            bpRegistration, bpos.getBlockPoolId(), reports);
+        numRPCs = 1;
+        numReportsSent = reports.length;
         if (cmd != null) {
           cmds.add(cmd);
         }
+      } else {
+        // Send one block report per message.
+        for (StorageBlockReport report : reports) {
+          StorageBlockReport singleReport[] = { report };
+          DatanodeCommand cmd = bpNamenode.blockReport(
+              bpRegistration, bpos.getBlockPoolId(), singleReport);
+          numReportsSent++;
+          numRPCs++;
+          if (cmd != null) {
+            cmds.add(cmd);
+          }
+        }
       }
+      success = true;
+    } finally {
+      // Log the block report processing stats from Datanode perspective
+      long brSendCost = now() - brSendStartTime;
+      long brCreateCost = brSendStartTime - brCreateStartTime;
+      dn.getMetrics().addBlockReport(brSendCost);
+      final int nCmds = cmds.size();
+      LOG.info((success ? "S" : "Uns") +
+          "uccessfully sent " + numReportsSent +
+          " of " + reports.length +
+          " blockreports for " + totalBlockCount +
+          " total blocks using " + numRPCs +
+          " RPCs. This took " + brCreateCost +
+          " msec to generate and " + brSendCost +
+          " msecs for RPC and NN processing." +
+          " Got back " +
+          ((nCmds == 0) ? "no commands" :
+              ((nCmds == 1) ? "one command: " + cmds.get(0) :
+                  (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) +
+          ".");
     }
-
-    // Log the block report processing stats from Datanode perspective
-    long brSendCost = now() - brSendStartTime;
-    long brCreateCost = brSendStartTime - brCreateStartTime;
-    dn.getMetrics().addBlockReport(brSendCost);
-    LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount +
-        " blocks total. Took " + brCreateCost +
-        " msec to generate and " + brSendCost +
-        " msecs for RPC and NN processing. " +
-        " Got back commands " +
-            (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds)));
-
     scheduleNextBlockReport(startTime);
     return cmds.size() == 0 ? null : cmds;
   }
@@ -719,12 +736,6 @@ private void offerService() throws Exception {
         DatanodeCommand cmd = cacheReport();
         processCommand(new DatanodeCommand[]{ cmd });
 
-        // Now safe to start scanning the block pool.
-        // If it has already been started, this is a no-op.
-        if (dn.blockScanner != null) {
-          dn.blockScanner.addBlockPool(bpos.getBlockPoolId());
-        }
-
         //
         // There is no work to do;  sleep until hearbeat timer elapses, 
         // or work arrives, and then iterate again.
@@ -968,7 +979,6 @@ int putMissingBlockInfos(ReceivedDeletedBlockInfo[] blockArray) {
 
     /**
      * Add pending incremental block report for a single block.
-     * @param blockID
      * @param blockInfo
      */
     void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java
index 51a61343ab66a..94493aa1d9dd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockMetadataHeader.java
@@ -162,7 +162,7 @@ public static BlockMetadataHeader readHeader(File file) throws IOException {
    * The current file position will be altered by this method.
    * If an error occurs, the file is <em>not</em> closed.
    */
-  static BlockMetadataHeader readHeader(RandomAccessFile raf) throws IOException {
+  public static BlockMetadataHeader readHeader(RandomAccessFile raf) throws IOException {
     byte[] buf = new byte[getHeaderSize()];
     raf.seek(0);
     raf.readFully(buf, 0, buf.length);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
deleted file mode 100644
index 61f1e7e7cabe0..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
+++ /dev/null
@@ -1,833 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import java.io.DataOutputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
-import org.apache.hadoop.hdfs.util.DataTransferThrottler;
-import org.apache.hadoop.util.GSet;
-import org.apache.hadoop.util.LightWeightGSet;
-import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.Time;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Scans the block files under a block pool and verifies that the
- * files are not corrupt.
- * This keeps track of blocks and their last verification times.
- * Currently it does not modify the metadata for block.
- */
-
-class BlockPoolSliceScanner {
-  
-  public static final Log LOG = LogFactory.getLog(BlockPoolSliceScanner.class);
-  
-  private static final String DATA_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
-
-  private static final int MAX_SCAN_RATE = 8 * 1024 * 1024; // 8MB per sec
-  private static final int MIN_SCAN_RATE = 1 * 1024 * 1024; // 1MB per sec
-  private static final long DEFAULT_SCAN_PERIOD_HOURS = 21*24L; // three weeks
-
-  private static final String VERIFICATION_PREFIX = "dncp_block_verification.log";
-
-  private final String blockPoolId;
-  private final long scanPeriod;
-  private final AtomicLong lastScanTime = new AtomicLong();
-
-  private final DataNode datanode;
-  private final FsDatasetSpi<? extends FsVolumeSpi> dataset;
-  
-  private final SortedSet<BlockScanInfo> blockInfoSet
-      = new TreeSet<BlockScanInfo>(BlockScanInfo.LAST_SCAN_TIME_COMPARATOR);
-
-  private final SortedSet<BlockScanInfo> newBlockInfoSet =
-      new TreeSet<BlockScanInfo>(BlockScanInfo.LAST_SCAN_TIME_COMPARATOR);
-
-  private final GSet<Block, BlockScanInfo> blockMap
-      = new LightWeightGSet<Block, BlockScanInfo>(
-          LightWeightGSet.computeCapacity(0.5, "BlockMap"));
-  
-  // processedBlocks keeps track of which blocks are scanned
-  // since the last run.
-  private volatile HashMap<Long, Integer> processedBlocks;
-  
-  private long totalScans = 0;
-  private long totalScanErrors = 0;
-  private long totalTransientErrors = 0;
-  private final AtomicInteger totalBlocksScannedInLastRun = new AtomicInteger(); // Used for test only
-  
-  private long currentPeriodStart = Time.monotonicNow();
-  private long bytesLeft = 0; // Bytes to scan in this period
-  private long totalBytesToScan = 0;
-  private boolean isNewPeriod = true;
-  
-  private final LogFileHandler verificationLog;
-  
-  private final DataTransferThrottler throttler = new DataTransferThrottler(
-       200, MAX_SCAN_RATE);
-  
-  private static enum ScanType {
-    VERIFICATION_SCAN,     // scanned as part of periodic verfication
-    NONE,
-  }
-
-  // Extend Block because in the DN process there's a 1-to-1 correspondence of
-  // BlockScanInfo to Block instances, so by extending rather than containing
-  // Block, we can save a bit of Object overhead (about 24 bytes per block
-  // replica.)
-  static class BlockScanInfo extends Block
-      implements LightWeightGSet.LinkedElement {
-
-    /** Compare the info by the last scan time. */
-    static final Comparator<BlockScanInfo> LAST_SCAN_TIME_COMPARATOR
-        = new Comparator<BlockPoolSliceScanner.BlockScanInfo>() {
-
-      @Override
-      public int compare(BlockScanInfo left, BlockScanInfo right) {
-        final long l = left.lastScanTime;
-        final long r = right.lastScanTime;
-        // compare blocks itself if scantimes are same to avoid.
-        // because TreeMap uses comparator if available to check existence of
-        // the object. 
-        return l < r? -1: l > r? 1: left.compareTo(right); 
-      }
-    };
-
-    long lastScanTime = 0;
-    ScanType lastScanType = ScanType.NONE; 
-    boolean lastScanOk = true;
-    private LinkedElement next;
-    
-    BlockScanInfo(Block block) {
-      super(block);
-    }
-    
-    @Override
-    public int hashCode() {
-      return super.hashCode();
-    }
-    
-    @Override
-    public boolean equals(Object that) {
-      if (this == that) {
-        return true;
-      }
-      return super.equals(that);
-    }
-    
-    long getLastScanTime() {
-      return (lastScanType == ScanType.NONE) ? 0 : lastScanTime;
-    }
-
-    @Override
-    public void setNext(LinkedElement next) {
-      this.next = next;
-    }
-
-    @Override
-    public LinkedElement getNext() {
-      return next;
-    }
-  }
-  
-  BlockPoolSliceScanner(String bpid, DataNode datanode,
-      FsDatasetSpi<? extends FsVolumeSpi> dataset, Configuration conf) {
-    this.datanode = datanode;
-    this.dataset = dataset;
-    this.blockPoolId  = bpid;
-    
-    long hours = conf.getInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 
-                             DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT);
-    if (hours <= 0) {
-      hours = DEFAULT_SCAN_PERIOD_HOURS;
-    }
-    this.scanPeriod = hours * 3600 * 1000;
-    LOG.info("Periodic Block Verification Scanner initialized with interval "
-        + hours + " hours for block pool " + bpid);
-
-    // get the list of blocks and arrange them in random order
-    List<FinalizedReplica> arr = dataset.getFinalizedBlocksOnPersistentStorage(blockPoolId);
-    Collections.shuffle(arr);
-    
-    long scanTime = -1;
-    for (Block block : arr) {
-      BlockScanInfo info = new BlockScanInfo( block );
-      info.lastScanTime = scanTime--; 
-      //still keep 'info.lastScanType' to NONE.
-      addBlockInfo(info, false);
-    }
-
-    RollingLogs rollingLogs = null;
-    try {
-       rollingLogs = dataset.createRollingLogs(blockPoolId, VERIFICATION_PREFIX);
-    } catch (IOException e) {
-      LOG.warn("Could not open verfication log. " +
-               "Verification times are not stored.");
-    }
-    verificationLog = rollingLogs == null? null: new LogFileHandler(rollingLogs);
-  }
-  
-  String getBlockPoolId() {
-    return blockPoolId;
-  }
-  
-  private void updateBytesToScan(long len, long lastScanTime) {
-    // len could be negative when a block is deleted.
-    totalBytesToScan += len;
-    if ( lastScanTime < currentPeriodStart ) {
-      bytesLeft += len;
-    }
-    // Should we change throttler bandwidth every time bytesLeft changes?
-    // not really required.
-  }
-
-  /**
-   * Add the BlockScanInfo to sorted set of blockScanInfo
-   * @param info BlockScanInfo to be added
-   * @param isNewBlock true if the block is the new Block, false if
-   *          BlockScanInfo is being updated with new scanTime
-   */
-  private synchronized void addBlockInfo(BlockScanInfo info,
-      boolean isNewBlock) {
-    boolean added = false;
-    if (isNewBlock) {
-      // check whether the block already present
-      boolean exists = blockInfoSet.contains(info);
-      added = !exists && newBlockInfoSet.add(info);
-    } else {
-      added = blockInfoSet.add(info);
-    }
-    blockMap.put(info);
-    
-    if (added) {
-      updateBytesToScan(info.getNumBytes(), info.lastScanTime);
-    }
-  }
-
-  private synchronized void delBlockInfo(BlockScanInfo info) {
-    boolean exists = blockInfoSet.remove(info);
-    if (!exists){
-      exists = newBlockInfoSet.remove(info);
-    }
-    blockMap.remove(info);
-
-    if (exists) {
-      updateBytesToScan(-info.getNumBytes(), info.lastScanTime);
-    }
-  }
-
-  /** Update blockMap by the given LogEntry */
-  private synchronized void updateBlockInfo(LogEntry e) {
-    BlockScanInfo info = blockMap.get(new Block(e.blockId, 0, e.genStamp));
-    
-    if(info != null && e.verificationTime > 0 && 
-        info.lastScanTime < e.verificationTime) {
-      delBlockInfo(info);
-      info.lastScanTime = e.verificationTime;
-      info.lastScanType = ScanType.VERIFICATION_SCAN;
-      addBlockInfo(info, false);
-    }
-  }
-
-  private synchronized long getNewBlockScanTime() {
-    /* If there are a lot of blocks, this returns a random time with in 
-     * the scan period. Otherwise something sooner.
-     */
-    long period = Math.min(scanPeriod, 
-                           Math.max(blockMap.size(),1) * 600 * 1000L);
-    int periodInt = Math.abs((int)period);
-    return Time.monotonicNow() - scanPeriod +
-        DFSUtil.getRandom().nextInt(periodInt);
-  }
-
-  /** Adds block to list of blocks */
-  synchronized void addBlock(ExtendedBlock block) {
-    BlockScanInfo info = blockMap.get(block.getLocalBlock());
-    if ( info != null ) {
-      LOG.warn("Adding an already existing block " + block);
-      delBlockInfo(info);
-    }
-    
-    info = new BlockScanInfo(block.getLocalBlock());    
-    info.lastScanTime = getNewBlockScanTime();
-    
-    addBlockInfo(info, true);
-    adjustThrottler();
-  }
-  
-  /** Deletes the block from internal structures */
-  synchronized void deleteBlock(Block block) {
-    BlockScanInfo info = blockMap.get(block);
-    if (info != null) {
-      delBlockInfo(info);
-    }
-  }
-
-  @VisibleForTesting
-  long getTotalScans() {
-    return totalScans;
-  }
-
-  /** @return the last scan time for the block pool. */
-  long getLastScanTime() {
-    return lastScanTime.get();
-  }
-
-  /** @return the last scan time the given block. */
-  synchronized long getLastScanTime(Block block) {
-    BlockScanInfo info = blockMap.get(block);
-    return info == null? 0: info.lastScanTime;
-  }
-
-  /** Deletes blocks from internal structures */
-  void deleteBlocks(Block[] blocks) {
-    for ( Block b : blocks ) {
-      deleteBlock(b);
-    }
-  }
-  
-  private synchronized void updateScanStatus(BlockScanInfo info,
-                                             ScanType type,
-                                             boolean scanOk) {
-    delBlockInfo(info);
-
-    long now = Time.monotonicNow();
-    info.lastScanType = type;
-    info.lastScanTime = now;
-    info.lastScanOk = scanOk;
-    addBlockInfo(info, false);
-        
-    // Don't update meta data if the verification failed.
-    if (!scanOk) {
-      return;
-    }
-    
-    if (verificationLog != null) {
-      verificationLog.append(now, info.getGenerationStamp(),
-          info.getBlockId());
-    }
-  }
-  
-  private void handleScanFailure(ExtendedBlock block) {
-    LOG.info("Reporting bad " + block);
-    try {
-      datanode.reportBadBlocks(block);
-    } catch (IOException ie) {
-      // it is bad, but not bad enough to shutdown the scanner
-      LOG.warn("Cannot report bad " + block.getBlockId());
-    }
-  }
-  
-  static private class LogEntry {
-
-    long blockId = -1;
-    long verificationTime = -1;
-    long genStamp = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
-    
-    /**
-     * The format consists of single line with multiple entries. each 
-     * entry is in the form : name="value".
-     * This simple text and easily extendable and easily parseable with a
-     * regex.
-     */
-    private static final Pattern entryPattern =
-      Pattern.compile("\\G\\s*([^=\\p{Space}]+)=\"(.*?)\"\\s*");
-    
-    static String toString(long verificationTime, long genStamp, long blockId,
-        DateFormat dateFormat) {
-      return "\ndate=\"" + dateFormat.format(new Date(verificationTime))
-          + "\"\t time=\"" + verificationTime
-          + "\"\t genstamp=\"" + genStamp
-          + "\"\t id=\"" + blockId + "\"";
-    }
-
-    static LogEntry parseEntry(String line) {
-      LogEntry entry = new LogEntry();
-      
-      Matcher matcher = entryPattern.matcher(line);
-      while (matcher.find()) {
-        String name = matcher.group(1);
-        String value = matcher.group(2);
-        
-        try {
-          if (name.equals("id")) {
-            entry.blockId = Long.parseLong(value);
-          } else if (name.equals("time")) {
-            entry.verificationTime = Long.parseLong(value);
-          } else if (name.equals("genstamp")) {
-            entry.genStamp = Long.parseLong(value);
-          }
-        } catch(NumberFormatException nfe) {
-          LOG.warn("Cannot parse line: " + line, nfe);
-          return null;
-        }
-      }
-      
-      return entry;
-    }
-  }
-  
-  private synchronized void adjustThrottler() {
-    long timeLeft = Math.max(1L,
-        currentPeriodStart + scanPeriod - Time.monotonicNow());
-    long bw = Math.max((bytesLeft * 1000) / timeLeft, MIN_SCAN_RATE);
-    throttler.setBandwidth(Math.min(bw, MAX_SCAN_RATE));
-  }
-  
-  @VisibleForTesting
-  void verifyBlock(ExtendedBlock block) {
-    BlockSender blockSender = null;
-
-    /* In case of failure, attempt to read second time to reduce
-     * transient errors. How do we flush block data from kernel 
-     * buffers before the second read? 
-     */
-    for (int i=0; i<2; i++) {
-      boolean second = (i > 0);
-      
-      try {
-        adjustThrottler();
-        
-        blockSender = new BlockSender(block, 0, -1, false, true, true, 
-            datanode, null, CachingStrategy.newDropBehind());
-
-        DataOutputStream out = 
-                new DataOutputStream(new IOUtils.NullOutputStream());
-        
-        blockSender.sendBlock(out, null, throttler);
-
-        LOG.info((second ? "Second " : "") +
-                 "Verification succeeded for " + block);
-        
-        if ( second ) {
-          totalTransientErrors++;
-        }
-        
-        updateScanStatus((BlockScanInfo)block.getLocalBlock(),
-            ScanType.VERIFICATION_SCAN, true);
-
-        return;
-      } catch (IOException e) {
-        updateScanStatus((BlockScanInfo)block.getLocalBlock(),
-            ScanType.VERIFICATION_SCAN, false);
-
-        // If the block does not exists anymore, then its not an error
-        if (!dataset.contains(block)) {
-          LOG.info(block + " is no longer in the dataset");
-          deleteBlock(block.getLocalBlock());
-          return;
-        }
-
-        // If the block exists, the exception may due to a race with write:
-        // The BlockSender got an old block path in rbw. BlockReceiver removed
-        // the rbw block from rbw to finalized but BlockSender tried to open the
-        // file before BlockReceiver updated the VolumeMap. The state of the
-        // block can be changed again now, so ignore this error here. If there
-        // is a block really deleted by mistake, DirectoryScan should catch it.
-        if (e instanceof FileNotFoundException ) {
-          LOG.info("Verification failed for " + block +
-              " - may be due to race with write");
-          deleteBlock(block.getLocalBlock());
-          return;
-        }
-
-        LOG.warn((second ? "Second " : "First ") + "Verification failed for "
-            + block, e);
-        
-        if (second) {
-          totalScanErrors++;
-          datanode.getMetrics().incrBlockVerificationFailures();
-          handleScanFailure(block);
-          return;
-        } 
-      } finally {
-        IOUtils.closeStream(blockSender);
-        datanode.getMetrics().incrBlocksVerified();
-        totalScans++;
-      }
-    }
-  }
-  
-  private synchronized long getEarliestScanTime() {
-    if (!blockInfoSet.isEmpty()) {
-      return blockInfoSet.first().lastScanTime;
-    }
-    return Long.MAX_VALUE; 
-  }
-  
-  private synchronized boolean isFirstBlockProcessed() {
-    if (!blockInfoSet.isEmpty()) {
-      long blockId = blockInfoSet.first().getBlockId();
-      if ((processedBlocks.get(blockId) != null)
-          && (processedBlocks.get(blockId) == 1)) {
-        return true;
-      }
-    }
-    return false;
-  }
-  
-  // Picks one block and verifies it
-  private void verifyFirstBlock() {
-    BlockScanInfo block = null;
-    synchronized (this) {
-      if (!blockInfoSet.isEmpty()) {
-        block = blockInfoSet.first();
-      }
-    }
-    if ( block != null ) {
-      verifyBlock(new ExtendedBlock(blockPoolId, block));
-      processedBlocks.put(block.getBlockId(), 1);
-    }
-  }
-  
-  // Used for tests only
-  int getBlocksScannedInLastRun() {
-    return totalBlocksScannedInLastRun.get();
-  }
-
-  /**
-   * Reads the current and previous log files (if any) and marks the blocks
-   * processed if they were processed within last scan period. Copies the log
-   * records of recently scanned blocks from previous to current file. 
-   * Returns false if the process was interrupted because the thread is marked 
-   * to exit.
-   */
-  private boolean assignInitialVerificationTimes() {
-    //First updates the last verification times from the log file.
-    if (verificationLog != null) {
-      long now = Time.monotonicNow();
-      RollingLogs.LineIterator logIterator = null;
-      try {
-        logIterator = verificationLog.logs.iterator(false);
-        // update verification times from the verificationLog.
-        while (logIterator.hasNext()) {
-          if (!datanode.shouldRun
-              || datanode.blockScanner.blockScannerThread.isInterrupted()) {
-            return false;
-          }
-          LogEntry entry = LogEntry.parseEntry(logIterator.next());
-          if (entry != null) {
-            updateBlockInfo(entry);
-            if (now - entry.verificationTime < scanPeriod) {
-              BlockScanInfo info = blockMap.get(new Block(entry.blockId, 0,
-                  entry.genStamp));
-              if (info != null) {
-                if (processedBlocks.get(entry.blockId) == null) {
-                  if (isNewPeriod) {
-                    updateBytesLeft(-info.getNumBytes());
-                  }
-                  processedBlocks.put(entry.blockId, 1);
-                }
-                if (logIterator.isLastReadFromPrevious()) {
-                  // write the log entry to current file
-                  // so that the entry is preserved for later runs.
-                  verificationLog.append(entry.verificationTime, entry.genStamp,
-                      entry.blockId);
-                }
-              }
-            }
-          }
-        }
-      } catch (IOException e) {
-        LOG.warn("Failed to read previous verification times.", e);
-      } finally {
-        IOUtils.closeStream(logIterator);
-      }
-      isNewPeriod = false;
-    }
-    
-    
-    /* Before this loop, entries in blockInfoSet that are not
-     * updated above have lastScanTime of <= 0 . Loop until first entry has
-     * lastModificationTime > 0.
-     */    
-    synchronized (this) {
-      final int numBlocks = Math.max(blockMap.size(), 1);
-      // Initially spread the block reads over half of scan period
-      // so that we don't keep scanning the blocks too quickly when restarted.
-      long verifyInterval = Math.min(scanPeriod/(2L * numBlocks), 10*60*1000L);
-      long lastScanTime = Time.monotonicNow() - scanPeriod;
-
-      if (!blockInfoSet.isEmpty()) {
-        BlockScanInfo info;
-        while ((info =  blockInfoSet.first()).lastScanTime < 0) {
-          delBlockInfo(info);        
-          info.lastScanTime = lastScanTime;
-          lastScanTime += verifyInterval;
-          addBlockInfo(info, false);
-        }
-      }
-    }
-    
-    return true;
-  }
-  
-  private synchronized void updateBytesLeft(long len) {
-    bytesLeft += len;
-  }
-  
-  private synchronized void startNewPeriod() {
-    LOG.info("Starting a new period : work left in prev period : "
-        + String.format("%.2f%%", totalBytesToScan == 0 ? 0
-            : (bytesLeft * 100.0) / totalBytesToScan));
-
-    // reset the byte counts :
-    bytesLeft = totalBytesToScan;
-    currentPeriodStart = Time.monotonicNow();
-    isNewPeriod = true;
-  }
-  
-  private synchronized boolean workRemainingInCurrentPeriod() {
-    if (bytesLeft <= 0 && Time.monotonicNow() < currentPeriodStart + scanPeriod) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Skipping scan since bytesLeft=" + bytesLeft + ", Start=" +
-                  currentPeriodStart + ", period=" + scanPeriod + ", now=" +
-                  Time.monotonicNow() + " " + blockPoolId);
-      }
-      return false;
-    } else {
-      return true;
-    }
-  }
-
-  void scanBlockPoolSlice() {
-    if (!workRemainingInCurrentPeriod()) {
-      return;
-    }
-
-    // Create a new processedBlocks structure
-    processedBlocks = new HashMap<Long, Integer>();
-    if (!assignInitialVerificationTimes()) {
-      return;
-    }
-    // Start scanning
-    try {
-      scan();
-    } finally {
-      totalBlocksScannedInLastRun.set(processedBlocks.size());
-      lastScanTime.set(Time.monotonicNow());
-    }
-  }
-
-  /**
-   * Shuts down this BlockPoolSliceScanner and releases any internal resources.
-   */
-  void shutdown() {
-    if (verificationLog != null) {
-      verificationLog.close();
-    }
-  }
-  
-  private void scan() {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Starting to scan blockpool: " + blockPoolId);
-    }
-    try {
-      adjustThrottler();
-        
-      while (datanode.shouldRun
-          && !datanode.blockScanner.blockScannerThread.isInterrupted()
-          && datanode.isBPServiceAlive(blockPoolId)) {
-        long now = Time.monotonicNow();
-        synchronized (this) {
-          if ( now >= (currentPeriodStart + scanPeriod)) {
-            startNewPeriod();
-          }
-        }
-        if (((now - getEarliestScanTime()) >= scanPeriod)
-            || ((!blockInfoSet.isEmpty()) && !(this.isFirstBlockProcessed()))) {
-          verifyFirstBlock();
-        } else {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("All remaining blocks were processed recently, "
-                + "so this run is complete");
-          }
-          break;
-        }
-      }
-    } catch (RuntimeException e) {
-      LOG.warn("RuntimeException during BlockPoolScanner.scan()", e);
-      throw e;
-    } finally {
-      rollVerificationLogs();
-      rollNewBlocksInfo();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Done scanning block pool: " + blockPoolId);
-      }
-    }
-  }
-
-  // add new blocks to scan in next iteration
-  private synchronized void rollNewBlocksInfo() {
-    for (BlockScanInfo newBlock : newBlockInfoSet) {
-      blockInfoSet.add(newBlock);
-    }
-    newBlockInfoSet.clear();
-  }
-
-  private synchronized void rollVerificationLogs() {
-    if (verificationLog != null) {
-      try {
-        verificationLog.logs.roll();
-      } catch (IOException ex) {
-        LOG.warn("Received exception: ", ex);
-        verificationLog.close();
-      }
-    }
-  }
-
-  
-  synchronized void printBlockReport(StringBuilder buffer, 
-                                     boolean summaryOnly) {
-    long oneHour = 3600*1000;
-    long oneDay = 24*oneHour;
-    long oneWeek = 7*oneDay;
-    long fourWeeks = 4*oneWeek;
-    
-    int inOneHour = 0;
-    int inOneDay = 0;
-    int inOneWeek = 0;
-    int inFourWeeks = 0;
-    int inScanPeriod = 0;
-    int neverScanned = 0;
-    
-    DateFormat dateFormat = new SimpleDateFormat(DATA_FORMAT);
-    
-    int total = blockInfoSet.size();
-    
-    long now = Time.monotonicNow();
-    
-    Date date = new Date();
-    
-    for(Iterator<BlockScanInfo> it = blockInfoSet.iterator(); it.hasNext();) {
-      BlockScanInfo info = it.next();
-      
-      long scanTime = info.getLastScanTime();
-      long diff = now - scanTime;
-      
-      if (diff <= oneHour) inOneHour++;
-      if (diff <= oneDay) inOneDay++;
-      if (diff <= oneWeek) inOneWeek++;
-      if (diff <= fourWeeks) inFourWeeks++;
-      if (diff <= scanPeriod) inScanPeriod++;      
-      if (scanTime <= 0) neverScanned++;
-      
-      if (!summaryOnly) {
-        date.setTime(scanTime);
-        String scanType = 
-          (info.lastScanType == ScanType.VERIFICATION_SCAN) ? "local" : "none"; 
-        buffer.append(String.format("%-26s : status : %-6s type : %-6s" +
-                                    " scan time : " +
-                                    "%-15d %s%n", info,
-                                    (info.lastScanOk ? "ok" : "failed"),
-                                    scanType, scanTime,
-                                    (scanTime <= 0) ? "not yet verified" : 
-                                      dateFormat.format(date)));
-      }
-    }
-    
-    double pctPeriodLeft = (scanPeriod + currentPeriodStart - now)
-                           *100.0/scanPeriod;
-    double pctProgress = (totalBytesToScan == 0) ? 100 :
-                         (totalBytesToScan-bytesLeft)*100.0/totalBytesToScan;
-                         
-    buffer.append(String.format("%nTotal Blocks                 : %6d" +
-                                "%nVerified in last hour        : %6d" +
-                                "%nVerified in last day         : %6d" +
-                                "%nVerified in last week        : %6d" +
-                                "%nVerified in last four weeks  : %6d" +
-                                "%nVerified in SCAN_PERIOD      : %6d" +
-                                "%nNot yet verified             : %6d" +
-                                "%nVerified since restart       : %6d" +
-                                "%nScans since restart          : %6d" +
-                                "%nScan errors since restart    : %6d" +
-                                "%nTransient scan errors        : %6d" +
-                                "%nCurrent scan rate limit KBps : %6d" +
-                                "%nProgress this period         : %6.0f%%" +
-                                "%nTime left in cur period      : %6.2f%%" +
-                                "%n", 
-                                total, inOneHour, inOneDay, inOneWeek,
-                                inFourWeeks, inScanPeriod, neverScanned,
-                                totalScans, totalScans, 
-                                totalScanErrors, totalTransientErrors, 
-                                Math.round(throttler.getBandwidth()/1024.0),
-                                pctProgress, pctPeriodLeft));
-  }
-  
-  /**
-   * This class takes care of log file used to store the last verification
-   * times of the blocks.
-   */
-  private static class LogFileHandler {
-    private final DateFormat dateFormat = new SimpleDateFormat(DATA_FORMAT);
-
-    private final RollingLogs logs;
-
-    private LogFileHandler(RollingLogs logs)  {
-      this.logs = logs;
-    }
-
-    void append(long verificationTime, long genStamp, long blockId) {
-      final String m = LogEntry.toString(verificationTime, genStamp, blockId,
-          dateFormat);
-      try {
-        logs.appender().append(m);
-      } catch (IOException e) {
-        LOG.warn("Failed to append to " + logs + ", m=" + m, e);
-      }
-    }
-
-    void close() {
-      try {
-        logs.appender().close();
-      } catch (IOException e) {
-        LOG.warn("Failed to close the appender of " + logs, e);
-      }
-    }
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
index 8c819a754de7d..4076a8b4c621d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
@@ -275,15 +275,19 @@ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOExce
   }
 
   /**
-   * Remove storage directories.
-   * @param storageDirs a set of storage directories to be removed.
+   * Remove block pool level storage directory.
+   * @param absPathToRemove the absolute path of the root for the block pool
+   *                        level storage to remove.
    */
-  void removeVolumes(Set<File> storageDirs) {
+  void remove(File absPathToRemove) {
+    Preconditions.checkArgument(absPathToRemove.isAbsolute());
+    LOG.info("Removing block level storage: " + absPathToRemove);
     for (Iterator<StorageDirectory> it = this.storageDirs.iterator();
          it.hasNext(); ) {
       StorageDirectory sd = it.next();
-      if (storageDirs.contains(sd.getRoot())) {
+      if (sd.getRoot().getAbsoluteFile().equals(absPathToRemove)) {
         it.remove();
+        break;
       }
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 2e388f9b9c6e2..65b4f2d074a04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -26,9 +26,10 @@
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileOutputStream;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -47,10 +48,8 @@
 import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.util.DataTransferThrottler;
 import org.apache.hadoop.io.IOUtils;
@@ -123,6 +122,8 @@ class BlockReceiver implements Closeable {
 
   private boolean syncOnClose;
   private long restartBudget;
+  /** the reference of the volume where the block receiver writes to */
+  private ReplicaHandler replicaHandler;
 
   /**
    * for replaceBlock response
@@ -177,48 +178,42 @@ class BlockReceiver implements Closeable {
       // Open local disk out
       //
       if (isDatanode) { //replication or move
-        replicaInfo = datanode.data.createTemporary(storageType, block);
+        replicaHandler = datanode.data.createTemporary(storageType, block);
       } else {
         switch (stage) {
         case PIPELINE_SETUP_CREATE:
-          replicaInfo = datanode.data.createRbw(storageType, block, allowLazyPersist);
+          replicaHandler = datanode.data.createRbw(storageType, block, allowLazyPersist);
           datanode.notifyNamenodeReceivingBlock(
-              block, replicaInfo.getStorageUuid());
+              block, replicaHandler.getReplica().getStorageUuid());
           break;
         case PIPELINE_SETUP_STREAMING_RECOVERY:
-          replicaInfo = datanode.data.recoverRbw(
+          replicaHandler = datanode.data.recoverRbw(
               block, newGs, minBytesRcvd, maxBytesRcvd);
           block.setGenerationStamp(newGs);
           break;
         case PIPELINE_SETUP_APPEND:
-          replicaInfo = datanode.data.append(block, newGs, minBytesRcvd);
-          if (datanode.blockScanner != null) { // remove from block scanner
-            datanode.blockScanner.deleteBlock(block.getBlockPoolId(),
-                block.getLocalBlock());
-          }
+          replicaHandler = datanode.data.append(block, newGs, minBytesRcvd);
           block.setGenerationStamp(newGs);
           datanode.notifyNamenodeReceivingBlock(
-              block, replicaInfo.getStorageUuid());
+              block, replicaHandler.getReplica().getStorageUuid());
           break;
         case PIPELINE_SETUP_APPEND_RECOVERY:
-          replicaInfo = datanode.data.recoverAppend(block, newGs, minBytesRcvd);
-          if (datanode.blockScanner != null) { // remove from block scanner
-            datanode.blockScanner.deleteBlock(block.getBlockPoolId(),
-                block.getLocalBlock());
-          }
+          replicaHandler = datanode.data.recoverAppend(block, newGs, minBytesRcvd);
           block.setGenerationStamp(newGs);
           datanode.notifyNamenodeReceivingBlock(
-              block, replicaInfo.getStorageUuid());
+              block, replicaHandler.getReplica().getStorageUuid());
           break;
         case TRANSFER_RBW:
         case TRANSFER_FINALIZED:
           // this is a transfer destination
-          replicaInfo = datanode.data.createTemporary(storageType, block);
+          replicaHandler =
+              datanode.data.createTemporary(storageType, block);
           break;
         default: throw new IOException("Unsupported stage " + stage + 
               " while receiving block " + block + " from " + inAddr);
         }
       }
+      replicaInfo = replicaHandler.getReplica();
       this.dropCacheBehindWrites = (cachingStrategy.getDropBehind() == null) ?
         datanode.getDnConf().dropCacheBehindWrites :
           cachingStrategy.getDropBehind();
@@ -337,6 +332,10 @@ public void close() throws IOException {
     finally{
       IOUtils.closeStream(out);
     }
+    if (replicaHandler != null) {
+      IOUtils.cleanup(null, replicaHandler);
+      replicaHandler = null;
+    }
     if (measuredFlushTime) {
       datanode.metrics.addFlushNanos(flushTotalNanos);
     }
@@ -836,9 +835,8 @@ void receiveBlock(
               LOG.warn("Failed to delete restart meta file: " +
                   restartMeta.getPath());
             }
-            FileWriter out = null;
-            try {
-              out = new FileWriter(restartMeta);
+            try (Writer out = new OutputStreamWriter(
+                new FileOutputStream(restartMeta), "UTF-8")) {
               // write out the current time.
               out.write(Long.toString(Time.now() + restartBudget));
               out.flush();
@@ -949,15 +947,12 @@ private Checksum computePartialChunkCrc(long blkoff, long ckoff)
     //
     byte[] buf = new byte[sizePartialChunk];
     byte[] crcbuf = new byte[checksumSize];
-    ReplicaInputStreams instr = null;
-    try { 
-      instr = datanode.data.getTmpInputStreams(block, blkoff, ckoff);
+    try (ReplicaInputStreams instr =
+        datanode.data.getTmpInputStreams(block, blkoff, ckoff)) {
       IOUtils.readFully(instr.getDataIn(), buf, 0, sizePartialChunk);
 
       // open meta file and read in crc value computer earlier
       IOUtils.readFully(instr.getChecksumIn(), crcbuf, 0, crcbuf.length);
-    } finally {
-      IOUtils.closeStream(instr);
     }
 
     // compute crc of partial chunk from data read in the block file.
@@ -983,9 +978,7 @@ private Checksum computePartialChunkCrc(long blkoff, long ckoff)
   private static enum PacketResponderType {
     NON_PIPELINE, LAST_IN_PIPELINE, HAS_DOWNSTREAM_IN_PIPELINE
   }
-  
-  private static final Status[] MIRROR_ERROR_STATUS = {Status.SUCCESS, Status.ERROR};
-  
+
   /**
    * Processes responses from downstream datanodes in the pipeline
    * and sends back replies to the originator.
@@ -1089,7 +1082,7 @@ void sendOOBResponse(final Status ackStatus) throws IOException,
       LOG.info("Sending an out of band ack of type " + ackStatus);
       try {
         sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L,
-            ackStatus);
+            PipelineAck.combineHeader(datanode.getECN(), ackStatus));
       } finally {
         // Let others send ack. Unless there are miltiple OOB send
         // calls, there can be only one waiter, the responder thread.
@@ -1172,7 +1165,8 @@ public void run() {
               if (oobStatus != null) {
                 LOG.info("Relaying an out of band ack of type " + oobStatus);
                 sendAckUpstream(ack, PipelineAck.UNKOWN_SEQNO, 0L, 0L,
-                    Status.SUCCESS);
+                    PipelineAck.combineHeader(datanode.getECN(),
+                      Status.SUCCESS));
                 continue;
               }
               seqno = ack.getSeqno();
@@ -1243,33 +1237,13 @@ public void run() {
 
           if (lastPacketInBlock) {
             // Finalize the block and close the block file
-            try {
-              finalizeBlock(startTime);
-            } catch (ReplicaNotFoundException e) {
-              // Verify that the exception is due to volume removal.
-              FsVolumeSpi volume;
-              synchronized (datanode.data) {
-                volume = datanode.data.getVolume(block);
-              }
-              if (volume == null) {
-                // ReplicaInfo has been removed due to the corresponding data
-                // volume has been removed. Don't need to check disk error.
-                LOG.info(myString
-                    + ": BlockReceiver is interrupted because the block pool "
-                    + block.getBlockPoolId() + " has been removed.", e);
-                sendAckUpstream(ack, expected, totalAckTimeNanos, 0,
-                    Status.OOB_INTERRUPTED);
-                running = false;
-                receiverThread.interrupt();
-                continue;
-              }
-              throw e;
-            }
+            finalizeBlock(startTime);
           }
 
+          Status myStatus = pkt != null ? pkt.ackStatus : Status.SUCCESS;
           sendAckUpstream(ack, expected, totalAckTimeNanos,
-              (pkt != null ? pkt.offsetInBlock : 0), 
-              (pkt != null ? pkt.ackStatus : Status.SUCCESS));
+            (pkt != null ? pkt.offsetInBlock : 0),
+            PipelineAck.combineHeader(datanode.getECN(), myStatus));
           if (pkt != null) {
             // remove the packet from the ack queue
             removeAckHead();
@@ -1329,11 +1303,11 @@ private void finalizeBlock(long startTime) throws IOException {
      * @param totalAckTimeNanos total ack time including all the downstream
      *          nodes
      * @param offsetInBlock offset in block for the data in packet
-     * @param myStatus the local ack status
+     * @param myHeader the local ack header
      */
     private void sendAckUpstream(PipelineAck ack, long seqno,
         long totalAckTimeNanos, long offsetInBlock,
-        Status myStatus) throws IOException {
+        int myHeader) throws IOException {
       try {
         // Wait for other sender to finish. Unless there is an OOB being sent,
         // the responder won't have to wait.
@@ -1347,7 +1321,7 @@ private void sendAckUpstream(PipelineAck ack, long seqno,
         try {
           if (!running) return;
           sendAckUpstreamUnprotected(ack, seqno, totalAckTimeNanos,
-              offsetInBlock, myStatus);
+              offsetInBlock, myHeader);
         } finally {
           synchronized(this) {
             sending = false;
@@ -1367,32 +1341,34 @@ private void sendAckUpstream(PipelineAck ack, long seqno,
      * @param totalAckTimeNanos total ack time including all the downstream
      *          nodes
      * @param offsetInBlock offset in block for the data in packet
-     * @param myStatus the local ack status
+     * @param myHeader the local ack header
      */
     private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
-        long totalAckTimeNanos, long offsetInBlock, Status myStatus)
+        long totalAckTimeNanos, long offsetInBlock, int myHeader)
         throws IOException {
-      Status[] replies = null;
+      final int[] replies;
       if (ack == null) {
         // A new OOB response is being sent from this node. Regardless of
         // downstream nodes, reply should contain one reply.
-        replies = new Status[1];
-        replies[0] = myStatus;
+        replies = new int[] { myHeader };
       } else if (mirrorError) { // ack read error
-        replies = MIRROR_ERROR_STATUS;
+        int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS);
+        int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR);
+        replies = new int[] {h, h1};
       } else {
         short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack
             .getNumOfReplies();
-        replies = new Status[1 + ackLen];
-        replies[0] = myStatus;
-        for (int i = 0; i < ackLen; i++) {
+        replies = new int[ackLen + 1];
+        replies[0] = myHeader;
+        for (int i = 0; i < ackLen; ++i) {
           replies[i + 1] = ack.getReply(i);
         }
         // If the mirror has reported that it received a corrupt packet,
-        // do self-destruct to mark myself bad, instead of making the 
+        // do self-destruct to mark myself bad, instead of making the
         // mirror node bad. The mirror is guaranteed to be good without
         // corrupt data on disk.
-        if (ackLen > 0 && replies[1] == Status.ERROR_CHECKSUM) {
+        if (ackLen > 0 && PipelineAck.getStatusFromHeader(replies[1]) ==
+          Status.ERROR_CHECKSUM) {
           throw new IOException("Shutting down writer and responder "
               + "since the down streams reported the data sent by this "
               + "thread is corrupt");
@@ -1418,7 +1394,8 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
       }
 
       // If a corruption was detected in the received data, terminate after
-      // sending ERROR_CHECKSUM back. 
+      // sending ERROR_CHECKSUM back.
+      Status myStatus = PipelineAck.getStatusFromHeader(myHeader);
       if (myStatus == Status.ERROR_CHECKSUM) {
         throw new IOException("Shutting down writer and responder "
             + "due to a checksum error in received data. The error "
@@ -1438,7 +1415,7 @@ private void removeAckHead() {
       }
     }
   }
-  
+
   /**
    * This information is cached by the Datanode in the ackQueue.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
new file mode 100644
index 0000000000000..7429fff9adbc0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -0,0 +1,308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.ScanResultHandler;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@InterfaceAudience.Private
+public class BlockScanner {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(BlockScanner.class);
+
+  /**
+   * The DataNode that this scanner is associated with.
+   */
+  private final DataNode datanode;
+
+  /**
+   * Maps Storage IDs to VolumeScanner objects.
+   */
+  private final TreeMap<String, VolumeScanner> scanners =
+      new TreeMap<String, VolumeScanner>();
+
+  /**
+   * The scanner configuration.
+   */
+  private final Conf conf;
+
+  /**
+   * The cached scanner configuration.
+   */
+  static class Conf {
+    // These are a few internal configuration keys used for unit tests.
+    // They can't be set unless the static boolean allowUnitTestSettings has
+    // been set to true.
+
+    @VisibleForTesting
+    static final String INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS =
+        "internal.dfs.datanode.scan.period.ms.key";
+
+    @VisibleForTesting
+    static final String INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER =
+        "internal.volume.scanner.scan.result.handler";
+
+    @VisibleForTesting
+    static final String INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS =
+        "internal.dfs.block.scanner.max_staleness.ms";
+
+    @VisibleForTesting
+    static final long INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS_DEFAULT =
+        TimeUnit.MILLISECONDS.convert(15, TimeUnit.MINUTES);
+
+    @VisibleForTesting
+    static final String INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS =
+        "dfs.block.scanner.cursor.save.interval.ms";
+
+    @VisibleForTesting
+    static final long
+        INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS_DEFAULT =
+            TimeUnit.MILLISECONDS.convert(10, TimeUnit.MINUTES);
+
+    static boolean allowUnitTestSettings = false;
+    final long targetBytesPerSec;
+    final long maxStalenessMs;
+    final long scanPeriodMs;
+    final long cursorSaveMs;
+    final Class<? extends ScanResultHandler> resultHandler;
+
+    private static long getUnitTestLong(Configuration conf, String key,
+                                        long defVal) {
+      if (allowUnitTestSettings) {
+        return conf.getLong(key, defVal);
+      } else {
+        return defVal;
+      }
+    }
+
+    @SuppressWarnings("unchecked")
+    Conf(Configuration conf) {
+      this.targetBytesPerSec = Math.max(0L, conf.getLong(
+          DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND,
+          DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT));
+      this.maxStalenessMs = Math.max(0L, getUnitTestLong(conf,
+          INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS,
+          INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS_DEFAULT));
+      this.scanPeriodMs = Math.max(0L,
+          getUnitTestLong(conf, INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS,
+              TimeUnit.MILLISECONDS.convert(conf.getLong(
+                  DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,
+                  DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT), TimeUnit.HOURS)));
+      this.cursorSaveMs = Math.max(0L, getUnitTestLong(conf,
+          INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS,
+          INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS_DEFAULT));
+      if (allowUnitTestSettings) {
+        this.resultHandler = (Class<? extends ScanResultHandler>)
+            conf.getClass(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+                          ScanResultHandler.class);
+      } else {
+        this.resultHandler = ScanResultHandler.class;
+      }
+    }
+  }
+
+  public BlockScanner(DataNode datanode, Configuration conf) {
+    this.datanode = datanode;
+    this.conf = new Conf(conf);
+    if (isEnabled()) {
+      LOG.info("Initialized block scanner with targetBytesPerSec {}",
+          this.conf.targetBytesPerSec);
+    } else {
+      LOG.info("Disabled block scanner.");
+    }
+  }
+
+  /**
+   * Returns true if the block scanner is enabled.<p/>
+   *
+   * If the block scanner is disabled, no volume scanners will be created, and
+   * no threads will start.
+   */
+  public boolean isEnabled() {
+    return (conf.scanPeriodMs) > 0 && (conf.targetBytesPerSec > 0);
+  }
+
+ /**
+  * Set up a scanner for the given block pool and volume.
+  *
+  * @param ref              A reference to the volume.
+  */
+  public synchronized void addVolumeScanner(FsVolumeReference ref) {
+    boolean success = false;
+    try {
+      FsVolumeSpi volume = ref.getVolume();
+      if (!isEnabled()) {
+        LOG.debug("Not adding volume scanner for {}, because the block " +
+            "scanner is disabled.", volume.getBasePath());
+        return;
+      }
+      VolumeScanner scanner = scanners.get(volume.getStorageID());
+      if (scanner != null) {
+        LOG.error("Already have a scanner for volume {}.",
+            volume.getBasePath());
+        return;
+      }
+      LOG.debug("Adding scanner for volume {} (StorageID {})",
+          volume.getBasePath(), volume.getStorageID());
+      scanner = new VolumeScanner(conf, datanode, ref);
+      scanner.start();
+      scanners.put(volume.getStorageID(), scanner);
+      success = true;
+    } finally {
+      if (!success) {
+        // If we didn't create a new VolumeScanner object, we don't
+        // need this reference to the volume.
+        IOUtils.cleanup(null, ref);
+      }
+    }
+  }
+
+  /**
+   * Stops and removes a volume scanner.<p/>
+   *
+   * This function will block until the volume scanner has stopped.
+   *
+   * @param volume           The volume to remove.
+   */
+  public synchronized void removeVolumeScanner(FsVolumeSpi volume) {
+    if (!isEnabled()) {
+      LOG.debug("Not removing volume scanner for {}, because the block " +
+          "scanner is disabled.", volume.getStorageID());
+      return;
+    }
+    VolumeScanner scanner = scanners.get(volume.getStorageID());
+    if (scanner == null) {
+      LOG.warn("No scanner found to remove for volumeId {}",
+          volume.getStorageID());
+      return;
+    }
+    LOG.info("Removing scanner for volume {} (StorageID {})",
+        volume.getBasePath(), volume.getStorageID());
+    scanner.shutdown();
+    scanners.remove(volume.getStorageID());
+    Uninterruptibles.joinUninterruptibly(scanner, 5, TimeUnit.MINUTES);
+  }
+
+  /**
+   * Stops and removes all volume scanners.<p/>
+   *
+   * This function will block until all the volume scanners have stopped.
+   */
+  public synchronized void removeAllVolumeScanners() {
+    for (Entry<String, VolumeScanner> entry : scanners.entrySet()) {
+      entry.getValue().shutdown();
+    }
+    for (Entry<String, VolumeScanner> entry : scanners.entrySet()) {
+      Uninterruptibles.joinUninterruptibly(entry.getValue(),
+          5, TimeUnit.MINUTES);
+    }
+    scanners.clear();
+  }
+
+  /**
+   * Enable scanning a given block pool id.
+   *
+   * @param bpid        The block pool id to enable scanning for.
+   */
+  synchronized void enableBlockPoolId(String bpid) {
+    Preconditions.checkNotNull(bpid);
+    for (VolumeScanner scanner : scanners.values()) {
+      scanner.enableBlockPoolId(bpid);
+    }
+  }
+
+  /**
+   * Disable scanning a given block pool id.
+   *
+   * @param bpid        The block pool id to disable scanning for.
+   */
+  synchronized void disableBlockPoolId(String bpid) {
+    Preconditions.checkNotNull(bpid);
+    for (VolumeScanner scanner : scanners.values()) {
+      scanner.disableBlockPoolId(bpid);
+    }
+  }
+
+  @VisibleForTesting
+  synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) {
+    VolumeScanner scanner = scanners.get(volumeId);
+    if (scanner == null) {
+      return null;
+    }
+    return scanner.getStatistics();
+  }
+
+  synchronized void printStats(StringBuilder p) {
+    // print out all bpids that we're scanning ?
+    for (Entry<String, VolumeScanner> entry : scanners.entrySet()) {
+      entry.getValue().printStats(p);
+    }
+  }
+
+  @InterfaceAudience.Private
+  public static class Servlet extends HttpServlet {
+    private static final long serialVersionUID = 1L;
+
+    @Override
+    public void doGet(HttpServletRequest request,
+                      HttpServletResponse response) throws IOException {
+      response.setContentType("text/plain");
+
+      DataNode datanode = (DataNode)
+          getServletContext().getAttribute("datanode");
+      BlockScanner blockScanner = datanode.getBlockScanner();
+
+      StringBuilder buffer = new StringBuilder(8 * 1024);
+      if (!blockScanner.isEnabled()) {
+        LOG.warn("Periodic block scanner is not running");
+        buffer.append("Periodic block scanner is not running. " +
+            "Please check the datanode log if this is unexpected.");
+      } else {
+        buffer.append("Block Scanner Statistics\n\n");
+        blockScanner.printStats(buffer);
+      }
+      String resp = buffer.toString();
+      LOG.trace("Returned Servlet info {}", resp);
+      response.getWriter().write(resp);
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 27d3e5ce2a1d7..c016e62e7f24c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
 import org.apache.hadoop.hdfs.util.DataTransferThrottler;
 import org.apache.hadoop.io.IOUtils;
@@ -45,12 +46,12 @@
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.net.SocketOutputStream;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import org.htrace.Sampler;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
 
 /**
  * Reads a block from the disk and sends it to a recipient.
@@ -143,6 +144,8 @@ class BlockSender implements java.io.Closeable {
   
   /** The file descriptor of the block being sent */
   private FileDescriptor blockInFd;
+  /** The reference to the volume where the block is located */
+  private FsVolumeReference volumeRef;
 
   // Cache-management related fields
   private final long readaheadLength;
@@ -257,6 +260,9 @@ class BlockSender implements java.io.Closeable {
       this.transferToAllowed = datanode.getDnConf().transferToAllowed &&
         (!is32Bit || length <= Integer.MAX_VALUE);
 
+      // Obtain a reference before reading data
+      this.volumeRef = datanode.data.getVolume(block).obtainReference();
+
       /* 
        * (corruptChecksumOK, meta_file_exist): operation
        * True,   True: will verify checksum  
@@ -420,6 +426,10 @@ public void close() throws IOException {
       blockIn = null;
       blockInFd = null;
     }
+    if (volumeRef != null) {
+      IOUtils.cleanup(null, volumeRef);
+      volumeRef = null;
+    }
     // throw IOException if there is any
     if(ioe!= null) {
       throw ioe;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ChunkChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ChunkChecksum.java
index cb69f8272349c..714445b8f0be6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ChunkChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ChunkChecksum.java
@@ -26,12 +26,12 @@
  *     the checksum applies for the last chunk, or bytes 512 - 1023
  */
 
-class ChunkChecksum {
+public class ChunkChecksum {
   private final long dataLength;
   // can be null if not available
   private final byte[] checksum;
 
-  ChunkChecksum(long dataLength, byte[] checksum) {
+  public ChunkChecksum(long dataLength, byte[] checksum) {
     this.dataLength = dataLength;
     this.checksum = checksum;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
deleted file mode 100644
index bee362582eea1..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import java.io.IOException;
-import java.util.TreeMap;
-
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * DataBlockScanner manages block scanning for all the block pools. For each
- * block pool a {@link BlockPoolSliceScanner} is created which runs in a separate
- * thread to scan the blocks for that block pool. When a {@link BPOfferService}
- * becomes alive or dies, blockPoolScannerMap in this class is updated.
- */
-@InterfaceAudience.Private
-public class DataBlockScanner implements Runnable {
-  public static final Log LOG = LogFactory.getLog(DataBlockScanner.class);
-  private final DataNode datanode;
-  private final FsDatasetSpi<? extends FsVolumeSpi> dataset;
-  private final Configuration conf;
-  
-  static final int SLEEP_PERIOD_MS = 5 * 1000;
-
-  /**
-   * Map to find the BlockPoolScanner for a given block pool id. This is updated
-   * when a BPOfferService becomes alive or dies.
-   */
-  private final TreeMap<String, BlockPoolSliceScanner> blockPoolScannerMap = 
-    new TreeMap<String, BlockPoolSliceScanner>();
-  Thread blockScannerThread = null;
-  
-  DataBlockScanner(DataNode datanode,
-      FsDatasetSpi<? extends FsVolumeSpi> dataset,
-      Configuration conf) {
-    this.datanode = datanode;
-    this.dataset = dataset;
-    this.conf = conf;
-  }
-  
-  @Override
-  public void run() {
-    String currentBpId = "";
-    boolean firstRun = true;
-    while (datanode.shouldRun && !Thread.interrupted()) {
-      //Sleep everytime except in the first iteration.
-      if (!firstRun) {
-        try {
-          Thread.sleep(SLEEP_PERIOD_MS);
-        } catch (InterruptedException ex) {
-          // Interrupt itself again to set the interrupt status
-          blockScannerThread.interrupt();
-          continue;
-        }
-      } else {
-        firstRun = false;
-      }
-      
-      BlockPoolSliceScanner bpScanner = getNextBPScanner(currentBpId);
-      if (bpScanner == null) {
-        // Possible if thread is interrupted
-        continue;
-      }
-      currentBpId = bpScanner.getBlockPoolId();
-      // If BPOfferService for this pool is not alive, don't process it
-      if (!datanode.isBPServiceAlive(currentBpId)) {
-        LOG.warn("Block Pool " + currentBpId + " is not alive");
-        // Remove in case BP service died abruptly without proper shutdown
-        removeBlockPool(currentBpId);
-        continue;
-      }
-      bpScanner.scanBlockPoolSlice();
-    }
-
-    // Call shutdown for each allocated BlockPoolSliceScanner.
-    for (BlockPoolSliceScanner bpss: blockPoolScannerMap.values()) {
-      bpss.shutdown();
-    }
-  }
-
-  // Wait for at least one block pool to be up
-  private void waitForInit() {
-    while ((getBlockPoolSetSize() < datanode.getAllBpOs().length)
-        || (getBlockPoolSetSize() < 1)) {
-      try {
-        Thread.sleep(SLEEP_PERIOD_MS);
-      } catch (InterruptedException e) {
-        blockScannerThread.interrupt();
-        return;
-      }
-    }
-  }
-  
-  /**
-   * Find next block pool id to scan. There should be only one current
-   * verification log file. Find which block pool contains the current
-   * verification log file and that is used as the starting block pool id. If no
-   * current files are found start with first block-pool in the blockPoolSet.
-   * However, if more than one current files are found, the one with latest 
-   * modification time is used to find the next block pool id.
-   */
-  private BlockPoolSliceScanner getNextBPScanner(String currentBpId) {
-    
-    String nextBpId = null;
-    while (datanode.shouldRun && !blockScannerThread.isInterrupted()) {
-      waitForInit();
-      synchronized (this) {
-        if (getBlockPoolSetSize() > 0) {          
-          // Find nextBpId by the minimum of the last scan time
-          long lastScanTime = 0;
-          for (String bpid : blockPoolScannerMap.keySet()) {
-            final long t = getBPScanner(bpid).getLastScanTime();
-            if (t != 0L) {
-              if (bpid == null || t < lastScanTime) {
-                lastScanTime =  t;
-                nextBpId = bpid;
-              }
-            }
-          }
-          
-          // nextBpId can still be null if no current log is found,
-          // find nextBpId sequentially.
-          if (nextBpId == null) {
-            nextBpId = blockPoolScannerMap.higherKey(currentBpId);
-            if (nextBpId == null) {
-              nextBpId = blockPoolScannerMap.firstKey();
-            }
-          }
-          if (nextBpId != null) {
-            return getBPScanner(nextBpId);
-          }
-        }
-      }
-      LOG.warn("No block pool is up, going to wait");
-      try {
-        Thread.sleep(5000);
-      } catch (InterruptedException ex) {
-        LOG.warn("Received exception: " + ex);
-        blockScannerThread.interrupt();
-        return null;
-      }
-    }
-    return null;
-  }
-
-  private synchronized int getBlockPoolSetSize() {
-    return blockPoolScannerMap.size();
-  }
-  
-  @VisibleForTesting
-  synchronized BlockPoolSliceScanner getBPScanner(String bpid) {
-    return blockPoolScannerMap.get(bpid);
-  }
-  
-  private synchronized String[] getBpIdList() {
-    return blockPoolScannerMap.keySet().toArray(
-        new String[blockPoolScannerMap.keySet().size()]);
-  }
-  
-  public void addBlock(ExtendedBlock block) {
-    BlockPoolSliceScanner bpScanner = getBPScanner(block.getBlockPoolId());
-    if (bpScanner != null) {
-      bpScanner.addBlock(block);
-    } else {
-      LOG.warn("No block pool scanner found for block pool id: "
-          + block.getBlockPoolId());
-    }
-  }
-  
-  boolean isInitialized(String bpid) {
-    return getBPScanner(bpid) != null;
-  }
-
-  public synchronized void printBlockReport(StringBuilder buffer,
-      boolean summary) {
-    String[] bpIdList = getBpIdList();
-    if (bpIdList == null || bpIdList.length == 0) {
-      buffer.append("Periodic block scanner is not yet initialized. "
-          + "Please check back again after some time.");
-      return;
-    }
-    for (String bpid : bpIdList) {
-      BlockPoolSliceScanner bpScanner = getBPScanner(bpid);
-      buffer.append("\n\nBlock report for block pool: "+bpid + "\n");
-      bpScanner.printBlockReport(buffer, summary);
-      buffer.append("\n");
-    }
-  }
-  
-  public void deleteBlock(String poolId, Block toDelete) {
-    BlockPoolSliceScanner bpScanner = getBPScanner(poolId);
-    if (bpScanner != null) {
-      bpScanner.deleteBlock(toDelete);
-    } else {
-      LOG.warn("No block pool scanner found for block pool id: "
-          + poolId);
-    }
-  }
-
-  public void deleteBlocks(String poolId, Block[] toDelete) {
-    BlockPoolSliceScanner bpScanner = getBPScanner(poolId);
-    if (bpScanner != null) {
-      bpScanner.deleteBlocks(toDelete);
-    } else {
-      LOG.warn("No block pool scanner found for block pool id: "
-          + poolId);
-    }
-  }
-  
-  public void shutdown() {
-    synchronized (this) {
-      if (blockScannerThread != null) {
-        blockScannerThread.interrupt();
-      }
-    }
-
-    // We cannot join within the synchronized block, because it would create a
-    // deadlock situation.  blockScannerThread calls other synchronized methods.
-    if (blockScannerThread != null) {
-      try {
-        blockScannerThread.join();
-      } catch (InterruptedException e) {
-        // shutting down anyway
-      }
-    }
-  }
-
-  public synchronized void addBlockPool(String blockPoolId) {
-    if (blockPoolScannerMap.get(blockPoolId) != null) {
-      return;
-    }
-    BlockPoolSliceScanner bpScanner = new BlockPoolSliceScanner(blockPoolId,
-        datanode, dataset, conf);
-    blockPoolScannerMap.put(blockPoolId, bpScanner);
-    LOG.info("Added bpid=" + blockPoolId + " to blockPoolScannerMap, new size="
-        + blockPoolScannerMap.size());
-  }
-  
-  public synchronized void removeBlockPool(String blockPoolId) {
-    BlockPoolSliceScanner bpss = blockPoolScannerMap.remove(blockPoolId);
-    if (bpss != null) {
-      bpss.shutdown();
-    }
-    LOG.info("Removed bpid="+blockPoolId+" from blockPoolScannerMap");
-  }
-  
-  @VisibleForTesting
-  long getBlocksScannedInLastRun(String bpid) throws IOException {
-    BlockPoolSliceScanner bpScanner = getBPScanner(bpid);
-    if (bpScanner == null) {
-      throw new IOException("Block Pool: "+bpid+" is not running");
-    } else {
-      return bpScanner.getBlocksScannedInLastRun();
-    }
-  }
-
-  @VisibleForTesting
-  long getTotalScans(String bpid) throws IOException {
-    BlockPoolSliceScanner bpScanner = getBPScanner(bpid);
-    if (bpScanner == null) {
-      throw new IOException("Block Pool: "+bpid+" is not running");
-    } else {
-      return bpScanner.getTotalScans();
-    }
-  }
-
-  public void start() {
-    blockScannerThread = new Thread(this);
-    blockScannerThread.setDaemon(true);
-    blockScannerThread.start();
-  }
-  
-  @InterfaceAudience.Private
-  public static class Servlet extends HttpServlet {
-    private static final long serialVersionUID = 1L;
-
-    @Override
-    public void doGet(HttpServletRequest request, 
-                      HttpServletResponse response) throws IOException {
-      response.setContentType("text/plain");
-      
-      DataNode datanode = (DataNode) getServletContext().getAttribute("datanode");
-      DataBlockScanner blockScanner = datanode.blockScanner;
-      
-      boolean summary = (request.getParameter("listblocks") == null);
-      
-      StringBuilder buffer = new StringBuilder(8*1024);
-      if (blockScanner == null) {
-        LOG.warn("Periodic block scanner is not running");
-        buffer.append("Periodic block scanner is not running. " +
-                      "Please check the datanode log if this is unexpected.");
-      } else {
-        blockScanner.printBlockReport(buffer, summary);
-      }
-      response.getWriter().write(buffer.toString()); // extra copy!
-    }
-  }
-
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 13c32d530adf2..ed3ec7df9c8e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -124,6 +124,7 @@
 import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
 import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
 import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
 import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
@@ -316,7 +317,7 @@ public static InetSocketAddress createSocketAddr(String target) {
   BlockPoolTokenSecretManager blockPoolTokenSecretManager;
   private boolean hasAnyBlockPoolRegistered = false;
   
-  volatile DataBlockScanner blockScanner = null;
+  private final BlockScanner blockScanner;
   private DirectoryScanner directoryScanner = null;
   
   /** Activated plug-ins. */
@@ -333,6 +334,7 @@ public static InetSocketAddress createSocketAddr(String target) {
   private Configuration conf;
   private final String confVersion;
   private final long maxNumberOfBlocksToLog;
+  private final boolean pipelineSupportECN;
 
   private final List<String> usersWithLocalPathAccess;
   private final boolean connectToDnViaHostname;
@@ -365,6 +367,8 @@ public static InetSocketAddress createSocketAddr(String target) {
     this.usersWithLocalPathAccess = null;
     this.connectToDnViaHostname = false;
     this.getHdfsBlockLocationsEnabled = false;
+    this.blockScanner = new BlockScanner(this, conf);
+    this.pipelineSupportECN = false;
   }
 
   /**
@@ -375,6 +379,7 @@ public static InetSocketAddress createSocketAddr(String target) {
            final List<StorageLocation> dataDirs,
            final SecureResources resources) throws IOException {
     super(conf);
+    this.blockScanner = new BlockScanner(this, conf);
     this.lastDiskErrorCheck = 0;
     this.maxNumberOfBlocksToLog = conf.getLong(DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
         DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT);
@@ -392,6 +397,9 @@ public static InetSocketAddress createSocketAddr(String target) {
     this.isPermissionEnabled = conf.getBoolean(
         DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,
         DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT);
+    this.pipelineSupportECN = conf.getBoolean(
+        DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED,
+        DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT);
 
     confVersion = "core-" +
         conf.get("hadoop.common.configuration.version", "UNSPECIFIED") +
@@ -466,6 +474,19 @@ public Collection<String> getReconfigurableProperties() {
     return reconfigurable;
   }
 
+  /**
+   * The ECN bit for the DataNode. The DataNode should return:
+   * <ul>
+   *   <li>ECN.DISABLED when ECN is disabled.</li>
+   *   <li>ECN.SUPPORTED when ECN is enabled but the DN still has capacity.</li>
+   *   <li>ECN.CONGESTED when ECN is enabled and the DN is congested.</li>
+   * </ul>
+   */
+  public PipelineAck.ECN getECN() {
+    return pipelineSupportECN ? PipelineAck.ECN.SUPPORTED : PipelineAck.ECN
+      .DISABLED;
+  }
+
   /**
    * Contains the StorageLocations for changed data volumes.
    */
@@ -580,7 +601,8 @@ public IOException call() {
           try {
             IOException ioe = ioExceptionFuture.get();
             if (ioe != null) {
-              errorMessageBuilder.append(String.format("FAILED TO ADD: %s: %s\n",
+              errorMessageBuilder.append(
+                  String.format("FAILED TO ADD: %s: %s%n",
                   volume, ioe.getMessage()));
               LOG.error("Failed to add volume: " + volume, ioe);
             } else {
@@ -588,8 +610,9 @@ public IOException call() {
               LOG.info("Successfully added volume: " + volume);
             }
           } catch (Exception e) {
-            errorMessageBuilder.append(String.format("FAILED to ADD: %s: %s\n",
-                volume, e.getMessage()));
+            errorMessageBuilder.append(
+                String.format("FAILED to ADD: %s: %s%n", volume,
+                              e.toString()));
           }
         }
       }
@@ -669,7 +692,8 @@ private void startInfoServer(Configuration conf)
     this.infoServer.setAttribute("datanode", this);
     this.infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
     this.infoServer.addServlet(null, "/blockScannerReport",
-                               DataBlockScanner.Servlet.class);
+                               BlockScanner.Servlet.class);
+
     this.infoServer.start();
     InetSocketAddress jettyAddr = infoServer.getConnectorAddress(0);
 
@@ -770,56 +794,12 @@ private void checkSuperuserPrivilege() throws IOException, AccessControlExceptio
     // Not a superuser.
     throw new AccessControlException();
   }
-  
-/**
- * Initialize the datanode's periodic scanners:
- *     {@link DataBlockScanner}
- *     {@link DirectoryScanner}
- * They report results on a per-blockpool basis but do their scanning 
- * on a per-Volume basis to minimize competition for disk iops.
- * 
- * @param conf - Configuration has the run intervals and other 
- *               parameters for these periodic scanners
- */
-  private void initPeriodicScanners(Configuration conf) {
-    initDataBlockScanner(conf);
-    initDirectoryScanner(conf);
-  }
-  
+
   private void shutdownPeriodicScanners() {
     shutdownDirectoryScanner();
-    shutdownDataBlockScanner();
-  }
-  
-  /**
-   * See {@link DataBlockScanner}
-   */
-  private synchronized void initDataBlockScanner(Configuration conf) {
-    if (blockScanner != null) {
-      return;
-    }
-    String reason = null;
-    assert data != null;
-    if (conf.getInt(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,
-                    DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT) < 0) {
-      reason = "verification is turned off by configuration";
-    } else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) {
-      reason = "verifcation is not supported by SimulatedFSDataset";
-    } 
-    if (reason == null) {
-      blockScanner = new DataBlockScanner(this, data, conf);
-      blockScanner.start();
-    } else {
-      LOG.info("Periodic Block Verification scan disabled because " + reason);
-    }
+    blockScanner.removeAllVolumeScanners();
   }
-  
-  private void shutdownDataBlockScanner() {
-    if (blockScanner != null) {
-      blockScanner.shutdown();
-    }
-  }
-  
+
   /**
    * See {@link DirectoryScanner}
    */
@@ -1248,9 +1228,8 @@ void shutdownBlockPool(BPOfferService bpos) {
       // registering anywhere. If that's the case, we wouldn't have
       // a block pool id
       String bpId = bpos.getBlockPoolId();
-      if (blockScanner != null) {
-        blockScanner.removeBlockPool(bpId);
-      }
+
+      blockScanner.disableBlockPoolId(bpId);
 
       if (data != null) {
         data.shutdownBlockPool(bpId);
@@ -1294,9 +1273,9 @@ void initBlockPool(BPOfferService bpos) throws IOException {
     // failures.
     checkDiskError();
 
-    initPeriodicScanners(conf);
-    
+    initDirectoryScanner(conf);
     data.addBlockPool(nsInfo.getBlockPoolID(), conf);
+    blockScanner.enableBlockPoolId(bpos.getBlockPoolId());
   }
 
   BPOfferService[] getAllBpOs() {
@@ -1616,9 +1595,13 @@ public void shutdown() {
     // in order to avoid any further acceptance of requests, but the peers
     // for block writes are not closed until the clients are notified.
     if (dataXceiverServer != null) {
-      xserver.sendOOBToPeers();
-      ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();
-      this.dataXceiverServer.interrupt();
+      try {
+        xserver.sendOOBToPeers();
+        ((DataXceiverServer) this.dataXceiverServer.getRunnable()).kill();
+        this.dataXceiverServer.interrupt();
+      } catch (Throwable e) {
+        // Ignore, since the out of band messaging is advisory.
+      }
     }
 
     // Interrupt the checkDiskErrorThread and terminate it.
@@ -2162,10 +2145,6 @@ void closeBlock(ExtendedBlock block, String delHint, String storageUuid) {
       LOG.warn("Cannot find BPOfferService for reporting block received for bpid="
           + block.getBlockPoolId());
     }
-    FsVolumeSpi volume = getFSDataset().getVolume(block);
-    if (blockScanner != null && !volume.isTransientStorage()) {
-      blockScanner.addBlock(block);
-    }
   }
 
   /** Start a single datanode daemon and wait for it to finish.
@@ -2439,8 +2418,9 @@ public FsDatasetSpi<?> getFSDataset() {
     return data;
   }
 
+  @VisibleForTesting
   /** @return the block scanner. */
-  public DataBlockScanner getBlockScanner() {
+  public BlockScanner getBlockScanner() {
     return blockScanner;
   }
 
@@ -2524,14 +2504,16 @@ private static ReplicaRecoveryInfo callInitReplicaRecovery(
    */
   @Override // InterDatanodeProtocol
   public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock,
-      final long recoveryId, final long newLength) throws IOException {
+      final long recoveryId, final long newBlockId, final long newLength)
+      throws IOException {
     final String storageID = data.updateReplicaUnderRecovery(oldBlock,
-        recoveryId, newLength);
+        recoveryId, newBlockId, newLength);
     // Notify the namenode of the updated block info. This is important
     // for HA, since otherwise the standby node may lose track of the
     // block locations until the next block report.
     ExtendedBlock newBlock = new ExtendedBlock(oldBlock);
     newBlock.setGenerationStamp(recoveryId);
+    newBlock.setBlockId(newBlockId);
     newBlock.setNumBytes(newLength);
     notifyNamenodeReceivedBlock(newBlock, "", storageID);
     return storageID;
@@ -2553,10 +2535,12 @@ static class BlockRecord {
       this.rInfo = rInfo;
     }
 
-    void updateReplicaUnderRecovery(String bpid, long recoveryId, long newLength 
-        ) throws IOException {
+    void updateReplicaUnderRecovery(String bpid, long recoveryId,
+                                    long newBlockId, long newLength)
+        throws IOException {
       final ExtendedBlock b = new ExtendedBlock(bpid, rInfo);
-      storageID = datanode.updateReplicaUnderRecovery(b, recoveryId, newLength);
+      storageID = datanode.updateReplicaUnderRecovery(b, recoveryId, newBlockId,
+          newLength);
     }
 
     @Override
@@ -2638,8 +2622,12 @@ void syncBlock(RecoveringBlock rBlock,
     final String bpid = block.getBlockPoolId();
     DatanodeProtocolClientSideTranslatorPB nn =
       getActiveNamenodeForBP(block.getBlockPoolId());
-    
+
     long recoveryId = rBlock.getNewGenerationStamp();
+    boolean isTruncateRecovery = rBlock.getNewBlock() != null;
+    long blockId = (isTruncateRecovery) ?
+        rBlock.getNewBlock().getBlockId() : block.getBlockId();
+
     if (LOG.isDebugEnabled()) {
       LOG.debug("block=" + block + ", (length=" + block.getNumBytes()
           + "), syncList=" + syncList);
@@ -2673,7 +2661,7 @@ void syncBlock(RecoveringBlock rBlock,
     // Calculate list of nodes that will participate in the recovery
     // and the new block size
     List<BlockRecord> participatingList = new ArrayList<BlockRecord>();
-    final ExtendedBlock newBlock = new ExtendedBlock(bpid, block.getBlockId(),
+    final ExtendedBlock newBlock = new ExtendedBlock(bpid, blockId,
         -1, recoveryId);
     switch(bestState) {
     case FINALIZED:
@@ -2703,12 +2691,15 @@ void syncBlock(RecoveringBlock rBlock,
     case TEMPORARY:
       assert false : "bad replica state: " + bestState;
     }
+    if(isTruncateRecovery)
+      newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
 
     List<DatanodeID> failedList = new ArrayList<DatanodeID>();
     final List<BlockRecord> successList = new ArrayList<BlockRecord>();
     for(BlockRecord r : participatingList) {
       try {
-        r.updateReplicaUnderRecovery(bpid, recoveryId, newBlock.getNumBytes());
+        r.updateReplicaUnderRecovery(bpid, recoveryId, blockId,
+            newBlock.getNumBytes());
         successList.add(r);
       } catch (IOException e) {
         InterDatanodeProtocol.LOG.warn("Failed to updateBlock (newblock="
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index c90ef954d923f..754df2c7353ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ComparisonChain;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.Futures;
@@ -57,13 +58,16 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
@@ -138,11 +142,20 @@ public synchronized void setDatanodeUuid(String newDatanodeUuid) {
     this.datanodeUuid = newDatanodeUuid;
   }
 
-  /** Create an ID for this storage. */
-  public synchronized void createStorageID(StorageDirectory sd) {
-    if (sd.getStorageUuid() == null) {
+  /** Create an ID for this storage.
+   * @return true if a new storage ID was generated.
+   * */
+  public synchronized boolean createStorageID(
+      StorageDirectory sd, boolean regenerateStorageIds) {
+    final String oldStorageID = sd.getStorageUuid();
+    if (oldStorageID == null || regenerateStorageIds) {
       sd.setStorageUuid(DatanodeStorage.generateUuid());
+      LOG.info("Generated new storageID " + sd.getStorageUuid() +
+          " for directory " + sd.getRoot() +
+          (oldStorageID == null ? "" : (" to replace " + oldStorageID)));
+      return true;
     }
+    return false;
   }
 
   /**
@@ -409,15 +422,21 @@ synchronized void removeVolumes(Collection<StorageLocation> locations)
       dataDirs.add(sl.getFile());
     }
 
-    for (BlockPoolSliceStorage bpsStorage : this.bpStorageMap.values()) {
-      bpsStorage.removeVolumes(dataDirs);
-    }
-
     StringBuilder errorMsgBuilder = new StringBuilder();
     for (Iterator<StorageDirectory> it = this.storageDirs.iterator();
          it.hasNext(); ) {
       StorageDirectory sd = it.next();
       if (dataDirs.contains(sd.getRoot())) {
+        // Remove the block pool level storage first.
+        for (Map.Entry<String, BlockPoolSliceStorage> entry :
+            this.bpStorageMap.entrySet()) {
+          String bpid = entry.getKey();
+          BlockPoolSliceStorage bpsStorage = entry.getValue();
+          File bpRoot =
+              BlockPoolSliceStorage.getBpRoot(bpid, sd.getCurrentDir());
+          bpsStorage.remove(bpRoot.getAbsoluteFile());
+        }
+
         it.remove();
         try {
           sd.unlock();
@@ -425,7 +444,7 @@ synchronized void removeVolumes(Collection<StorageLocation> locations)
           LOG.warn(String.format(
             "I/O error attempting to unlock storage directory %s.",
             sd.getRoot()), e);
-          errorMsgBuilder.append(String.format("Failed to remove %s: %s\n",
+          errorMsgBuilder.append(String.format("Failed to remove %s: %s%n",
               sd.getRoot(), e.getMessage()));
         }
       }
@@ -673,20 +692,25 @@ private void doTransition( DataNode datanode,
           + sd.getRoot().getCanonicalPath() + ": namenode clusterID = "
           + nsInfo.getClusterID() + "; datanode clusterID = " + getClusterID());
     }
-    
-    // After addition of the federation feature, ctime check is only 
-    // meaningful at BlockPoolSliceStorage level. 
 
-    // regular start up. 
+    // Clusters previously upgraded from layout versions earlier than
+    // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a
+    // new storage ID. We check for that and fix it now.
+    boolean haveValidStorageId =
+        DataNodeLayoutVersion.supports(
+            LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, layoutVersion) &&
+            DatanodeStorage.isValidStorageId(sd.getStorageUuid());
+
+    // regular start up.
     if (this.layoutVersion == HdfsConstants.DATANODE_LAYOUT_VERSION) {
-      createStorageID(sd);
+      createStorageID(sd, !haveValidStorageId);
       return; // regular startup
     }
-    
+
     // do upgrade
     if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION) {
       doUpgrade(datanode, sd, nsInfo);  // upgrade
-      createStorageID(sd);
+      createStorageID(sd, !haveValidStorageId);
       return;
     }
     
@@ -979,10 +1003,10 @@ private void linkAllBlocks(DataNode datanode, File fromDir, File fromBbwDir,
   }
 
   private static class LinkArgs {
-    public File src;
-    public File dst;
+    File src;
+    File dst;
 
-    public LinkArgs(File src, File dst) {
+    LinkArgs(File src, File dst) {
       this.src = src;
       this.dst = dst;
     }
@@ -999,9 +1023,19 @@ static void linkBlocks(DataNode datanode, File from, File to, int oldLV,
       upgradeToIdBasedLayout = true;
     }
 
-    final List<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList();
+    final ArrayList<LinkArgs> idBasedLayoutSingleLinks = Lists.newArrayList();
     linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
         idBasedLayoutSingleLinks);
+
+    // Detect and remove duplicate entries.
+    final ArrayList<LinkArgs> duplicates =
+        findDuplicateEntries(idBasedLayoutSingleLinks);
+    if (!duplicates.isEmpty()) {
+      LOG.error("There are " + duplicates.size() + " duplicate block " +
+          "entries within the same volume.");
+      removeDuplicateEntries(idBasedLayoutSingleLinks, duplicates);
+    }
+
     int numLinkWorkers = datanode.getConf().getInt(
         DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY,
         DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS);
@@ -1028,7 +1062,162 @@ public Void call() throws IOException {
       Futures.get(f, IOException.class);
     }
   }
-  
+
+  /**
+   * Find duplicate entries with an array of LinkArgs.
+   * Duplicate entries are entries with the same last path component.
+   */
+  static ArrayList<LinkArgs> findDuplicateEntries(ArrayList<LinkArgs> all) {
+    // Find duplicates by sorting the list by the final path component.
+    Collections.sort(all, new Comparator<LinkArgs>() {
+      /**
+       * Compare two LinkArgs objects, such that objects with the same
+       * terminal source path components are grouped together.
+       */
+      @Override
+      public int compare(LinkArgs a, LinkArgs b) {
+        return ComparisonChain.start().
+            compare(a.src.getName(), b.src.getName()).
+            compare(a.src, b.src).
+            compare(a.dst, b.dst).
+            result();
+      }
+    });
+    final ArrayList<LinkArgs> duplicates = Lists.newArrayList();
+    Long prevBlockId = null;
+    boolean prevWasMeta = false;
+    boolean addedPrev = false;
+    for (int i = 0; i < all.size(); i++) {
+      LinkArgs args = all.get(i);
+      long blockId = Block.getBlockId(args.src.getName());
+      boolean isMeta = Block.isMetaFilename(args.src.getName());
+      if ((prevBlockId == null) ||
+          (prevBlockId.longValue() != blockId)) {
+        prevBlockId = blockId;
+        addedPrev = false;
+      } else if (isMeta == prevWasMeta) {
+        // If we saw another file for the same block ID previously,
+        // and it had the same meta-ness as this file, we have a
+        // duplicate.
+        duplicates.add(args);
+        if (!addedPrev) {
+          duplicates.add(all.get(i - 1));
+        }
+        addedPrev = true;
+      } else {
+        addedPrev = false;
+      }
+      prevWasMeta = isMeta;
+    }
+    return duplicates;
+  }
+
+  /**
+   * Remove duplicate entries from the list.
+   * We do this by choosing:
+   * 1. the entries with the highest genstamp (this takes priority),
+   * 2. the entries with the longest block files,
+   * 3. arbitrarily, if neither #1 nor #2 gives a clear winner.
+   *
+   * Block and metadata files form a pair-- if you take a metadata file from
+   * one subdirectory, you must also take the block file from that
+   * subdirectory.
+   */
+  private static void removeDuplicateEntries(ArrayList<LinkArgs> all,
+                                             ArrayList<LinkArgs> duplicates) {
+    // Maps blockId -> metadata file with highest genstamp
+    TreeMap<Long, List<LinkArgs>> highestGenstamps =
+        new TreeMap<Long, List<LinkArgs>>();
+    for (LinkArgs duplicate : duplicates) {
+      if (!Block.isMetaFilename(duplicate.src.getName())) {
+        continue;
+      }
+      long blockId = Block.getBlockId(duplicate.src.getName());
+      List<LinkArgs> prevHighest = highestGenstamps.get(blockId);
+      if (prevHighest == null) {
+        List<LinkArgs> highest = new LinkedList<LinkArgs>();
+        highest.add(duplicate);
+        highestGenstamps.put(blockId, highest);
+        continue;
+      }
+      long prevGenstamp =
+          Block.getGenerationStamp(prevHighest.get(0).src.getName());
+      long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+      if (genstamp < prevGenstamp) {
+        continue;
+      }
+      if (genstamp > prevGenstamp) {
+        prevHighest.clear();
+      }
+      prevHighest.add(duplicate);
+    }
+
+    // Remove data / metadata entries that don't have the highest genstamp
+    // from the duplicates list.
+    for (Iterator<LinkArgs> iter = duplicates.iterator(); iter.hasNext(); ) {
+      LinkArgs duplicate = iter.next();
+      long blockId = Block.getBlockId(duplicate.src.getName());
+      List<LinkArgs> highest = highestGenstamps.get(blockId);
+      if (highest != null) {
+        boolean found = false;
+        for (LinkArgs high : highest) {
+          if (high.src.getParent().equals(duplicate.src.getParent())) {
+            found = true;
+            break;
+          }
+        }
+        if (!found) {
+          LOG.warn("Unexpectedly low genstamp on " +
+                   duplicate.src.getAbsolutePath() + ".");
+          iter.remove();
+        }
+      }
+    }
+
+    // Find the longest block files
+    // We let the "last guy win" here, since we're only interested in
+    // preserving one block file / metadata file pair.
+    TreeMap<Long, LinkArgs> longestBlockFiles = new TreeMap<Long, LinkArgs>();
+    for (LinkArgs duplicate : duplicates) {
+      if (Block.isMetaFilename(duplicate.src.getName())) {
+        continue;
+      }
+      long blockId = Block.getBlockId(duplicate.src.getName());
+      LinkArgs prevLongest = longestBlockFiles.get(blockId);
+      if (prevLongest == null) {
+        longestBlockFiles.put(blockId, duplicate);
+        continue;
+      }
+      long blockLength = duplicate.src.length();
+      long prevBlockLength = prevLongest.src.length();
+      if (blockLength < prevBlockLength) {
+        LOG.warn("Unexpectedly short length on " +
+            duplicate.src.getAbsolutePath() + ".");
+        continue;
+      }
+      if (blockLength > prevBlockLength) {
+        LOG.warn("Unexpectedly short length on " +
+            prevLongest.src.getAbsolutePath() + ".");
+      }
+      longestBlockFiles.put(blockId, duplicate);
+    }
+
+    // Remove data / metadata entries that aren't the longest, or weren't
+    // arbitrarily selected by us.
+    for (Iterator<LinkArgs> iter = all.iterator(); iter.hasNext(); ) {
+      LinkArgs args = iter.next();
+      long blockId = Block.getBlockId(args.src.getName());
+      LinkArgs bestDuplicate = longestBlockFiles.get(blockId);
+      if (bestDuplicate == null) {
+        continue; // file has no duplicates
+      }
+      if (!bestDuplicate.src.getParent().equals(args.src.getParent())) {
+        LOG.warn("Discarding " + args.src.getAbsolutePath() + ".");
+        iter.remove();
+      }
+    }
+  }
+
   static void linkBlocksHelper(File from, File to, int oldLV, HardLink hl,
   boolean upgradeToIdBasedLayout, File blockRoot,
       List<LinkArgs> idBasedLayoutSingleLinks) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index bbc23e6d1ee5d..3a2723f16bd4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -192,10 +192,17 @@ public void run() {
           HdfsConstants.SMALL_BUFFER_SIZE);
         socketOut = saslStreams.out;
       } catch (InvalidMagicNumberException imne) {
-        LOG.info("Failed to read expected encryption handshake from client " +
-            "at " + peer.getRemoteAddressString() + ". Perhaps the client " +
-            "is running an older version of Hadoop which does not support " +
-            "encryption");
+        if (imne.isHandshake4Encryption()) {
+          LOG.info("Failed to read expected encryption handshake from client " +
+              "at " + peer.getRemoteAddressString() + ". Perhaps the client " +
+              "is running an older version of Hadoop which does not support " +
+              "encryption");
+        } else {
+          LOG.info("Failed to read expected SASL data transfer protection " +
+              "handshake from client at " + peer.getRemoteAddressString() + 
+              ". Perhaps the client is running an older version of Hadoop " +
+              "which does not support SASL data transfer protection");
+        }
         return;
       }
       
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaHandler.java
new file mode 100644
index 0000000000000..b563d7f9e9ee2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaHandler.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * This class includes a replica being actively written and the reference to
+ * the fs volume where this replica is located.
+ */
+public class ReplicaHandler implements Closeable {
+  private final ReplicaInPipelineInterface replica;
+  private final FsVolumeReference volumeReference;
+
+  public ReplicaHandler(
+      ReplicaInPipelineInterface replica, FsVolumeReference reference) {
+    this.replica = replica;
+    this.volumeReference = reference;
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (this.volumeReference != null) {
+      volumeReference.close();
+    }
+  }
+
+  public ReplicaInPipelineInterface getReplica() {
+    return replica;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
new file mode 100644
index 0000000000000..ce0a8755f2f3a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -0,0 +1,676 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import java.io.DataOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner.Conf;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.BlockIterator;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * VolumeScanner scans a single volume.  Each VolumeScanner has its own thread.<p/>
+ * They are all managed by the DataNode's BlockScanner.
+ */
+public class VolumeScanner extends Thread {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(VolumeScanner.class);
+
+  /**
+   * Number of seconds in a minute.
+   */
+  private final static int SECONDS_PER_MINUTE = 60;
+
+  /**
+   * Number of minutes in an hour.
+   */
+  private final static int MINUTES_PER_HOUR = 60;
+
+  /**
+   * Name of the block iterator used by this scanner.
+   */
+  private final static String BLOCK_ITERATOR_NAME = "scanner";
+
+  /**
+   * The configuration.
+   */
+  private final Conf conf;
+
+  /**
+   * The DataNode this VolumEscanner is associated with.
+   */
+  private final DataNode datanode;
+
+  /**
+   * A reference to the volume that we're scanning.
+   */
+  private final FsVolumeReference ref;
+
+  /**
+   * The volume that we're scanning.
+   */
+  final FsVolumeSpi volume;
+
+  /**
+   * The number of scanned bytes in each minute of the last hour.<p/>
+   *
+   * This array is managed as a circular buffer.  We take the monotonic time and
+   * divide it up into one-minute periods.  Each entry in the array represents
+   * how many bytes were scanned during that period.
+   */
+  private final long scannedBytes[] = new long[MINUTES_PER_HOUR];
+
+  /**
+   * The sum of all the values of scannedBytes.
+   */
+  private long scannedBytesSum = 0;
+
+  /**
+   * The throttler to use with BlockSender objects.
+   */
+  private final DataTransferThrottler throttler = new DataTransferThrottler(1);
+
+  /**
+   * The null output stream to use with BlockSender objects.
+   */
+  private final DataOutputStream nullStream =
+      new DataOutputStream(new IOUtils.NullOutputStream());
+
+  /**
+   * The block iterators associated with this VolumeScanner.<p/>
+   *
+   * Each block pool has its own BlockIterator.
+   */
+  private final List<BlockIterator> blockIters =
+      new LinkedList<BlockIterator>();
+
+  /**
+   * The current block iterator, or null if there is none.
+   */
+  private BlockIterator curBlockIter = null;
+
+  /**
+   * True if the thread is stopping.<p/>
+   * Protected by this object's lock.
+   */
+  private boolean stopping = false;
+
+  /**
+   * The monotonic minute that the volume scanner was started on.
+   */
+  private long startMinute = 0;
+
+  /**
+   * The current minute, in monotonic terms.
+   */
+  private long curMinute = 0;
+
+  /**
+   * Handles scan results.
+   */
+  private final ScanResultHandler resultHandler;
+
+  private final Statistics stats = new Statistics();
+
+  static class Statistics {
+    long bytesScannedInPastHour = 0;
+    long blocksScannedInCurrentPeriod = 0;
+    long blocksScannedSinceRestart = 0;
+    long scansSinceRestart = 0;
+    long scanErrorsSinceRestart = 0;
+    long nextBlockPoolScanStartMs = -1;
+    long blockPoolPeriodEndsMs = -1;
+    ExtendedBlock lastBlockScanned = null;
+    boolean eof = false;
+
+    Statistics() {
+    }
+
+    Statistics(Statistics other) {
+      this.bytesScannedInPastHour = other.bytesScannedInPastHour;
+      this.blocksScannedInCurrentPeriod = other.blocksScannedInCurrentPeriod;
+      this.blocksScannedSinceRestart = other.blocksScannedSinceRestart;
+      this.scansSinceRestart = other.scansSinceRestart;
+      this.scanErrorsSinceRestart = other.scanErrorsSinceRestart;
+      this.nextBlockPoolScanStartMs = other.nextBlockPoolScanStartMs;
+      this.blockPoolPeriodEndsMs = other.blockPoolPeriodEndsMs;
+      this.lastBlockScanned = other.lastBlockScanned;
+      this.eof = other.eof;
+    }
+
+    @Override
+    public String toString() {
+      return new StringBuilder().
+          append("Statistics{").
+          append("bytesScannedInPastHour=").append(bytesScannedInPastHour).
+          append(", blocksScannedInCurrentPeriod=").
+              append(blocksScannedInCurrentPeriod).
+          append(", blocksScannedSinceRestart=").
+              append(blocksScannedSinceRestart).
+          append(", scansSinceRestart=").append(scansSinceRestart).
+          append(", scanErrorsSinceRestart=").append(scanErrorsSinceRestart).
+          append(", nextBlockPoolScanStartMs=").append(nextBlockPoolScanStartMs).
+          append(", blockPoolPeriodEndsMs=").append(blockPoolPeriodEndsMs).
+          append(", lastBlockScanned=").append(lastBlockScanned).
+          append(", eof=").append(eof).
+          append("}").toString();
+    }
+  }
+
+  private static double positiveMsToHours(long ms) {
+    if (ms <= 0) {
+      return 0;
+    } else {
+      return TimeUnit.HOURS.convert(ms, TimeUnit.MILLISECONDS);
+    }
+  }
+
+  public void printStats(StringBuilder p) {
+    p.append("Block scanner information for volume " +
+        volume.getStorageID() + " with base path " + volume.getBasePath() +
+        "%n");
+    synchronized (stats) {
+      p.append(String.format("Bytes verified in last hour       : %57d%n",
+          stats.bytesScannedInPastHour));
+      p.append(String.format("Blocks scanned in current period  : %57d%n",
+          stats.blocksScannedInCurrentPeriod));
+      p.append(String.format("Blocks scanned since restart      : %57d%n",
+          stats.blocksScannedSinceRestart));
+      p.append(String.format("Block pool scans since restart    : %57d%n",
+          stats.scansSinceRestart));
+      p.append(String.format("Block scan errors since restart   : %57d%n",
+          stats.scanErrorsSinceRestart));
+      if (stats.nextBlockPoolScanStartMs > 0) {
+        p.append(String.format("Hours until next block pool scan  : %57.3f%n",
+            positiveMsToHours(stats.nextBlockPoolScanStartMs -
+                Time.monotonicNow())));
+      }
+      if (stats.blockPoolPeriodEndsMs > 0) {
+        p.append(String.format("Hours until possible pool rescan  : %57.3f%n",
+            positiveMsToHours(stats.blockPoolPeriodEndsMs -
+                Time.now())));
+      }
+      p.append(String.format("Last block scanned                : %57s%n",
+          ((stats.lastBlockScanned == null) ? "none" :
+          stats.lastBlockScanned.toString())));
+      p.append(String.format("More blocks to scan in period     : %57s%n",
+          !stats.eof));
+      p.append("%n");
+    }
+  }
+
+  static class ScanResultHandler {
+    private VolumeScanner scanner;
+
+    public void setup(VolumeScanner scanner) {
+      LOG.trace("Starting VolumeScanner {}",
+          scanner.volume.getBasePath());
+      this.scanner = scanner;
+    }
+
+    public void handle(ExtendedBlock block, IOException e) {
+      FsVolumeSpi volume = scanner.volume;
+      if (e == null) {
+        LOG.trace("Successfully scanned {} on {}", block, volume.getBasePath());
+        return;
+      }
+      // If the block does not exist anymore, then it's not an error.
+      if (!volume.getDataset().contains(block)) {
+        LOG.debug("Volume {}: block {} is no longer in the dataset.",
+            volume.getBasePath(), block);
+        return;
+      }
+      // If the block exists, the exception may due to a race with write:
+      // The BlockSender got an old block path in rbw. BlockReceiver removed
+      // the rbw block from rbw to finalized but BlockSender tried to open the
+      // file before BlockReceiver updated the VolumeMap. The state of the
+      // block can be changed again now, so ignore this error here. If there
+      // is a block really deleted by mistake, DirectoryScan should catch it.
+      if (e instanceof FileNotFoundException ) {
+        LOG.info("Volume {}: verification failed for {} because of " +
+                "FileNotFoundException.  This may be due to a race with write.",
+            volume.getBasePath(), block);
+        return;
+      }
+      LOG.warn("Reporting bad {} on {}", block, volume.getBasePath());
+      try {
+        scanner.datanode.reportBadBlocks(block);
+      } catch (IOException ie) {
+        // This is bad, but not bad enough to shut down the scanner.
+        LOG.warn("Cannot report bad " + block.getBlockId(), e);
+      }
+    }
+  }
+
+  VolumeScanner(Conf conf, DataNode datanode, FsVolumeReference ref) {
+    this.conf = conf;
+    this.datanode = datanode;
+    this.ref = ref;
+    this.volume = ref.getVolume();
+    ScanResultHandler handler;
+    try {
+      handler = conf.resultHandler.newInstance();
+    } catch (Throwable e) {
+      LOG.error("unable to instantiate {}", conf.resultHandler, e);
+      handler = new ScanResultHandler();
+    }
+    this.resultHandler = handler;
+    setName("VolumeScannerThread(" + volume.getBasePath() + ")");
+    setDaemon(true);
+  }
+
+  private void saveBlockIterator(BlockIterator iter) {
+    try {
+      iter.save();
+    } catch (IOException e) {
+      LOG.warn("{}: error saving {}.", this, iter, e);
+    }
+  }
+
+  private void expireOldScannedBytesRecords(long monotonicMs) {
+    long newMinute =
+        TimeUnit.MINUTES.convert(monotonicMs, TimeUnit.MILLISECONDS);
+    if (curMinute == newMinute) {
+      return;
+    }
+    // If a minute or more has gone past since we last updated the scannedBytes
+    // array, zero out the slots corresponding to those minutes.
+    for (long m = curMinute + 1; m <= newMinute; m++) {
+      int slotIdx = (int)(m % MINUTES_PER_HOUR);
+      LOG.trace("{}: updateScannedBytes is zeroing out slotIdx {}.  " +
+              "curMinute = {}; newMinute = {}", this, slotIdx,
+              curMinute, newMinute);
+      scannedBytesSum -= scannedBytes[slotIdx];
+      scannedBytes[slotIdx] = 0;
+    }
+    curMinute = newMinute;
+  }
+
+  /**
+   * Find a usable block iterator.<p/>
+   *
+   * We will consider available block iterators in order.  This property is
+   * important so that we don't keep rescanning the same block pool id over
+   * and over, while other block pools stay unscanned.<p/>
+   *
+   * A block pool is always ready to scan if the iterator is not at EOF.  If
+   * the iterator is at EOF, the block pool will be ready to scan when
+   * conf.scanPeriodMs milliseconds have elapsed since the iterator was last
+   * rewound.<p/>
+   *
+   * @return                     0 if we found a usable block iterator; the
+   *                               length of time we should delay before
+   *                               checking again otherwise.
+   */
+  private synchronized long findNextUsableBlockIter() {
+    int numBlockIters = blockIters.size();
+    if (numBlockIters == 0) {
+      LOG.debug("{}: no block pools are registered.", this);
+      return Long.MAX_VALUE;
+    }
+    int curIdx;
+    if (curBlockIter == null) {
+      curIdx = 0;
+    } else {
+      curIdx = blockIters.indexOf(curBlockIter);
+      Preconditions.checkState(curIdx >= 0);
+    }
+    // Note that this has to be wall-clock time, not monotonic time.  This is
+    // because the time saved in the cursor file is a wall-clock time.  We do
+    // not want to save a monotonic time in the cursor file, because it resets
+    // every time the machine reboots (on most platforms).
+    long nowMs = Time.now();
+    long minTimeoutMs = Long.MAX_VALUE;
+    for (int i = 0; i < numBlockIters; i++) {
+      int idx = (curIdx + i + 1) % numBlockIters;
+      BlockIterator iter = blockIters.get(idx);
+      if (!iter.atEnd()) {
+        LOG.info("Now scanning bpid {} on volume {}",
+            iter.getBlockPoolId(), volume.getBasePath());
+        curBlockIter = iter;
+        return 0L;
+      }
+      long iterStartMs = iter.getIterStartMs();
+      long waitMs = (iterStartMs + conf.scanPeriodMs) - nowMs;
+      if (waitMs <= 0) {
+        iter.rewind();
+        LOG.info("Now rescanning bpid {} on volume {}, after more than " +
+            "{} hour(s)", iter.getBlockPoolId(), volume.getBasePath(),
+            TimeUnit.HOURS.convert(conf.scanPeriodMs, TimeUnit.MILLISECONDS));
+        curBlockIter = iter;
+        return 0L;
+      }
+      minTimeoutMs = Math.min(minTimeoutMs, waitMs);
+    }
+    LOG.info("{}: no suitable block pools found to scan.  Waiting {} ms.",
+        this, minTimeoutMs);
+    return minTimeoutMs;
+  }
+
+  /**
+   * Scan a block.
+   *
+   * @param cblock               The block to scan.
+   * @param bytesPerSec          The bytes per second to scan at.
+   *
+   * @return                     The length of the block that was scanned, or
+   *                               -1 if the block could not be scanned.
+   */
+  private long scanBlock(ExtendedBlock cblock, long bytesPerSec) {
+    // 'cblock' has a valid blockId and block pool id, but we don't yet know the
+    // genstamp the block is supposed to have.  Ask the FsDatasetImpl for this
+    // information.
+    ExtendedBlock block = null;
+    try {
+      Block b = volume.getDataset().getStoredBlock(
+          cblock.getBlockPoolId(), cblock.getBlockId());
+      if (b == null) {
+        LOG.info("FileNotFound while finding block {} on volume {}",
+            cblock, volume.getBasePath());
+      } else {
+        block = new ExtendedBlock(cblock.getBlockPoolId(), b);
+      }
+    } catch (FileNotFoundException e) {
+      LOG.info("FileNotFoundException while finding block {} on volume {}",
+          cblock, volume.getBasePath());
+    } catch (IOException e) {
+      LOG.warn("I/O error while finding block {} on volume {}",
+            cblock, volume.getBasePath());
+    }
+    if (block == null) {
+      return -1; // block not found.
+    }
+    BlockSender blockSender = null;
+    try {
+      blockSender = new BlockSender(block, 0, -1,
+          false, true, true, datanode, null,
+          CachingStrategy.newDropBehind());
+      throttler.setBandwidth(bytesPerSec);
+      long bytesRead = blockSender.sendBlock(nullStream, null, throttler);
+      resultHandler.handle(block, null);
+      return bytesRead;
+    } catch (IOException e) {
+      resultHandler.handle(block, e);
+    } finally {
+      IOUtils.cleanup(null, blockSender);
+    }
+    return -1;
+  }
+
+  @VisibleForTesting
+  static boolean calculateShouldScan(String storageId, long targetBytesPerSec,
+                   long scannedBytesSum, long startMinute, long curMinute) {
+    long runMinutes = curMinute - startMinute;
+    long effectiveBytesPerSec;
+    if (runMinutes <= 0) {
+      // avoid division by zero
+      effectiveBytesPerSec = scannedBytesSum;
+    } else {
+      if (runMinutes > MINUTES_PER_HOUR) {
+        // we only keep an hour's worth of rate information
+        runMinutes = MINUTES_PER_HOUR;
+      }
+      effectiveBytesPerSec = scannedBytesSum /
+          (SECONDS_PER_MINUTE * runMinutes);
+    }
+
+    boolean shouldScan = effectiveBytesPerSec <= targetBytesPerSec;
+    LOG.trace("{}: calculateShouldScan: effectiveBytesPerSec = {}, and " +
+        "targetBytesPerSec = {}.  startMinute = {}, curMinute = {}, " +
+        "shouldScan = {}",
+        storageId, effectiveBytesPerSec, targetBytesPerSec,
+        startMinute, curMinute, shouldScan);
+    return shouldScan;
+  }
+
+  /**
+   * Run an iteration of the VolumeScanner loop.
+   *
+   * @return     The number of milliseconds to delay before running the loop
+   *               again, or 0 to re-run the loop immediately.
+   */
+  private long runLoop() {
+    long bytesScanned = -1;
+    boolean scanError = false;
+    ExtendedBlock block = null;
+    try {
+      long monotonicMs = Time.monotonicNow();
+      expireOldScannedBytesRecords(monotonicMs);
+
+      if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec,
+          scannedBytesSum, startMinute, curMinute)) {
+        // If neededBytesPerSec is too low, then wait few seconds for some old
+        // scannedBytes records to expire.
+        return 30000L;
+      }
+
+      // Find a usable block pool to scan.
+      if ((curBlockIter == null) || curBlockIter.atEnd()) {
+        long timeout = findNextUsableBlockIter();
+        if (timeout > 0) {
+          LOG.trace("{}: no block pools are ready to scan yet.  Waiting " +
+              "{} ms.", this, timeout);
+          synchronized (stats) {
+            stats.nextBlockPoolScanStartMs = Time.monotonicNow() + timeout;
+          }
+          return timeout;
+        }
+        synchronized (stats) {
+          stats.scansSinceRestart++;
+          stats.blocksScannedInCurrentPeriod = 0;
+          stats.nextBlockPoolScanStartMs = -1;
+        }
+        return 0L;
+      }
+
+      try {
+        block = curBlockIter.nextBlock();
+      } catch (IOException e) {
+        // There was an error listing the next block in the volume.  This is a
+        // serious issue.
+        LOG.warn("{}: nextBlock error on {}", this, curBlockIter);
+        // On the next loop iteration, curBlockIter#eof will be set to true, and
+        // we will pick a different block iterator.
+        return 0L;
+      }
+      if (block == null) {
+        // The BlockIterator is at EOF.
+        LOG.info("{}: finished scanning block pool {}",
+            this, curBlockIter.getBlockPoolId());
+        saveBlockIterator(curBlockIter);
+        return 0;
+      }
+      long saveDelta = monotonicMs - curBlockIter.getLastSavedMs();
+      if (saveDelta >= conf.cursorSaveMs) {
+        LOG.debug("{}: saving block iterator {} after {} ms.",
+            this, curBlockIter, saveDelta);
+        saveBlockIterator(curBlockIter);
+      }
+      bytesScanned = scanBlock(block, conf.targetBytesPerSec);
+      if (bytesScanned >= 0) {
+        scannedBytesSum += bytesScanned;
+        scannedBytes[(int)(curMinute % MINUTES_PER_HOUR)] += bytesScanned;
+      } else {
+        scanError = true;
+      }
+      return 0L;
+    } finally {
+      synchronized (stats) {
+        stats.bytesScannedInPastHour = scannedBytesSum;
+        if (bytesScanned >= 0) {
+          stats.blocksScannedInCurrentPeriod++;
+          stats.blocksScannedSinceRestart++;
+        }
+        if (scanError) {
+          stats.scanErrorsSinceRestart++;
+        }
+        if (block != null) {
+          stats.lastBlockScanned = block;
+        }
+        if (curBlockIter == null) {
+          stats.eof = true;
+          stats.blockPoolPeriodEndsMs = -1;
+        } else {
+          stats.eof = curBlockIter.atEnd();
+          stats.blockPoolPeriodEndsMs =
+              curBlockIter.getIterStartMs() + conf.scanPeriodMs;
+        }
+      }
+    }
+  }
+
+  @Override
+  public void run() {
+    // Record the minute on which the scanner started.
+    this.startMinute =
+        TimeUnit.MINUTES.convert(Time.monotonicNow(), TimeUnit.MILLISECONDS);
+    this.curMinute = startMinute;
+    try {
+      LOG.trace("{}: thread starting.", this);
+      resultHandler.setup(this);
+      try {
+        long timeout = 0;
+        while (true) {
+          // Take the lock to check if we should stop.
+          synchronized (this) {
+            if (stopping) {
+              break;
+            }
+            if (timeout > 0) {
+              wait(timeout);
+              if (stopping) {
+                break;
+              }
+            }
+          }
+          timeout = runLoop();
+        }
+      } catch (InterruptedException e) {
+        // We are exiting because of an InterruptedException,
+        // probably sent by VolumeScanner#shutdown.
+        LOG.trace("{} exiting because of InterruptedException.", this);
+      } catch (Throwable e) {
+        LOG.error("{} exiting because of exception ", this, e);
+      }
+      LOG.info("{} exiting.", this);
+      // Save the current position of all block iterators and close them.
+      for (BlockIterator iter : blockIters) {
+        saveBlockIterator(iter);
+        IOUtils.cleanup(null, iter);
+      }
+    } finally {
+      // When the VolumeScanner exits, release the reference we were holding
+      // on the volume.  This will allow the volume to be removed later.
+      IOUtils.cleanup(null, ref);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "VolumeScanner(" + volume.getBasePath() +
+        ", " + volume.getStorageID() + ")";
+  }
+
+  /**
+   * Shut down this scanner.
+   */
+  public synchronized void shutdown() {
+    stopping = true;
+    notify();
+    this.interrupt();
+  }
+
+  /**
+   * Allow the scanner to scan the given block pool.
+   *
+   * @param bpid       The block pool id.
+   */
+  public synchronized void enableBlockPoolId(String bpid) {
+    for (BlockIterator iter : blockIters) {
+      if (iter.getBlockPoolId().equals(bpid)) {
+        LOG.warn("{}: already enabled scanning on block pool {}", this, bpid);
+        return;
+      }
+    }
+    BlockIterator iter = null;
+    try {
+      // Load a block iterator for the next block pool on the volume.
+      iter = volume.loadBlockIterator(bpid, BLOCK_ITERATOR_NAME);
+      LOG.trace("{}: loaded block iterator for {}.", this, bpid);
+    } catch (FileNotFoundException e) {
+      LOG.debug("{}: failed to load block iterator: " + e.getMessage(), this);
+    } catch (IOException e) {
+      LOG.warn("{}: failed to load block iterator.", this, e);
+    }
+    if (iter == null) {
+      iter = volume.newBlockIterator(bpid, BLOCK_ITERATOR_NAME);
+      LOG.trace("{}: created new block iterator for {}.", this, bpid);
+    }
+    iter.setMaxStalenessMs(conf.maxStalenessMs);
+    blockIters.add(iter);
+    notify();
+  }
+
+  /**
+   * Disallow the scanner from scanning the given block pool.
+   *
+   * @param bpid       The block pool id.
+   */
+  public synchronized void disableBlockPoolId(String bpid) {
+    Iterator<BlockIterator> i = blockIters.iterator();
+    while (i.hasNext()) {
+      BlockIterator iter = i.next();
+      if (iter.getBlockPoolId().equals(bpid)) {
+        LOG.trace("{}: disabling scanning on block pool {}", this, bpid);
+        i.remove();
+        IOUtils.cleanup(null, iter);
+        if (curBlockIter == iter) {
+          curBlockIter = null;
+        }
+        notify();
+        return;
+      }
+    }
+    LOG.warn("{}: can't remove block pool {}, because it was never " +
+        "added.", this, bpid);
+  }
+
+  @VisibleForTesting
+  Statistics getStatistics() {
+    synchronized (stats) {
+      return new Statistics(stats);
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 462ad31c03c65..162e3064eadfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.Replica;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaHandler;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
@@ -89,24 +90,30 @@ public boolean isSimulated() {
     }
   }
 
-  /**
-   * Create rolling logs.
-   *
-   * @param prefix the prefix of the log names.
-   * @return rolling logs
-   */
-  public RollingLogs createRollingLogs(String bpid, String prefix
-      ) throws IOException;
-
   /** @return a list of volumes. */
   public List<V> getVolumes();
 
-  /** Add an array of StorageLocation to FsDataset. */
+  /**
+   * Add a new volume to the FsDataset.<p/>
+   *
+   * If the FSDataset supports block scanning, this function registers
+   * the new volume with the block scanner.
+   *
+   * @param location      The storage location for the new volume.
+   * @param nsInfos       Namespace information for the new volume.
+   */
   public void addVolume(
       final StorageLocation location,
       final List<NamespaceInfo> nsInfos) throws IOException;
 
-  /** Removes a collection of volumes from FsDataset. */
+  /**
+   * Removes a collection of volumes from FsDataset.
+   *
+   * If the FSDataset supports block scanning, this function removes
+   * the volumes from the block scanner.
+   *
+   * @param volumes      The storage locations of the volumes to remove.
+   */
   public void removeVolumes(Collection<StorageLocation> volumes);
 
   /** @return a storage with the given storage ID */
@@ -198,7 +205,7 @@ public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff,
    * @return the meta info of the replica which is being written to
    * @throws IOException if an error occurs
    */
-  public ReplicaInPipelineInterface createTemporary(StorageType storageType,
+  public ReplicaHandler createTemporary(StorageType storageType,
       ExtendedBlock b) throws IOException;
 
   /**
@@ -208,7 +215,7 @@ public ReplicaInPipelineInterface createTemporary(StorageType storageType,
    * @return the meta info of the replica which is being written to
    * @throws IOException if an error occurs
    */
-  public ReplicaInPipelineInterface createRbw(StorageType storageType,
+  public ReplicaHandler createRbw(StorageType storageType,
       ExtendedBlock b, boolean allowLazyPersist) throws IOException;
 
   /**
@@ -221,7 +228,7 @@ public ReplicaInPipelineInterface createRbw(StorageType storageType,
    * @return the meta info of the replica which is being written to
    * @throws IOException if an error occurs
    */
-  public ReplicaInPipelineInterface recoverRbw(ExtendedBlock b, 
+  public ReplicaHandler recoverRbw(ExtendedBlock b,
       long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException;
 
   /**
@@ -241,7 +248,7 @@ public ReplicaInPipelineInterface convertTemporaryToRbw(
    * @return the meata info of the replica which is being written to
    * @throws IOException
    */
-  public ReplicaInPipelineInterface append(ExtendedBlock b, long newGS,
+  public ReplicaHandler append(ExtendedBlock b, long newGS,
       long expectedBlockLen) throws IOException;
 
   /**
@@ -254,8 +261,8 @@ public ReplicaInPipelineInterface append(ExtendedBlock b, long newGS,
    * @return the meta info of the replica which is being written to
    * @throws IOException
    */
-  public ReplicaInPipelineInterface recoverAppend(ExtendedBlock b, long newGS,
-      long expectedBlockLen) throws IOException;
+  public ReplicaHandler recoverAppend(
+      ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException;
   
   /**
    * Recover a failed pipeline close
@@ -418,7 +425,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock
    * @return the ID of storage that stores the block
    */
   public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
-      long recoveryId, long newLength) throws IOException;
+      long recoveryId, long newBlockId, long newLength) throws IOException;
 
   /**
    * add new block pool ID
@@ -503,7 +510,7 @@ public void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block,
    * Callback from RamDiskAsyncLazyPersistService upon async lazy persist task end
    */
    public void onCompleteLazyPersist(String bpId, long blockId,
-      long creationTime, File[] savedFiles, FsVolumeImpl targetVolume);
+      long creationTime, File[] savedFiles, V targetVolume);
 
    /**
     * Callback from RamDiskAsyncLazyPersistService upon async lazy persist task fail
@@ -513,6 +520,6 @@ public void onCompleteLazyPersist(String bpId, long blockId,
     /**
      * Move block from one storage to another storage
      */
-    public ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block,
+   public ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block,
         StorageType targetStorageType) throws IOException;
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeReference.java
new file mode 100644
index 0000000000000..e61a059f9423e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeReference.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * This is the interface for holding reference count as AutoClosable resource.
+ * It increases the reference count by one in the constructor, and decreases
+ * the reference count by one in {@link #close()}.
+ *
+ * <pre>
+ *  {@code
+ *    try (FsVolumeReference ref = volume.obtainReference()) {
+ *      // Do IOs on the volume
+ *      volume.createRwb(...);
+ *      ...
+ *    }
+ *  }
+ * </pre>
+ */
+public interface FsVolumeReference extends Closeable {
+  /**
+   * Descrese the reference count of the volume.
+   * @throws IOException it never throws IOException.
+   */
+  @Override
+  public void close() throws IOException;
+
+  /** Returns the underlying volume object */
+  public FsVolumeSpi getVolume();
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 8ebf2b4a8bf2e..1355e319ed011 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -17,15 +17,27 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset;
 
+import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
 
 import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 
 /**
  * This is an interface for the underlying volume.
  */
 public interface FsVolumeSpi {
+  /**
+   * Obtain a reference object that had increased 1 reference count of the
+   * volume.
+   *
+   * It is caller's responsibility to close {@link FsVolumeReference} to decrease
+   * the reference count on the volume.
+   */
+  FsVolumeReference obtainReference() throws ClosedChannelException;
+
   /** @return the StorageUuid of the volume */
   public String getStorageID();
 
@@ -59,4 +71,112 @@ public interface FsVolumeSpi {
    * Release disk space previously reserved for RBW block.
    */
   public void releaseReservedSpace(long bytesToRelease);
+
+  /**
+   * BlockIterator will return ExtendedBlock entries from a block pool in
+   * this volume.  The entries will be returned in sorted order.<p/>
+   *
+   * BlockIterator objects themselves do not always have internal
+   * synchronization, so they can only safely be used by a single thread at a
+   * time.<p/>
+   *
+   * Closing the iterator does not save it.  You must call save to save it.
+   */
+  public interface BlockIterator extends Closeable {
+    /**
+     * Get the next block.<p/>
+     *
+     * Note that this block may be removed in between the time we list it,
+     * and the time the caller tries to use it, or it may represent a stale
+     * entry.  Callers should handle the case where the returned block no
+     * longer exists.
+     *
+     * @return               The next block, or null if there are no
+     *                         more blocks.  Null if there was an error
+     *                         determining the next block.
+     *
+     * @throws IOException   If there was an error getting the next block in
+     *                         this volume.  In this case, EOF will be set on
+     *                         the iterator.
+     */
+    public ExtendedBlock nextBlock() throws IOException;
+
+    /**
+     * Returns true if we got to the end of the block pool.
+     */
+    public boolean atEnd();
+
+    /**
+     * Repositions the iterator at the beginning of the block pool.
+     */
+    public void rewind();
+
+    /**
+     * Save this block iterator to the underlying volume.
+     * Any existing saved block iterator with this name will be overwritten.
+     * maxStalenessMs will not be saved.
+     *
+     * @throws IOException   If there was an error when saving the block
+     *                         iterator.
+     */
+    public void save() throws IOException;
+
+    /**
+     * Set the maximum staleness of entries that we will return.<p/>
+     *
+     * A maximum staleness of 0 means we will never return stale entries; a
+     * larger value will allow us to reduce resource consumption in exchange
+     * for returning more potentially stale entries.  Even with staleness set
+     * to 0, consumers of this API must handle race conditions where block
+     * disappear before they can be processed.
+     */
+    public void setMaxStalenessMs(long maxStalenessMs);
+
+    /**
+     * Get the wall-clock time, measured in milliseconds since the Epoch,
+     * when this iterator was created.
+     */
+    public long getIterStartMs();
+
+    /**
+     * Get the wall-clock time, measured in milliseconds since the Epoch,
+     * when this iterator was last saved.  Returns iterStartMs if the
+     * iterator was never saved.
+     */
+    public long getLastSavedMs();
+
+    /**
+     * Get the id of the block pool which this iterator traverses.
+     */
+    public String getBlockPoolId();
+  }
+
+  /**
+   * Create a new block iterator.  It will start at the beginning of the
+   * block set.
+   *
+   * @param bpid             The block pool id to iterate over.
+   * @param name             The name of the block iterator to create.
+   *
+   * @return                 The new block iterator.
+   */
+  public BlockIterator newBlockIterator(String bpid, String name);
+
+  /**
+   * Load a saved block iterator.
+   *
+   * @param bpid             The block pool id to iterate over.
+   * @param name             The name of the block iterator to load.
+   *
+   * @return                 The saved block iterator.
+   * @throws IOException     If there was an IO error loading the saved
+   *                           block iterator.
+   */
+  public BlockIterator loadBlockIterator(String bpid, String name)
+      throws IOException;
+
+  /**
+   * Get the FSDatasetSpi which this volume is a part of.
+   */
+  public FsDatasetSpi getDataset();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java
index 6bd71992d4f59..227179d4f8493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java
@@ -18,8 +18,6 @@
 package org.apache.hadoop.hdfs.server.datanode.fsdataset;
 
 import java.io.Closeable;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
 import java.io.InputStream;
 
 import org.apache.hadoop.io.IOUtils;
@@ -30,11 +28,14 @@
 public class ReplicaInputStreams implements Closeable {
   private final InputStream dataIn;
   private final InputStream checksumIn;
+  private final FsVolumeReference volumeRef;
 
   /** Create an object with a data input stream and a checksum input stream. */
-  public ReplicaInputStreams(FileDescriptor dataFd, FileDescriptor checksumFd) {
-    this.dataIn = new FileInputStream(dataFd);
-    this.checksumIn = new FileInputStream(checksumFd);
+  public ReplicaInputStreams(InputStream dataStream, InputStream checksumStream,
+      FsVolumeReference volumeRef) {
+    this.volumeRef = volumeRef;
+    this.dataIn = dataStream;
+    this.checksumIn = checksumStream;
   }
 
   /** @return the data input stream. */
@@ -51,5 +52,6 @@ public InputStream getChecksumIn() {
   public void close() {
     IOUtils.closeStream(dataIn);
     IOUtils.closeStream(checksumIn);
+    IOUtils.cleanup(null, volumeRef);
   }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java
deleted file mode 100644
index 5d547701e322b..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.fsdataset;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Iterator;
-
-/**
- * Rolling logs consist of a current log and a set of previous logs.
- *
- * The implementation should support a single appender and multiple readers.
- */
-public interface RollingLogs {
-  /**
-   * To iterate the lines of the logs.
-   */
-  public interface LineIterator extends Iterator<String>, Closeable {
-    /** Is the iterator iterating the previous? */
-    public boolean isPrevious();
-
-    /**
-     * Is the last read entry from previous? This should be called after
-     * reading.
-     */
-    public boolean isLastReadFromPrevious();
-  }
-
-  /**
-   * To append text to the logs.
-   */
-  public interface Appender extends Appendable, Closeable {
-  }
-
-  /**
-   * Create an iterator to iterate the lines in the logs.
-   * 
-   * @param skipPrevious Should it skip reading the previous log? 
-   * @return a new iterator.
-   */
-  public LineIterator iterator(boolean skipPrevious) throws IOException;
-
-  /**
-   * @return the only appender to append text to the logs.
-   *   The same object is returned if it is invoked multiple times.
-   */
-  public Appender appender();
-
-  /**
-   * Roll current to previous.
-   *
-   * @return true if the rolling succeeded.
-   *   When it returns false, it is not equivalent to an error. 
-   *   It means that the rolling cannot be performed at the moment,
-   *   e.g. the logs are being read.
-   */
-  public boolean roll() throws IOException;
-}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 77cdb91ea4549..5a69e1e4d666d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -22,10 +22,13 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStreamWriter;
 import java.io.RandomAccessFile;
+import java.io.Writer;
 import java.util.Scanner;
 
 import org.apache.commons.io.FileUtils;
@@ -186,7 +189,7 @@ long loadDfsUsed() {
     Scanner sc;
 
     try {
-      sc = new Scanner(new File(currentDir, DU_CACHE_FILE));
+      sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
     } catch (FileNotFoundException fnfe) {
       return -1;
     }
@@ -227,23 +230,18 @@ void saveDfsUsed() {
         outFile.getParent());
     }
 
-    FileWriter out = null;
     try {
       long used = getDfsUsed();
-      if (used > 0) {
-        out = new FileWriter(outFile);
+      try (Writer out = new OutputStreamWriter(
+          new FileOutputStream(outFile), "UTF-8")) {
         // mtime is written last, so that truncated writes won't be valid.
         out.write(Long.toString(used) + " " + Long.toString(Time.now()));
         out.flush();
-        out.close();
-        out = null;
       }
     } catch (IOException ioe) {
       // If write failed, the volume might be bad. Since the cache file is
       // not critical, log the error and continue.
       FsDatasetImpl.LOG.warn("Failed to write dfsUsed to " + outFile, ioe);
-    } finally {
-      IOUtils.cleanup(null, out);
     }
   }
 
@@ -447,7 +445,7 @@ void addToReplicasMap(ReplicaMap volumeMap, File dir,
             File.pathSeparator + "." + file.getName() + ".restart");
         Scanner sc = null;
         try {
-          sc = new Scanner(restartMeta);
+          sc = new Scanner(restartMeta, "UTF-8");
           // The restart meta file exists
           if (sc.hasNextLong() && (sc.nextLong() > Time.now())) {
             // It didn't expire. Load the replica as a RBW.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index bee7bf70c3eb4..13e854f0b7c91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -20,6 +20,7 @@
 
 import java.io.File;
 import java.io.FileDescriptor;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -31,7 +32,9 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIOException;
 
@@ -200,13 +203,13 @@ public void run() {
    * Delete the block file and meta file from the disk asynchronously, adjust
    * dfsUsed statistics accordingly.
    */
-  void deleteAsync(FsVolumeImpl volume, File blockFile, File metaFile,
+  void deleteAsync(FsVolumeReference volumeRef, File blockFile, File metaFile,
       ExtendedBlock block, String trashDirectory) {
     LOG.info("Scheduling " + block.getLocalBlock()
         + " file " + blockFile + " for deletion");
     ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask(
-        volume, blockFile, metaFile, block, trashDirectory);
-    execute(volume.getCurrentDir(), deletionTask);
+        volumeRef, blockFile, metaFile, block, trashDirectory);
+    execute(((FsVolumeImpl) volumeRef.getVolume()).getCurrentDir(), deletionTask);
   }
   
   /** A task for deleting a block file and its associated meta file, as well
@@ -216,15 +219,17 @@ void deleteAsync(FsVolumeImpl volume, File blockFile, File metaFile,
    *  files are deleted immediately.
    */
   class ReplicaFileDeleteTask implements Runnable {
+    final FsVolumeReference volumeRef;
     final FsVolumeImpl volume;
     final File blockFile;
     final File metaFile;
     final ExtendedBlock block;
     final String trashDirectory;
     
-    ReplicaFileDeleteTask(FsVolumeImpl volume, File blockFile,
+    ReplicaFileDeleteTask(FsVolumeReference volumeRef, File blockFile,
         File metaFile, ExtendedBlock block, String trashDirectory) {
-      this.volume = volume;
+      this.volumeRef = volumeRef;
+      this.volume = (FsVolumeImpl) volumeRef.getVolume();
       this.blockFile = blockFile;
       this.metaFile = metaFile;
       this.block = block;
@@ -281,6 +286,7 @@ public void run() {
         LOG.info("Deleted " + block.getBlockPoolId() + " "
             + block.getLocalBlock() + " file " + blockFile);
       }
+      IOUtils.cleanup(null, volumeRef);
     }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 2c6f409e050e0..9a981fe390a87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -29,6 +29,7 @@
 import java.io.InputStream;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,15 +46,13 @@
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.ExtendedBlockId;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.ExtendedBlockId;
 import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -66,7 +65,6 @@
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
-import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
@@ -74,6 +72,7 @@
 import org.apache.hadoop.hdfs.server.datanode.Replica;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaHandler;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
@@ -82,14 +81,14 @@
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdfs.server.datanode.UnexpectedReplicaStateException;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
-import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica;
 import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -107,6 +106,9 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
 /**************************************************
  * FSDataset manages a set of data blocks.  Each block
  * has a unique name and an extent on disk.
@@ -126,7 +128,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
 
   @Override // FsDatasetSpi
   public List<FsVolumeImpl> getVolumes() {
-    return volumes.volumes;
+    return volumes.getVolumes();
   }
 
   @Override
@@ -137,21 +139,26 @@ public DatanodeStorage getStorage(final String storageUuid) {
   @Override // FsDatasetSpi
   public StorageReport[] getStorageReports(String bpid)
       throws IOException {
-    StorageReport[] reports;
+    List<StorageReport> reports;
     synchronized (statsLock) {
-      reports = new StorageReport[volumes.volumes.size()];
-      int i = 0;
-      for (FsVolumeImpl volume : volumes.volumes) {
-        reports[i++] = new StorageReport(volume.toDatanodeStorage(),
-                                         false,
-                                         volume.getCapacity(),
-                                         volume.getDfsUsed(),
-                                         volume.getAvailable(),
-                                         volume.getBlockPoolUsed(bpid));
+      List<FsVolumeImpl> curVolumes = getVolumes();
+      reports = new ArrayList<>(curVolumes.size());
+      for (FsVolumeImpl volume : curVolumes) {
+        try (FsVolumeReference ref = volume.obtainReference()) {
+          StorageReport sr = new StorageReport(volume.toDatanodeStorage(),
+              false,
+              volume.getCapacity(),
+              volume.getDfsUsed(),
+              volume.getAvailable(),
+              volume.getBlockPoolUsed(bpid));
+          reports.add(sr);
+        } catch (ClosedChannelException e) {
+          continue;
+        }
       }
     }
 
-    return reports;
+    return reports.toArray(new StorageReport[reports.size()]);
   }
 
   @Override
@@ -276,7 +283,8 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
             DFSConfigKeys.DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY,
             RoundRobinVolumeChoosingPolicy.class,
             VolumeChoosingPolicy.class), conf);
-    volumes = new FsVolumeList(volsFailed, blockChooserImpl);
+    volumes = new FsVolumeList(volsFailed, datanode.getBlockScanner(),
+        blockChooserImpl);
     asyncDiskService = new FsDatasetAsyncDiskService(datanode);
     asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode);
 
@@ -304,6 +312,7 @@ private void addVolume(Collection<StorageLocation> dataLocations,
     // storageMap and asyncDiskService, consistent.
     FsVolumeImpl fsVolume = new FsVolumeImpl(
         this, sd.getStorageUuid(), dir, this.conf, storageType);
+    FsVolumeReference ref = fsVolume.obtainReference();
     ReplicaMap tempVolumeMap = new ReplicaMap(this);
     fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker);
 
@@ -314,7 +323,7 @@ private void addVolume(Collection<StorageLocation> dataLocations,
               DatanodeStorage.State.NORMAL,
               storageType));
       asyncDiskService.addVolume(sd.getCurrentDir());
-      volumes.addVolume(fsVolume);
+      volumes.addVolume(ref);
     }
 
     LOG.info("Added volume - " + dir + ", StorageType: " + storageType);
@@ -334,7 +343,7 @@ public void addVolume(final StorageLocation location,
 
     StorageType storageType = location.getStorageType();
     final FsVolumeImpl fsVolume = new FsVolumeImpl(
-        this, sd.getStorageUuid(), dir, this.conf, storageType);
+        this, sd.getStorageUuid(), sd.getCurrentDir(), this.conf, storageType);
     final ReplicaMap tempVolumeMap = new ReplicaMap(fsVolume);
     ArrayList<IOException> exceptions = Lists.newArrayList();
 
@@ -353,6 +362,7 @@ public void addVolume(final StorageLocation location,
       throw MultipleIOException.createIOException(exceptions);
     }
 
+    final FsVolumeReference ref = fsVolume.obtainReference();
     setupAsyncLazyPersistThread(fsVolume);
 
     builder.build();
@@ -363,7 +373,7 @@ public void addVolume(final StorageLocation location,
               DatanodeStorage.State.NORMAL,
               storageType));
       asyncDiskService.addVolume(sd.getCurrentDir());
-      volumes.addVolume(fsVolume);
+      volumes.addVolume(ref);
     }
     LOG.info("Added volume - " + dir + ", StorageType: " + storageType);
   }
@@ -377,19 +387,19 @@ public void addVolume(final StorageLocation location,
    */
   @Override
   public synchronized void removeVolumes(Collection<StorageLocation> volumes) {
-    Set<File> volumeSet = new HashSet<File>();
+    Set<String> volumeSet = new HashSet<>();
     for (StorageLocation sl : volumes) {
-      volumeSet.add(sl.getFile());
+      volumeSet.add(sl.getFile().getAbsolutePath());
     }
     for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) {
       Storage.StorageDirectory sd = dataStorage.getStorageDir(idx);
-      if (volumeSet.contains(sd.getRoot())) {
-        String volume = sd.getRoot().toString();
+      String volume = sd.getRoot().getAbsolutePath();
+      if (volumeSet.contains(volume)) {
         LOG.info("Removing " + volume + " from FsDataset.");
 
         // Disable the volume from the service.
         asyncDiskService.removeVolume(sd.getCurrentDir());
-        this.volumes.removeVolume(volume);
+        this.volumes.removeVolume(sd.getRoot());
 
         // Removed all replica information for the blocks on the volume. Unlike
         // updating the volumeMap in addVolume(), this operation does not scan
@@ -399,15 +409,14 @@ public synchronized void removeVolumes(Collection<StorageLocation> volumes) {
           for (Iterator<ReplicaInfo> it = volumeMap.replicas(bpid).iterator();
               it.hasNext(); ) {
             ReplicaInfo block = it.next();
-            if (block.getVolume().getBasePath().equals(volume)) {
+            String absBasePath =
+                  new File(block.getVolume().getBasePath()).getAbsolutePath();
+            if (absBasePath.equals(volume)) {
               invalidate(bpid, block);
               blocks.add(block);
               it.remove();
             }
           }
-          // Delete blocks from the block scanner in batch.
-          datanode.getBlockScanner().deleteBlocks(bpid,
-              blocks.toArray(new Block[blocks.size()]));
         }
 
         storageMap.remove(sd.getStorageUuid());
@@ -562,18 +571,12 @@ public InputStream getBlockInputStream(ExtendedBlock b,
     if (isNativeIOAvailable) {
       return NativeIO.getShareDeleteFileInputStream(blockFile, seekOffset);
     } else {
-      RandomAccessFile blockInFile;
       try {
-        blockInFile = new RandomAccessFile(blockFile, "r");
+        return openAndSeek(blockFile, seekOffset);
       } catch (FileNotFoundException fnfe) {
         throw new IOException("Block " + b + " is not valid. " +
             "Expected block file at " + blockFile + " does not exist.");
       }
-
-      if (seekOffset > 0) {
-        blockInFile.seek(seekOffset);
-      }
-      return new FileInputStream(blockInFile.getFD());
     }
   }
 
@@ -618,20 +621,38 @@ private ReplicaInfo getReplicaInfo(String bpid, long blkid)
    * Returns handles to the block file and its metadata file
    */
   @Override // FsDatasetSpi
-  public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, 
-                          long blkOffset, long ckoff) throws IOException {
+  public synchronized ReplicaInputStreams getTmpInputStreams(ExtendedBlock b,
+      long blkOffset, long metaOffset) throws IOException {
     ReplicaInfo info = getReplicaInfo(b);
-    File blockFile = info.getBlockFile();
-    RandomAccessFile blockInFile = new RandomAccessFile(blockFile, "r");
-    if (blkOffset > 0) {
-      blockInFile.seek(blkOffset);
+    FsVolumeReference ref = info.getVolume().obtainReference();
+    try {
+      InputStream blockInStream = openAndSeek(info.getBlockFile(), blkOffset);
+      try {
+        InputStream metaInStream = openAndSeek(info.getMetaFile(), metaOffset);
+        return new ReplicaInputStreams(blockInStream, metaInStream, ref);
+      } catch (IOException e) {
+        IOUtils.cleanup(null, blockInStream);
+        throw e;
+      }
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
     }
-    File metaFile = info.getMetaFile();
-    RandomAccessFile metaInFile = new RandomAccessFile(metaFile, "r");
-    if (ckoff > 0) {
-      metaInFile.seek(ckoff);
+  }
+
+  private static FileInputStream openAndSeek(File file, long offset)
+      throws IOException {
+    RandomAccessFile raf = null;
+    try {
+      raf = new RandomAccessFile(file, "r");
+      if (offset > 0) {
+        raf.seek(offset);
+      }
+      return new FileInputStream(raf.getFD());
+    } catch(IOException ioe) {
+      IOUtils.cleanup(null, raf);
+      throw ioe;
     }
-    return new ReplicaInputStreams(blockInFile.getFD(), metaInFile.getFD());
   }
 
   static File moveBlockFiles(Block b, File srcfile, File destdir)
@@ -669,6 +690,12 @@ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta,
     final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId);
     final File dstFile = new File(destDir, srcFile.getName());
     final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp);
+    return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum);
+  }
+
+  static File[] copyBlockFiles(File srcMeta, File srcFile, File dstMeta,
+                               File dstFile, boolean calculateChecksum)
+      throws IOException {
     if (calculateChecksum) {
       computeChecksum(srcMeta, dstMeta, srcFile);
     } else {
@@ -725,29 +752,29 @@ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
           + replicaInfo.getVolume().getStorageType());
     }
 
-    FsVolumeImpl targetVolume = volumes.getNextVolume(targetStorageType,
-        block.getNumBytes());
-    File oldBlockFile = replicaInfo.getBlockFile();
-    File oldMetaFile = replicaInfo.getMetaFile();
-
-    // Copy files to temp dir first
-    File[] blockFiles = copyBlockFiles(block.getBlockId(),
-        block.getGenerationStamp(), oldMetaFile, oldBlockFile,
-        targetVolume.getTmpDir(block.getBlockPoolId()),
-        replicaInfo.isOnTransientStorage());
+    try (FsVolumeReference volumeRef = volumes.getNextVolume(
+        targetStorageType, block.getNumBytes())) {
+      File oldBlockFile = replicaInfo.getBlockFile();
+      File oldMetaFile = replicaInfo.getMetaFile();
+      FsVolumeImpl targetVolume = (FsVolumeImpl) volumeRef.getVolume();
+      // Copy files to temp dir first
+      File[] blockFiles = copyBlockFiles(block.getBlockId(),
+          block.getGenerationStamp(), oldMetaFile, oldBlockFile,
+          targetVolume.getTmpDir(block.getBlockPoolId()),
+          replicaInfo.isOnTransientStorage());
 
-    ReplicaInfo newReplicaInfo = new ReplicaInPipeline(
-        replicaInfo.getBlockId(), replicaInfo.getGenerationStamp(),
-        targetVolume, blockFiles[0].getParentFile(), 0);
-    newReplicaInfo.setNumBytes(blockFiles[1].length());
-    // Finalize the copied files
-    newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
+      ReplicaInfo newReplicaInfo = new ReplicaInPipeline(
+          replicaInfo.getBlockId(), replicaInfo.getGenerationStamp(),
+          targetVolume, blockFiles[0].getParentFile(), 0);
+      newReplicaInfo.setNumBytes(blockFiles[1].length());
+      // Finalize the copied files
+      newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
 
-    removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
-        oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+      removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
+          oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+    }
 
     // Replace the old block if any to reschedule the scanning.
-    datanode.getBlockScanner().addBlock(block);
     return replicaInfo;
   }
 
@@ -769,7 +796,6 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
     final byte[] crcs = new byte[checksum.getChecksumSize(data.length)];
 
     DataOutputStream metaOut = null;
-    InputStream dataIn = null;
     try {
       File parentFile = dstMeta.getParentFile();
       if (parentFile != null) {
@@ -782,22 +808,23 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
           new FileOutputStream(dstMeta), HdfsConstants.SMALL_BUFFER_SIZE));
       BlockMetadataHeader.writeHeader(metaOut, checksum);
 
-      dataIn = isNativeIOAvailable ?
+      int offset = 0;
+      try (InputStream dataIn = isNativeIOAvailable ?
           NativeIO.getShareDeleteFileInputStream(blockFile) :
-          new FileInputStream(blockFile);
+          new FileInputStream(blockFile)) {
 
-      int offset = 0;
-      for(int n; (n = dataIn.read(data, offset, data.length - offset)) != -1; ) {
-        if (n > 0) {
-          n += offset;
-          offset = n % checksum.getBytesPerChecksum();
-          final int length = n - offset;
+        for (int n; (n = dataIn.read(data, offset, data.length - offset)) != -1; ) {
+          if (n > 0) {
+            n += offset;
+            offset = n % checksum.getBytesPerChecksum();
+            final int length = n - offset;
 
-          if (length > 0) {
-            checksum.calculateChunkedSums(data, 0, length, crcs, 0);
-            metaOut.write(crcs, 0, checksum.getChecksumSize(length));
+            if (length > 0) {
+              checksum.calculateChunkedSums(data, 0, length, crcs, 0);
+              metaOut.write(crcs, 0, checksum.getChecksumSize(length));
 
-            System.arraycopy(data, length, data, 0, offset);
+              System.arraycopy(data, length, data, 0, offset);
+            }
           }
         }
       }
@@ -806,7 +833,7 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
       checksum.calculateChunkedSums(data, 0, offset, crcs, 0);
       metaOut.write(crcs, 0, 4);
     } finally {
-      IOUtils.cleanup(LOG, dataIn, metaOut);
+      IOUtils.cleanup(LOG, metaOut);
     }
   }
 
@@ -863,7 +890,7 @@ static private void truncateBlock(File blockFile, File metaFile,
 
 
   @Override  // FsDatasetSpi
-  public synchronized ReplicaInPipeline append(ExtendedBlock b,
+  public synchronized ReplicaHandler append(ExtendedBlock b,
       long newGS, long expectedBlockLen) throws IOException {
     // If the block was successfully finalized because all packets
     // were successfully processed at the Datanode but the ack for
@@ -888,8 +915,16 @@ public synchronized ReplicaInPipeline append(ExtendedBlock b,
           " expected length is " + expectedBlockLen);
     }
 
-    return append(b.getBlockPoolId(), (FinalizedReplica)replicaInfo, newGS,
-        b.getNumBytes());
+    FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
+    ReplicaBeingWritten replica = null;
+    try {
+      replica = append(b.getBlockPoolId(), (FinalizedReplica)replicaInfo, newGS,
+          b.getNumBytes());
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
+    }
+    return new ReplicaHandler(replica, ref);
   }
   
   /** Append to a finalized replica
@@ -1010,22 +1045,30 @@ private ReplicaInfo recoverCheck(ExtendedBlock b, long newGS,
     
     return replicaInfo;
   }
-  
+
   @Override  // FsDatasetSpi
-  public synchronized ReplicaInPipeline recoverAppend(ExtendedBlock b,
-      long newGS, long expectedBlockLen) throws IOException {
+  public synchronized ReplicaHandler recoverAppend(
+      ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException {
     LOG.info("Recover failed append to " + b);
 
     ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
 
-    // change the replica's state/gs etc.
-    if (replicaInfo.getState() == ReplicaState.FINALIZED ) {
-      return append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo, newGS, 
-          b.getNumBytes());
-    } else { //RBW
-      bumpReplicaGS(replicaInfo, newGS);
-      return (ReplicaBeingWritten)replicaInfo;
+    FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
+    ReplicaBeingWritten replica;
+    try {
+      // change the replica's state/gs etc.
+      if (replicaInfo.getState() == ReplicaState.FINALIZED) {
+        replica = append(b.getBlockPoolId(), (FinalizedReplica) replicaInfo,
+                         newGS, b.getNumBytes());
+      } else { //RBW
+        bumpReplicaGS(replicaInfo, newGS);
+        replica = (ReplicaBeingWritten) replicaInfo;
+      }
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
     }
+    return new ReplicaHandler(replica, ref);
   }
 
   @Override // FsDatasetSpi
@@ -1073,8 +1116,9 @@ private void bumpReplicaGS(ReplicaInfo replicaInfo,
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipeline createRbw(StorageType storageType,
-      ExtendedBlock b, boolean allowLazyPersist) throws IOException {
+  public synchronized ReplicaHandler createRbw(
+      StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
+      throws IOException {
     ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
         b.getBlockId());
     if (replicaInfo != null) {
@@ -1083,15 +1127,15 @@ public synchronized ReplicaInPipeline createRbw(StorageType storageType,
       " and thus cannot be created.");
     }
     // create a new block
-    FsVolumeImpl v;
+    FsVolumeReference ref;
     while (true) {
       try {
         if (allowLazyPersist) {
           // First try to place the block on a transient volume.
-          v = volumes.getNextTransientVolume(b.getNumBytes());
+          ref = volumes.getNextTransientVolume(b.getNumBytes());
           datanode.getMetrics().incrRamDiskBlocksWrite();
         } else {
-          v = volumes.getNextVolume(storageType, b.getNumBytes());
+          ref = volumes.getNextVolume(storageType, b.getNumBytes());
         }
       } catch (DiskOutOfSpaceException de) {
         if (allowLazyPersist) {
@@ -1103,18 +1147,25 @@ public synchronized ReplicaInPipeline createRbw(StorageType storageType,
       }
       break;
     }
+    FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
     // create an rbw file to hold block in the designated volume
-    File f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
+    File f;
+    try {
+      f = v.createRbwFile(b.getBlockPoolId(), b.getLocalBlock());
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
+    }
+
     ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), 
         b.getGenerationStamp(), v, f.getParentFile(), b.getNumBytes());
     volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
-
-    return newReplicaInfo;
+    return new ReplicaHandler(newReplicaInfo, ref);
   }
-  
+
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipeline recoverRbw(ExtendedBlock b,
-      long newGS, long minBytesRcvd, long maxBytesRcvd)
+  public synchronized ReplicaHandler recoverRbw(
+      ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
       throws IOException {
     LOG.info("Recover RBW replica " + b);
 
@@ -1153,20 +1204,25 @@ public synchronized ReplicaInPipeline recoverRbw(ExtendedBlock b,
           minBytesRcvd + ", " + maxBytesRcvd + "].");
     }
 
-    // Truncate the potentially corrupt portion.
-    // If the source was client and the last node in the pipeline was lost,
-    // any corrupt data written after the acked length can go unnoticed. 
-    if (numBytes > bytesAcked) {
-      final File replicafile = rbw.getBlockFile();
-      truncateBlock(replicafile, rbw.getMetaFile(), numBytes, bytesAcked);
-      rbw.setNumBytes(bytesAcked);
-      rbw.setLastChecksumAndDataLen(bytesAcked, null);
-    }
+    FsVolumeReference ref = rbw.getVolume().obtainReference();
+    try {
+      // Truncate the potentially corrupt portion.
+      // If the source was client and the last node in the pipeline was lost,
+      // any corrupt data written after the acked length can go unnoticed.
+      if (numBytes > bytesAcked) {
+        final File replicafile = rbw.getBlockFile();
+        truncateBlock(replicafile, rbw.getMetaFile(), numBytes, bytesAcked);
+        rbw.setNumBytes(bytesAcked);
+        rbw.setLastChecksumAndDataLen(bytesAcked, null);
+      }
 
-    // bump the replica's generation stamp to newGS
-    bumpReplicaGS(rbw, newGS);
-    
-    return rbw;
+      // bump the replica's generation stamp to newGS
+      bumpReplicaGS(rbw, newGS);
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
+    }
+    return new ReplicaHandler(rbw, ref);
   }
   
   @Override // FsDatasetSpi
@@ -1231,8 +1287,8 @@ public synchronized ReplicaInPipeline convertTemporaryToRbw(
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipeline createTemporary(StorageType storageType,
-      ExtendedBlock b) throws IOException {
+  public synchronized ReplicaHandler createTemporary(
+      StorageType storageType, ExtendedBlock b) throws IOException {
     ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId());
     if (replicaInfo != null) {
       if (replicaInfo.getGenerationStamp() < b.getGenerationStamp()
@@ -1247,14 +1303,22 @@ public synchronized ReplicaInPipeline createTemporary(StorageType storageType,
             " and thus cannot be created.");
       }
     }
-    
-    FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
+
+    FsVolumeReference ref = volumes.getNextVolume(storageType, b.getNumBytes());
+    FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
     // create a temporary file to hold block in the designated volume
-    File f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock());
+    File f;
+    try {
+      f = v.createTmpFile(b.getBlockPoolId(), b.getLocalBlock());
+    } catch (IOException e) {
+      IOUtils.cleanup(null, ref);
+      throw e;
+    }
+
     ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(), 
         b.getGenerationStamp(), v, f.getParentFile(), 0);
     volumeMap.add(b.getBlockPoolId(), newReplicaInfo);
-    return newReplicaInfo;
+    return new ReplicaHandler(newReplicaInfo, ref);
   }
 
   /**
@@ -1393,7 +1457,8 @@ public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
     Map<String, ArrayList<ReplicaInfo>> uc =
         new HashMap<String, ArrayList<ReplicaInfo>>();
 
-    for (FsVolumeSpi v : volumes.volumes) {
+    List<FsVolumeImpl> curVolumes = getVolumes();
+    for (FsVolumeSpi v : curVolumes) {
       finalized.put(v.getStorageID(), new ArrayList<ReplicaInfo>());
       uc.put(v.getStorageID(), new ArrayList<ReplicaInfo>());
     }
@@ -1420,7 +1485,7 @@ public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
       }
     }
 
-    for (FsVolumeImpl v : volumes.volumes) {
+    for (FsVolumeImpl v : curVolumes) {
       ArrayList<ReplicaInfo> finalizedList = finalized.get(v.getStorageID());
       ArrayList<ReplicaInfo> ucList = uc.get(v.getStorageID());
       blockReportsMap.put(v.toDatanodeStorage(),
@@ -1599,11 +1664,6 @@ public void invalidate(String bpid, Block invalidBlks[]) throws IOException {
         }
         f = info.getBlockFile();
         v = (FsVolumeImpl)info.getVolume();
-        if (f == null) {
-          errors.add("Failed to delete replica " + invalidBlks[i]
-              +  ": File not found, volume=" + v);
-          continue;
-        }
         if (v == null) {
           errors.add("Failed to delete replica " + invalidBlks[i]
               +  ". No volume for this replica, file=" + f);
@@ -1641,10 +1701,15 @@ public void invalidate(String bpid, Block invalidBlks[]) throws IOException {
       // Delete the block asynchronously to make sure we can do it fast enough.
       // It's ok to unlink the block file before the uncache operation
       // finishes.
-      asyncDiskService.deleteAsync(v, f,
-          FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
-          new ExtendedBlock(bpid, invalidBlks[i]),
-          dataStorage.getTrashDirectoryForBlockFile(bpid, f));
+      try {
+        asyncDiskService.deleteAsync(v.obtainReference(), f,
+            FsDatasetUtil.getMetaFile(f, invalidBlks[i].getGenerationStamp()),
+            new ExtendedBlock(bpid, invalidBlks[i]),
+            dataStorage.getTrashDirectoryForBlockFile(bpid, f));
+      } catch (ClosedChannelException e) {
+        LOG.warn("Volume " + v + " is closed, ignore the deletion task for " +
+            "block " + invalidBlks[i]);
+      }
     }
     if (!errors.isEmpty()) {
       StringBuilder b = new StringBuilder("Failed to delete ")
@@ -1944,10 +2009,6 @@ public void checkAndUpdate(String bpid, long blockId, File diskFile,
           // Block is in memory and not on the disk
           // Remove the block from volumeMap
           volumeMap.remove(bpid, blockId);
-          final DataBlockScanner blockScanner = datanode.getBlockScanner();
-          if (blockScanner != null) {
-            blockScanner.deleteBlock(bpid, new Block(blockId));
-          }
           if (vol.isTransientStorage()) {
             ramDiskReplicaTracker.discardReplica(bpid, blockId, true);
           }
@@ -1970,12 +2031,7 @@ public void checkAndUpdate(String bpid, long blockId, File diskFile,
         ReplicaInfo diskBlockInfo = new FinalizedReplica(blockId, 
             diskFile.length(), diskGS, vol, diskFile.getParentFile());
         volumeMap.add(bpid, diskBlockInfo);
-        final DataBlockScanner blockScanner = datanode.getBlockScanner();
-        if (!vol.isTransientStorage()) {
-          if (blockScanner != null) {
-            blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo));
-          }
-        } else {
+        if (vol.isTransientStorage()) {
           ramDiskReplicaTracker.addReplica(bpid, blockId, (FsVolumeImpl) vol);
         }
         LOG.warn("Added missing block to memory " + diskBlockInfo);
@@ -2160,6 +2216,7 @@ static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
   public synchronized String updateReplicaUnderRecovery(
                                     final ExtendedBlock oldBlock,
                                     final long recoveryId,
+                                    final long newBlockId,
                                     final long newlength) throws IOException {
     //get replica
     final String bpid = oldBlock.getBlockPoolId();
@@ -2192,13 +2249,26 @@ public synchronized String updateReplicaUnderRecovery(
 
     //update replica
     final FinalizedReplica finalized = updateReplicaUnderRecovery(oldBlock
-        .getBlockPoolId(), (ReplicaUnderRecovery) replica, recoveryId, newlength);
-    assert finalized.getBlockId() == oldBlock.getBlockId()
-        && finalized.getGenerationStamp() == recoveryId
-        && finalized.getNumBytes() == newlength
-        : "Replica information mismatched: oldBlock=" + oldBlock
-            + ", recoveryId=" + recoveryId + ", newlength=" + newlength
-            + ", finalized=" + finalized;
+        .getBlockPoolId(), (ReplicaUnderRecovery) replica, recoveryId,
+        newBlockId, newlength);
+
+    boolean copyTruncate = newBlockId != oldBlock.getBlockId();
+    if(!copyTruncate) {
+      assert finalized.getBlockId() == oldBlock.getBlockId()
+          && finalized.getGenerationStamp() == recoveryId
+          && finalized.getNumBytes() == newlength
+          : "Replica information mismatched: oldBlock=" + oldBlock
+              + ", recoveryId=" + recoveryId + ", newlength=" + newlength
+              + ", newBlockId=" + newBlockId + ", finalized=" + finalized;
+    } else {
+      assert finalized.getBlockId() == oldBlock.getBlockId()
+          && finalized.getGenerationStamp() == oldBlock.getGenerationStamp()
+          && finalized.getNumBytes() == oldBlock.getNumBytes()
+          : "Finalized and old information mismatched: oldBlock=" + oldBlock
+              + ", genStamp=" + oldBlock.getGenerationStamp()
+              + ", len=" + oldBlock.getNumBytes()
+              + ", finalized=" + finalized;
+    }
 
     //check replica files after update
     checkReplicaFiles(finalized);
@@ -2211,6 +2281,7 @@ private FinalizedReplica updateReplicaUnderRecovery(
                                           String bpid,
                                           ReplicaUnderRecovery rur,
                                           long recoveryId,
+                                          long newBlockId,
                                           long newlength) throws IOException {
     //check recovery id
     if (rur.getRecoveryID() != recoveryId) {
@@ -2218,26 +2289,65 @@ private FinalizedReplica updateReplicaUnderRecovery(
           + ", rur=" + rur);
     }
 
+    boolean copyOnTruncate = newBlockId > 0L && rur.getBlockId() != newBlockId;
+    File blockFile;
+    File metaFile;
     // bump rur's GS to be recovery id
-    bumpReplicaGS(rur, recoveryId);
+    if(!copyOnTruncate) {
+      bumpReplicaGS(rur, recoveryId);
+      blockFile = rur.getBlockFile();
+      metaFile = rur.getMetaFile();
+    } else {
+      File[] copiedReplicaFiles =
+          copyReplicaWithNewBlockIdAndGS(rur, bpid, newBlockId, recoveryId);
+      blockFile = copiedReplicaFiles[1];
+      metaFile = copiedReplicaFiles[0];
+    }
 
     //update length
-    final File replicafile = rur.getBlockFile();
     if (rur.getNumBytes() < newlength) {
       throw new IOException("rur.getNumBytes() < newlength = " + newlength
           + ", rur=" + rur);
     }
     if (rur.getNumBytes() > newlength) {
       rur.unlinkBlock(1);
-      truncateBlock(replicafile, rur.getMetaFile(), rur.getNumBytes(), newlength);
-      // update RUR with the new length
-      rur.setNumBytes(newlength);
+      truncateBlock(blockFile, metaFile, rur.getNumBytes(), newlength);
+      if(!copyOnTruncate) {
+        // update RUR with the new length
+        rur.setNumBytes(newlength);
+      } else {
+        // Copying block to a new block with new blockId.
+        // Not truncating original block.
+        ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(
+            newBlockId, recoveryId, rur.getVolume(), blockFile.getParentFile(),
+            newlength);
+        newReplicaInfo.setNumBytes(newlength);
+        volumeMap.add(bpid, newReplicaInfo);
+        finalizeReplica(bpid, newReplicaInfo);
+      }
    }
 
     // finalize the block
     return finalizeReplica(bpid, rur);
   }
 
+  private File[] copyReplicaWithNewBlockIdAndGS(
+      ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS)
+      throws IOException {
+    String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId;
+    try (FsVolumeReference ref = volumes.getNextVolume(
+        replicaInfo.getVolume().getStorageType(), replicaInfo.getNumBytes())) {
+      FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
+      final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir();
+      final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId);
+      final File dstBlockFile = new File(destDir, blockFileName);
+      final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS);
+      return copyBlockFiles(replicaInfo.getMetaFile(),
+          replicaInfo.getBlockFile(),
+          dstMetaFile, dstBlockFile, true);
+    }
+  }
+
   @Override // FsDatasetSpi
   public synchronized long getReplicaVisibleLength(final ExtendedBlock block)
   throws IOException {
@@ -2288,12 +2398,14 @@ private static class VolumeInfo {
 
   private Collection<VolumeInfo> getVolumeInfo() {
     Collection<VolumeInfo> info = new ArrayList<VolumeInfo>();
-    for (FsVolumeImpl volume : volumes.volumes) {
+    for (FsVolumeImpl volume : getVolumes()) {
       long used = 0;
       long free = 0;
-      try {
+      try (FsVolumeReference ref = volume.obtainReference()) {
         used = volume.getDfsUsed();
         free = volume.getAvailable();
+      } catch (ClosedChannelException e) {
+        continue;
       } catch (IOException e) {
         LOG.warn(e.getMessage());
         used = 0;
@@ -2322,17 +2434,26 @@ public Map<String, Object> getVolumeInfoMap() {
   @Override //FsDatasetSpi
   public synchronized void deleteBlockPool(String bpid, boolean force)
       throws IOException {
+    List<FsVolumeImpl> curVolumes = getVolumes();
     if (!force) {
-      for (FsVolumeImpl volume : volumes.volumes) {
-        if (!volume.isBPDirEmpty(bpid)) {
-          LOG.warn(bpid + " has some block files, cannot delete unless forced");
-          throw new IOException("Cannot delete block pool, "
-              + "it contains some block files");
+      for (FsVolumeImpl volume : curVolumes) {
+        try (FsVolumeReference ref = volume.obtainReference()) {
+          if (!volume.isBPDirEmpty(bpid)) {
+            LOG.warn(bpid + " has some block files, cannot delete unless forced");
+            throw new IOException("Cannot delete block pool, "
+                + "it contains some block files");
+          }
+        } catch (ClosedChannelException e) {
+          // ignore.
         }
       }
     }
-    for (FsVolumeImpl volume : volumes.volumes) {
-      volume.deleteBPDirectories(bpid, force);
+    for (FsVolumeImpl volume : curVolumes) {
+      try (FsVolumeReference ref = volume.obtainReference()) {
+        volume.deleteBPDirectories(bpid, force);
+      } catch (ClosedChannelException e) {
+        // ignore.
+      }
     }
   }
   
@@ -2349,13 +2470,14 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
   @Override // FsDatasetSpi
   public HdfsBlocksMetadata getHdfsBlocksMetadata(String poolId,
       long[] blockIds) throws IOException {
+    List<FsVolumeImpl> curVolumes = getVolumes();
     // List of VolumeIds, one per volume on the datanode
-    List<byte[]> blocksVolumeIds = new ArrayList<byte[]>(volumes.volumes.size());
+    List<byte[]> blocksVolumeIds = new ArrayList<>(curVolumes.size());
     // List of indexes into the list of VolumeIds, pointing at the VolumeId of
     // the volume that the block is on
     List<Integer> blocksVolumeIndexes = new ArrayList<Integer>(blockIds.length);
     // Initialize the list of VolumeIds simply by enumerating the volumes
-    for (int i = 0; i < volumes.volumes.size(); i++) {
+    for (int i = 0; i < curVolumes.size(); i++) {
       blocksVolumeIds.add(ByteBuffer.allocate(4).putInt(i).array());
     }
     // Determine the index of the VolumeId of each block's volume, by comparing 
@@ -2368,7 +2490,7 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(String poolId,
       int volumeIndex = 0;
       if (info != null) {
         FsVolumeSpi blockVolume = info.getVolume();
-        for (FsVolumeImpl volume : volumes.volumes) {
+        for (FsVolumeImpl volume : curVolumes) {
           // This comparison of references should be safe
           if (blockVolume == volume) {
             isValid = true;
@@ -2412,23 +2534,6 @@ public void clearRollingUpgradeMarker(String bpid) throws IOException {
     dataStorage.clearRollingUpgradeMarker(bpid);
   }
 
-  @Override
-  public RollingLogs createRollingLogs(String bpid, String prefix
-      ) throws IOException {
-    String dir = null;
-    final List<FsVolumeImpl> volumes = getVolumes();
-    for (FsVolumeImpl vol : volumes) {
-      String bpDir = vol.getPath(bpid);
-      if (RollingLogsImpl.isFilePresent(bpDir, prefix)) {
-        dir = bpDir;
-        break;
-      }
-    }
-    if (dir == null) {
-      dir = volumes.get(0).getPath(bpid);
-    }
-    return new RollingLogsImpl(dir, prefix);
-  }
 
   @Override
   public void onCompleteLazyPersist(String bpId, long blockId,
@@ -2564,6 +2669,7 @@ public LazyWriter(Configuration conf) {
      */
     private boolean saveNextReplica() {
       RamDiskReplica block = null;
+      FsVolumeReference targetReference;
       FsVolumeImpl targetVolume;
       ReplicaInfo replicaInfo;
       boolean succeeded = false;
@@ -2581,8 +2687,9 @@ private boolean saveNextReplica() {
             if (replicaInfo != null &&
                 replicaInfo.getVolume().isTransientStorage()) {
               // Pick a target volume to persist the block.
-              targetVolume = volumes.getNextVolume(
+              targetReference = volumes.getNextVolume(
                   StorageType.DEFAULT, replicaInfo.getNumBytes());
+              targetVolume = (FsVolumeImpl) targetReference.getVolume();
 
               ramDiskReplicaTracker.recordStartLazyPersist(
                   block.getBlockPoolId(), block.getBlockId(), targetVolume);
@@ -2598,7 +2705,7 @@ private boolean saveNextReplica() {
                   block.getBlockPoolId(), block.getBlockId(),
                   replicaInfo.getGenerationStamp(), block.getCreationTime(),
                   replicaInfo.getMetaFile(), replicaInfo.getBlockFile(),
-                  targetVolume);
+                  targetReference);
             }
           }
         }
@@ -2621,10 +2728,14 @@ private boolean transientFreeSpaceBelowThreshold() throws IOException {
 
       // Don't worry about fragmentation for now. We don't expect more than one
       // transient volume per DN.
-      for (FsVolumeImpl v : volumes.volumes) {
-        if (v.isTransientStorage()) {
-          capacity += v.getCapacity();
-          free += v.getAvailable();
+      for (FsVolumeImpl v : getVolumes()) {
+        try (FsVolumeReference ref = v.obtainReference()) {
+          if (v.isTransientStorage()) {
+            capacity += v.getCapacity();
+            free += v.getAvailable();
+          }
+        } catch (ClosedChannelException e) {
+          // ignore.
         }
       }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 48427ad052a03..5ce2710347806 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -17,8 +17,18 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+import java.io.OutputStreamWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -31,6 +41,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.DF;
@@ -38,13 +50,24 @@
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.util.CloseableReferenceCount;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.Time;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The underlying volume used to store replica.
@@ -54,6 +77,9 @@
 @InterfaceAudience.Private
 @VisibleForTesting
 public class FsVolumeImpl implements FsVolumeSpi {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(FsVolumeImpl.class);
+
   private final FsDatasetImpl dataset;
   private final String storageID;
   private final StorageType storageType;
@@ -62,6 +88,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
   private final File currentDir;    // <StorageDirectory>/current
   private final DF usage;           
   private final long reserved;
+  private CloseableReferenceCount reference = new CloseableReferenceCount();
 
   // Disk space reserved for open blocks.
   private AtomicLong reservedForRbw;
@@ -99,6 +126,10 @@ protected ThreadPoolExecutor initializeCacheExecutor(File parent) {
     if (storageType.isTransient()) {
       return null;
     }
+    if (dataset.datanode == null) {
+      // FsVolumeImpl is used in test.
+      return null;
+    }
 
     final int maxNumThreads = dataset.datanode.getConf().getInt(
         DFSConfigKeys.DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY,
@@ -116,7 +147,114 @@ protected ThreadPoolExecutor initializeCacheExecutor(File parent) {
     executor.allowCoreThreadTimeOut(true);
     return executor;
   }
-  
+
+  private void printReferenceTraceInfo(String op) {
+    StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+    for (StackTraceElement ste : stack) {
+      switch (ste.getMethodName()) {
+      case "getDfsUsed":
+      case "getBlockPoolUsed":
+      case "getAvailable":
+      case "getVolumeMap":
+        return;
+      default:
+        break;
+      }
+    }
+    FsDatasetImpl.LOG.trace("Reference count: " + op + " " + this + ": " +
+        this.reference.getReferenceCount());
+    FsDatasetImpl.LOG.trace(
+        Joiner.on("\n").join(Thread.currentThread().getStackTrace()));
+  }
+
+  /**
+   * Increase the reference count. The caller must increase the reference count
+   * before issuing IOs.
+   *
+   * @throws IOException if the volume is already closed.
+   */
+  private void reference() throws ClosedChannelException {
+    this.reference.reference();
+    if (FsDatasetImpl.LOG.isTraceEnabled()) {
+      printReferenceTraceInfo("incr");
+    }
+  }
+
+  /**
+   * Decrease the reference count.
+   */
+  private void unreference() {
+    if (FsDatasetImpl.LOG.isTraceEnabled()) {
+      printReferenceTraceInfo("desc");
+    }
+    if (FsDatasetImpl.LOG.isDebugEnabled()) {
+      if (reference.getReferenceCount() <= 0) {
+        FsDatasetImpl.LOG.debug("Decrease reference count <= 0 on " + this +
+          Joiner.on("\n").join(Thread.currentThread().getStackTrace()));
+      }
+    }
+    checkReference();
+    this.reference.unreference();
+  }
+
+  private static class FsVolumeReferenceImpl implements FsVolumeReference {
+    private final FsVolumeImpl volume;
+
+    FsVolumeReferenceImpl(FsVolumeImpl volume) throws ClosedChannelException {
+      this.volume = volume;
+      volume.reference();
+    }
+
+    /**
+     * Decreases the reference count.
+     * @throws IOException it never throws IOException.
+     */
+    @Override
+    public void close() throws IOException {
+      volume.unreference();
+    }
+
+    @Override
+    public FsVolumeSpi getVolume() {
+      return this.volume;
+    }
+  }
+
+  @Override
+  public FsVolumeReference obtainReference() throws ClosedChannelException {
+    return new FsVolumeReferenceImpl(this);
+  }
+
+  private void checkReference() {
+    Preconditions.checkState(reference.getReferenceCount() > 0);
+  }
+
+  /**
+   * Close this volume and wait all other threads to release the reference count
+   * on this volume.
+   * @throws IOException if the volume is closed or the waiting is interrupted.
+   */
+  void closeAndWait() throws IOException {
+    try {
+      this.reference.setClosed();
+    } catch (ClosedChannelException e) {
+      throw new IOException("The volume has already closed.", e);
+    }
+    final int SLEEP_MILLIS = 500;
+    while (this.reference.getReferenceCount() > 0) {
+      if (FsDatasetImpl.LOG.isDebugEnabled()) {
+        FsDatasetImpl.LOG.debug(String.format(
+            "The reference count for %s is %d, wait to be 0.",
+            this, reference.getReferenceCount()));
+      }
+      try {
+        Thread.sleep(SLEEP_MILLIS);
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
   File getCurrentDir() {
     return currentDir;
   }
@@ -250,6 +388,7 @@ public String[] getBlockPoolList() {
    * the block is finalized.
    */
   File createTmpFile(String bpid, Block b) throws IOException {
+    checkReference();
     return getBlockPoolSlice(bpid).createTmpFile(b);
   }
 
@@ -277,11 +416,338 @@ public void releaseReservedSpace(long bytesToRelease) {
     }
   }
 
+  private enum SubdirFilter implements FilenameFilter {
+    INSTANCE;
+
+    @Override
+    public boolean accept(File dir, String name) {
+      return name.startsWith("subdir");
+    }
+  }
+
+  private enum BlockFileFilter implements FilenameFilter {
+    INSTANCE;
+
+    @Override
+    public boolean accept(File dir, String name) {
+      return !name.endsWith(".meta") && name.startsWith("blk_");
+    }
+  }
+
+  @VisibleForTesting
+  public static String nextSorted(List<String> arr, String prev) {
+    int res = 0;
+    if (prev != null) {
+      res = Collections.binarySearch(arr, prev);
+      if (res < 0) {
+        res = -1 - res;
+      } else {
+        res++;
+      }
+    }
+    if (res >= arr.size()) {
+      return null;
+    }
+    return arr.get(res);
+  }
+
+  private static class BlockIteratorState {
+    BlockIteratorState() {
+      lastSavedMs = iterStartMs = Time.now();
+      curFinalizedDir = null;
+      curFinalizedSubDir = null;
+      curEntry = null;
+      atEnd = false;
+    }
+
+    // The wall-clock ms since the epoch at which this iterator was last saved.
+    @JsonProperty
+    private long lastSavedMs;
+
+    // The wall-clock ms since the epoch at which this iterator was created.
+    @JsonProperty
+    private long iterStartMs;
+
+    @JsonProperty
+    private String curFinalizedDir;
+
+    @JsonProperty
+    private String curFinalizedSubDir;
+
+    @JsonProperty
+    private String curEntry;
+
+    @JsonProperty
+    private boolean atEnd;
+  }
+
+  /**
+   * A BlockIterator implementation for FsVolumeImpl.
+   */
+  private class BlockIteratorImpl implements FsVolumeSpi.BlockIterator {
+    private final File bpidDir;
+    private final String name;
+    private final String bpid;
+    private long maxStalenessMs = 0;
+
+    private List<String> cache;
+    private long cacheMs;
+
+    private BlockIteratorState state;
+
+    BlockIteratorImpl(String bpid, String name) {
+      this.bpidDir = new File(currentDir, bpid);
+      this.name = name;
+      this.bpid = bpid;
+      rewind();
+    }
+
+    /**
+     * Get the next subdirectory within the block pool slice.
+     *
+     * @return         The next subdirectory within the block pool slice, or
+     *                   null if there are no more.
+     */
+    private String getNextSubDir(String prev, File dir)
+          throws IOException {
+      List<String> children =
+          IOUtils.listDirectory(dir, SubdirFilter.INSTANCE);
+      cache = null;
+      cacheMs = 0;
+      if (children.size() == 0) {
+        LOG.trace("getNextSubDir({}, {}): no subdirectories found in {}",
+            storageID, bpid, dir.getAbsolutePath());
+        return null;
+      }
+      Collections.sort(children);
+      String nextSubDir = nextSorted(children, prev);
+      if (nextSubDir == null) {
+        LOG.trace("getNextSubDir({}, {}): no more subdirectories found in {}",
+            storageID, bpid, dir.getAbsolutePath());
+      } else {
+        LOG.trace("getNextSubDir({}, {}): picking next subdirectory {} " +
+            "within {}", storageID, bpid, nextSubDir, dir.getAbsolutePath());
+      }
+      return nextSubDir;
+    }
+
+    private String getNextFinalizedDir() throws IOException {
+      File dir = Paths.get(
+          bpidDir.getAbsolutePath(), "current", "finalized").toFile();
+      return getNextSubDir(state.curFinalizedDir, dir);
+    }
+
+    private String getNextFinalizedSubDir() throws IOException {
+      if (state.curFinalizedDir == null) {
+        return null;
+      }
+      File dir = Paths.get(
+          bpidDir.getAbsolutePath(), "current", "finalized",
+              state.curFinalizedDir).toFile();
+      return getNextSubDir(state.curFinalizedSubDir, dir);
+    }
+
+    private List<String> getSubdirEntries() throws IOException {
+      if (state.curFinalizedSubDir == null) {
+        return null; // There are no entries in the null subdir.
+      }
+      long now = Time.monotonicNow();
+      if (cache != null) {
+        long delta = now - cacheMs;
+        if (delta < maxStalenessMs) {
+          return cache;
+        } else {
+          LOG.trace("getSubdirEntries({}, {}): purging entries cache for {} " +
+            "after {} ms.", storageID, bpid, state.curFinalizedSubDir, delta);
+          cache = null;
+        }
+      }
+      File dir = Paths.get(bpidDir.getAbsolutePath(), "current", "finalized",
+                    state.curFinalizedDir, state.curFinalizedSubDir).toFile();
+      List<String> entries =
+          IOUtils.listDirectory(dir, BlockFileFilter.INSTANCE);
+      if (entries.size() == 0) {
+        entries = null;
+      } else {
+        Collections.sort(entries);
+      }
+      if (entries == null) {
+        LOG.trace("getSubdirEntries({}, {}): no entries found in {}",
+            storageID, bpid, dir.getAbsolutePath());
+      } else {
+        LOG.trace("getSubdirEntries({}, {}): listed {} entries in {}", 
+            storageID, bpid, entries.size(), dir.getAbsolutePath());
+      }
+      cache = entries;
+      cacheMs = now;
+      return cache;
+    }
+
+    /**
+     * Get the next block.<p/>
+     *
+     * Each volume has a hierarchical structure.<p/>
+     *
+     * <code>
+     * BPID B0
+     *   finalized/
+     *     subdir0
+     *       subdir0
+     *         blk_000
+     *         blk_001
+     *       ...
+     *     subdir1
+     *       subdir0
+     *         ...
+     *   rbw/
+     * </code>
+     *
+     * When we run out of entries at one level of the structure, we search
+     * progressively higher levels.  For example, when we run out of blk_
+     * entries in a subdirectory, we search for the next subdirectory.
+     * And so on.
+     */
+    @Override
+    public ExtendedBlock nextBlock() throws IOException {
+      if (state.atEnd) {
+        return null;
+      }
+      try {
+        while (true) {
+          List<String> entries = getSubdirEntries();
+          if (entries != null) {
+            state.curEntry = nextSorted(entries, state.curEntry);
+            if (state.curEntry == null) {
+              LOG.trace("nextBlock({}, {}): advancing from {} to next " +
+                  "subdirectory.", storageID, bpid, state.curFinalizedSubDir);
+            } else {
+              ExtendedBlock block =
+                  new ExtendedBlock(bpid, Block.filename2id(state.curEntry));
+              LOG.trace("nextBlock({}, {}): advancing to {}",
+                  storageID, bpid, block);
+              return block;
+            }
+          }
+          state.curFinalizedSubDir = getNextFinalizedSubDir();
+          if (state.curFinalizedSubDir == null) {
+            state.curFinalizedDir = getNextFinalizedDir();
+            if (state.curFinalizedDir == null) {
+              state.atEnd = true;
+              return null;
+            }
+          }
+        }
+      } catch (IOException e) {
+        state.atEnd = true;
+        LOG.error("nextBlock({}, {}): I/O error", storageID, bpid, e);
+        throw e;
+      }
+    }
+
+    @Override
+    public boolean atEnd() {
+      return state.atEnd;
+    }
+
+    @Override
+    public void rewind() {
+      cache = null;
+      cacheMs = 0;
+      state = new BlockIteratorState();
+    }
+
+    @Override
+    public void save() throws IOException {
+      state.lastSavedMs = Time.now();
+      boolean success = false;
+      ObjectMapper mapper = new ObjectMapper();
+      try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
+                new FileOutputStream(getTempSaveFile(), false), "UTF-8"))) {
+        mapper.writerWithDefaultPrettyPrinter().writeValue(writer, state);
+        success = true;
+      } finally {
+        if (!success) {
+          if (getTempSaveFile().delete()) {
+            LOG.debug("save({}, {}): error deleting temporary file.",
+                storageID, bpid);
+          }
+        }
+      }
+      Files.move(getTempSaveFile().toPath(), getSaveFile().toPath(),
+          StandardCopyOption.ATOMIC_MOVE);
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("save({}, {}): saved {}", storageID, bpid,
+            mapper.writerWithDefaultPrettyPrinter().writeValueAsString(state));
+      }
+    }
+
+    public void load() throws IOException {
+      ObjectMapper mapper = new ObjectMapper();
+      File file = getSaveFile();
+      this.state = mapper.reader(BlockIteratorState.class).readValue(file);
+      LOG.trace("load({}, {}): loaded iterator {} from {}: {}", storageID,
+          bpid, name, file.getAbsoluteFile(),
+          mapper.writerWithDefaultPrettyPrinter().writeValueAsString(state));
+    }
+
+    File getSaveFile() {
+      return new File(bpidDir, name + ".cursor");
+    }
+
+    File getTempSaveFile() {
+      return new File(bpidDir, name + ".cursor.tmp");
+    }
+
+    @Override
+    public void setMaxStalenessMs(long maxStalenessMs) {
+      this.maxStalenessMs = maxStalenessMs;
+    }
+
+    @Override
+    public void close() throws IOException {
+      // No action needed for this volume implementation.
+    }
+
+    @Override
+    public long getIterStartMs() {
+      return state.iterStartMs;
+    }
+
+    @Override
+    public long getLastSavedMs() {
+      return state.lastSavedMs;
+    }
+
+    @Override
+    public String getBlockPoolId() {
+      return bpid;
+    }
+  }
+
+  @Override
+  public BlockIterator newBlockIterator(String bpid, String name) {
+    return new BlockIteratorImpl(bpid, name);
+  }
+
+  @Override
+  public BlockIterator loadBlockIterator(String bpid, String name)
+      throws IOException {
+    BlockIteratorImpl iter = new BlockIteratorImpl(bpid, name);
+    iter.load();
+    return iter;
+  }
+
+  @Override
+  public FsDatasetSpi getDataset() {
+    return dataset;
+  }
+
   /**
    * RBW files. They get moved to the finalized block directory when
    * the block is finalized.
    */
   File createRbwFile(String bpid, Block b) throws IOException {
+    checkReference();
     reserveSpaceForRbw(b.getNumBytes());
     return getBlockPoolSlice(bpid).createRbwFile(b);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 837ddf720afb0..ae2f5b4cba7f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -17,104 +17,150 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 
+import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.Time;
 
 class FsVolumeList {
-  /**
-   * Read access to this unmodifiable list is not synchronized.
-   * This list is replaced on modification holding "this" lock.
-   */
-  volatile List<FsVolumeImpl> volumes = null;
+  private final AtomicReference<FsVolumeImpl[]> volumes =
+      new AtomicReference<>(new FsVolumeImpl[0]);
+  private Object checkDirsMutex = new Object();
 
   private final VolumeChoosingPolicy<FsVolumeImpl> blockChooser;
+  private final BlockScanner blockScanner;
   private volatile int numFailedVolumes;
 
-  FsVolumeList(int failedVols,
+  FsVolumeList(int failedVols, BlockScanner blockScanner,
       VolumeChoosingPolicy<FsVolumeImpl> blockChooser) {
     this.blockChooser = blockChooser;
+    this.blockScanner = blockScanner;
     this.numFailedVolumes = failedVols;
   }
   
   int numberOfFailedVolumes() {
     return numFailedVolumes;
   }
-  
+
+  /**
+   * Return an immutable list view of all the volumes.
+   */
+  List<FsVolumeImpl> getVolumes() {
+    return Collections.unmodifiableList(Arrays.asList(volumes.get()));
+  }
+
+  private FsVolumeReference chooseVolume(List<FsVolumeImpl> list, long blockSize)
+      throws IOException {
+    while (true) {
+      FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize);
+      try {
+        return volume.obtainReference();
+      } catch (ClosedChannelException e) {
+        FsDatasetImpl.LOG.warn("Chosen a closed volume: " + volume);
+        // blockChooser.chooseVolume returns DiskOutOfSpaceException when the list
+        // is empty, indicating that all volumes are closed.
+        list.remove(volume);
+      }
+    }
+  }
+
   /** 
-   * Get next volume. Synchronized to ensure {@link #curVolume} is updated
-   * by a single thread and next volume is chosen with no concurrent
-   * update to {@link #volumes}.
+   * Get next volume.
+   *
    * @param blockSize free space needed on the volume
    * @param storageType the desired {@link StorageType} 
    * @return next volume to store the block in.
    */
-  synchronized FsVolumeImpl getNextVolume(StorageType storageType,
-      long blockSize) throws IOException {
-    final List<FsVolumeImpl> list = new ArrayList<FsVolumeImpl>(volumes.size());
-    for(FsVolumeImpl v : volumes) {
+  FsVolumeReference getNextVolume(StorageType storageType, long blockSize)
+      throws IOException {
+    // Get a snapshot of currently available volumes.
+    final FsVolumeImpl[] curVolumes = volumes.get();
+    final List<FsVolumeImpl> list = new ArrayList<>(curVolumes.length);
+    for(FsVolumeImpl v : curVolumes) {
       if (v.getStorageType() == storageType) {
         list.add(v);
       }
     }
-    return blockChooser.chooseVolume(list, blockSize);
+    return chooseVolume(list, blockSize);
   }
 
   /**
-   * Get next volume. Synchronized to ensure {@link #curVolume} is updated
-   * by a single thread and next volume is chosen with no concurrent
-   * update to {@link #volumes}.
+   * Get next volume.
+   *
    * @param blockSize free space needed on the volume
    * @return next volume to store the block in.
    */
-  synchronized FsVolumeImpl getNextTransientVolume(
-      long blockSize) throws IOException {
-    final List<FsVolumeImpl> list = new ArrayList<FsVolumeImpl>(volumes.size());
-    for(FsVolumeImpl v : volumes) {
+  FsVolumeReference getNextTransientVolume(long blockSize) throws IOException {
+    // Get a snapshot of currently available volumes.
+    final List<FsVolumeImpl> curVolumes = getVolumes();
+    final List<FsVolumeImpl> list = new ArrayList<>(curVolumes.size());
+    for(FsVolumeImpl v : curVolumes) {
       if (v.isTransientStorage()) {
         list.add(v);
       }
     }
-    return blockChooser.chooseVolume(list, blockSize);
+    return chooseVolume(list, blockSize);
   }
 
   long getDfsUsed() throws IOException {
     long dfsUsed = 0L;
-    for (FsVolumeImpl v : volumes) {
-      dfsUsed += v.getDfsUsed();
+    for (FsVolumeImpl v : volumes.get()) {
+      try(FsVolumeReference ref = v.obtainReference()) {
+        dfsUsed += v.getDfsUsed();
+      } catch (ClosedChannelException e) {
+        // ignore.
+      }
     }
     return dfsUsed;
   }
 
   long getBlockPoolUsed(String bpid) throws IOException {
     long dfsUsed = 0L;
-    for (FsVolumeImpl v : volumes) {
-      dfsUsed += v.getBlockPoolUsed(bpid);
+    for (FsVolumeImpl v : volumes.get()) {
+      try (FsVolumeReference ref = v.obtainReference()) {
+        dfsUsed += v.getBlockPoolUsed(bpid);
+      } catch (ClosedChannelException e) {
+        // ignore.
+      }
     }
     return dfsUsed;
   }
 
   long getCapacity() {
     long capacity = 0L;
-    for (FsVolumeImpl v : volumes) {
-      capacity += v.getCapacity();
+    for (FsVolumeImpl v : volumes.get()) {
+      try (FsVolumeReference ref = v.obtainReference()) {
+        capacity += v.getCapacity();
+      } catch (IOException e) {
+        // ignore.
+      }
     }
     return capacity;
   }
     
   long getRemaining() throws IOException {
     long remaining = 0L;
-    for (FsVolumeSpi vol : volumes) {
-      remaining += vol.getAvailable();
+    for (FsVolumeSpi vol : volumes.get()) {
+      try (FsVolumeReference ref = vol.obtainReference()) {
+        remaining += vol.getAvailable();
+      } catch (ClosedChannelException e) {
+        // ignore
+      }
     }
     return remaining;
   }
@@ -127,10 +173,10 @@ void getAllVolumesMap(final String bpid,
     final List<IOException> exceptions = Collections.synchronizedList(
         new ArrayList<IOException>());
     List<Thread> replicaAddingThreads = new ArrayList<Thread>();
-    for (final FsVolumeImpl v : volumes) {
+    for (final FsVolumeImpl v : volumes.get()) {
       Thread t = new Thread() {
         public void run() {
-          try {
+          try (FsVolumeReference ref = v.obtainReference()) {
             FsDatasetImpl.LOG.info("Adding replicas to map for block pool " +
                 bpid + " on volume " + v + "...");
             long startTime = Time.monotonicNow();
@@ -138,6 +184,9 @@ public void run() {
             long timeTaken = Time.monotonicNow() - startTime;
             FsDatasetImpl.LOG.info("Time to add replicas to map for block pool"
                 + " " + bpid + " on volume " + v + ": " + timeTaken + "ms");
+          } catch (ClosedChannelException e) {
+            FsDatasetImpl.LOG.info("The volume " + v + " is closed while " +
+                "addng replicas, ignored.");
           } catch (IOException ioe) {
             FsDatasetImpl.LOG.info("Caught exception while adding replicas " +
                 "from " + v + ". Will throw later.", ioe);
@@ -167,76 +216,135 @@ public void run() {
    * Calls {@link FsVolumeImpl#checkDirs()} on each volume, removing any
    * volumes from the active list that result in a DiskErrorException.
    * 
-   * This method is synchronized to allow only one instance of checkDirs() 
-   * call
+   * Use checkDirsMutext to allow only one instance of checkDirs() call
+   *
    * @return list of all the removed volumes.
    */
-  synchronized List<FsVolumeImpl> checkDirs() {
-    ArrayList<FsVolumeImpl> removedVols = null;
-    
-    // Make a copy of volumes for performing modification 
-    final List<FsVolumeImpl> volumeList = new ArrayList<FsVolumeImpl>(volumes);
+  List<FsVolumeImpl> checkDirs() {
+    synchronized(checkDirsMutex) {
+      ArrayList<FsVolumeImpl> removedVols = null;
+      
+      // Make a copy of volumes for performing modification 
+      final List<FsVolumeImpl> volumeList = getVolumes();
 
-    for(Iterator<FsVolumeImpl> i = volumeList.iterator(); i.hasNext(); ) {
-      final FsVolumeImpl fsv = i.next();
-      try {
-        fsv.checkDirs();
-      } catch (DiskErrorException e) {
-        FsDatasetImpl.LOG.warn("Removing failed volume " + fsv + ": ",e);
-        if (removedVols == null) {
-          removedVols = new ArrayList<FsVolumeImpl>(1);
+      for(Iterator<FsVolumeImpl> i = volumeList.iterator(); i.hasNext(); ) {
+        final FsVolumeImpl fsv = i.next();
+        try (FsVolumeReference ref = fsv.obtainReference()) {
+          fsv.checkDirs();
+        } catch (DiskErrorException e) {
+          FsDatasetImpl.LOG.warn("Removing failed volume " + fsv + ": ", e);
+          if (removedVols == null) {
+            removedVols = new ArrayList<>(1);
+          }
+          removedVols.add(fsv);
+          removeVolume(fsv);
+          numFailedVolumes++;
+        } catch (ClosedChannelException e) {
+          FsDatasetImpl.LOG.debug("Caught exception when obtaining " +
+            "reference count on closed volume", e);
+        } catch (IOException e) {
+          FsDatasetImpl.LOG.error("Unexpected IOException", e);
         }
-        removedVols.add(fsv);
-        fsv.shutdown(); 
-        i.remove(); // Remove the volume
-        numFailedVolumes++;
       }
-    }
-    
-    if (removedVols != null && removedVols.size() > 0) {
-      // Replace volume list
-      volumes = Collections.unmodifiableList(volumeList);
-      FsDatasetImpl.LOG.warn("Completed checkDirs. Removed " + removedVols.size()
-          + " volumes. Current volumes: " + this);
-    }
+      
+      if (removedVols != null && removedVols.size() > 0) {
+        FsDatasetImpl.LOG.warn("Completed checkDirs. Removed " + removedVols.size()
+            + " volumes. Current volumes: " + this);
+      }
 
-    return removedVols;
+      return removedVols;
+    }
   }
 
   @Override
   public String toString() {
-    return volumes.toString();
+    return Arrays.toString(volumes.get());
   }
 
   /**
    * Dynamically add new volumes to the existing volumes that this DN manages.
-   * @param newVolume the instance of new FsVolumeImpl.
+   *
+   * @param ref       a reference to the new FsVolumeImpl instance.
+   */
+  void addVolume(FsVolumeReference ref) {
+    while (true) {
+      final FsVolumeImpl[] curVolumes = volumes.get();
+      final List<FsVolumeImpl> volumeList = Lists.newArrayList(curVolumes);
+      volumeList.add((FsVolumeImpl)ref.getVolume());
+      if (volumes.compareAndSet(curVolumes,
+          volumeList.toArray(new FsVolumeImpl[volumeList.size()]))) {
+        break;
+      } else {
+        if (FsDatasetImpl.LOG.isDebugEnabled()) {
+          FsDatasetImpl.LOG.debug(
+              "The volume list has been changed concurrently, " +
+                  "retry to remove volume: " + ref.getVolume().getStorageID());
+        }
+      }
+    }
+    if (blockScanner != null) {
+      blockScanner.addVolumeScanner(ref);
+    }
+    FsDatasetImpl.LOG.info("Added new volume: " +
+        ref.getVolume().getStorageID());
+  }
+
+  /**
+   * Dynamically remove a volume in the list.
+   * @param target the volume instance to be removed.
    */
-  synchronized void addVolume(FsVolumeImpl newVolume) {
-    // Make a copy of volumes to add new volumes.
-    final List<FsVolumeImpl> volumeList = volumes == null ?
-        new ArrayList<FsVolumeImpl>() :
-        new ArrayList<FsVolumeImpl>(volumes);
-    volumeList.add(newVolume);
-    volumes = Collections.unmodifiableList(volumeList);
-    FsDatasetImpl.LOG.info("Added new volume: " + newVolume.toString());
+  private void removeVolume(FsVolumeImpl target) {
+    while (true) {
+      final FsVolumeImpl[] curVolumes = volumes.get();
+      final List<FsVolumeImpl> volumeList = Lists.newArrayList(curVolumes);
+      if (volumeList.remove(target)) {
+        if (volumes.compareAndSet(curVolumes,
+            volumeList.toArray(new FsVolumeImpl[volumeList.size()]))) {
+          if (blockScanner != null) {
+            blockScanner.removeVolumeScanner(target);
+          }
+          try {
+            target.closeAndWait();
+          } catch (IOException e) {
+            FsDatasetImpl.LOG.warn(
+                "Error occurs when waiting volume to close: " + target, e);
+          }
+          target.shutdown();
+          FsDatasetImpl.LOG.info("Removed volume: " + target);
+          break;
+        } else {
+          if (FsDatasetImpl.LOG.isDebugEnabled()) {
+            FsDatasetImpl.LOG.debug(
+                "The volume list has been changed concurrently, " +
+                "retry to remove volume: " + target);
+          }
+        }
+      } else {
+        if (FsDatasetImpl.LOG.isDebugEnabled()) {
+          FsDatasetImpl.LOG.debug("Volume " + target +
+              " does not exist or is removed by others.");
+        }
+        break;
+      }
+    }
   }
 
   /**
-   * Dynamically remove volume to the list.
+   * Dynamically remove volume in the list.
    * @param volume the volume to be removed.
    */
-  synchronized void removeVolume(String volume) {
+  void removeVolume(File volume) {
     // Make a copy of volumes to remove one volume.
-    final List<FsVolumeImpl> volumeList = new ArrayList<FsVolumeImpl>(volumes);
+    final FsVolumeImpl[] curVolumes = volumes.get();
+    final List<FsVolumeImpl> volumeList = Lists.newArrayList(curVolumes);
     for (Iterator<FsVolumeImpl> it = volumeList.iterator(); it.hasNext(); ) {
       FsVolumeImpl fsVolume = it.next();
-      if (fsVolume.getBasePath().equals(volume)) {
-        fsVolume.shutdown();
-        it.remove();
-        volumes = Collections.unmodifiableList(volumeList);
-        FsDatasetImpl.LOG.info("Removed volume: " + volume);
-        break;
+      String basePath, targetPath;
+      basePath = new File(fsVolume.getBasePath()).getAbsolutePath();
+      targetPath = volume.getAbsolutePath();
+      if (basePath.equals(targetPath)) {
+        // Make sure the removed volume is the one in the curVolumes.
+        removeVolume(fsVolume);
       }
     }
   }
@@ -247,10 +355,10 @@ void addBlockPool(final String bpid, final Configuration conf) throws IOExceptio
     final List<IOException> exceptions = Collections.synchronizedList(
         new ArrayList<IOException>());
     List<Thread> blockPoolAddingThreads = new ArrayList<Thread>();
-    for (final FsVolumeImpl v : volumes) {
+    for (final FsVolumeImpl v : volumes.get()) {
       Thread t = new Thread() {
         public void run() {
-          try {
+          try (FsVolumeReference ref = v.obtainReference()) {
             FsDatasetImpl.LOG.info("Scanning block pool " + bpid +
                 " on volume " + v + "...");
             long startTime = Time.monotonicNow();
@@ -258,6 +366,8 @@ public void run() {
             long timeTaken = Time.monotonicNow() - startTime;
             FsDatasetImpl.LOG.info("Time taken to scan block pool " + bpid +
                 " on " + v + ": " + timeTaken + "ms");
+          } catch (ClosedChannelException e) {
+            // ignore.
           } catch (IOException ioe) {
             FsDatasetImpl.LOG.info("Caught exception while scanning " + v +
                 ". Will throw later.", ioe);
@@ -285,13 +395,13 @@ public void run() {
   }
   
   void removeBlockPool(String bpid) {
-    for (FsVolumeImpl v : volumes) {
+    for (FsVolumeImpl v : volumes.get()) {
       v.shutdownBlockPool(bpid);
     }
   }
 
   void shutdown() {
-    for (FsVolumeImpl volume : volumes) {
+    for (FsVolumeImpl volume : volumes.get()) {
       if(volume != null) {
         volume.shutdown();
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
index c9aba8a292a5b..cf8de0a5efc1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
@@ -21,7 +21,9 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 
+import javax.ws.rs.HEAD;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
@@ -175,13 +177,14 @@ synchronized void shutdown() {
   void submitLazyPersistTask(String bpId, long blockId,
       long genStamp, long creationTime,
       File metaFile, File blockFile,
-      FsVolumeImpl targetVolume) throws IOException {
+      FsVolumeReference target) throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("LazyWriter schedule async task to persist RamDisk block pool id: "
           + bpId + " block id: " + blockId);
     }
 
-    File lazyPersistDir  = targetVolume.getLazyPersistDir(bpId);
+    FsVolumeImpl volume = (FsVolumeImpl)target.getVolume();
+    File lazyPersistDir  = volume.getLazyPersistDir(bpId);
     if (!lazyPersistDir.exists() && !lazyPersistDir.mkdirs()) {
       FsDatasetImpl.LOG.warn("LazyWriter failed to create " + lazyPersistDir);
       throw new IOException("LazyWriter fail to find or create lazy persist dir: "
@@ -190,8 +193,8 @@ void submitLazyPersistTask(String bpId, long blockId,
 
     ReplicaLazyPersistTask lazyPersistTask = new ReplicaLazyPersistTask(
         bpId, blockId, genStamp, creationTime, blockFile, metaFile,
-        targetVolume, lazyPersistDir);
-    execute(targetVolume.getCurrentDir(), lazyPersistTask);
+        target, lazyPersistDir);
+    execute(volume.getCurrentDir(), lazyPersistTask);
   }
 
   class ReplicaLazyPersistTask implements Runnable {
@@ -201,13 +204,13 @@ class ReplicaLazyPersistTask implements Runnable {
     final long creationTime;
     final File blockFile;
     final File metaFile;
-    final FsVolumeImpl targetVolume;
+    final FsVolumeReference targetVolume;
     final File lazyPersistDir;
 
     ReplicaLazyPersistTask(String bpId, long blockId,
         long genStamp, long creationTime,
         File blockFile, File metaFile,
-        FsVolumeImpl targetVolume, File lazyPersistDir) {
+        FsVolumeReference targetVolume, File lazyPersistDir) {
       this.bpId = bpId;
       this.blockId = blockId;
       this.genStamp = genStamp;
@@ -229,14 +232,15 @@ public String toString() {
     @Override
     public void run() {
       boolean succeeded = false;
-      try {
+      final FsDatasetImpl dataset = (FsDatasetImpl)datanode.getFSDataset();
+      try (FsVolumeReference ref = this.targetVolume) {
         // No FsDatasetImpl lock for the file copy
         File targetFiles[] = FsDatasetImpl.copyBlockFiles(
             blockId, genStamp, metaFile, blockFile, lazyPersistDir, true);
 
         // Lock FsDataSetImpl during onCompleteLazyPersist callback
-        datanode.getFSDataset().onCompleteLazyPersist(bpId, blockId,
-            creationTime, targetFiles, targetVolume);
+        dataset.onCompleteLazyPersist(bpId, blockId,
+                creationTime, targetFiles, (FsVolumeImpl)ref.getVolume());
         succeeded = true;
       } catch (Exception e){
         FsDatasetImpl.LOG.warn(
@@ -244,7 +248,7 @@ public void run() {
             + bpId + "block Id: " + blockId, e);
       } finally {
         if (!succeeded) {
-          datanode.getFSDataset().onFailLazyPersist(bpId, blockId);
+          dataset.onFailLazyPersist(bpId, blockId);
         }
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java
deleted file mode 100644
index 121127d849e3a..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
-
-import com.google.common.base.Charsets;
-
-class RollingLogsImpl implements RollingLogs {
-  private static final String CURR_SUFFIX = ".curr";
-  private static final String PREV_SUFFIX = ".prev";
-
-  static boolean isFilePresent(String dir, String filePrefix) {
-    return new File(dir, filePrefix + CURR_SUFFIX).exists() ||
-           new File(dir, filePrefix + PREV_SUFFIX).exists();
-  }
-
-  private final File curr;
-  private final File prev;
-  private PrintWriter out; //require synchronized access
-
-  private final Appender appender = new Appender() {
-    @Override
-    public Appendable append(CharSequence csq) {
-      synchronized(RollingLogsImpl.this) {
-        if (out == null) {
-          throw new IllegalStateException(RollingLogsImpl.this
-              + " is not yet opened.");
-        }
-        out.print(csq);
-        out.flush();
-      }
-      return this;
-    }
-
-    @Override
-    public Appendable append(char c) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Appendable append(CharSequence csq, int start, int end) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void close() {
-      synchronized(RollingLogsImpl.this) {
-        if (out != null) {
-          out.close();
-          out = null;
-        }
-      }
-    }
-  };
-
-
-  private final AtomicInteger numReaders = new AtomicInteger();
-
-  RollingLogsImpl(String dir, String filePrefix) throws FileNotFoundException{
-    curr = new File(dir, filePrefix + CURR_SUFFIX);
-    prev = new File(dir, filePrefix + PREV_SUFFIX);
-    out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(
-        curr, true), Charsets.UTF_8));
-  }
-
-  @Override
-  public Reader iterator(boolean skipPrevFile) throws IOException {
-    numReaders.incrementAndGet(); 
-    return new Reader(skipPrevFile);
-  }
-
-  @Override
-  public Appender appender() {
-    return appender;
-  }
-
-  @Override
-  public boolean roll() throws IOException {
-    if (numReaders.get() > 0) {
-      return false;
-    }
-    if (!prev.delete() && prev.exists()) {
-      throw new IOException("Failed to delete " + prev);
-    }
-
-    synchronized(this) {
-      appender.close();
-      final boolean renamed = curr.renameTo(prev);
-      out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(
-          curr, true), Charsets.UTF_8));
-      if (!renamed) {
-        throw new IOException("Failed to rename " + curr + " to " + prev);
-      }
-    }
-    return true;
-  }
-
-  @Override
-  public String toString() {
-    return curr.toString();
-  }
-  
-  /**
-   * This is used to read the lines in order.
-   * If the data is not read completely (i.e, untill hasNext() returns
-   * false), it needs to be explicitly 
-   */
-  private class Reader implements RollingLogs.LineIterator {
-    private File file;
-    private File lastReadFile;
-    private BufferedReader reader;
-    private String line;
-    private boolean closed = false;
-    
-    private Reader(boolean skipPrevFile) throws IOException {
-      reader = null;
-      file = skipPrevFile? curr : prev;
-      readNext();        
-    }
-
-    @Override
-    public boolean isPrevious() {
-      return file == prev;
-    }
-
-    @Override
-    public boolean isLastReadFromPrevious() {
-      return lastReadFile == prev;
-    }
-
-    private boolean openFile() throws IOException {
-
-      for(int i=0; i<2; i++) {
-        if (reader != null || i > 0) {
-          // move to next file
-          file = isPrevious()? curr : null;
-        }
-        if (file == null) {
-          return false;
-        }
-        if (file.exists()) {
-          break;
-        }
-      }
-      
-      if (reader != null ) {
-        reader.close();
-        reader = null;
-      }
-      
-      reader = new BufferedReader(new InputStreamReader(new FileInputStream(
-          file), Charsets.UTF_8));
-      return true;
-    }
-    
-    // read next line if possible.
-    private void readNext() throws IOException {
-      line = null;
-      try {
-        if (reader != null && (line = reader.readLine()) != null) {
-          return;
-        }
-        // move to the next file.
-        if (openFile()) {
-          readNext();
-        }
-      } finally {
-        if (!hasNext()) {
-          close();
-        }
-      }
-    }
-    
-    @Override
-    public boolean hasNext() {
-      return line != null;
-    }
-
-    @Override
-    public String next() {
-      String curLine = line;
-      try {
-        lastReadFile = file;
-        readNext();
-      } catch (IOException e) {
-        DataBlockScanner.LOG.warn("Failed to read next line.", e);
-      }
-      return curLine;
-    }
-
-    @Override
-    public void remove() {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (!closed) {
-        try {
-          if (reader != null) {
-            reader.close();
-          }
-        } finally {
-          file = null;
-          reader = null;
-          closed = true;
-          final int n = numReaders.decrementAndGet();
-          assert(n >= 0);
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index fea40d7ce9644..a7bb4907d9cff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
 
+import com.google.common.base.Charsets;
 import com.sun.jersey.api.ParamException;
 import com.sun.jersey.api.container.ContainerException;
 import io.netty.buffer.Unpooled;
@@ -39,7 +40,7 @@
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
 import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
 
 class ExceptionHandler {
   static Log LOG = WebHdfsHandler.LOG;
@@ -82,11 +83,11 @@ static DefaultFullHttpResponse exceptionCaught(Throwable cause) {
       s = INTERNAL_SERVER_ERROR;
     }
 
-    final byte[] js = JsonUtil.toJsonString(e).getBytes();
+    final byte[] js = JsonUtil.toJsonString(e).getBytes(Charsets.UTF_8);
     DefaultFullHttpResponse resp =
       new DefaultFullHttpResponse(HTTP_1_1, s, Unpooled.wrappedBuffer(js));
 
-    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON);
+    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
     resp.headers().set(CONTENT_LENGTH, js.length);
     return resp;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index cf7021828836b..be1faecebddc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -29,6 +29,7 @@
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.QueryStringDecoder;
 import io.netty.handler.stream.ChunkedStream;
+import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -77,7 +78,8 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
   public static final int WEBHDFS_PREFIX_LENGTH = WEBHDFS_PREFIX.length();
   public static final String APPLICATION_OCTET_STREAM =
     "application/octet-stream";
-  public static final String APPLICATION_JSON = "application/json";
+  public static final String APPLICATION_JSON_UTF8 =
+      "application/json; charset=utf-8";
 
   private final Configuration conf;
   private final Configuration confForCreate;
@@ -174,7 +176,8 @@ private void onAppend(ChannelHandlerContext ctx) throws IOException {
     final int bufferSize = params.bufferSize();
 
     DFSClient dfsClient = newDfsClient(nnId, conf);
-    OutputStream out = dfsClient.append(path, bufferSize, null, null);
+    OutputStream out = dfsClient.append(path, bufferSize,
+        EnumSet.of(CreateFlag.APPEND), null, null);
     DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, OK);
     resp.headers().set(CONTENT_LENGTH, 0);
     ctx.pipeline().replace(this, HdfsWriter.class.getSimpleName(),
@@ -224,11 +227,11 @@ private void onGetFileChecksum(ChannelHandlerContext ctx) throws IOException {
     } finally {
       IOUtils.cleanup(LOG, dfsclient);
     }
-    final byte[] js = JsonUtil.toJsonString(checksum).getBytes();
+    final byte[] js = JsonUtil.toJsonString(checksum).getBytes(Charsets.UTF_8);
     DefaultFullHttpResponse resp =
       new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(js));
 
-    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON);
+    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
     resp.headers().set(CONTENT_LENGTH, js.length);
     resp.headers().set(CONNECTION, CLOSE);
     ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 108eb386a7457..a22f920099a40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -48,8 +48,10 @@
 import org.apache.hadoop.util.ToolRunner;
 
 import java.io.BufferedReader;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.URI;
 import java.text.DateFormat;
 import java.util.*;
@@ -579,7 +581,8 @@ private static Options buildCliOptions() {
 
     private static String[] readPathFile(String file) throws IOException {
       List<String> list = Lists.newArrayList();
-      BufferedReader reader = new BufferedReader(new FileReader(file));
+      BufferedReader reader = new BufferedReader(
+          new InputStreamReader(new FileInputStream(file), "UTF-8"));
       try {
         String line;
         while ((line = reader.readLine()) != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index e097b05bc803d..97d4759742908 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -18,8 +18,11 @@
 
 package org.apache.hadoop.hdfs.server.namenode;
 
+import java.util.Arrays;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -28,9 +31,10 @@
  * Feature that represents the ACLs of the inode.
  */
 @InterfaceAudience.Private
-public class AclFeature implements INode.Feature {
+public class AclFeature implements INode.Feature, ReferenceCounter {
   public static final ImmutableList<AclEntry> EMPTY_ENTRY_LIST =
     ImmutableList.of();
+  private int refCount = 0;
 
   private final int [] entries;
 
@@ -56,4 +60,35 @@ int getEntryAt(int pos) {
         "Invalid position for AclEntry");
     return entries[pos];
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == null) {
+      return false;
+    }
+    if (getClass() != o.getClass()) {
+      return false;
+    }
+    return Arrays.equals(entries, ((AclFeature) o).entries);
+  }
+
+  @Override
+  public int hashCode() {
+    return Arrays.hashCode(entries);
+  }
+
+  @Override
+  public int getRefCount() {
+    return refCount;
+  }
+
+  @Override
+  public int incrementAndGetRefCount() {
+    return ++refCount;
+  }
+
+  @Override
+  public int decrementAndGetRefCount() {
+    return (refCount > 0) ? --refCount : 0;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index a86604686fd56..4f6ce3ad4adca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.fs.permission.ScopedAclEntries;
 import org.apache.hadoop.hdfs.protocol.AclException;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap;
 
 /**
  * AclStorage contains utility methods that define how ACL data is stored in the
@@ -61,7 +62,10 @@
  * {@link AclTransformation}.
  */
 @InterfaceAudience.Private
-final class AclStorage {
+public final class AclStorage {
+
+  private final static ReferenceCountMap<AclFeature> UNIQUE_ACL_FEATURES =
+      new ReferenceCountMap<AclFeature>();
 
   /**
    * If a default ACL is defined on a parent directory, then copies that default
@@ -359,4 +363,28 @@ private static FsPermission createFsPermissionForMinimalAcl(
       accessEntries.get(2).getPermission(),
       existingPerm.getStickyBit());
   }
+
+  @VisibleForTesting
+  public static ReferenceCountMap<AclFeature> getUniqueAclFeatures() {
+    return UNIQUE_ACL_FEATURES;
+  }
+
+  /**
+   * Add reference for the said AclFeature
+   * 
+   * @param aclFeature
+   * @return Referenced AclFeature
+   */
+  public static AclFeature addAclFeature(AclFeature aclFeature) {
+    return UNIQUE_ACL_FEATURES.put(aclFeature);
+  }
+
+  /**
+   * Remove reference to the AclFeature
+   * 
+   * @param aclFeature
+   */
+  public static void removeAclFeature(AclFeature aclFeature) {
+    UNIQUE_ACL_FEATURES.remove(aclFeature);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index d253543741270..1900b4082de0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -139,14 +139,6 @@ void recoverCreateRead() throws IOException {
     }
   }
 
-  /**
-   * Save meta-data into fsimage files.
-   * and create empty edits.
-   */
-  void saveCheckpoint() throws IOException {
-    saveNamespace(namesystem);
-  }
-
   /**
    * Receive a batch of edits from the NameNode.
    * 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
index 827c6b5d07eb1..b7f8a38455e14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
@@ -411,7 +411,7 @@ private static NamespaceInfo handshake(NamenodeProtocol namenode)
       errorMsg = "Incompatible build versions: active name-node BV = " 
         + nsInfo.getBuildVersion() + "; backup node BV = "
         + Storage.getBuildVersion();
-      LOG.fatal(errorMsg);
+      LOG.error(errorMsg);
       throw new IOException(errorMsg);
     }
     assert HdfsConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() :
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 135979f50dc81..3fe748d0eb926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -199,9 +199,9 @@ String getKeyName(final INodesInPath iip) {
   private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) {
     assert dir.hasReadLock();
     Preconditions.checkNotNull(iip);
-    final INode[] inodes = iip.getINodes();
-    for (int i = inodes.length - 1; i >= 0; i--) {
-      final INode inode = inodes[i];
+    List<INode> inodes = iip.getReadOnlyINodes();
+    for (int i = inodes.size() - 1; i >= 0; i--) {
+      final INode inode = inodes.get(i);
       if (inode != null) {
         final EncryptionZoneInt ezi = encryptionZones.get(inode.getId());
         if (ezi != null) {
@@ -249,6 +249,10 @@ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
     final boolean dstInEZ = (dstEZI != null);
     if (srcInEZ) {
       if (!dstInEZ) {
+        if (srcEZI.getINodeId() == srcIIP.getLastINode().getId()) {
+          // src is ez root and dest is not in an ez. Allow the rename.
+          return;
+        }
         throw new IOException(
             src + " can't be moved from an encryption zone.");
       }
@@ -259,9 +263,7 @@ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
       }
     }
 
-    if (srcInEZ || dstInEZ) {
-      Preconditions.checkState(srcEZI != null, "couldn't find src EZ?");
-      Preconditions.checkState(dstEZI != null, "couldn't find dst EZ?");
+    if (srcInEZ) {
       if (srcEZI != dstEZI) {
         final String srcEZPath = getFullPathName(srcEZI);
         final String dstEZPath = getFullPathName(dstEZI);
@@ -311,7 +313,8 @@ XAttr createEncryptionZone(String src, CipherSuite suite,
     xattrs.add(ezXAttr);
     // updating the xattr will call addEncryptionZone,
     // done this way to handle edit log loading
-    dir.unprotectedSetXAttrs(src, xattrs, EnumSet.of(XAttrSetFlag.CREATE));
+    FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs,
+                                      EnumSet.of(XAttrSetFlag.CREATE));
     return ezXAttr;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index ac899aab362ae..dff1c2e337da0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -41,12 +41,12 @@ static HdfsFileStatus modifyAclEntries(
     FSPermissionChecker pc = fsd.getPermissionChecker();
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     src = fsd.resolvePath(pc, src, pathComponents);
+    INodesInPath iip;
     fsd.writeLock();
     try {
-      INodesInPath iip = fsd.getINodesInPath4Write(
-          FSDirectory.normalizePath(src), true);
+      iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
       fsd.checkOwner(pc, iip);
-      INode inode = FSDirectory.resolveLastINode(src, iip);
+      INode inode = FSDirectory.resolveLastINode(iip);
       int snapshotId = iip.getLatestSnapshotId();
       List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
       List<AclEntry> newAcl = AclTransformation.mergeAclEntries(
@@ -56,7 +56,7 @@ static HdfsFileStatus modifyAclEntries(
     } finally {
       fsd.writeUnlock();
     }
-    return fsd.getAuditFileInfo(src, false);
+    return fsd.getAuditFileInfo(iip);
   }
 
   static HdfsFileStatus removeAclEntries(
@@ -67,12 +67,12 @@ static HdfsFileStatus removeAclEntries(
     FSPermissionChecker pc = fsd.getPermissionChecker();
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     src = fsd.resolvePath(pc, src, pathComponents);
+    INodesInPath iip;
     fsd.writeLock();
     try {
-      INodesInPath iip = fsd.getINodesInPath4Write(
-          FSDirectory.normalizePath(src), true);
+      iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
       fsd.checkOwner(pc, iip);
-      INode inode = FSDirectory.resolveLastINode(src, iip);
+      INode inode = FSDirectory.resolveLastINode(iip);
       int snapshotId = iip.getLatestSnapshotId();
       List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
       List<AclEntry> newAcl = AclTransformation.filterAclEntriesByAclSpec(
@@ -82,7 +82,7 @@ static HdfsFileStatus removeAclEntries(
     } finally {
       fsd.writeUnlock();
     }
-    return fsd.getAuditFileInfo(src, false);
+    return fsd.getAuditFileInfo(iip);
   }
 
   static HdfsFileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
@@ -92,12 +92,12 @@ static HdfsFileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
     FSPermissionChecker pc = fsd.getPermissionChecker();
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     src = fsd.resolvePath(pc, src, pathComponents);
+    INodesInPath iip;
     fsd.writeLock();
     try {
-      INodesInPath iip = fsd.getINodesInPath4Write(
-          FSDirectory.normalizePath(src), true);
+      iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
       fsd.checkOwner(pc, iip);
-      INode inode = FSDirectory.resolveLastINode(src, iip);
+      INode inode = FSDirectory.resolveLastINode(iip);
       int snapshotId = iip.getLatestSnapshotId();
       List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
       List<AclEntry> newAcl = AclTransformation.filterDefaultAclEntries(
@@ -107,7 +107,7 @@ static HdfsFileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
     } finally {
       fsd.writeUnlock();
     }
-    return fsd.getAuditFileInfo(src, false);
+    return fsd.getAuditFileInfo(iip);
   }
 
   static HdfsFileStatus removeAcl(FSDirectory fsd, final String srcArg)
@@ -117,16 +117,17 @@ static HdfsFileStatus removeAcl(FSDirectory fsd, final String srcArg)
     FSPermissionChecker pc = fsd.getPermissionChecker();
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     src = fsd.resolvePath(pc, src, pathComponents);
+    INodesInPath iip;
     fsd.writeLock();
     try {
-      INodesInPath iip = fsd.getINodesInPath4Write(src);
+      iip = fsd.getINodesInPath4Write(src);
       fsd.checkOwner(pc, iip);
-      unprotectedRemoveAcl(fsd, src);
+      unprotectedRemoveAcl(fsd, iip);
     } finally {
       fsd.writeUnlock();
     }
     fsd.getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST);
-    return fsd.getAuditFileInfo(src, false);
+    return fsd.getAuditFileInfo(iip);
   }
 
   static HdfsFileStatus setAcl(
@@ -137,16 +138,17 @@ static HdfsFileStatus setAcl(
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     FSPermissionChecker pc = fsd.getPermissionChecker();
     src = fsd.resolvePath(pc, src, pathComponents);
+    INodesInPath iip;
     fsd.writeLock();
     try {
-      INodesInPath iip = fsd.getINodesInPath4Write(src);
+      iip = fsd.getINodesInPath4Write(src);
       fsd.checkOwner(pc, iip);
-      List<AclEntry> newAcl = unprotectedSetAcl(fsd, src, aclSpec);
+      List<AclEntry> newAcl = unprotectedSetAcl(fsd, src, aclSpec, false);
       fsd.getEditLog().logSetAcl(src, newAcl);
     } finally {
       fsd.writeUnlock();
     }
-    return fsd.getAuditFileInfo(src, false);
+    return fsd.getAuditFileInfo(iip);
   }
 
   static AclStatus getAclStatus(
@@ -168,12 +170,14 @@ static AclStatus getAclStatus(
       if (fsd.isPermissionEnabled()) {
         fsd.checkTraverse(pc, iip);
       }
-      INode inode = FSDirectory.resolveLastINode(srcs, iip);
+      INode inode = FSDirectory.resolveLastINode(iip);
       int snapshotId = iip.getPathSnapshotId();
       List<AclEntry> acl = AclStorage.readINodeAcl(inode, snapshotId);
+      FsPermission fsPermission = inode.getFsPermission(snapshotId);
       return new AclStatus.Builder()
           .owner(inode.getUserName()).group(inode.getGroupName())
-          .stickyBit(inode.getFsPermission(snapshotId).getStickyBit())
+          .stickyBit(fsPermission.getStickyBit())
+          .setPermission(fsPermission)
           .addEntries(acl).build();
     } finally {
       fsd.readUnlock();
@@ -181,22 +185,25 @@ static AclStatus getAclStatus(
   }
 
   static List<AclEntry> unprotectedSetAcl(
-      FSDirectory fsd, String src, List<AclEntry> aclSpec)
+      FSDirectory fsd, String src, List<AclEntry> aclSpec, boolean fromEdits)
       throws IOException {
+    assert fsd.hasWriteLock();
+    final INodesInPath iip = fsd.getINodesInPath4Write(
+        FSDirectory.normalizePath(src), true);
+
     // ACL removal is logged to edits as OP_SET_ACL with an empty list.
     if (aclSpec.isEmpty()) {
-      unprotectedRemoveAcl(fsd, src);
+      unprotectedRemoveAcl(fsd, iip);
       return AclFeature.EMPTY_ENTRY_LIST;
     }
 
-    assert fsd.hasWriteLock();
-    INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath
-        (src), true);
-    INode inode = FSDirectory.resolveLastINode(src, iip);
+    INode inode = FSDirectory.resolveLastINode(iip);
     int snapshotId = iip.getLatestSnapshotId();
-    List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
-    List<AclEntry> newAcl = AclTransformation.replaceAclEntries(existingAcl,
-      aclSpec);
+    List<AclEntry> newAcl = aclSpec;
+    if (!fromEdits) {
+      List<AclEntry> existingAcl = AclStorage.readINodeLogicalAcl(inode);
+      newAcl = AclTransformation.replaceAclEntries(existingAcl, aclSpec);
+    }
     AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
     return newAcl;
   }
@@ -210,12 +217,10 @@ private static void checkAclsConfigFlag(FSDirectory fsd) throws AclException {
     }
   }
 
-  private static void unprotectedRemoveAcl(FSDirectory fsd, String src)
+  private static void unprotectedRemoveAcl(FSDirectory fsd, INodesInPath iip)
       throws IOException {
     assert fsd.hasWriteLock();
-    INodesInPath iip = fsd.getINodesInPath4Write(
-        FSDirectory.normalizePath(src), true);
-    INode inode = FSDirectory.resolveLastINode(src, iip);
+    INode inode = FSDirectory.resolveLastINode(iip);
     int snapshotId = iip.getLatestSnapshotId();
     AclFeature f = inode.getAclFeature();
     if (f == null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
new file mode 100644
index 0000000000000..6c1890ecc6096
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -0,0 +1,459 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
+
+public class FSDirAttrOp {
+  static HdfsFileStatus setPermission(
+      FSDirectory fsd, final String srcArg, FsPermission permission)
+      throws IOException {
+    String src = srcArg;
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      iip = fsd.getINodesInPath4Write(src);
+      fsd.checkOwner(pc, iip);
+      unprotectedSetPermission(fsd, src, permission);
+    } finally {
+      fsd.writeUnlock();
+    }
+    fsd.getEditLog().logSetPermissions(src, permission);
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static HdfsFileStatus setOwner(
+      FSDirectory fsd, String src, String username, String group)
+      throws IOException {
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      iip = fsd.getINodesInPath4Write(src);
+      fsd.checkOwner(pc, iip);
+      if (!pc.isSuperUser()) {
+        if (username != null && !pc.getUser().equals(username)) {
+          throw new AccessControlException("Non-super user cannot change owner");
+        }
+        if (group != null && !pc.containsGroup(group)) {
+          throw new AccessControlException("User does not belong to " + group);
+        }
+      }
+      unprotectedSetOwner(fsd, src, username, group);
+    } finally {
+      fsd.writeUnlock();
+    }
+    fsd.getEditLog().logSetOwner(src, username, group);
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static HdfsFileStatus setTimes(
+      FSDirectory fsd, String src, long mtime, long atime)
+      throws IOException {
+    if (!fsd.isAccessTimeSupported() && atime != -1) {
+      throw new IOException(
+          "Access time for hdfs is not configured. " +
+              " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY
+              + " configuration parameter.");
+    }
+
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      iip = fsd.getINodesInPath4Write(src);
+      // Write access is required to set access and modification times
+      if (fsd.isPermissionEnabled()) {
+        fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+      }
+      final INode inode = iip.getLastINode();
+      if (inode == null) {
+        throw new FileNotFoundException("File/Directory " + src +
+                                            " does not exist.");
+      }
+      boolean changed = unprotectedSetTimes(fsd, inode, mtime, atime, true,
+                                            iip.getLatestSnapshotId());
+      if (changed) {
+        fsd.getEditLog().logTimes(src, mtime, atime);
+      }
+    } finally {
+      fsd.writeUnlock();
+    }
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static boolean setReplication(
+      FSDirectory fsd, BlockManager bm, String src, final short replication)
+      throws IOException {
+    bm.verifyReplication(src, replication, null);
+    final boolean isFile;
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      final INodesInPath iip = fsd.getINodesInPath4Write(src);
+      if (fsd.isPermissionEnabled()) {
+        fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+      }
+
+      final short[] blockRepls = new short[2]; // 0: old, 1: new
+      final Block[] blocks = unprotectedSetReplication(fsd, src, replication,
+                                                       blockRepls);
+      isFile = blocks != null;
+      if (isFile) {
+        fsd.getEditLog().logSetReplication(src, replication);
+        bm.setReplication(blockRepls[0], blockRepls[1], src, blocks);
+      }
+    } finally {
+      fsd.writeUnlock();
+    }
+    return isFile;
+  }
+
+  static HdfsFileStatus setStoragePolicy(
+      FSDirectory fsd, BlockManager bm, String src, final String policyName)
+      throws IOException {
+    if (!fsd.isStoragePolicyEnabled()) {
+      throw new IOException(
+          "Failed to set storage policy since "
+              + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
+    }
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      src = FSDirectory.resolvePath(src, pathComponents, fsd);
+      iip = fsd.getINodesInPath4Write(src);
+
+      if (fsd.isPermissionEnabled()) {
+        fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+      }
+
+      // get the corresponding policy and make sure the policy name is valid
+      BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
+      if (policy == null) {
+        throw new HadoopIllegalArgumentException(
+            "Cannot find a block policy with the name " + policyName);
+      }
+      unprotectedSetStoragePolicy(fsd, bm, iip, policy.getId());
+      fsd.getEditLog().logSetStoragePolicy(src, policy.getId());
+    } finally {
+      fsd.writeUnlock();
+    }
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static BlockStoragePolicy[] getStoragePolicies(BlockManager bm)
+      throws IOException {
+    return bm.getStoragePolicies();
+  }
+
+  static long getPreferredBlockSize(FSDirectory fsd, String src)
+      throws IOException {
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    fsd.readLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      final INodesInPath iip = fsd.getINodesInPath(src, false);
+      if (fsd.isPermissionEnabled()) {
+        fsd.checkTraverse(pc, iip);
+      }
+      return INodeFile.valueOf(iip.getLastINode(), src)
+          .getPreferredBlockSize();
+    } finally {
+      fsd.readUnlock();
+    }
+  }
+
+  /**
+   * Set the namespace quota and diskspace quota for a directory.
+   *
+   * Note: This does not support ".inodes" relative path.
+   */
+  static void setQuota(FSDirectory fsd, String src, long nsQuota, long dsQuota)
+      throws IOException {
+    if (fsd.isPermissionEnabled()) {
+      FSPermissionChecker pc = fsd.getPermissionChecker();
+      pc.checkSuperuserPrivilege();
+    }
+
+    fsd.writeLock();
+    try {
+      INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, dsQuota);
+      if (changed != null) {
+        final Quota.Counts q = changed.getQuotaCounts();
+        fsd.getEditLog().logSetQuota(
+            src, q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE));
+      }
+    } finally {
+      fsd.writeUnlock();
+    }
+  }
+
+  static void unprotectedSetPermission(
+      FSDirectory fsd, String src, FsPermission permissions)
+      throws FileNotFoundException, UnresolvedLinkException,
+             QuotaExceededException, SnapshotAccessControlException {
+    assert fsd.hasWriteLock();
+    final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true);
+    final INode inode = inodesInPath.getLastINode();
+    if (inode == null) {
+      throw new FileNotFoundException("File does not exist: " + src);
+    }
+    int snapshotId = inodesInPath.getLatestSnapshotId();
+    inode.setPermission(permissions, snapshotId);
+  }
+
+  static void unprotectedSetOwner(
+      FSDirectory fsd, String src, String username, String groupname)
+      throws FileNotFoundException, UnresolvedLinkException,
+      QuotaExceededException, SnapshotAccessControlException {
+    assert fsd.hasWriteLock();
+    final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true);
+    INode inode = inodesInPath.getLastINode();
+    if (inode == null) {
+      throw new FileNotFoundException("File does not exist: " + src);
+    }
+    if (username != null) {
+      inode = inode.setUser(username, inodesInPath.getLatestSnapshotId());
+    }
+    if (groupname != null) {
+      inode.setGroup(groupname, inodesInPath.getLatestSnapshotId());
+    }
+  }
+
+  static boolean setTimes(
+      FSDirectory fsd, INode inode, long mtime, long atime, boolean force,
+      int latestSnapshotId) throws QuotaExceededException {
+    fsd.writeLock();
+    try {
+      return unprotectedSetTimes(fsd, inode, mtime, atime, force,
+                                 latestSnapshotId);
+    } finally {
+      fsd.writeUnlock();
+    }
+  }
+
+  static boolean unprotectedSetTimes(
+      FSDirectory fsd, String src, long mtime, long atime, boolean force)
+      throws UnresolvedLinkException, QuotaExceededException {
+    assert fsd.hasWriteLock();
+    final INodesInPath i = fsd.getINodesInPath(src, true);
+    return unprotectedSetTimes(fsd, i.getLastINode(), mtime, atime,
+                               force, i.getLatestSnapshotId());
+  }
+
+  /**
+   * See {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long)}
+   * for the contract.
+   * Sets quota for for a directory.
+   * @return INodeDirectory if any of the quotas have changed. null otherwise.
+   * @throws FileNotFoundException if the path does not exist.
+   * @throws PathIsNotDirectoryException if the path is not a directory.
+   * @throws QuotaExceededException if the directory tree size is
+   *                                greater than the given quota
+   * @throws UnresolvedLinkException if a symlink is encountered in src.
+   * @throws SnapshotAccessControlException if path is in RO snapshot
+   */
+  static INodeDirectory unprotectedSetQuota(
+      FSDirectory fsd, String src, long nsQuota, long dsQuota)
+      throws FileNotFoundException, PathIsNotDirectoryException,
+      QuotaExceededException, UnresolvedLinkException,
+      SnapshotAccessControlException {
+    assert fsd.hasWriteLock();
+    // sanity check
+    if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
+         nsQuota != HdfsConstants.QUOTA_RESET) ||
+        (dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET &&
+          dsQuota != HdfsConstants.QUOTA_RESET)) {
+      throw new IllegalArgumentException("Illegal value for nsQuota or " +
+                                         "dsQuota : " + nsQuota + " and " +
+                                         dsQuota);
+    }
+
+    String srcs = FSDirectory.normalizePath(src);
+    final INodesInPath iip = fsd.getINodesInPath4Write(srcs, true);
+    INodeDirectory dirNode = INodeDirectory.valueOf(iip.getLastINode(), srcs);
+    if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
+      throw new IllegalArgumentException("Cannot clear namespace quota on root.");
+    } else { // a directory inode
+      final Quota.Counts oldQuota = dirNode.getQuotaCounts();
+      final long oldNsQuota = oldQuota.get(Quota.NAMESPACE);
+      final long oldDsQuota = oldQuota.get(Quota.DISKSPACE);
+      if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
+        nsQuota = oldNsQuota;
+      }
+      if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
+        dsQuota = oldDsQuota;
+      }
+      if (oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
+        return null;
+      }
+
+      final int latest = iip.getLatestSnapshotId();
+      dirNode.recordModification(latest);
+      dirNode.setQuota(nsQuota, dsQuota);
+      return dirNode;
+    }
+  }
+
+  static Block[] unprotectedSetReplication(
+      FSDirectory fsd, String src, short replication, short[] blockRepls)
+      throws QuotaExceededException, UnresolvedLinkException,
+             SnapshotAccessControlException {
+    assert fsd.hasWriteLock();
+
+    final INodesInPath iip = fsd.getINodesInPath4Write(src, true);
+    final INode inode = iip.getLastINode();
+    if (inode == null || !inode.isFile()) {
+      return null;
+    }
+    INodeFile file = inode.asFile();
+    final short oldBR = file.getBlockReplication();
+
+    // before setFileReplication, check for increasing block replication.
+    // if replication > oldBR, then newBR == replication.
+    // if replication < oldBR, we don't know newBR yet.
+    if (replication > oldBR) {
+      long dsDelta = (replication - oldBR)*(file.diskspaceConsumed()/oldBR);
+      fsd.updateCount(iip, 0, dsDelta, true);
+    }
+
+    file.setFileReplication(replication, iip.getLatestSnapshotId());
+
+    final short newBR = file.getBlockReplication();
+    // check newBR < oldBR case.
+    if (newBR < oldBR) {
+      long dsDelta = (newBR - oldBR)*(file.diskspaceConsumed()/newBR);
+      fsd.updateCount(iip, 0, dsDelta, true);
+    }
+
+    if (blockRepls != null) {
+      blockRepls[0] = oldBR;
+      blockRepls[1] = newBR;
+    }
+    return file.getBlocks();
+  }
+
+  static void unprotectedSetStoragePolicy(
+      FSDirectory fsd, BlockManager bm, INodesInPath iip, byte policyId)
+      throws IOException {
+    assert fsd.hasWriteLock();
+    final INode inode = iip.getLastINode();
+    if (inode == null) {
+      throw new FileNotFoundException("File/Directory does not exist: "
+          + iip.getPath());
+    }
+    final int snapshotId = iip.getLatestSnapshotId();
+    if (inode.isFile()) {
+      BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
+      if (newPolicy.isCopyOnCreateFile()) {
+        throw new HadoopIllegalArgumentException(
+            "Policy " + newPolicy + " cannot be set after file creation.");
+      }
+
+      BlockStoragePolicy currentPolicy =
+          bm.getStoragePolicy(inode.getLocalStoragePolicyID());
+
+      if (currentPolicy != null && currentPolicy.isCopyOnCreateFile()) {
+        throw new HadoopIllegalArgumentException(
+            "Existing policy " + currentPolicy.getName() +
+                " cannot be changed after file creation.");
+      }
+      inode.asFile().setStoragePolicyID(policyId, snapshotId);
+    } else if (inode.isDirectory()) {
+      setDirStoragePolicy(fsd, inode.asDirectory(), policyId, snapshotId);
+    } else {
+      throw new FileNotFoundException(iip.getPath()
+          + " is not a file or directory");
+    }
+  }
+
+  private static void setDirStoragePolicy(
+      FSDirectory fsd, INodeDirectory inode, byte policyId,
+      int latestSnapshotId) throws IOException {
+    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
+    XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
+    List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
+                                                        Arrays.asList(xAttr),
+                                                        EnumSet.of(
+                                                            XAttrSetFlag.CREATE,
+                                                            XAttrSetFlag.REPLACE));
+    XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
+  }
+
+  private static boolean unprotectedSetTimes(
+      FSDirectory fsd, INode inode, long mtime, long atime, boolean force,
+      int latest) throws QuotaExceededException {
+    assert fsd.hasWriteLock();
+    boolean status = false;
+    if (mtime != -1) {
+      inode = inode.setModificationTime(mtime, latest);
+      status = true;
+    }
+    if (atime != -1) {
+      long inodeTime = inode.getAccessTime();
+
+      // if the last access time update was within the last precision interval, then
+      // no need to store access time
+      if (atime <= inodeTime + fsd.getFSNamesystem().getAccessTimePrecision()
+          && !force) {
+        status =  false;
+      } else {
+        inode.setAccessTime(atime, latest);
+        status = true;
+      }
+    }
+    return status;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index c2e0f088207a4..ecfd2e1a61911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -19,11 +19,10 @@
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -33,202 +32,171 @@
 import static org.apache.hadoop.util.Time.now;
 
 class FSDirConcatOp {
-  static HdfsFileStatus concat(
-    FSDirectory fsd, String target, String[] srcs,
+
+  static HdfsFileStatus concat(FSDirectory fsd, String target, String[] srcs,
     boolean logRetryCache) throws IOException {
     Preconditions.checkArgument(!target.isEmpty(), "Target file name is empty");
     Preconditions.checkArgument(srcs != null && srcs.length > 0,
       "No sources given");
     assert srcs != null;
-
-    FSDirectory.LOG.debug("concat {} to {}", Arrays.toString(srcs), target);
-    // We require all files be in the same directory
-    String trgParent =
-      target.substring(0, target.lastIndexOf(Path.SEPARATOR_CHAR));
-    for (String s : srcs) {
-      String srcParent = s.substring(0, s.lastIndexOf(Path.SEPARATOR_CHAR));
-      if (!srcParent.equals(trgParent)) {
-        throw new IllegalArgumentException(
-           "Sources and target are not in the same directory");
-      }
+    if (FSDirectory.LOG.isDebugEnabled()) {
+      FSDirectory.LOG.debug("concat {} to {}", Arrays.toString(srcs), target);
     }
 
-    final INodesInPath trgIip = fsd.getINodesInPath4Write(target);
+    final INodesInPath targetIIP = fsd.getINodesInPath4Write(target);
     // write permission for the target
+    FSPermissionChecker pc = null;
     if (fsd.isPermissionEnabled()) {
-      FSPermissionChecker pc = fsd.getPermissionChecker();
-      fsd.checkPathAccess(pc, trgIip, FsAction.WRITE);
-
-      // and srcs
-      for(String aSrc: srcs) {
-        final INodesInPath srcIip = fsd.getINodesInPath4Write(aSrc);
-        fsd.checkPathAccess(pc, srcIip, FsAction.READ); // read the file
-        fsd.checkParentAccess(pc, srcIip, FsAction.WRITE); // for delete
-      }
+      pc = fsd.getPermissionChecker();
+      fsd.checkPathAccess(pc, targetIIP, FsAction.WRITE);
     }
 
-    // to make sure no two files are the same
-    Set<INode> si = new HashSet<INode>();
+    // check the target
+    verifyTargetFile(fsd, target, targetIIP);
+    // check the srcs
+    INodeFile[] srcFiles = verifySrcFiles(fsd, srcs, targetIIP, pc);
 
-    // we put the following prerequisite for the operation
-    // replication and blocks sizes should be the same for ALL the blocks
+    if(NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* NameSystem.concat: " +
+          Arrays.toString(srcs) + " to " + target);
+    }
+
+    long timestamp = now();
+    fsd.writeLock();
+    try {
+      unprotectedConcat(fsd, targetIIP, srcFiles, timestamp);
+    } finally {
+      fsd.writeUnlock();
+    }
+    fsd.getEditLog().logConcat(target, srcs, timestamp, logRetryCache);
+    return fsd.getAuditFileInfo(targetIIP);
+  }
 
+  private static void verifyTargetFile(FSDirectory fsd, final String target,
+      final INodesInPath targetIIP) throws IOException {
     // check the target
-    if (fsd.getEZForPath(trgIip) != null) {
+    if (fsd.getEZForPath(targetIIP) != null) {
       throw new HadoopIllegalArgumentException(
           "concat can not be called for files in an encryption zone.");
     }
-    final INodeFile trgInode = INodeFile.valueOf(trgIip.getLastINode(), target);
-    if(trgInode.isUnderConstruction()) {
+    final INodeFile targetINode = INodeFile.valueOf(targetIIP.getLastINode(),
+        target);
+    if(targetINode.isUnderConstruction()) {
       throw new HadoopIllegalArgumentException("concat: target file "
           + target + " is under construction");
     }
-    // per design target shouldn't be empty and all the blocks same size
-    if(trgInode.numBlocks() == 0) {
-      throw new HadoopIllegalArgumentException("concat: target file "
-          + target + " is empty");
-    }
-    if (trgInode.isWithSnapshot()) {
-      throw new HadoopIllegalArgumentException("concat: target file "
-          + target + " is in a snapshot");
-    }
-
-    long blockSize = trgInode.getPreferredBlockSize();
-
-    // check the end block to be full
-    final BlockInfo last = trgInode.getLastBlock();
-    if(blockSize != last.getNumBytes()) {
-      throw new HadoopIllegalArgumentException("The last block in " + target
-          + " is not full; last block size = " + last.getNumBytes()
-          + " but file block size = " + blockSize);
-    }
-
-    si.add(trgInode);
-    final short repl = trgInode.getFileReplication();
+  }
 
+  private static INodeFile[] verifySrcFiles(FSDirectory fsd, String[] srcs,
+      INodesInPath targetIIP, FSPermissionChecker pc) throws IOException {
+    // to make sure no two files are the same
+    Set<INodeFile> si = new HashSet<>();
+    final INodeFile targetINode = targetIIP.getLastINode().asFile();
+    final INodeDirectory targetParent = targetINode.getParent();
     // now check the srcs
-    boolean endSrc = false; // final src file doesn't have to have full end block
-    for(int i=0; i< srcs.length; i++) {
-      String src = srcs[i];
-      if(i== srcs.length-1)
-        endSrc=true;
-
-      final INodeFile srcInode = INodeFile.valueOf(fsd.getINode4Write(src), src);
-      if(src.isEmpty()
-          || srcInode.isUnderConstruction()
-          || srcInode.numBlocks() == 0) {
-        throw new HadoopIllegalArgumentException("concat: source file " + src
-            + " is invalid or empty or underConstruction");
+    for(String src : srcs) {
+      final INodesInPath iip = fsd.getINodesInPath4Write(src);
+      // permission check for srcs
+      if (pc != null) {
+        fsd.checkPathAccess(pc, iip, FsAction.READ); // read the file
+        fsd.checkParentAccess(pc, iip, FsAction.WRITE); // for delete
       }
-
-      // check replication and blocks size
-      if(repl != srcInode.getBlockReplication()) {
-        throw new HadoopIllegalArgumentException("concat: the source file "
-            + src + " and the target file " + target
-            + " should have the same replication: source replication is "
-            + srcInode.getBlockReplication()
-            + " but target replication is " + repl);
+      final INode srcINode = iip.getLastINode();
+      final INodeFile srcINodeFile = INodeFile.valueOf(srcINode, src);
+      // make sure the src file and the target file are in the same dir
+      if (srcINodeFile.getParent() != targetParent) {
+        throw new HadoopIllegalArgumentException("Source file " + src
+            + " is not in the same directory with the target "
+            + targetIIP.getPath());
       }
-
-      //boolean endBlock=false;
-      // verify that all the blocks are of the same length as target
-      // should be enough to check the end blocks
-      final BlockInfo[] srcBlocks = srcInode.getBlocks();
-      int idx = srcBlocks.length-1;
-      if(endSrc)
-        idx = srcBlocks.length-2; // end block of endSrc is OK not to be full
-      if(idx >= 0 && srcBlocks[idx].getNumBytes() != blockSize) {
-        throw new HadoopIllegalArgumentException("concat: the source file "
-            + src + " and the target file " + target
-            + " should have the same blocks sizes: target block size is "
-            + blockSize + " but the size of source block " + idx + " is "
-            + srcBlocks[idx].getNumBytes());
+      // make sure all the source files are not in snapshot
+      if (srcINode.isInLatestSnapshot(iip.getLatestSnapshotId())) {
+        throw new SnapshotException("Concat: the source file " + src
+            + " is in snapshot");
       }
-
-      si.add(srcInode);
+      // check if the file has other references.
+      if (srcINode.isReference() && ((INodeReference.WithCount)
+          srcINode.asReference().getReferredINode()).getReferenceCount() > 1) {
+        throw new SnapshotException("Concat: the source file " + src
+            + " is referred by some other reference in some snapshot.");
+      }
+      if (srcINode == targetINode) {
+        throw new HadoopIllegalArgumentException("concat: the src file " + src
+            + " is the same with the target file " + targetIIP.getPath());
+      }
+      if(srcINodeFile.isUnderConstruction() || srcINodeFile.numBlocks() == 0) {
+        throw new HadoopIllegalArgumentException("concat: source file " + src
+            + " is invalid or empty or underConstruction");
+      }
+      si.add(srcINodeFile);
     }
 
     // make sure no two files are the same
-    if(si.size() < srcs.length+1) { // trg + srcs
+    if(si.size() < srcs.length) {
       // it means at least two files are the same
       throw new HadoopIllegalArgumentException(
           "concat: at least two of the source files are the same");
     }
+    return si.toArray(new INodeFile[si.size()]);
+  }
 
-    if(NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* NameSystem.concat: " +
-          Arrays.toString(srcs) + " to " + target);
+  private static long computeQuotaDelta(INodeFile target, INodeFile[] srcList) {
+    long delta = 0;
+    short targetRepl = target.getBlockReplication();
+    for (INodeFile src : srcList) {
+      if (targetRepl != src.getBlockReplication()) {
+        delta += src.computeFileSize() *
+            (targetRepl - src.getBlockReplication());
+      }
     }
+    return delta;
+  }
 
-    long timestamp = now();
-    fsd.writeLock();
-    try {
-      unprotectedConcat(fsd, target, srcs, timestamp);
-    } finally {
-      fsd.writeUnlock();
+  private static void verifyQuota(FSDirectory fsd, INodesInPath targetIIP,
+      long delta) throws QuotaExceededException {
+    if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
+      // Do not check quota if editlog is still being processed
+      return;
     }
-    fsd.getEditLog().logConcat(target, srcs, timestamp, logRetryCache);
-    return fsd.getAuditFileInfo(target, false);
+    FSDirectory.verifyQuota(targetIIP, targetIIP.length() - 1, 0, delta, null);
   }
 
   /**
    * Concat all the blocks from srcs to trg and delete the srcs files
    * @param fsd FSDirectory
-   * @param target target file to move the blocks to
-   * @param srcs list of file to move the blocks from
    */
-  static void unprotectedConcat(
-    FSDirectory fsd, String target, String[] srcs, long timestamp)
-    throws IOException {
+  static void unprotectedConcat(FSDirectory fsd, INodesInPath targetIIP,
+      INodeFile[] srcList, long timestamp) throws IOException {
     assert fsd.hasWriteLock();
     if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* FSNamesystem.concat to "+target);
+      NameNode.stateChangeLog.debug("DIR* FSNamesystem.concat to "
+          + targetIIP.getPath());
     }
-    // do the move
-
-    final INodesInPath trgIIP = fsd.getINodesInPath4Write(target, true);
-    final INode[] trgINodes = trgIIP.getINodes();
-    final INodeFile trgInode = trgIIP.getLastINode().asFile();
-    INodeDirectory trgParent = trgINodes[trgINodes.length-2].asDirectory();
-    final int trgLatestSnapshot = trgIIP.getLatestSnapshotId();
-
-    final INodeFile [] allSrcInodes = new INodeFile[srcs.length];
-    for(int i = 0; i < srcs.length; i++) {
-      final INodesInPath iip = fsd.getINodesInPath4Write(srcs[i]);
-      final int latest = iip.getLatestSnapshotId();
-      final INode inode = iip.getLastINode();
-
-      // check if the file in the latest snapshot
-      if (inode.isInLatestSnapshot(latest)) {
-        throw new SnapshotException("Concat: the source file " + srcs[i]
-            + " is in snapshot " + latest);
-      }
 
-      // check if the file has other references.
-      if (inode.isReference() && ((INodeReference.WithCount)
-          inode.asReference().getReferredINode()).getReferenceCount() > 1) {
-        throw new SnapshotException("Concat: the source file " + srcs[i]
-            + " is referred by some other reference in some snapshot.");
-      }
+    final INodeFile trgInode = targetIIP.getLastINode().asFile();
+    long delta = computeQuotaDelta(trgInode, srcList);
+    verifyQuota(fsd, targetIIP, delta);
 
-      allSrcInodes[i] = inode.asFile();
-    }
-    trgInode.concatBlocks(allSrcInodes);
+    // the target file can be included in a snapshot
+    trgInode.recordModification(targetIIP.getLatestSnapshotId());
+    INodeDirectory trgParent = targetIIP.getINode(-2).asDirectory();
+    trgInode.concatBlocks(srcList);
 
     // since we are in the same dir - we can use same parent to remove files
     int count = 0;
-    for(INodeFile nodeToRemove: allSrcInodes) {
-      if(nodeToRemove == null) continue;
-
-      nodeToRemove.setBlocks(null);
-      trgParent.removeChild(nodeToRemove, trgLatestSnapshot);
-      fsd.getINodeMap().remove(nodeToRemove);
-      count++;
+    for (INodeFile nodeToRemove : srcList) {
+      if(nodeToRemove != null) {
+        nodeToRemove.setBlocks(null);
+        nodeToRemove.getParent().removeChild(nodeToRemove);
+        fsd.getINodeMap().remove(nodeToRemove);
+        count++;
+      }
     }
 
-    trgInode.setModificationTime(timestamp, trgLatestSnapshot);
-    trgParent.updateModificationTime(timestamp, trgLatestSnapshot);
+    trgInode.setModificationTime(timestamp, targetIIP.getLatestSnapshotId());
+    trgParent.updateModificationTime(timestamp, targetIIP.getLatestSnapshotId());
     // update quota on the parent directory ('count' files removed, 0 space)
-    FSDirectory.unprotectedUpdateCount(trgIIP, trgINodes.length - 1, -count, 0);
+    FSDirectory.unprotectedUpdateCount(targetIIP, targetIIP.length() - 1,
+        -count, delta);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
new file mode 100644
index 0000000000000..47c641cf9baa5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+import org.apache.hadoop.util.ChunkedArrayList;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
+import static org.apache.hadoop.util.Time.now;
+
+class FSDirDeleteOp {
+  /**
+   * Delete the target directory and collect the blocks under it
+   *
+   * @param iip the INodesInPath instance containing all the INodes for the path
+   * @param collectedBlocks Blocks under the deleted directory
+   * @param removedINodes INodes that should be removed from inodeMap
+   * @return the number of files that have been removed
+   */
+  static long delete(
+      FSDirectory fsd, INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
+      List<INode> removedINodes, long mtime) throws IOException {
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + iip.getPath());
+    }
+    final long filesRemoved;
+    fsd.writeLock();
+    try {
+      if (!deleteAllowed(iip, iip.getPath()) ) {
+        filesRemoved = -1;
+      } else {
+        List<INodeDirectory> snapshottableDirs = new ArrayList<>();
+        FSDirSnapshotOp.checkSnapshot(iip.getLastINode(), snapshottableDirs);
+        filesRemoved = unprotectedDelete(fsd, iip, collectedBlocks,
+                                         removedINodes, mtime);
+        fsd.getFSNamesystem().removeSnapshottableDirs(snapshottableDirs);
+      }
+    } finally {
+      fsd.writeUnlock();
+    }
+    return filesRemoved;
+  }
+
+  /**
+   * Remove a file/directory from the namespace.
+   * <p>
+   * For large directories, deletion is incremental. The blocks under
+   * the directory are collected and deleted a small number at a time holding
+   * the {@link FSNamesystem} lock.
+   * <p>
+   * For small directory or file the deletion is done in one shot.
+   *
+   */
+  static BlocksMapUpdateInfo delete(
+      FSNamesystem fsn, String src, boolean recursive, boolean logRetryCache)
+      throws IOException {
+    FSDirectory fsd = fsn.getFSDirectory();
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+    src = fsd.resolvePath(pc, src, pathComponents);
+    final INodesInPath iip = fsd.getINodesInPath4Write(src, false);
+    if (!recursive && fsd.isNonEmptyDirectory(iip)) {
+      throw new PathIsNotEmptyDirectoryException(src + " is non empty");
+    }
+    if (fsd.isPermissionEnabled()) {
+      fsd.checkPermission(pc, iip, false, null, FsAction.WRITE, null,
+                          FsAction.ALL, true);
+    }
+
+    return deleteInternal(fsn, src, iip, logRetryCache);
+  }
+
+  /**
+   * Delete a path from the name space
+   * Update the count at each ancestor directory with quota
+   * <br>
+   * Note: This is to be used by
+   * {@link org.apache.hadoop.hdfs.server.namenode.FSEditLog} only.
+   * <br>
+   * @param src a string representation of a path to an inode
+   * @param mtime the time the inode is removed
+   */
+  static void deleteForEditLog(FSDirectory fsd, String src, long mtime)
+      throws IOException {
+    assert fsd.hasWriteLock();
+    FSNamesystem fsn = fsd.getFSNamesystem();
+    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+    List<INode> removedINodes = new ChunkedArrayList<>();
+
+    final INodesInPath iip = fsd.getINodesInPath4Write(
+        FSDirectory.normalizePath(src), false);
+    if (!deleteAllowed(iip, src)) {
+      return;
+    }
+    List<INodeDirectory> snapshottableDirs = new ArrayList<>();
+    FSDirSnapshotOp.checkSnapshot(iip.getLastINode(), snapshottableDirs);
+    long filesRemoved = unprotectedDelete(
+        fsd, iip, collectedBlocks, removedINodes, mtime);
+    fsn.removeSnapshottableDirs(snapshottableDirs);
+
+    if (filesRemoved >= 0) {
+      fsn.removeLeasesAndINodes(src, removedINodes, false);
+      fsn.removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+    }
+  }
+
+  /**
+   * Remove a file/directory from the namespace.
+   * <p>
+   * For large directories, deletion is incremental. The blocks under
+   * the directory are collected and deleted a small number at a time holding
+   * the {@link org.apache.hadoop.hdfs.server.namenode.FSNamesystem} lock.
+   * <p>
+   * For small directory or file the deletion is done in one shot.
+   */
+  static BlocksMapUpdateInfo deleteInternal(
+      FSNamesystem fsn, String src, INodesInPath iip, boolean logRetryCache)
+      throws IOException {
+    assert fsn.hasWriteLock();
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
+    }
+
+    FSDirectory fsd = fsn.getFSDirectory();
+    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+    List<INode> removedINodes = new ChunkedArrayList<>();
+
+    long mtime = now();
+    // Unlink the target directory from directory tree
+    long filesRemoved = delete(
+        fsd, iip, collectedBlocks, removedINodes, mtime);
+    if (filesRemoved < 0) {
+      return null;
+    }
+    fsd.getEditLog().logDelete(src, mtime, logRetryCache);
+    incrDeletedFileCount(filesRemoved);
+
+    fsn.removeLeasesAndINodes(src, removedINodes, true);
+    fsd.getEditLog().logSync();
+
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
+                                        + src +" is removed");
+    }
+    return collectedBlocks;
+  }
+
+  static void incrDeletedFileCount(long count) {
+    NameNode.getNameNodeMetrics().incrFilesDeleted(count);
+  }
+
+  private static boolean deleteAllowed(final INodesInPath iip,
+      final String src) {
+    if (iip.length() < 1 || iip.getLastINode() == null) {
+      if (NameNode.stateChangeLog.isDebugEnabled()) {
+        NameNode.stateChangeLog.debug(
+            "DIR* FSDirectory.unprotectedDelete: failed to remove "
+                + src + " because it does not exist");
+      }
+      return false;
+    } else if (iip.length() == 1) { // src is the root
+      NameNode.stateChangeLog.warn(
+          "DIR* FSDirectory.unprotectedDelete: failed to remove " + src +
+              " because the root is not allowed to be deleted");
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Delete a path from the name space
+   * Update the count at each ancestor directory with quota
+   * @param iip the inodes resolved from the path
+   * @param collectedBlocks blocks collected from the deleted path
+   * @param removedINodes inodes that should be removed from inodeMap
+   * @param mtime the time the inode is removed
+   * @return the number of inodes deleted; 0 if no inodes are deleted.
+   */
+  private static long unprotectedDelete(
+      FSDirectory fsd, INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
+      List<INode> removedINodes, long mtime) {
+    assert fsd.hasWriteLock();
+
+    // check if target node exists
+    INode targetNode = iip.getLastINode();
+    if (targetNode == null) {
+      return -1;
+    }
+
+    // record modification
+    final int latestSnapshot = iip.getLatestSnapshotId();
+    targetNode.recordModification(latestSnapshot);
+
+    // Remove the node from the namespace
+    long removed = fsd.removeLastINode(iip);
+    if (removed == -1) {
+      return -1;
+    }
+
+    // set the parent's modification time
+    final INodeDirectory parent = targetNode.getParent();
+    parent.updateModificationTime(mtime, latestSnapshot);
+
+    fsd.updateCountForDelete(targetNode, iip);
+    if (removed == 0) {
+      return 0;
+    }
+
+    // collect block and update quota
+    if (!targetNode.isInLatestSnapshot(latestSnapshot)) {
+      targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+    } else {
+      Quota.Counts counts = targetNode.cleanSubtree(CURRENT_STATE_ID,
+          latestSnapshot, collectedBlocks, removedINodes);
+      removed = counts.get(Quota.NAMESPACE);
+      fsd.updateCountNoQuotaCheck(iip, iip.length() - 1,
+          -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+    }
+
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
+          + iip.getPath() + " is removed");
+    }
+    return removed;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index af9e925e08232..f51427f2b30b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -17,9 +17,10 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.InvalidPathException;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.FsAction;
@@ -29,20 +30,20 @@
 import org.apache.hadoop.hdfs.protocol.AclException;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 
 import java.io.IOException;
+import java.util.AbstractMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.hadoop.util.Time.now;
 
 class FSDirMkdirOp {
-  static HdfsFileStatus mkdirs(
-      FSNamesystem fsn, String src, PermissionStatus permissions,
-      boolean createParent) throws IOException {
+
+  static HdfsFileStatus mkdirs(FSNamesystem fsn, String src,
+      PermissionStatus permissions, boolean createParent) throws IOException {
     FSDirectory fsd = fsn.getFSDirectory();
-    final String srcArg = src;
     if(NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
     }
@@ -50,188 +51,194 @@ static HdfsFileStatus mkdirs(
       throw new InvalidPathException(src);
     }
     FSPermissionChecker pc = fsd.getPermissionChecker();
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath
-        (src);
-    src = fsd.resolvePath(pc, src, pathComponents);
-    INodesInPath iip = fsd.getINodesInPath4Write(src);
-    if (fsd.isPermissionEnabled()) {
-      fsd.checkTraverse(pc, iip);
-    }
-
-    if (!isDirMutable(fsd, iip)) {
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      INodesInPath iip = fsd.getINodesInPath4Write(src);
       if (fsd.isPermissionEnabled()) {
-        fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+        fsd.checkTraverse(pc, iip);
       }
 
-      if (!createParent) {
-        fsd.verifyParentDir(iip, src);
+      final INode lastINode = iip.getLastINode();
+      if (lastINode != null && lastINode.isFile()) {
+        throw new FileAlreadyExistsException("Path is not a directory: " + src);
       }
 
-      // validate that we have enough inodes. This is, at best, a
-      // heuristic because the mkdirs() operation might need to
-      // create multiple inodes.
-      fsn.checkFsObjectLimit();
+      INodesInPath existing = lastINode != null ? iip : iip.getExistingINodes();
+      if (lastINode == null) {
+        if (fsd.isPermissionEnabled()) {
+          fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+        }
 
-      if (!mkdirsRecursively(fsd, src, permissions, false, now())) {
-        throw new IOException("Failed to create directory: " + src);
+        if (!createParent) {
+          fsd.verifyParentDir(iip, src);
+        }
+
+        // validate that we have enough inodes. This is, at best, a
+        // heuristic because the mkdirs() operation might need to
+        // create multiple inodes.
+        fsn.checkFsObjectLimit();
+
+        List<String> nonExisting = iip.getPath(existing.length(),
+            iip.length() - existing.length());
+        int length = nonExisting.size();
+        if (length > 1) {
+          List<String> ancestors = nonExisting.subList(0, length - 1);
+          // Ensure that the user can traversal the path by adding implicit
+          // u+wx permission to all ancestor directories
+          existing = createChildrenDirectories(fsd, existing, ancestors,
+              addImplicitUwx(permissions, permissions));
+          if (existing == null) {
+            throw new IOException("Failed to create directory: " + src);
+          }
+        }
+
+        if ((existing = createChildrenDirectories(fsd, existing,
+            nonExisting.subList(length - 1, length), permissions)) == null) {
+          throw new IOException("Failed to create directory: " + src);
+        }
       }
+      return fsd.getAuditFileInfo(existing);
+    } finally {
+      fsd.writeUnlock();
     }
-    return fsd.getAuditFileInfo(srcArg, false);
   }
 
-  static INode unprotectedMkdir(
-      FSDirectory fsd, long inodeId, String src,
-      PermissionStatus permissions, List<AclEntry> aclEntries, long timestamp)
-      throws QuotaExceededException, UnresolvedLinkException, AclException {
-    assert fsd.hasWriteLock();
-    byte[][] components = INode.getPathComponents(src);
-    INodesInPath iip = fsd.getExistingPathINodes(components);
-    INode[] inodes = iip.getINodes();
-    final int pos = inodes.length - 1;
-    unprotectedMkdir(fsd, inodeId, iip, pos, components[pos], permissions,
-        aclEntries, timestamp);
-    return inodes[pos];
+  /**
+   * For a given absolute path, create all ancestors as directories along the
+   * path. All ancestors inherit their parent's permission plus an implicit
+   * u+wx permission. This is used by create() and addSymlink() for
+   * implicitly creating all directories along the path.
+   *
+   * For example, path="/foo/bar/spam", "/foo" is an existing directory,
+   * "/foo/bar" is not existing yet, the function will create directory bar.
+   *
+   * @return a tuple which contains both the new INodesInPath (with all the
+   * existing and newly created directories) and the last component in the
+   * relative path. Or return null if there are errors.
+   */
+  static Map.Entry<INodesInPath, String> createAncestorDirectories(
+      FSDirectory fsd, INodesInPath iip, PermissionStatus permission)
+      throws IOException {
+    final String last = new String(iip.getLastLocalName(), Charsets.UTF_8);
+    INodesInPath existing = iip.getExistingINodes();
+    List<String> children = iip.getPath(existing.length(),
+        iip.length() - existing.length());
+    int size = children.size();
+    if (size > 1) { // otherwise all ancestors have been created
+      List<String> directories = children.subList(0, size - 1);
+      INode parentINode = existing.getLastINode();
+      // Ensure that the user can traversal the path by adding implicit
+      // u+wx permission to all ancestor directories
+      existing = createChildrenDirectories(fsd, existing, directories,
+          addImplicitUwx(parentINode.getPermissionStatus(), permission));
+      if (existing == null) {
+        return null;
+      }
+    }
+    return new AbstractMap.SimpleImmutableEntry<>(existing, last);
   }
 
   /**
-   * Create a directory
-   * If ancestor directories do not exist, automatically create them.
-
+   * Create the directory {@code parent} / {@code children} and all ancestors
+   * along the path.
+   *
    * @param fsd FSDirectory
-   * @param src string representation of the path to the directory
-   * @param permissions the permission of the directory
-   * @param inheritPermission
-   *   if the permission of the directory should inherit from its parent or not.
-   *   u+wx is implicitly added to the automatically created directories,
-   *   and to the given directory if inheritPermission is true
-   * @param now creation time
-   * @return true if the operation succeeds false otherwise
-   * @throws QuotaExceededException if directory creation violates
-   *                                any quota limit
-   * @throws UnresolvedLinkException if a symlink is encountered in src.
-   * @throws SnapshotAccessControlException if path is in RO snapshot
+   * @param existing The INodesInPath instance containing all the existing
+   *                 ancestral INodes
+   * @param children The relative path from the parent towards children,
+   *                 starting with "/"
+   * @param perm the permission of the directory. Note that all ancestors
+   *             created along the path has implicit {@code u+wx} permissions.
+   *
+   * @return {@link INodesInPath} which contains all inodes to the
+   * target directory, After the execution parentPath points to the path of
+   * the returned INodesInPath. The function return null if the operation has
+   * failed.
    */
-  static boolean mkdirsRecursively(
-      FSDirectory fsd, String src, PermissionStatus permissions,
-      boolean inheritPermission, long now)
-      throws FileAlreadyExistsException, QuotaExceededException,
-             UnresolvedLinkException, SnapshotAccessControlException,
-             AclException {
-    src = FSDirectory.normalizePath(src);
-    String[] names = INode.getPathNames(src);
-    byte[][] components = INode.getPathComponents(names);
-    final int lastInodeIndex = components.length - 1;
+  private static INodesInPath createChildrenDirectories(FSDirectory fsd,
+      INodesInPath existing, List<String> children, PermissionStatus perm)
+      throws IOException {
+    assert fsd.hasWriteLock();
 
-    fsd.writeLock();
-    try {
-      INodesInPath iip = fsd.getExistingPathINodes(components);
-      if (iip.isSnapshot()) {
-        throw new SnapshotAccessControlException(
-                "Modification on RO snapshot is disallowed");
-      }
-      INode[] inodes = iip.getINodes();
-
-      // find the index of the first null in inodes[]
-      StringBuilder pathbuilder = new StringBuilder();
-      int i = 1;
-      for(; i < inodes.length && inodes[i] != null; i++) {
-        pathbuilder.append(Path.SEPARATOR).append(names[i]);
-        if (!inodes[i].isDirectory()) {
-          throw new FileAlreadyExistsException(
-                  "Parent path is not a directory: "
-                  + pathbuilder + " "+inodes[i].getLocalName());
-        }
+    for (String component : children) {
+      existing = createSingleDirectory(fsd, existing, component, perm);
+      if (existing == null) {
+        return null;
       }
+    }
+    return existing;
+  }
 
-      // default to creating parent dirs with the given perms
-      PermissionStatus parentPermissions = permissions;
-
-      // if not inheriting and it's the last inode, there's no use in
-      // computing perms that won't be used
-      if (inheritPermission || (i < lastInodeIndex)) {
-        // if inheriting (ie. creating a file or symlink), use the parent dir,
-        // else the supplied permissions
-        // NOTE: the permissions of the auto-created directories violate posix
-        FsPermission parentFsPerm = inheritPermission
-                ? inodes[i-1].getFsPermission() : permissions.getPermission();
-
-        // ensure that the permissions allow user write+execute
-        if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) {
-          parentFsPerm = new FsPermission(
-                  parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE),
-                  parentFsPerm.getGroupAction(),
-                  parentFsPerm.getOtherAction()
-          );
-        }
+  static void mkdirForEditLog(FSDirectory fsd, long inodeId, String src,
+      PermissionStatus permissions, List<AclEntry> aclEntries, long timestamp)
+      throws QuotaExceededException, UnresolvedLinkException, AclException,
+      FileAlreadyExistsException {
+    assert fsd.hasWriteLock();
+    INodesInPath iip = fsd.getINodesInPath(src, false);
+    final byte[] localName = iip.getLastLocalName();
+    final INodesInPath existing = iip.getParentINodesInPath();
+    Preconditions.checkState(existing.getLastINode() != null);
+    unprotectedMkdir(fsd, inodeId, existing, localName, permissions, aclEntries,
+        timestamp);
+  }
 
-        if (!parentPermissions.getPermission().equals(parentFsPerm)) {
-          parentPermissions = new PermissionStatus(
-                  parentPermissions.getUserName(),
-                  parentPermissions.getGroupName(),
-                  parentFsPerm
-          );
-          // when inheriting, use same perms for entire path
-          if (inheritPermission) permissions = parentPermissions;
-        }
-      }
+  private static INodesInPath createSingleDirectory(FSDirectory fsd,
+      INodesInPath existing, String localName, PermissionStatus perm)
+      throws IOException {
+    assert fsd.hasWriteLock();
+    existing = unprotectedMkdir(fsd, fsd.allocateNewInodeId(), existing,
+        localName.getBytes(Charsets.UTF_8), perm, null, now());
+    if (existing == null) {
+      return null;
+    }
 
-      // create directories beginning from the first null index
-      for(; i < inodes.length; i++) {
-        pathbuilder.append(Path.SEPARATOR).append(names[i]);
-        unprotectedMkdir(fsd, fsd.allocateNewInodeId(), iip, i, components[i],
-            (i < lastInodeIndex) ? parentPermissions : permissions, null, now);
-        if (inodes[i] == null) {
-          return false;
-        }
-        // Directory creation also count towards FilesCreated
-        // to match count of FilesDeleted metric.
-        NameNode.getNameNodeMetrics().incrFilesCreated();
-
-        final String cur = pathbuilder.toString();
-        fsd.getEditLog().logMkDir(cur, inodes[i]);
-        if(NameNode.stateChangeLog.isDebugEnabled()) {
-          NameNode.stateChangeLog.debug(
-                  "mkdirs: created directory " + cur);
-        }
-      }
-    } finally {
-      fsd.writeUnlock();
+    final INode newNode = existing.getLastINode();
+    // Directory creation also count towards FilesCreated
+    // to match count of FilesDeleted metric.
+    NameNode.getNameNodeMetrics().incrFilesCreated();
+
+    String cur = existing.getPath();
+    fsd.getEditLog().logMkDir(cur, newNode);
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("mkdirs: created directory " + cur);
     }
-    return true;
+    return existing;
   }
 
-  /**
-   * Check whether the path specifies a directory
-   * @throws SnapshotAccessControlException if path is in RO snapshot
-   */
-  private static boolean isDirMutable(FSDirectory fsd, INodesInPath iip)
-      throws SnapshotAccessControlException {
-    fsd.readLock();
-    try {
-      INode node = iip.getLastINode();
-      return node != null && node.isDirectory();
-    } finally {
-      fsd.readUnlock();
-    }
+  private static PermissionStatus addImplicitUwx(PermissionStatus parentPerm,
+      PermissionStatus perm) {
+    FsPermission p = parentPerm.getPermission();
+    FsPermission ancestorPerm = new FsPermission(
+        p.getUserAction().or(FsAction.WRITE_EXECUTE),
+        p.getGroupAction(),
+        p.getOtherAction());
+    return new PermissionStatus(perm.getUserName(), perm.getGroupName(),
+        ancestorPerm);
   }
 
-  /** create a directory at index pos.
-   * The parent path to the directory is at [0, pos-1].
-   * All ancestors exist. Newly created one stored at index pos.
+  /**
+   * create a directory at path specified by parent
    */
-  private static void unprotectedMkdir(
-      FSDirectory fsd, long inodeId, INodesInPath inodesInPath, int pos,
-      byte[] name, PermissionStatus permission, List<AclEntry> aclEntries,
-      long timestamp)
-      throws QuotaExceededException, AclException {
+  private static INodesInPath unprotectedMkdir(FSDirectory fsd, long inodeId,
+      INodesInPath parent, byte[] name, PermissionStatus permission,
+      List<AclEntry> aclEntries, long timestamp)
+      throws QuotaExceededException, AclException, FileAlreadyExistsException {
     assert fsd.hasWriteLock();
+    assert parent.getLastINode() != null;
+    if (!parent.getLastINode().isDirectory()) {
+      throw new FileAlreadyExistsException("Parent path is not a directory: " +
+          parent.getPath() + " " + DFSUtil.bytes2String(name));
+    }
     final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
         timestamp);
-    if (fsd.addChild(inodesInPath, pos, dir, true)) {
-      if (aclEntries != null) {
-        AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
-      }
-      inodesInPath.setINode(pos, dir);
+
+    INodesInPath iip = fsd.addLastINode(parent, dir, true);
+    if (iip != null && aclEntries != null) {
+      AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
     }
+    return iip;
   }
 }
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index 9f3983a50c182..6a58093dd4ece 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.Options;
@@ -25,14 +26,14 @@
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.FSLimitException;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.util.ChunkedArrayList;
+import org.apache.hadoop.util.Time;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -42,9 +43,11 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.hadoop.util.Time.now;
+import static org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
+import static org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
 
 class FSDirRenameOp {
+  @Deprecated
   static RenameOldResult renameToInt(
       FSDirectory fsd, final String srcArg, final String dstArg,
       boolean logRetryCache)
@@ -66,9 +69,10 @@ static RenameOldResult renameToInt(
     src = fsd.resolvePath(pc, src, srcComponents);
     dst = fsd.resolvePath(pc, dst, dstComponents);
     @SuppressWarnings("deprecation")
-    final boolean status = renameToInternal(fsd, pc, src, dst, logRetryCache);
+    final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
     if (status) {
-      resultingStat = fsd.getAuditFileInfo(dst, false);
+      INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
+      resultingStat = fsd.getAuditFileInfo(dstIIP);
     }
     return new RenameOldResult(status, resultingStat);
   }
@@ -77,45 +81,58 @@ static RenameOldResult renameToInt(
    * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
    * dstInodes[dstInodes.length-1]
    */
-  static void verifyQuotaForRename(FSDirectory fsd,
-      INode[] src, INode[] dst)
-      throws QuotaExceededException {
+  private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src,
+      INodesInPath dst) throws QuotaExceededException {
     if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
       // Do not check quota if edits log is still being processed
       return;
     }
     int i = 0;
-    while(src[i] == dst[i]) { i++; }
+    while(src.getINode(i) == dst.getINode(i)) { i++; }
     // src[i - 1] is the last common ancestor.
 
-    final Quota.Counts delta = src[src.length - 1].computeQuotaUsage();
+    final Quota.Counts delta = src.getLastINode().computeQuotaUsage();
 
     // Reduce the required quota by dst that is being removed
-    final int dstIndex = dst.length - 1;
-    if (dst[dstIndex] != null) {
-      delta.subtract(dst[dstIndex].computeQuotaUsage());
+    final INode dstINode = dst.getLastINode();
+    if (dstINode != null) {
+      delta.subtract(dstINode.computeQuotaUsage());
     }
-    FSDirectory.verifyQuota(dst, dstIndex, delta.get(Quota.NAMESPACE),
-        delta.get(Quota.DISKSPACE), src[i - 1]);
+    FSDirectory.verifyQuota(dst, dst.length() - 1, delta.get(Quota.NAMESPACE),
+        delta.get(Quota.DISKSPACE), src.getINode(i - 1));
   }
 
   /**
    * Checks file system limits (max component length and max directory items)
    * during a rename operation.
    */
-  static void verifyFsLimitsForRename(FSDirectory fsd,
-      INodesInPath srcIIP,
+  static void verifyFsLimitsForRename(FSDirectory fsd, INodesInPath srcIIP,
       INodesInPath dstIIP)
-      throws FSLimitException.PathComponentTooLongException,
-          FSLimitException.MaxDirectoryItemsExceededException {
+      throws PathComponentTooLongException, MaxDirectoryItemsExceededException {
     byte[] dstChildName = dstIIP.getLastLocalName();
-    INode[] dstInodes = dstIIP.getINodes();
-    int pos = dstInodes.length - 1;
-    fsd.verifyMaxComponentLength(dstChildName, dstInodes, pos);
+    final String parentPath = dstIIP.getParentPath();
+    fsd.verifyMaxComponentLength(dstChildName, parentPath);
     // Do not enforce max directory items if renaming within same directory.
     if (srcIIP.getINode(-2) != dstIIP.getINode(-2)) {
-      fsd.verifyMaxDirItems(dstInodes, pos);
+      fsd.verifyMaxDirItems(dstIIP.getINode(-2).asDirectory(), parentPath);
+    }
+  }
+
+  /**
+   * <br>
+   * Note: This is to be used by {@link FSEditLogLoader} only.
+   * <br>
+   */
+  @Deprecated
+  @SuppressWarnings("deprecation")
+  static boolean renameForEditLog(FSDirectory fsd, String src, String dst,
+      long timestamp) throws IOException {
+    if (fsd.isDir(dst)) {
+      dst += Path.SEPARATOR + new Path(src).getName();
     }
+    final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+    final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+    return unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp);
   }
 
   /**
@@ -129,24 +146,19 @@ static void verifyFsLimitsForRename(FSDirectory fsd,
    * boolean, Options.Rename...)}
    */
   @Deprecated
-  static boolean unprotectedRenameTo(
-      FSDirectory fsd, String src, String dst, long timestamp)
+  static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+      final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp)
       throws IOException {
     assert fsd.hasWriteLock();
-    INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
     final INode srcInode = srcIIP.getLastINode();
     try {
-      validateRenameSource(src, srcIIP);
+      validateRenameSource(srcIIP);
     } catch (SnapshotException e) {
       throw e;
     } catch (IOException ignored) {
       return false;
     }
 
-    if (fsd.isDir(dst)) {
-      dst += Path.SEPARATOR + new Path(src).getName();
-    }
-
     // validate the destination
     if (dst.equals(src)) {
       return true;
@@ -158,7 +170,6 @@ static boolean unprotectedRenameTo(
       return false;
     }
 
-    INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
     if (dstIIP.getLastINode() != null) {
       NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
           "failed to rename " + src + " to " + dst + " because destination " +
@@ -176,7 +187,7 @@ static boolean unprotectedRenameTo(
     fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
     // Ensure dst has quota to accommodate rename
     verifyFsLimitsForRename(fsd, srcIIP, dstIIP);
-    verifyQuotaForRename(fsd, srcIIP.getINodes(), dstIIP.getINodes());
+    verifyQuotaForRename(fsd, srcIIP, dstIIP);
 
     RenameOperation tx = new RenameOperation(fsd, src, dst, srcIIP, dstIIP);
 
@@ -184,11 +195,7 @@ static boolean unprotectedRenameTo(
 
     try {
       // remove src
-      final long removedSrc = fsd.removeLastINode(srcIIP);
-      if (removedSrc == -1) {
-        NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
-            + "failed to rename " + src + " to " + dst + " because the source" +
-            " can not be removed");
+      if (!tx.removeSrc4OldRename()) {
         return false;
       }
 
@@ -237,41 +244,56 @@ static Map.Entry<BlocksMapUpdateInfo, HdfsFileStatus> renameToInt(
     BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
     src = fsd.resolvePath(pc, src, srcComponents);
     dst = fsd.resolvePath(pc, dst, dstComponents);
-    renameToInternal(fsd, pc, src, dst, logRetryCache, collectedBlocks,
-        options);
-    HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dst, false);
+    renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
+    INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
+    HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dstIIP);
 
-    return new AbstractMap.SimpleImmutableEntry<BlocksMapUpdateInfo,
-        HdfsFileStatus>(collectedBlocks, resultingStat);
+    return new AbstractMap.SimpleImmutableEntry<>(
+        collectedBlocks, resultingStat);
   }
 
   /**
-   * @see #unprotectedRenameTo(FSDirectory, String, String, long,
-   * org.apache.hadoop.fs.Options.Rename...)
+   * @see {@link #unprotectedRenameTo(FSDirectory, String, String, INodesInPath,
+   * INodesInPath, long, BlocksMapUpdateInfo, Options.Rename...)}
    */
-  static void renameTo(
-      FSDirectory fsd, String src, String dst, long mtime,
-      BlocksMapUpdateInfo collectedBlocks, Options.Rename... options)
-      throws IOException {
+  static void renameTo(FSDirectory fsd, FSPermissionChecker pc, String src,
+      String dst, BlocksMapUpdateInfo collectedBlocks, boolean logRetryCache,
+      Options.Rename... options) throws IOException {
+    final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+    final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+    if (fsd.isPermissionEnabled()) {
+      // Rename does not operate on link targets
+      // Do not resolveLink when checking permissions of src and dst
+      // Check write access to parent of src
+      fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+          false);
+      // Check write access to ancestor of dst
+      fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
+          false);
+    }
+
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
           + dst);
     }
+    final long mtime = Time.now();
     fsd.writeLock();
     try {
-      if (unprotectedRenameTo(fsd, src, dst, mtime, collectedBlocks, options)) {
-        fsd.getFSNamesystem().incrDeletedFileCount(1);
+      if (unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, mtime,
+          collectedBlocks, options)) {
+        FSDirDeleteOp.incrDeletedFileCount(1);
       }
     } finally {
       fsd.writeUnlock();
     }
+    fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
   }
 
   /**
    * Rename src to dst.
    * <br>
    * Note: This is to be used by {@link org.apache.hadoop.hdfs.server
-   * .namenode.FSEditLog} only.
+   * .namenode.FSEditLogLoader} only.
    * <br>
    *
    * @param fsd       FSDirectory
@@ -280,12 +302,14 @@ static void renameTo(
    * @param timestamp modification time
    * @param options   Rename options
    */
-  static boolean unprotectedRenameTo(
+  static boolean renameForEditLog(
       FSDirectory fsd, String src, String dst, long timestamp,
       Options.Rename... options)
       throws IOException {
     BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
-    boolean ret = unprotectedRenameTo(fsd, src, dst, timestamp,
+    final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+    final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+    boolean ret = unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp,
         collectedBlocks, options);
     if (!collectedBlocks.getToDeleteList().isEmpty()) {
       fsd.getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
@@ -304,9 +328,10 @@ static boolean unprotectedRenameTo(
    * @param timestamp       modification time
    * @param collectedBlocks blocks to be removed
    * @param options         Rename options
+   * @return whether a file/directory gets overwritten in the dst path
    */
-  static boolean unprotectedRenameTo(
-      FSDirectory fsd, String src, String dst, long timestamp,
+  static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+      final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp,
       BlocksMapUpdateInfo collectedBlocks, Options.Rename... options)
       throws IOException {
     assert fsd.hasWriteLock();
@@ -314,9 +339,8 @@ static boolean unprotectedRenameTo(
         && Arrays.asList(options).contains(Options.Rename.OVERWRITE);
 
     final String error;
-    final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
     final INode srcInode = srcIIP.getLastINode();
-    validateRenameSource(src, srcIIP);
+    validateRenameSource(srcIIP);
 
     // validate the destination
     if (dst.equals(src)) {
@@ -325,8 +349,7 @@ static boolean unprotectedRenameTo(
     }
     validateDestination(src, dst, srcInode);
 
-    INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
-    if (dstIIP.getINodes().length == 1) {
+    if (dstIIP.length() == 1) {
       error = "rename destination cannot be the root";
       NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
           error);
@@ -335,7 +358,7 @@ static boolean unprotectedRenameTo(
 
     fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
     final INode dstInode = dstIIP.getLastINode();
-    List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
+    List<INodeDirectory> snapshottableDirs = new ArrayList<>();
     if (dstInode != null) { // Destination exists
       validateOverwrite(src, dst, overwrite, srcInode, dstInode);
       FSDirSnapshotOp.checkSnapshot(dstInode, snapshottableDirs);
@@ -357,27 +380,19 @@ static boolean unprotectedRenameTo(
 
     // Ensure dst has quota to accommodate rename
     verifyFsLimitsForRename(fsd, srcIIP, dstIIP);
-    verifyQuotaForRename(fsd, srcIIP.getINodes(), dstIIP.getINodes());
+    verifyQuotaForRename(fsd, srcIIP, dstIIP);
 
     RenameOperation tx = new RenameOperation(fsd, src, dst, srcIIP, dstIIP);
 
     boolean undoRemoveSrc = true;
-    final long removedSrc = fsd.removeLastINode(srcIIP);
-    if (removedSrc == -1) {
-      error = "Failed to rename " + src + " to " + dst +
-          " because the source can not be removed";
-      NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
-          error);
-      throw new IOException(error);
-    }
+    tx.removeSrc();
 
     boolean undoRemoveDst = false;
-    INode removedDst = null;
     long removedNum = 0;
     try {
-      if (dstInode != null) { // dst exists remove it
-        if ((removedNum = fsd.removeLastINode(dstIIP)) != -1) {
-          removedDst = dstIIP.getLastINode();
+      if (dstInode != null) { // dst exists, remove it
+        removedNum = tx.removeDst();
+        if (removedNum != -1) {
           undoRemoveDst = true;
         }
       }
@@ -394,22 +409,10 @@ static boolean unprotectedRenameTo(
 
         // Collect the blocks and remove the lease for previous dst
         boolean filesDeleted = false;
-        if (removedDst != null) {
+        if (undoRemoveDst) {
           undoRemoveDst = false;
           if (removedNum > 0) {
-            List<INode> removedINodes = new ChunkedArrayList<INode>();
-            if (!removedDst.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
-              removedDst.destroyAndCollectBlocks(collectedBlocks,
-                  removedINodes);
-              filesDeleted = true;
-            } else {
-              filesDeleted = removedDst.cleanSubtree(
-                  Snapshot.CURRENT_STATE_ID, dstIIP.getLatestSnapshotId(),
-                  collectedBlocks, removedINodes, true)
-                  .get(Quota.NAMESPACE) >= 0;
-            }
-            fsd.getFSNamesystem().removePathAndBlocks(src, null,
-                removedINodes, false);
+            filesDeleted = tx.cleanDst(collectedBlocks);
           }
         }
 
@@ -426,23 +429,8 @@ static boolean unprotectedRenameTo(
       if (undoRemoveSrc) {
         tx.restoreSource();
       }
-
-      if (undoRemoveDst) {
-        // Rename failed - restore dst
-        if (dstParent.isDirectory() &&
-            dstParent.asDirectory().isWithSnapshot()) {
-          dstParent.asDirectory().undoRename4DstParent(removedDst,
-              dstIIP.getLatestSnapshotId());
-        } else {
-          fsd.addLastINodeNoQuotaCheck(dstIIP, removedDst);
-        }
-        assert removedDst != null;
-        if (removedDst.isReference()) {
-          final INodeReference removedDstRef = removedDst.asReference();
-          final INodeReference.WithCount wc = (INodeReference.WithCount)
-              removedDstRef.getReferredINode().asReference();
-          wc.addReference(removedDstRef);
-        }
+      if (undoRemoveDst) { // Rename failed - restore dst
+        tx.restoreDst();
       }
     }
     NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
@@ -451,59 +439,41 @@ static boolean unprotectedRenameTo(
   }
 
   /**
-   * @see #unprotectedRenameTo(FSDirectory, String, String, long)
    * @deprecated Use {@link #renameToInt(FSDirectory, String, String,
    * boolean, Options.Rename...)}
    */
   @Deprecated
   @SuppressWarnings("deprecation")
-  private static boolean renameTo(
-      FSDirectory fsd, String src, String dst, long mtime)
-      throws IOException {
+  private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
+      String src, String dst, boolean logRetryCache) throws IOException {
+    // Rename does not operate on link targets
+    // Do not resolveLink when checking permissions of src and dst
+    // Check write access to parent of src
+    final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+    // Note: We should not be doing this.  This is move() not renameTo().
+    final String actualDst = fsd.isDir(dst) ?
+        dst + Path.SEPARATOR + new Path(src).getName() : dst;
+    final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
+    if (fsd.isPermissionEnabled()) {
+      fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+          false);
+      // Check write access to ancestor of dst
+      fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
+          null, false);
+    }
+
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
           + dst);
     }
+    final long mtime = Time.now();
     boolean stat = false;
     fsd.writeLock();
     try {
-      stat = unprotectedRenameTo(fsd, src, dst, mtime);
+      stat = unprotectedRenameTo(fsd, src, actualDst, srcIIP, dstIIP, mtime);
     } finally {
       fsd.writeUnlock();
     }
-    return stat;
-  }
-
-  /**
-   * @deprecated See {@link #renameTo(FSDirectory, String, String, long)}
-   */
-  @Deprecated
-  private static boolean renameToInternal(
-      FSDirectory fsd, FSPermissionChecker pc, String src, String dst,
-      boolean logRetryCache)
-      throws IOException {
-    if (fsd.isPermissionEnabled()) {
-      //We should not be doing this.  This is move() not renameTo().
-      //but for now,
-      //NOTE: yes, this is bad!  it's assuming much lower level behavior
-      //      of rewriting the dst
-      String actualdst = fsd.isDir(dst) ? dst + Path.SEPARATOR + new Path
-          (src).getName() : dst;
-      // Rename does not operates on link targets
-      // Do not resolveLink when checking permissions of src and dst
-      // Check write access to parent of src
-      INodesInPath srcIIP = fsd.getINodesInPath(src, false);
-      fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
-          false);
-      INodesInPath dstIIP = fsd.getINodesInPath(actualdst, false);
-      // Check write access to ancestor of dst
-      fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
-          null, false);
-    }
-
-    long mtime = now();
-    @SuppressWarnings("deprecation")
-    final boolean stat = renameTo(fsd, src, dst, mtime);
     if (stat) {
       fsd.getEditLog().logRename(src, dst, mtime, logRetryCache);
       return true;
@@ -511,29 +481,6 @@ private static boolean renameToInternal(
     return false;
   }
 
-  private static void renameToInternal(
-      FSDirectory fsd, FSPermissionChecker pc, String src, String dst,
-      boolean logRetryCache, BlocksMapUpdateInfo collectedBlocks,
-      Options.Rename... options)
-      throws IOException {
-    if (fsd.isPermissionEnabled()) {
-      // Rename does not operates on link targets
-      // Do not resolveLink when checking permissions of src and dst
-      // Check write access to parent of src
-      INodesInPath srcIIP = fsd.getINodesInPath(src, false);
-      fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
-          false);
-      // Check write access to ancestor of dst
-      INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
-      fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
-          false);
-    }
-
-    long mtime = now();
-    renameTo(fsd, src, dst, mtime, collectedBlocks, options);
-    fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
-  }
-
   private static void validateDestination(
       String src, String dst, INode srcInode)
       throws IOException {
@@ -583,18 +530,18 @@ private static void validateOverwrite(
     }
   }
 
-  private static void validateRenameSource(String src, INodesInPath srcIIP)
+  private static void validateRenameSource(INodesInPath srcIIP)
       throws IOException {
     String error;
     final INode srcInode = srcIIP.getLastINode();
     // validate source
     if (srcInode == null) {
-      error = "rename source " + src + " is not found.";
+      error = "rename source " + srcIIP.getPath() + " is not found.";
       NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
           + error);
       throw new FileNotFoundException(error);
     }
-    if (srcIIP.getINodes().length == 1) {
+    if (srcIIP.length() == 1) {
       error = "rename source cannot be the root";
       NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
           + error);
@@ -607,8 +554,10 @@ private static void validateRenameSource(String src, INodesInPath srcIIP)
 
   private static class RenameOperation {
     private final FSDirectory fsd;
-    private final INodesInPath srcIIP;
-    private final INodesInPath dstIIP;
+    private INodesInPath srcIIP;
+    private final INodesInPath srcParentIIP;
+    private INodesInPath dstIIP;
+    private final INodesInPath dstParentIIP;
     private final String src;
     private final String dst;
     private final INodeReference.WithCount withCount;
@@ -619,27 +568,31 @@ private static class RenameOperation {
     private final boolean srcChildIsReference;
     private final Quota.Counts oldSrcCounts;
     private INode srcChild;
+    private INode oldDstChild;
 
     RenameOperation(FSDirectory fsd, String src, String dst,
                     INodesInPath srcIIP, INodesInPath dstIIP)
         throws QuotaExceededException {
       this.fsd = fsd;
-      this.srcIIP = srcIIP;
-      this.dstIIP = dstIIP;
       this.src = src;
       this.dst = dst;
-      srcChild = srcIIP.getLastINode();
+      this.srcIIP = srcIIP;
+      this.dstIIP = dstIIP;
+      this.srcParentIIP = srcIIP.getParentINodesInPath();
+      this.dstParentIIP = dstIIP.getParentINodesInPath();
+
+      srcChild = this.srcIIP.getLastINode();
       srcChildName = srcChild.getLocalNameBytes();
-      isSrcInSnapshot = srcChild.isInLatestSnapshot(srcIIP
-          .getLatestSnapshotId());
+      final int srcLatestSnapshotId = srcIIP.getLatestSnapshotId();
+      isSrcInSnapshot = srcChild.isInLatestSnapshot(srcLatestSnapshotId);
       srcChildIsReference = srcChild.isReference();
-      srcParent = srcIIP.getINode(-2).asDirectory();
+      srcParent = this.srcIIP.getINode(-2).asDirectory();
 
       // Record the snapshot on srcChild. After the rename, before any new
       // snapshot is taken on the dst tree, changes will be recorded in the
       // latest snapshot of the src tree.
       if (isSrcInSnapshot) {
-        srcChild.recordModification(srcIIP.getLatestSnapshotId());
+        srcChild.recordModification(srcLatestSnapshotId);
       }
 
       // check srcChild for reference
@@ -647,12 +600,12 @@ private static class RenameOperation {
           srcChild.asReference().getDstSnapshotId() : Snapshot.CURRENT_STATE_ID;
       oldSrcCounts = Quota.Counts.newInstance();
       if (isSrcInSnapshot) {
-        final INodeReference.WithName withName =
-            srcIIP.getINode(-2).asDirectory().replaceChild4ReferenceWithName(
-                srcChild, srcIIP.getLatestSnapshotId());
+        final INodeReference.WithName withName = srcParent
+            .replaceChild4ReferenceWithName(srcChild, srcLatestSnapshotId);
         withCount = (INodeReference.WithCount) withName.getReferredINode();
         srcChild = withName;
-        srcIIP.setLastINode(srcChild);
+        this.srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1,
+            srcChild);
         // get the counts before rename
         withCount.getReferredINode().computeQuotaUsage(oldSrcCounts, true);
       } else if (srcChildIsReference) {
@@ -664,9 +617,50 @@ private static class RenameOperation {
       }
     }
 
+    long removeSrc() throws IOException {
+      long removedNum = fsd.removeLastINode(srcIIP);
+      if (removedNum == -1) {
+        String error = "Failed to rename " + src + " to " + dst +
+            " because the source can not be removed";
+        NameNode.stateChangeLog.warn("DIR* FSDirRenameOp.unprotectedRenameTo:" +
+            error);
+        throw new IOException(error);
+      } else {
+        // update the quota count if necessary
+        fsd.updateCountForDelete(srcChild, srcIIP);
+        srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1, null);
+        return removedNum;
+      }
+    }
+
+    boolean removeSrc4OldRename() {
+      final long removedSrc = fsd.removeLastINode(srcIIP);
+      if (removedSrc == -1) {
+        NameNode.stateChangeLog.warn("DIR* FSDirRenameOp.unprotectedRenameTo: "
+            + "failed to rename " + src + " to " + dst + " because the source" +
+            " can not be removed");
+        return false;
+      } else {
+        // update the quota count if necessary
+        fsd.updateCountForDelete(srcChild, srcIIP);
+        srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1, null);
+        return true;
+      }
+    }
+
+    long removeDst() {
+      long removedNum = fsd.removeLastINode(dstIIP);
+      if (removedNum != -1) {
+        oldDstChild = dstIIP.getLastINode();
+        // update the quota count if necessary
+        fsd.updateCountForDelete(oldDstChild, dstIIP);
+        dstIIP = INodesInPath.replace(dstIIP, dstIIP.length() - 1, null);
+      }
+      return removedNum;
+    }
+
     boolean addSourceToDestination() {
-      final INode dstParent = dstIIP.getINode(-2);
-      srcChild = srcIIP.getLastINode();
+      final INode dstParent = dstParentIIP.getLastINode();
       final byte[] dstChildName = dstIIP.getLastLocalName();
       final INode toDst;
       if (withCount == null) {
@@ -674,16 +668,15 @@ boolean addSourceToDestination() {
         toDst = srcChild;
       } else {
         withCount.getReferredINode().setLocalName(dstChildName);
-        int dstSnapshotId = dstIIP.getLatestSnapshotId();
         toDst = new INodeReference.DstReference(dstParent.asDirectory(),
-            withCount, dstSnapshotId);
+            withCount, dstIIP.getLatestSnapshotId());
       }
-      return fsd.addLastINodeNoQuotaCheck(dstIIP, toDst);
+      return fsd.addLastINodeNoQuotaCheck(dstParentIIP, toDst) != null;
     }
 
     void updateMtimeAndLease(long timestamp) throws QuotaExceededException {
       srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshotId());
-      final INode dstParent = dstIIP.getINode(-2);
+      final INode dstParent = dstParentIIP.getLastINode();
       dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshotId());
       // update moved lease with new filename
       fsd.getFSNamesystem().unprotectedChangeLease(src, dst);
@@ -712,8 +705,41 @@ void restoreSource() throws QuotaExceededException {
       } else {
         // srcParent is not an INodeDirectoryWithSnapshot, we only need to add
         // the srcChild back
-        fsd.addLastINodeNoQuotaCheck(srcIIP, srcChild);
+        fsd.addLastINodeNoQuotaCheck(srcParentIIP, srcChild);
+      }
+    }
+
+    void restoreDst() throws QuotaExceededException {
+      Preconditions.checkState(oldDstChild != null);
+      final INodeDirectory dstParent = dstParentIIP.getLastINode().asDirectory();
+      if (dstParent.isWithSnapshot()) {
+        dstParent.undoRename4DstParent(oldDstChild, dstIIP.getLatestSnapshotId());
+      } else {
+        fsd.addLastINodeNoQuotaCheck(dstParentIIP, oldDstChild);
+      }
+      if (oldDstChild != null && oldDstChild.isReference()) {
+        final INodeReference removedDstRef = oldDstChild.asReference();
+        final INodeReference.WithCount wc = (INodeReference.WithCount)
+            removedDstRef.getReferredINode().asReference();
+        wc.addReference(removedDstRef);
+      }
+    }
+
+    boolean cleanDst(BlocksMapUpdateInfo collectedBlocks)
+        throws QuotaExceededException {
+      Preconditions.checkState(oldDstChild != null);
+      List<INode> removedINodes = new ChunkedArrayList<>();
+      final boolean filesDeleted;
+      if (!oldDstChild.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
+        oldDstChild.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+        filesDeleted = true;
+      } else {
+        filesDeleted = oldDstChild.cleanSubtree(Snapshot.CURRENT_STATE_ID,
+            dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes)
+            .get(Quota.NAMESPACE) >= 0;
       }
+      fsd.getFSNamesystem().removeLeasesAndINodes(src, removedINodes, false);
+      return filesDeleted;
     }
 
     void updateQuotasInSourceTree() throws QuotaExceededException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
index f295e060ca0d4..833bc30245a8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
@@ -29,7 +29,7 @@
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
+import org.apache.hadoop.util.ChunkedArrayList;
 
 import java.io.IOException;
 import java.util.List;
@@ -45,7 +45,7 @@ static void verifySnapshotName(FSDirectory fsd, String snapshotName,
     }
     final byte[] bytes = DFSUtil.string2Bytes(snapshotName);
     fsd.verifyINodeName(bytes);
-    fsd.verifyMaxComponentLength(bytes, path, 0);
+    fsd.verifyMaxComponentLength(bytes, path);
   }
 
   /** Allow snapshot on a directory. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index a8c3c16287757..cb3da193e8f4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
 import org.apache.hadoop.fs.FileEncryptionInfo;
@@ -50,7 +51,7 @@ static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
     FSPermissionChecker pc = fsd.getPermissionChecker();
     byte[][] pathComponents = FSDirectory
         .getPathComponentsForReservedPath(srcArg);
-    final String startAfterString = new String(startAfter);
+    final String startAfterString = new String(startAfter, Charsets.UTF_8);
     final String src = fsd.resolvePath(pc, srcArg, pathComponents);
     final INodesInPath iip = fsd.getINodesInPath(src, true);
 
@@ -71,14 +72,14 @@ static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
 
     boolean isSuperUser = true;
     if (fsd.isPermissionEnabled()) {
-      if (fsd.isDir(src)) {
+      if (iip.getLastINode() != null && iip.getLastINode().isDirectory()) {
         fsd.checkPathAccess(pc, iip, FsAction.READ_EXECUTE);
       } else {
         fsd.checkTraverse(pc, iip);
       }
       isSuperUser = pc.isSuperUser();
     }
-    return getListing(fsd, src, startAfter, needLocation, isSuperUser);
+    return getListing(fsd, iip, src, startAfter, needLocation, isSuperUser);
   }
 
   /**
@@ -122,9 +123,7 @@ static boolean isFileClosed(FSDirectory fsd, String src) throws IOException {
     if (fsd.isPermissionEnabled()) {
       fsd.checkTraverse(pc, iip);
     }
-    INode[] inodes = iip.getINodes();
-    return !INodeFile.valueOf(inodes[inodes.length - 1],
-        src).isUnderConstruction();
+    return !INodeFile.valueOf(iip.getLastINode(), src).isUnderConstruction();
   }
 
   static ContentSummary getContentSummary(
@@ -132,12 +131,17 @@ static ContentSummary getContentSummary(
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     FSPermissionChecker pc = fsd.getPermissionChecker();
     src = fsd.resolvePath(pc, src, pathComponents);
-    final INodesInPath iip = fsd.getINodesInPath(src, true);
+    final INodesInPath iip = fsd.getINodesInPath(src, false);
     if (fsd.isPermissionEnabled()) {
       fsd.checkPermission(pc, iip, false, null, null, null,
           FsAction.READ_EXECUTE);
     }
-    return getContentSummaryInt(fsd, src);
+    return getContentSummaryInt(fsd, iip);
+  }
+
+  private static byte getStoragePolicyID(byte inodePolicy, byte parentPolicy) {
+    return inodePolicy != BlockStoragePolicySuite.ID_UNSPECIFIED ? inodePolicy :
+        parentPolicy;
   }
 
   /**
@@ -149,14 +153,15 @@ static ContentSummary getContentSummary(
    * that at least this.lsLimit block locations are in the response
    *
    * @param fsd FSDirectory
+   * @param iip the INodesInPath instance containing all the INodes along the
+   *            path
    * @param src the directory name
    * @param startAfter the name to start listing after
    * @param needLocation if block locations are returned
    * @return a partial listing starting after startAfter
    */
-  private static DirectoryListing getListing(
-      FSDirectory fsd, String src, byte[] startAfter, boolean needLocation,
-      boolean isSuperUser)
+  private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip,
+      String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
       throws IOException {
     String srcs = FSDirectory.normalizePath(src);
     final boolean isRawPath = FSDirectory.isReservedRawName(src);
@@ -166,10 +171,8 @@ private static DirectoryListing getListing(
       if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
         return getSnapshotsListing(fsd, srcs, startAfter);
       }
-      final INodesInPath inodesInPath = fsd.getINodesInPath(srcs, true);
-      final INode[] inodes = inodesInPath.getINodes();
-      final int snapshot = inodesInPath.getPathSnapshotId();
-      final INode targetNode = inodes[inodes.length - 1];
+      final int snapshot = iip.getPathSnapshotId();
+      final INode targetNode = iip.getLastINode();
       if (targetNode == null)
         return null;
       byte parentStoragePolicy = isSuperUser ?
@@ -180,7 +183,7 @@ private static DirectoryListing getListing(
         return new DirectoryListing(
             new HdfsFileStatus[]{createFileStatus(fsd,
                 HdfsFileStatus.EMPTY_NAME, targetNode, needLocation,
-                parentStoragePolicy, snapshot, isRawPath, inodesInPath)}, 0);
+                parentStoragePolicy, snapshot, isRawPath, iip)}, 0);
       }
 
       final INodeDirectory dirInode = targetNode.asDirectory();
@@ -198,8 +201,8 @@ private static DirectoryListing getListing(
             cur.getLocalStoragePolicyID():
             BlockStoragePolicySuite.ID_UNSPECIFIED;
         listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), cur,
-            needLocation, fsd.getStoragePolicyID(curPolicy,
-                parentStoragePolicy), snapshot, isRawPath, inodesInPath);
+            needLocation, getStoragePolicyID(curPolicy,
+                parentStoragePolicy), snapshot, isRawPath, iip);
         listingCnt++;
         if (needLocation) {
             // Once we  hit lsLimit locations, stop.
@@ -261,30 +264,46 @@ private static DirectoryListing getSnapshotsListing(
   /** Get the file info for a specific file.
    * @param fsd FSDirectory
    * @param src The string representation of the path to the file
-   * @param resolveLink whether to throw UnresolvedLinkException
    * @param isRawPath true if a /.reserved/raw pathname was passed by the user
    * @param includeStoragePolicy whether to include storage policy
    * @return object containing information regarding the file
    *         or null if file not found
    */
+  static HdfsFileStatus getFileInfo(
+      FSDirectory fsd, INodesInPath src, boolean isRawPath,
+      boolean includeStoragePolicy)
+      throws IOException {
+    fsd.readLock();
+    try {
+      final INode i = src.getLastINode();
+      byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ?
+          i.getStoragePolicyID() : BlockStoragePolicySuite.ID_UNSPECIFIED;
+      return i == null ? null : createFileStatus(
+          fsd, HdfsFileStatus.EMPTY_NAME, i, policyId,
+          src.getPathSnapshotId(), isRawPath, src);
+    } finally {
+      fsd.readUnlock();
+    }
+  }
+
   static HdfsFileStatus getFileInfo(
       FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath,
       boolean includeStoragePolicy)
     throws IOException {
     String srcs = FSDirectory.normalizePath(src);
+    if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
+      if (fsd.getINode4DotSnapshot(srcs) != null) {
+        return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+            HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+            BlockStoragePolicySuite.ID_UNSPECIFIED);
+      }
+      return null;
+    }
+
     fsd.readLock();
     try {
-      if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
-        return getFileInfo4DotSnapshot(fsd, srcs);
-      }
-      final INodesInPath inodesInPath = fsd.getINodesInPath(srcs, resolveLink);
-      final INode[] inodes = inodesInPath.getINodes();
-      final INode i = inodes[inodes.length - 1];
-      byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ?
-          i.getStoragePolicyID() : BlockStoragePolicySuite.ID_UNSPECIFIED;
-      return i == null ? null : createFileStatus(fsd,
-          HdfsFileStatus.EMPTY_NAME, i, policyId,
-          inodesInPath.getPathSnapshotId(), isRawPath, inodesInPath);
+      final INodesInPath iip = fsd.getINodesInPath(srcs, resolveLink);
+      return getFileInfo(fsd, iip, isRawPath, includeStoragePolicy);
     } finally {
       fsd.readUnlock();
     }
@@ -405,7 +424,7 @@ private static HdfsLocatedFileStatus createLocatedFileStatus(
           fileNode.computeFileSizeNotIncludingLastUcBlock() : size;
 
       loc = fsd.getFSNamesystem().getBlockManager().createLocatedBlocks(
-          fileNode.getBlocks(), fileSize, isUc, 0L, size, false,
+          fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false,
           inSnapshot, feInfo);
       if (loc == null) {
         loc = new LocatedBlocks();
@@ -457,14 +476,13 @@ private static FsPermission getPermissionForFileStatus(
     return perm;
   }
 
-  private static ContentSummary getContentSummaryInt(
-      FSDirectory fsd, String src) throws IOException {
-    String srcs = FSDirectory.normalizePath(src);
+  private static ContentSummary getContentSummaryInt(FSDirectory fsd,
+      INodesInPath iip) throws IOException {
     fsd.readLock();
     try {
-      INode targetNode = fsd.getNode(srcs, false);
+      INode targetNode = iip.getLastINode();
       if (targetNode == null) {
-        throw new FileNotFoundException("File does not exist: " + srcs);
+        throw new FileNotFoundException("File does not exist: " + iip.getPath());
       }
       else {
         // Make it relinquish locks everytime contentCountLimit entries are
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
new file mode 100644
index 0000000000000..8970fbe18de41
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.hadoop.util.Time.now;
+
+class FSDirSymlinkOp {
+
+  static HdfsFileStatus createSymlinkInt(
+      FSNamesystem fsn, String target, final String linkArg,
+      PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
+      throws IOException {
+    FSDirectory fsd = fsn.getFSDirectory();
+    String link = linkArg;
+    if (!DFSUtil.isValidName(link)) {
+      throw new InvalidPathException("Invalid link name: " + link);
+    }
+    if (FSDirectory.isReservedName(target) || target.isEmpty()) {
+      throw new InvalidPathException("Invalid target name: " + target);
+    }
+
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* NameSystem.createSymlink: target="
+          + target + " link=" + link);
+    }
+
+    FSPermissionChecker pc = fsn.getPermissionChecker();
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(link);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      link = fsd.resolvePath(pc, link, pathComponents);
+      iip = fsd.getINodesInPath4Write(link, false);
+      if (!createParent) {
+        fsd.verifyParentDir(iip, link);
+      }
+      if (!fsd.isValidToCreate(link, iip)) {
+        throw new IOException(
+            "failed to create link " + link +
+                " either because the filename is invalid or the file exists");
+      }
+      if (fsd.isPermissionEnabled()) {
+        fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+      }
+      // validate that we have enough inodes.
+      fsn.checkFsObjectLimit();
+
+      // add symbolic link to namespace
+      addSymlink(fsd, link, iip, target, dirPerms, createParent, logRetryCache);
+    } finally {
+      fsd.writeUnlock();
+    }
+    NameNode.getNameNodeMetrics().incrCreateSymlinkOps();
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static INodeSymlink unprotectedAddSymlink(FSDirectory fsd, INodesInPath iip,
+      byte[] localName, long id, String target, long mtime, long atime,
+      PermissionStatus perm)
+      throws UnresolvedLinkException, QuotaExceededException {
+    assert fsd.hasWriteLock();
+    final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
+        target);
+    symlink.setLocalName(localName);
+    return fsd.addINode(iip, symlink) != null ? symlink : null;
+  }
+
+  /**
+   * Add the given symbolic link to the fs. Record it in the edits log.
+   */
+  private static INodeSymlink addSymlink(FSDirectory fsd, String path,
+      INodesInPath iip, String target, PermissionStatus dirPerms,
+      boolean createParent, boolean logRetryCache) throws IOException {
+    final long mtime = now();
+    final byte[] localName = iip.getLastLocalName();
+    if (createParent) {
+      Map.Entry<INodesInPath, String> e = FSDirMkdirOp
+          .createAncestorDirectories(fsd, iip, dirPerms);
+      if (e == null) {
+        return null;
+      }
+      iip = INodesInPath.append(e.getKey(), null, localName);
+    }
+    final String userName = dirPerms.getUserName();
+    long id = fsd.allocateNewInodeId();
+    PermissionStatus perm = new PermissionStatus(
+        userName, null, FsPermission.getDefault());
+    INodeSymlink newNode = unprotectedAddSymlink(fsd, iip.getExistingINodes(),
+        localName, id, target, mtime, mtime, perm);
+    if (newNode == null) {
+      NameNode.stateChangeLog.info("addSymlink: failed to add " + path);
+      return null;
+    }
+    fsd.getEditLog().logSymlink(path, target, mtime, mtime, newNode,
+        logRetryCache);
+
+    if(NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("addSymlink: " + path + " is added");
+    }
+    return newNode;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
new file mode 100644
index 0000000000000..45e63f2effe00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.XAttrHelper;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
+
+class FSDirXAttrOp {
+  private static final XAttr KEYID_XATTR =
+      XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
+  private static final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
+      XAttrHelper.buildXAttr(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER, null);
+
+  /**
+   * Set xattr for a file or directory.
+   *
+   * @param src
+   *          - path on which it sets the xattr
+   * @param xAttr
+   *          - xAttr details to set
+   * @param flag
+   *          - xAttrs flags
+   * @throws IOException
+   */
+  static HdfsFileStatus setXAttr(
+      FSDirectory fsd, String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
+      boolean logRetryCache)
+      throws IOException {
+    checkXAttrsConfigFlag(fsd);
+    checkXAttrSize(fsd, xAttr);
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    XAttrPermissionFilter.checkPermissionForApi(
+        pc, xAttr, FSDirectory.isReservedRawName(src));
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    src = fsd.resolvePath(pc, src, pathComponents);
+    List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+    xAttrs.add(xAttr);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      iip = fsd.getINodesInPath4Write(src);
+      checkXAttrChangeAccess(fsd, iip, xAttr, pc);
+      unprotectedSetXAttrs(fsd, src, xAttrs, flag);
+    } finally {
+      fsd.writeUnlock();
+    }
+    fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static List<XAttr> getXAttrs(FSDirectory fsd, final String srcArg,
+                               List<XAttr> xAttrs)
+      throws IOException {
+    String src = srcArg;
+    checkXAttrsConfigFlag(fsd);
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    final boolean isRawPath = FSDirectory.isReservedRawName(src);
+    boolean getAll = xAttrs == null || xAttrs.isEmpty();
+    if (!getAll) {
+      XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
+    }
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    src = fsd.resolvePath(pc, src, pathComponents);
+    final INodesInPath iip = fsd.getINodesInPath(src, true);
+    if (fsd.isPermissionEnabled()) {
+      fsd.checkPathAccess(pc, iip, FsAction.READ);
+    }
+    List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
+    List<XAttr> filteredAll = XAttrPermissionFilter.
+        filterXAttrsForApi(pc, all, isRawPath);
+
+    if (getAll) {
+      return filteredAll;
+    }
+    if (filteredAll == null || filteredAll.isEmpty()) {
+      return null;
+    }
+    List<XAttr> toGet = Lists.newArrayListWithCapacity(xAttrs.size());
+    for (XAttr xAttr : xAttrs) {
+      boolean foundIt = false;
+      for (XAttr a : filteredAll) {
+        if (xAttr.getNameSpace() == a.getNameSpace() && xAttr.getName().equals(
+            a.getName())) {
+          toGet.add(a);
+          foundIt = true;
+          break;
+        }
+      }
+      if (!foundIt) {
+        throw new IOException(
+            "At least one of the attributes provided was not found.");
+      }
+    }
+    return toGet;
+  }
+
+  static List<XAttr> listXAttrs(
+      FSDirectory fsd, String src) throws IOException {
+    FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
+    final FSPermissionChecker pc = fsd.getPermissionChecker();
+    final boolean isRawPath = FSDirectory.isReservedRawName(src);
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    src = fsd.resolvePath(pc, src, pathComponents);
+    final INodesInPath iip = fsd.getINodesInPath(src, true);
+    if (fsd.isPermissionEnabled()) {
+      /* To access xattr names, you need EXECUTE in the owning directory. */
+      fsd.checkParentAccess(pc, iip, FsAction.EXECUTE);
+    }
+    final List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
+    return XAttrPermissionFilter.
+        filterXAttrsForApi(pc, all, isRawPath);
+  }
+
+  /**
+   * Remove an xattr for a file or directory.
+   *
+   * @param src
+   *          - path to remove the xattr from
+   * @param xAttr
+   *          - xAttr to remove
+   * @throws IOException
+   */
+  static HdfsFileStatus removeXAttr(
+      FSDirectory fsd, String src, XAttr xAttr, boolean logRetryCache)
+      throws IOException {
+    FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
+    FSPermissionChecker pc = fsd.getPermissionChecker();
+    XAttrPermissionFilter.checkPermissionForApi(
+        pc, xAttr, FSDirectory.isReservedRawName(src));
+    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
+        src);
+
+    List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+    xAttrs.add(xAttr);
+    INodesInPath iip;
+    fsd.writeLock();
+    try {
+      src = fsd.resolvePath(pc, src, pathComponents);
+      iip = fsd.getINodesInPath4Write(src);
+      checkXAttrChangeAccess(fsd, iip, xAttr, pc);
+
+      List<XAttr> removedXAttrs = unprotectedRemoveXAttrs(fsd, src, xAttrs);
+      if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+        fsd.getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache);
+      } else {
+        throw new IOException(
+            "No matching attributes found for remove operation");
+      }
+    } finally {
+      fsd.writeUnlock();
+    }
+    return fsd.getAuditFileInfo(iip);
+  }
+
+  static List<XAttr> unprotectedRemoveXAttrs(
+      FSDirectory fsd, final String src, final List<XAttr> toRemove)
+      throws IOException {
+    assert fsd.hasWriteLock();
+    INodesInPath iip = fsd.getINodesInPath4Write(
+        FSDirectory.normalizePath(src), true);
+    INode inode = FSDirectory.resolveLastINode(iip);
+    int snapshotId = iip.getLatestSnapshotId();
+    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
+    List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
+    List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
+                                              removedXAttrs);
+    if (existingXAttrs.size() != newXAttrs.size()) {
+      XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
+      return removedXAttrs;
+    }
+    return null;
+  }
+
+  /**
+   * Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
+   * toFilter and puts them into filtered. Upon completion,
+   * toFilter contains the filter XAttrs that were not found, while
+   * fitleredXAttrs contains the XAttrs that were found.
+   *
+   * @param existingXAttrs Existing XAttrs to be filtered
+   * @param toFilter XAttrs to filter from the existing XAttrs
+   * @param filtered Return parameter, XAttrs that were filtered
+   * @return List of XAttrs that does not contain filtered XAttrs
+   */
+  @VisibleForTesting
+  static List<XAttr> filterINodeXAttrs(
+      final List<XAttr> existingXAttrs, final List<XAttr> toFilter,
+      final List<XAttr> filtered)
+    throws AccessControlException {
+    if (existingXAttrs == null || existingXAttrs.isEmpty() ||
+        toFilter == null || toFilter.isEmpty()) {
+      return existingXAttrs;
+    }
+
+    // Populate a new list with XAttrs that pass the filter
+    List<XAttr> newXAttrs =
+        Lists.newArrayListWithCapacity(existingXAttrs.size());
+    for (XAttr a : existingXAttrs) {
+      boolean add = true;
+      for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
+          ;) {
+        XAttr filter = it.next();
+        Preconditions.checkArgument(
+            !KEYID_XATTR.equalsIgnoreValue(filter),
+            "The encryption zone xattr should never be deleted.");
+        if (UNREADABLE_BY_SUPERUSER_XATTR.equalsIgnoreValue(filter)) {
+          throw new AccessControlException("The xattr '" +
+              SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' can not be deleted.");
+        }
+        if (a.equalsIgnoreValue(filter)) {
+          add = false;
+          it.remove();
+          filtered.add(filter);
+          break;
+        }
+      }
+      if (add) {
+        newXAttrs.add(a);
+      }
+    }
+
+    return newXAttrs;
+  }
+
+  static INode unprotectedSetXAttrs(
+      FSDirectory fsd, final String src, final List<XAttr> xAttrs,
+      final EnumSet<XAttrSetFlag> flag)
+      throws IOException {
+    assert fsd.hasWriteLock();
+    INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src),
+        true);
+    INode inode = FSDirectory.resolveLastINode(iip);
+    int snapshotId = iip.getLatestSnapshotId();
+    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
+    List<XAttr> newXAttrs = setINodeXAttrs(fsd, existingXAttrs, xAttrs, flag);
+    final boolean isFile = inode.isFile();
+
+    for (XAttr xattr : newXAttrs) {
+      final String xaName = XAttrHelper.getPrefixName(xattr);
+
+      /*
+       * If we're adding the encryption zone xattr, then add src to the list
+       * of encryption zones.
+       */
+      if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
+        final HdfsProtos.ZoneEncryptionInfoProto ezProto =
+            HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
+        fsd.ezManager.addEncryptionZone(inode.getId(),
+                                        PBHelper.convert(ezProto.getSuite()),
+                                        PBHelper.convert(
+                                            ezProto.getCryptoProtocolVersion()),
+                                        ezProto.getKeyName());
+      }
+
+      if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
+        throw new IOException("Can only set '" +
+            SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file.");
+      }
+    }
+
+    XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
+    return inode;
+  }
+
+  static List<XAttr> setINodeXAttrs(
+      FSDirectory fsd, final List<XAttr> existingXAttrs,
+      final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
+      throws IOException {
+    // Check for duplicate XAttrs in toSet
+    // We need to use a custom comparator, so using a HashSet is not suitable
+    for (int i = 0; i < toSet.size(); i++) {
+      for (int j = i + 1; j < toSet.size(); j++) {
+        if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
+          throw new IOException("Cannot specify the same XAttr to be set " +
+              "more than once");
+        }
+      }
+    }
+
+    // Count the current number of user-visible XAttrs for limit checking
+    int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
+
+    // The XAttr list is copied to an exactly-sized array when it's stored,
+    // so there's no need to size it precisely here.
+    int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
+    newSize += toSet.size();
+    List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
+
+    // Check if the XAttr already exists to validate with the provided flag
+    for (XAttr xAttr: toSet) {
+      boolean exist = false;
+      if (existingXAttrs != null) {
+        for (XAttr a : existingXAttrs) {
+          if (a.equalsIgnoreValue(xAttr)) {
+            exist = true;
+            break;
+          }
+        }
+      }
+      XAttrSetFlag.validate(xAttr.getName(), exist, flag);
+      // add the new XAttr since it passed validation
+      xAttrs.add(xAttr);
+      if (isUserVisible(xAttr)) {
+        userVisibleXAttrsNum++;
+      }
+    }
+
+    // Add the existing xattrs back in, if they weren't already set
+    if (existingXAttrs != null) {
+      for (XAttr existing : existingXAttrs) {
+        boolean alreadySet = false;
+        for (XAttr set : toSet) {
+          if (set.equalsIgnoreValue(existing)) {
+            alreadySet = true;
+            break;
+          }
+        }
+        if (!alreadySet) {
+          xAttrs.add(existing);
+          if (isUserVisible(existing)) {
+            userVisibleXAttrsNum++;
+          }
+        }
+      }
+    }
+
+    if (userVisibleXAttrsNum > fsd.getInodeXAttrsLimit()) {
+      throw new IOException("Cannot add additional XAttr to inode, "
+          + "would exceed limit of " + fsd.getInodeXAttrsLimit());
+    }
+
+    return xAttrs;
+  }
+
+  static List<XAttr> getXAttrs(FSDirectory fsd, INode inode, int snapshotId)
+      throws IOException {
+    fsd.readLock();
+    try {
+      return XAttrStorage.readINodeXAttrs(inode, snapshotId);
+    } finally {
+      fsd.readUnlock();
+    }
+  }
+
+  static XAttr unprotectedGetXAttrByName(
+      INode inode, int snapshotId, String xAttrName)
+      throws IOException {
+    List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
+    if (xAttrs == null) {
+      return null;
+    }
+    for (XAttr x : xAttrs) {
+      if (XAttrHelper.getPrefixName(x)
+          .equals(xAttrName)) {
+        return x;
+      }
+    }
+    return null;
+  }
+
+  private static void checkXAttrChangeAccess(
+      FSDirectory fsd, INodesInPath iip, XAttr xAttr,
+      FSPermissionChecker pc)
+      throws AccessControlException {
+    if (fsd.isPermissionEnabled() && xAttr.getNameSpace() == XAttr.NameSpace
+        .USER) {
+      final INode inode = iip.getLastINode();
+      if (inode != null &&
+          inode.isDirectory() &&
+          inode.getFsPermission().getStickyBit()) {
+        if (!pc.isSuperUser()) {
+          fsd.checkOwner(pc, iip);
+        }
+      } else {
+        fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+      }
+    }
+  }
+
+  /**
+   * Verifies that the combined size of the name and value of an xattr is within
+   * the configured limit. Setting a limit of zero disables this check.
+   */
+  private static void checkXAttrSize(FSDirectory fsd, XAttr xAttr) {
+    if (fsd.getXattrMaxSize() == 0) {
+      return;
+    }
+    int size = xAttr.getName().getBytes(Charsets.UTF_8).length;
+    if (xAttr.getValue() != null) {
+      size += xAttr.getValue().length;
+    }
+    if (size > fsd.getXattrMaxSize()) {
+      throw new HadoopIllegalArgumentException(
+          "The XAttr is too big. The maximum combined size of the"
+          + " name and value is " + fsd.getXattrMaxSize()
+          + ", but the total size is " + size);
+    }
+  }
+
+  private static void checkXAttrsConfigFlag(FSDirectory fsd) throws
+                                                             IOException {
+    if (!fsd.isXattrsEnabled()) {
+      throw new IOException(String.format(
+          "The XAttr operation has been rejected.  "
+              + "Support for XAttrs has been disabled by setting %s to false.",
+          DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY));
+    }
+  }
+
+  private static List<XAttr> getXAttrs(FSDirectory fsd,
+                                String src) throws IOException {
+    String srcs = FSDirectory.normalizePath(src);
+    fsd.readLock();
+    try {
+      INodesInPath iip = fsd.getINodesInPath(srcs, true);
+      INode inode = FSDirectory.resolveLastINode(iip);
+      int snapshotId = iip.getPathSnapshotId();
+      return XAttrStorage.readINodeXAttrs(inode, snapshotId);
+    } finally {
+      fsd.readUnlock();
+    }
+  }
+
+  private static boolean isUserVisible(XAttr xAttr) {
+    XAttr.NameSpace ns = xAttr.getNameSpace();
+    return ns == XAttr.NameSpace.USER || ns == XAttr.NameSpace.TRUSTED;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 82741ce0aa9ac..2a2c881b15cf4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -17,23 +17,11 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
-import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
-import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
-import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
-import static org.apache.hadoop.util.Time.now;
-
-import java.io.Closeable;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -43,7 +31,6 @@
 import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIsNotDirectoryException;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.XAttrSetFlag;
@@ -56,8 +43,6 @@
 import org.apache.hadoop.hdfs.XAttrHelper;
 import org.apache.hadoop.hdfs.protocol.AclException;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
 import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
@@ -67,25 +52,37 @@
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.util.ByteArray;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
+import static org.apache.hadoop.util.Time.now;
+
 /**
  * Both FSDirectory and FSNamesystem manage the state of the namespace.
  * FSDirectory is a pure in-memory data structure, all of whose operations
@@ -122,10 +119,6 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
   public final static String DOT_INODES_STRING = ".inodes";
   public final static byte[] DOT_INODES = 
       DFSUtil.string2Bytes(DOT_INODES_STRING);
-  private final XAttr KEYID_XATTR =
-      XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
-  private final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
-      XAttrHelper.buildXAttr(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER, null);
 
   INodeDirectory rootDir;
   private final FSNamesystem namesystem;
@@ -136,6 +129,7 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
   private final int contentCountLimit; // max content summary counts per run
   private final INodeMap inodeMap; // Synchronized by dirLock
   private long yieldCount = 0; // keep track of lock yield count.
+
   private final int inodeXAttrsLimit; //inode xattrs max limit
 
   // lock to protect the directory and BlockMap
@@ -148,6 +142,14 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
    * ACL-related operations.
    */
   private final boolean aclsEnabled;
+  private final boolean xattrsEnabled;
+  private final int xattrMaxSize;
+
+  // precision of access times.
+  private final long accessTimePrecision;
+  // whether setStoragePolicy is allowed.
+  private final boolean storagePolicyEnabled;
+
   private final String fsOwnerShortUserName;
   private final String supergroup;
   private final INodeId inodeId;
@@ -213,6 +215,27 @@ public int getWriteHoldCount() {
         DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
         DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
     LOG.info("ACLs enabled? " + aclsEnabled);
+    this.xattrsEnabled = conf.getBoolean(
+        DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
+        DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
+    LOG.info("XAttrs enabled? " + xattrsEnabled);
+    this.xattrMaxSize = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
+        DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
+    Preconditions.checkArgument(xattrMaxSize >= 0,
+                                "Cannot set a negative value for the maximum size of an xattr (%s).",
+                                DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
+    final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
+    LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);
+
+    this.accessTimePrecision = conf.getLong(
+        DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
+        DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);
+
+    this.storagePolicyEnabled =
+        conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
+                        DFS_STORAGE_POLICY_ENABLED_DEFAULT);
+
     int configuredLimit = conf.getInt(
         DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
     this.lsLimit = configuredLimit>0 ?
@@ -242,7 +265,7 @@ public int getWriteHoldCount() {
     Preconditions.checkArgument(
         maxDirItems > 0 && maxDirItems <= MAX_DIR_ITEMS, "Cannot set "
             + DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY
-            + " to a value less than 0 or greater than " + MAX_DIR_ITEMS);
+            + " to a value less than 1 or greater than " + MAX_DIR_ITEMS);
 
     int threshold = conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
@@ -274,6 +297,17 @@ boolean isPermissionEnabled() {
   boolean isAclsEnabled() {
     return aclsEnabled;
   }
+  boolean isXattrsEnabled() {
+    return xattrsEnabled;
+  }
+  int getXattrMaxSize() { return xattrMaxSize; }
+  boolean isStoragePolicyEnabled() {
+    return storagePolicyEnabled;
+  }
+  boolean isAccessTimeSupported() {
+    return accessTimePrecision > 0;
+  }
+
 
   int getLsLimit() {
     return lsLimit;
@@ -283,6 +317,10 @@ int getContentCountLimit() {
     return contentCountLimit;
   }
 
+  int getInodeXAttrsLimit() {
+    return inodeXAttrsLimit;
+  }
+
   FSEditLog getEditLog() {
     return editLog;
   }
@@ -318,95 +356,86 @@ void disableQuotaChecks() {
 
   private static INodeFile newINodeFile(long id, PermissionStatus permissions,
       long mtime, long atime, short replication, long preferredBlockSize) {
-    return newINodeFile(id, permissions, mtime, atime, replication, preferredBlockSize,
-        (byte)0);
+    return newINodeFile(id, permissions, mtime, atime, replication,
+        preferredBlockSize, (byte)0);
   }
 
   private static INodeFile newINodeFile(long id, PermissionStatus permissions,
       long mtime, long atime, short replication, long preferredBlockSize,
       byte storagePolicyId) {
     return new INodeFile(id, null, permissions, mtime, atime,
-        BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
+        BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
         storagePolicyId);
   }
 
   /**
    * Add the given filename to the fs.
-   * @throws FileAlreadyExistsException
-   * @throws QuotaExceededException
-   * @throws UnresolvedLinkException
-   * @throws SnapshotAccessControlException 
+   * @return the new INodesInPath instance that contains the new INode
    */
-  INodeFile addFile(String path, PermissionStatus permissions,
-                    short replication, long preferredBlockSize,
-                    String clientName, String clientMachine)
+  INodesInPath addFile(INodesInPath existing, String localName, PermissionStatus
+      permissions, short replication, long preferredBlockSize,
+      String clientName, String clientMachine)
     throws FileAlreadyExistsException, QuotaExceededException,
       UnresolvedLinkException, SnapshotAccessControlException, AclException {
 
     long modTime = now();
-    INodeFile newNode = newINodeFile(allocateNewInodeId(), permissions, modTime, modTime, replication, preferredBlockSize);
+    INodeFile newNode = newINodeFile(allocateNewInodeId(), permissions, modTime,
+        modTime, replication, preferredBlockSize);
+    newNode.setLocalName(localName.getBytes(Charsets.UTF_8));
     newNode.toUnderConstruction(clientName, clientMachine);
 
-    boolean added = false;
+    INodesInPath newiip;
     writeLock();
     try {
-      added = addINode(path, newNode);
+      newiip = addINode(existing, newNode);
     } finally {
       writeUnlock();
     }
-    if (!added) {
-      NameNode.stateChangeLog.info("DIR* addFile: failed to add " + path);
+    if (newiip == null) {
+      NameNode.stateChangeLog.info("DIR* addFile: failed to add " +
+          existing.getPath() + "/" + localName);
       return null;
     }
 
     if(NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* addFile: " + path + " is added");
-    }
-    return newNode;
-  }
-
-  INodeFile unprotectedAddFile( long id,
-                            String path, 
-                            PermissionStatus permissions,
-                            List<AclEntry> aclEntries,
-                            List<XAttr> xAttrs,
-                            short replication,
-                            long modificationTime,
-                            long atime,
-                            long preferredBlockSize,
-                            boolean underConstruction,
-                            String clientName,
-                            String clientMachine,
-                            byte storagePolicyId) {
+      NameNode.stateChangeLog.debug("DIR* addFile: " + localName + " is added");
+    }
+    return newiip;
+  }
+
+  INodeFile addFileForEditLog(long id, INodesInPath existing, byte[] localName,
+      PermissionStatus permissions, List<AclEntry> aclEntries,
+      List<XAttr> xAttrs, short replication, long modificationTime, long atime,
+      long preferredBlockSize, boolean underConstruction, String clientName,
+      String clientMachine, byte storagePolicyId) {
     final INodeFile newNode;
     assert hasWriteLock();
     if (underConstruction) {
       newNode = newINodeFile(id, permissions, modificationTime,
           modificationTime, replication, preferredBlockSize, storagePolicyId);
       newNode.toUnderConstruction(clientName, clientMachine);
-
     } else {
       newNode = newINodeFile(id, permissions, modificationTime, atime,
           replication, preferredBlockSize, storagePolicyId);
     }
 
+    newNode.setLocalName(localName);
     try {
-      if (addINode(path, newNode)) {
+      INodesInPath iip = addINode(existing, newNode);
+      if (iip != null) {
         if (aclEntries != null) {
-          AclStorage.updateINodeAcl(newNode, aclEntries,
-            Snapshot.CURRENT_STATE_ID);
+          AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
         }
         if (xAttrs != null) {
-          XAttrStorage.updateINodeXAttrs(newNode, xAttrs,
-              Snapshot.CURRENT_STATE_ID);
+          XAttrStorage.updateINodeXAttrs(newNode, xAttrs, CURRENT_STATE_ID);
         }
         return newNode;
       }
     } catch (IOException e) {
       if(NameNode.stateChangeLog.isDebugEnabled()) {
         NameNode.stateChangeLog.debug(
-            "DIR* FSDirectory.unprotectedAddFile: exception when add " + path
-                + " to the file system", e);
+            "DIR* FSDirectory.unprotectedAddFile: exception when add "
+                + existing.getPath() + " to the file system", e);
       }
     }
     return null;
@@ -415,21 +444,25 @@ INodeFile unprotectedAddFile( long id,
   /**
    * Add a block to the file. Returns a reference to the added block.
    */
-  BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
-      DatanodeStorageInfo[] targets) throws IOException {
+  BlockInfoContiguous addBlock(String path, INodesInPath inodesInPath, Block block,
+                     DatanodeStorageInfo[] targets,
+                     boolean isStriped) throws IOException {
     writeLock();
     try {
       final INodeFile fileINode = inodesInPath.getLastINode().asFile();
+      short numLocations = isStriped ?
+          HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS :
+          fileINode.getFileReplication();
       Preconditions.checkState(fileINode.isUnderConstruction());
 
       // check quota limits and updated space consumed
-      updateCount(inodesInPath, 0, fileINode.getBlockDiskspace(), true);
+      updateCount(inodesInPath, 0, fileINode.getPreferredBlockDiskspace(), true);
 
       // associate new last block for the file
-      BlockInfoUnderConstruction blockInfo =
-        new BlockInfoUnderConstruction(
+      BlockInfoContiguousUnderConstruction blockInfo =
+        new BlockInfoContiguousUnderConstruction(
             block,
-            fileINode.getFileReplication(),
+            numLocations,
             BlockUCState.UNDER_CONSTRUCTION,
             targets);
       getBlockManager().addBlockCollection(blockInfo, fileINode);
@@ -451,18 +484,18 @@ BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
    * Remove a block from the file.
    * @return Whether the block exists in the corresponding file
    */
-  boolean removeBlock(String path, INodeFile fileNode, Block block)
-      throws IOException {
+  boolean removeBlock(String path, INodesInPath iip, INodeFile fileNode,
+      Block block) throws IOException {
     Preconditions.checkArgument(fileNode.isUnderConstruction());
     writeLock();
     try {
-      return unprotectedRemoveBlock(path, fileNode, block);
+      return unprotectedRemoveBlock(path, iip, fileNode, block);
     } finally {
       writeUnlock();
     }
   }
   
-  boolean unprotectedRemoveBlock(String path,
+  boolean unprotectedRemoveBlock(String path, INodesInPath iip,
       INodeFile fileNode, Block block) throws IOException {
     // modify file-> block and blocksMap
     // fileNode should be under construction
@@ -479,8 +512,7 @@ boolean unprotectedRemoveBlock(String path,
     }
 
     // update space consumed
-    final INodesInPath iip = getINodesInPath4Write(path, true);
-    updateCount(iip, 0, -fileNode.getBlockDiskspace(), true);
+    updateCount(iip, 0, -fileNode.getPreferredBlockDiskspace(), true);
     return true;
   }
 
@@ -507,244 +539,13 @@ String resolvePath(FSPermissionChecker pc, String path, byte[][] pathComponents)
     return resolvePath(path, pathComponents, this);
   }
 
-  /**
-   * Set file replication
-   * 
-   * @param src file name
-   * @param replication new replication
-   * @param blockRepls block replications - output parameter
-   * @return array of file blocks
-   * @throws QuotaExceededException
-   * @throws SnapshotAccessControlException 
-   */
-  Block[] setReplication(String src, short replication, short[] blockRepls)
-      throws QuotaExceededException, UnresolvedLinkException,
-      SnapshotAccessControlException {
-    writeLock();
-    try {
-      return unprotectedSetReplication(src, replication, blockRepls);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  Block[] unprotectedSetReplication(String src, short replication,
-      short[] blockRepls) throws QuotaExceededException,
-      UnresolvedLinkException, SnapshotAccessControlException {
-    assert hasWriteLock();
-
-    final INodesInPath iip = getINodesInPath4Write(src, true);
-    final INode inode = iip.getLastINode();
-    if (inode == null || !inode.isFile()) {
-      return null;
-    }
-    INodeFile file = inode.asFile();
-    final short oldBR = file.getBlockReplication();
-
-    // before setFileReplication, check for increasing block replication.
-    // if replication > oldBR, then newBR == replication.
-    // if replication < oldBR, we don't know newBR yet. 
-    if (replication > oldBR) {
-      long dsDelta = (replication - oldBR)*(file.diskspaceConsumed()/oldBR);
-      updateCount(iip, 0, dsDelta, true);
-    }
-
-    file.setFileReplication(replication, iip.getLatestSnapshotId());
-    
-    final short newBR = file.getBlockReplication(); 
-    // check newBR < oldBR case. 
-    if (newBR < oldBR) {
-      long dsDelta = (newBR - oldBR)*(file.diskspaceConsumed()/newBR);
-      updateCount(iip, 0, dsDelta, true);
-    }
-
-    if (blockRepls != null) {
-      blockRepls[0] = oldBR;
-      blockRepls[1] = newBR;
-    }
-    return file.getBlocks();
-  }
-
-  /** Set block storage policy for a directory */
-  void setStoragePolicy(INodesInPath iip, byte policyId)
-      throws IOException {
-    writeLock();
-    try {
-      unprotectedSetStoragePolicy(iip, policyId);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  void unprotectedSetStoragePolicy(INodesInPath iip, byte policyId)
-      throws IOException {
-    assert hasWriteLock();
-    final INode inode = iip.getLastINode();
-    if (inode == null) {
-      throw new FileNotFoundException("File/Directory does not exist: "
-          + iip.getPath());
-    }
-    final int snapshotId = iip.getLatestSnapshotId();
-    if (inode.isFile()) {
-      BlockStoragePolicy newPolicy = getBlockManager().getStoragePolicy(policyId);
-      if (newPolicy.isCopyOnCreateFile()) {
-        throw new HadoopIllegalArgumentException(
-            "Policy " + newPolicy + " cannot be set after file creation.");
-      }
-
-      BlockStoragePolicy currentPolicy =
-          getBlockManager().getStoragePolicy(inode.getLocalStoragePolicyID());
-
-      if (currentPolicy != null && currentPolicy.isCopyOnCreateFile()) {
-        throw new HadoopIllegalArgumentException(
-            "Existing policy " + currentPolicy.getName() +
-                " cannot be changed after file creation.");
-      }
-      inode.asFile().setStoragePolicyID(policyId, snapshotId);
-    } else if (inode.isDirectory()) {
-      setDirStoragePolicy(inode.asDirectory(), policyId, snapshotId);  
-    } else {
-      throw new FileNotFoundException(iip.getPath()
-          + " is not a file or directory");
-    }
-  }
-
-  private void setDirStoragePolicy(INodeDirectory inode, byte policyId,
-      int latestSnapshotId) throws IOException {
-    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
-    XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
-    List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, Arrays.asList(xAttr),
-        EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
-    XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
-  }
-
-  /**
-   * @param path the file path
-   * @return the block size of the file. 
-   */
-  long getPreferredBlockSize(String path) throws UnresolvedLinkException,
-      FileNotFoundException, IOException {
-    readLock();
-    try {
-      return INodeFile.valueOf(getNode(path, false), path
-          ).getPreferredBlockSize();
-    } finally {
-      readUnlock();
-    }
-  }
-
-  void setPermission(String src, FsPermission permission)
-      throws FileNotFoundException, UnresolvedLinkException,
-      QuotaExceededException, SnapshotAccessControlException {
-    writeLock();
-    try {
-      unprotectedSetPermission(src, permission);
-    } finally {
-      writeUnlock();
-    }
-  }
-  
-  void unprotectedSetPermission(String src, FsPermission permissions)
-      throws FileNotFoundException, UnresolvedLinkException,
-      QuotaExceededException, SnapshotAccessControlException {
-    assert hasWriteLock();
-    final INodesInPath inodesInPath = getINodesInPath4Write(src, true);
-    final INode inode = inodesInPath.getLastINode();
-    if (inode == null) {
-      throw new FileNotFoundException("File does not exist: " + src);
-    }
-    int snapshotId = inodesInPath.getLatestSnapshotId();
-    inode.setPermission(permissions, snapshotId);
-  }
-
-  void setOwner(String src, String username, String groupname)
-      throws FileNotFoundException, UnresolvedLinkException,
-      QuotaExceededException, SnapshotAccessControlException {
-    writeLock();
-    try {
-      unprotectedSetOwner(src, username, groupname);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  void unprotectedSetOwner(String src, String username, String groupname)
-      throws FileNotFoundException, UnresolvedLinkException,
-      QuotaExceededException, SnapshotAccessControlException {
-    assert hasWriteLock();
-    final INodesInPath inodesInPath = getINodesInPath4Write(src, true);
-    INode inode = inodesInPath.getLastINode();
-    if (inode == null) {
-      throw new FileNotFoundException("File does not exist: " + src);
-    }
-    if (username != null) {
-      inode = inode.setUser(username, inodesInPath.getLatestSnapshotId());
-    }
-    if (groupname != null) {
-      inode.setGroup(groupname, inodesInPath.getLatestSnapshotId());
-    }
-  }
-
-  /**
-   * Delete the target directory and collect the blocks under it
-   * 
-   * @param src Path of a directory to delete
-   * @param collectedBlocks Blocks under the deleted directory
-   * @param removedINodes INodes that should be removed from {@link #inodeMap}
-   * @return the number of files that have been removed
-   */
-  long delete(String src, BlocksMapUpdateInfo collectedBlocks,
-              List<INode> removedINodes, long mtime) throws IOException {
-    if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
-    }
-    final long filesRemoved;
-    writeLock();
-    try {
-      final INodesInPath inodesInPath = getINodesInPath4Write(
-          normalizePath(src), false);
-      if (!deleteAllowed(inodesInPath, src) ) {
-        filesRemoved = -1;
-      } else {
-        List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
-        FSDirSnapshotOp.checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
-        filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
-            removedINodes, mtime);
-        namesystem.removeSnapshottableDirs(snapshottableDirs);
-      }
-    } finally {
-      writeUnlock();
-    }
-    return filesRemoved;
-  }
-  
-  private static boolean deleteAllowed(final INodesInPath iip,
-      final String src) {
-    final INode[] inodes = iip.getINodes(); 
-    if (inodes == null || inodes.length == 0
-        || inodes[inodes.length - 1] == null) {
-      if(NameNode.stateChangeLog.isDebugEnabled()) {
-        NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
-            + "failed to remove " + src + " because it does not exist");
-      }
-      return false;
-    } else if (inodes.length == 1) { // src is the root
-      NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedDelete: "
-          + "failed to remove " + src
-          + " because the root is not allowed to be deleted");
-      return false;
-    }
-    return true;
-  }
-  
   /**
    * @return true if the path is a non-empty directory; otherwise, return false.
    */
   boolean isNonEmptyDirectory(INodesInPath inodesInPath) {
     readLock();
     try {
-      final INode[] inodes = inodesInPath.getINodes();
-      final INode inode = inodes[inodes.length - 1];
+      final INode inode = inodesInPath.getLastINode();
       if (inode == null || !inode.isDirectory()) {
         //not found or not a directory
         return false;
@@ -757,178 +558,14 @@ boolean isNonEmptyDirectory(INodesInPath inodesInPath) {
   }
 
   /**
-   * Delete a path from the name space
-   * Update the count at each ancestor directory with quota
-   * <br>
-   * Note: This is to be used by {@link FSEditLog} only.
-   * <br>
-   * @param src a string representation of a path to an inode
-   * @param mtime the time the inode is removed
-   * @throws SnapshotAccessControlException if path is in RO snapshot
-   */
-  void unprotectedDelete(String src, long mtime) throws UnresolvedLinkException,
-      QuotaExceededException, SnapshotAccessControlException, IOException {
-    assert hasWriteLock();
-    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
-    List<INode> removedINodes = new ChunkedArrayList<INode>();
-
-    final INodesInPath inodesInPath = getINodesInPath4Write(
-        normalizePath(src), false);
-    long filesRemoved = -1;
-    if (deleteAllowed(inodesInPath, src)) {
-      List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
-      FSDirSnapshotOp.checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
-      filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
-          removedINodes, mtime);
-      namesystem.removeSnapshottableDirs(snapshottableDirs); 
-    }
-
-    if (filesRemoved >= 0) {
-      getFSNamesystem().removePathAndBlocks(src, collectedBlocks, 
-          removedINodes, false);
-    }
-  }
-  
-  /**
-   * Delete a path from the name space
-   * Update the count at each ancestor directory with quota
-   * @param iip the inodes resolved from the path
-   * @param collectedBlocks blocks collected from the deleted path
-   * @param removedINodes inodes that should be removed from {@link #inodeMap}
-   * @param mtime the time the inode is removed
-   * @return the number of inodes deleted; 0 if no inodes are deleted.
-   */ 
-  long unprotectedDelete(INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
-      List<INode> removedINodes, long mtime) throws QuotaExceededException {
-    assert hasWriteLock();
-
-    // check if target node exists
-    INode targetNode = iip.getLastINode();
-    if (targetNode == null) {
-      return -1;
-    }
-
-    // record modification
-    final int latestSnapshot = iip.getLatestSnapshotId();
-    targetNode.recordModification(latestSnapshot);
-
-    // Remove the node from the namespace
-    long removed = removeLastINode(iip);
-    if (removed == -1) {
-      return -1;
-    }
-
-    // set the parent's modification time
-    final INodeDirectory parent = targetNode.getParent();
-    parent.updateModificationTime(mtime, latestSnapshot);
-    if (removed == 0) {
-      return 0;
-    }
-    
-    // collect block
-    if (!targetNode.isInLatestSnapshot(latestSnapshot)) {
-      targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
-    } else {
-      Quota.Counts counts = targetNode.cleanSubtree(Snapshot.CURRENT_STATE_ID,
-          latestSnapshot, collectedBlocks, removedINodes, true);
-      parent.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
-          -counts.get(Quota.DISKSPACE), true);
-      removed = counts.get(Quota.NAMESPACE);
-    }
-    if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
-          + iip.getPath() + " is removed");
-    }
-    return removed;
-  }
-
-  byte getStoragePolicyID(byte inodePolicy, byte parentPolicy) {
-    return inodePolicy != BlockStoragePolicySuite.ID_UNSPECIFIED ? inodePolicy :
-        parentPolicy;
-  }
-
-  INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
-    Preconditions.checkArgument(
-        src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
-        "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
-    
-    final String dirPath = normalizePath(src.substring(0,
-        src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
-    
-    final INode node = this.getINode(dirPath);
-    if (node != null && node.isDirectory()
-        && node.asDirectory().isSnapshottable()) {
-      return node;
-    }
-    return null;
-  }
-
-  INodesInPath getExistingPathINodes(byte[][] components)
-      throws UnresolvedLinkException {
-    return INodesInPath.resolve(rootDir, components);
-  }
-
-  /**
-   * Get {@link INode} associated with the file / directory.
-   */
-  public INode getINode(String src) throws UnresolvedLinkException {
-    return getLastINodeInPath(src).getINode(0);
-  }
-
-  /**
-   * Get {@link INode} associated with the file / directory.
-   */
-  public INodesInPath getLastINodeInPath(String src)
-       throws UnresolvedLinkException {
-    readLock();
-    try {
-      return getLastINodeInPath(src, true);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
-   * Get {@link INode} associated with the file / directory.
-   */
-  public INodesInPath getINodesInPath4Write(String src
-      ) throws UnresolvedLinkException, SnapshotAccessControlException {
-    readLock();
-    try {
-      return getINodesInPath4Write(src, true);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /**
-   * Get {@link INode} associated with the file / directory.
-   * @throws SnapshotAccessControlException if path is in RO snapshot
-   */
-  public INode getINode4Write(String src) throws UnresolvedLinkException,
-      SnapshotAccessControlException {
-    readLock();
-    try {
-      return getINode4Write(src, true);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  /** 
    * Check whether the filepath could be created
    * @throws SnapshotAccessControlException if path is in RO snapshot
    */
-  boolean isValidToCreate(String src) throws UnresolvedLinkException,
-      SnapshotAccessControlException {
+  boolean isValidToCreate(String src, INodesInPath iip)
+      throws SnapshotAccessControlException {
     String srcs = normalizePath(src);
-    readLock();
-    try {
-      return srcs.startsWith("/") && !srcs.endsWith("/")
-              && getINode4Write(srcs, false) == null;
-    } finally {
-      readUnlock();
-    }
+    return srcs.startsWith("/") && !srcs.endsWith("/") &&
+        iip.getLastINode() == null;
   }
 
   /**
@@ -938,7 +575,7 @@ boolean isDir(String src) throws UnresolvedLinkException {
     src = normalizePath(src);
     readLock();
     try {
-      INode node = getNode(src, false);
+      INode node = getINode(src, false);
       return node != null && node.isDirectory();
     } finally {
       readUnlock();
@@ -947,31 +584,45 @@ boolean isDir(String src) throws UnresolvedLinkException {
 
   /** Updates namespace and diskspace consumed for all
    * directories until the parent directory of file represented by path.
-   * 
-   * @param path path for the file.
+   *
+   * @param iip the INodesInPath instance containing all the INodes for
+   *            updating quota usage
    * @param nsDelta the delta change of namespace
    * @param dsDelta the delta change of diskspace
    * @throws QuotaExceededException if the new count violates any quota limit
    * @throws FileNotFoundException if path does not exist.
    */
-  void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
+  void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta)
       throws QuotaExceededException, FileNotFoundException,
           UnresolvedLinkException, SnapshotAccessControlException {
     writeLock();
     try {
-      final INodesInPath iip = getINodesInPath4Write(path, false);
       if (iip.getLastINode() == null) {
-        throw new FileNotFoundException("Path not found: " + path);
+        throw new FileNotFoundException("Path not found: " + iip.getPath());
       }
       updateCount(iip, nsDelta, dsDelta, true);
     } finally {
       writeUnlock();
     }
   }
-  
-  private void updateCount(INodesInPath iip, long nsDelta, long dsDelta,
+
+  /**
+   * Update the quota usage after deletion. The quota update is only necessary
+   * when image/edits have been loaded and the file/dir to be deleted is not
+   * contained in snapshots.
+   */
+  void updateCountForDelete(final INode inode, final INodesInPath iip) {
+    if (getFSNamesystem().isImageLoaded() &&
+        !inode.isInLatestSnapshot(iip.getLatestSnapshotId())) {
+      Quota.Counts counts = inode.computeQuotaUsage();
+      unprotectedUpdateCount(iip, iip.length() - 1,
+          -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+    }
+  }
+
+  void updateCount(INodesInPath iip, long nsDelta, long dsDelta,
       boolean checkQuota) throws QuotaExceededException {
-    updateCount(iip, iip.getINodes().length - 1, nsDelta, dsDelta, checkQuota);
+    updateCount(iip, iip.length() - 1, nsDelta, dsDelta, checkQuota);
   }
 
   /** update count of each inode with quota
@@ -991,12 +642,11 @@ private void updateCount(INodesInPath iip, int numOfINodes,
       //still initializing. do not check or update quotas.
       return;
     }
-    final INode[] inodes = iip.getINodes();
-    if (numOfINodes > inodes.length) {
-      numOfINodes = inodes.length;
+    if (numOfINodes > iip.length()) {
+      numOfINodes = iip.length();
     }
     if (checkQuota && !skipQuotaCheck) {
-      verifyQuota(inodes, numOfINodes, nsDelta, dsDelta, null);
+      verifyQuota(iip, numOfINodes, nsDelta, dsDelta, null);
     }
     unprotectedUpdateCount(iip, numOfINodes, nsDelta, dsDelta);
   }
@@ -1005,8 +655,8 @@ private void updateCount(INodesInPath iip, int numOfINodes,
    * update quota of each inode and check to see if quota is exceeded. 
    * See {@link #updateCount(INodesInPath, long, long, boolean)}
    */ 
-  private void updateCountNoQuotaCheck(INodesInPath inodesInPath,
-      int numOfINodes, long nsDelta, long dsDelta) {
+  void updateCountNoQuotaCheck(INodesInPath inodesInPath, int numOfINodes,
+      long nsDelta, long dsDelta) {
     assert hasWriteLock();
     try {
       updateCount(inodesInPath, numOfINodes, nsDelta, dsDelta, false);
@@ -1019,11 +669,11 @@ private void updateCountNoQuotaCheck(INodesInPath inodesInPath,
    * updates quota without verification
    * callers responsibility is to make sure quota is not exceeded
    */
-  static void unprotectedUpdateCount(INodesInPath inodesInPath, int numOfINodes, long nsDelta, long dsDelta) {
-    final INode[] inodes = inodesInPath.getINodes();
+  static void unprotectedUpdateCount(INodesInPath inodesInPath,
+      int numOfINodes, long nsDelta, long dsDelta) {
     for(int i=0; i < numOfINodes; i++) {
-      if (inodes[i].isQuotaSet()) { // a directory with quota
-        inodes[i].asDirectory().getDirectoryWithQuotaFeature()
+      if (inodesInPath.getINode(i).isQuotaSet()) { // a directory with quota
+        inodesInPath.getINode(i).asDirectory().getDirectoryWithQuotaFeature()
             .addSpaceConsumed2Cache(nsDelta, dsDelta);
       }
     }
@@ -1082,17 +732,18 @@ static String getFullPathName(INode inode) {
 
   /**
    * Add the given child to the namespace.
-   * @param src The full path name of the child node.
+   * @param existing the INodesInPath containing all the ancestral INodes
+   * @param child the new INode to add
+   * @return a new INodesInPath instance containing the new child INode. Null
+   * if the adding fails.
    * @throws QuotaExceededException is thrown if it violates quota limit
    */
-  private boolean addINode(String src, INode child
-      ) throws QuotaExceededException, UnresolvedLinkException {
-    byte[][] components = INode.getPathComponents(src);
-    child.setLocalName(components[components.length-1]);
+  INodesInPath addINode(INodesInPath existing, INode child)
+      throws QuotaExceededException, UnresolvedLinkException {
     cacheName(child);
     writeLock();
     try {
-      return addLastINode(getExistingPathINodes(components), child, true);
+      return addLastINode(existing, child, true);
     } finally {
       writeUnlock();
     }
@@ -1102,7 +753,7 @@ private boolean addINode(String src, INode child
    * Verify quota for adding or moving a new INode with required 
    * namespace and diskspace to a given position.
    *  
-   * @param inodes INodes corresponding to a path
+   * @param iip INodes corresponding to a path
    * @param pos position where a new INode will be added
    * @param nsDelta needed namespace
    * @param dsDelta needed diskspace
@@ -1111,7 +762,7 @@ private boolean addINode(String src, INode child
    *          Pass null if a node is not being moved.
    * @throws QuotaExceededException if quota limit is exceeded.
    */
-  static void verifyQuota(INode[] inodes, int pos, long nsDelta,
+  static void verifyQuota(INodesInPath iip, int pos, long nsDelta,
       long dsDelta, INode commonAncestor) throws QuotaExceededException {
     if (nsDelta <= 0 && dsDelta <= 0) {
       // if quota is being freed or not being consumed
@@ -1119,18 +770,20 @@ static void verifyQuota(INode[] inodes, int pos, long nsDelta,
     }
 
     // check existing components in the path
-    for(int i = (pos > inodes.length? inodes.length: pos) - 1; i >= 0; i--) {
-      if (commonAncestor == inodes[i]) {
+    for(int i = (pos > iip.length() ? iip.length(): pos) - 1; i >= 0; i--) {
+      if (commonAncestor == iip.getINode(i)) {
         // Stop checking for quota when common ancestor is reached
         return;
       }
       final DirectoryWithQuotaFeature q
-          = inodes[i].asDirectory().getDirectoryWithQuotaFeature();
+          = iip.getINode(i).asDirectory().getDirectoryWithQuotaFeature();
       if (q != null) { // a directory with quota
         try {
           q.verifyQuota(nsDelta, dsDelta);
         } catch (QuotaExceededException e) {
-          e.setPathName(getFullPathName(inodes, i));
+          List<INode> inodes = iip.getReadOnlyINodes();
+          final String path = getFullPathName(inodes.toArray(new INode[inodes.size()]), i);
+          e.setPathName(path);
           throw e;
         }
       }
@@ -1152,22 +805,20 @@ void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
    * Verify child's name for fs limit.
    *
    * @param childName byte[] containing new child name
-   * @param parentPath Object either INode[] or String containing parent path
-   * @param pos int position of new child in path
+   * @param parentPath String containing parent path
    * @throws PathComponentTooLongException child's name is too long.
    */
-  void verifyMaxComponentLength(byte[] childName, Object parentPath,
-      int pos) throws PathComponentTooLongException {
+  void verifyMaxComponentLength(byte[] childName, String parentPath)
+      throws PathComponentTooLongException {
     if (maxComponentLength == 0) {
       return;
     }
 
     final int length = childName.length;
     if (length > maxComponentLength) {
-      final String p = parentPath instanceof INode[]?
-          getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
       final PathComponentTooLongException e = new PathComponentTooLongException(
-          maxComponentLength, length, p, DFSUtil.bytes2String(childName));
+          maxComponentLength, length, parentPath,
+          DFSUtil.bytes2String(childName));
       if (namesystem.isImageLoaded()) {
         throw e;
       } else {
@@ -1180,20 +831,16 @@ void verifyMaxComponentLength(byte[] childName, Object parentPath,
   /**
    * Verify children size for fs limit.
    *
-   * @param pathComponents INode[] containing full path of inodes to new child
-   * @param pos int position of new child in pathComponents
    * @throws MaxDirectoryItemsExceededException too many children.
    */
-  void verifyMaxDirItems(INode[] pathComponents, int pos)
+  void verifyMaxDirItems(INodeDirectory parent, String parentPath)
       throws MaxDirectoryItemsExceededException {
-
-    final INodeDirectory parent = pathComponents[pos-1].asDirectory();
-    final int count = parent.getChildrenList(Snapshot.CURRENT_STATE_ID).size();
+    final int count = parent.getChildrenList(CURRENT_STATE_ID).size();
     if (count >= maxDirItems) {
       final MaxDirectoryItemsExceededException e
           = new MaxDirectoryItemsExceededException(maxDirItems, count);
       if (namesystem.isImageLoaded()) {
-        e.setPathName(getFullPathName(pathComponents, pos - 1));
+        e.setPathName(parentPath);
         throw e;
       } else {
         // Do not throw if edits log is still being processed
@@ -1204,35 +851,28 @@ void verifyMaxDirItems(INode[] pathComponents, int pos)
   }
   
   /**
-   * The same as {@link #addChild(INodesInPath, int, INode, boolean)}
-   * with pos = length - 1.
+   * Add a child to the end of the path specified by INodesInPath.
+   * @return an INodesInPath instance containing the new INode
    */
-  private boolean addLastINode(INodesInPath inodesInPath,
-      INode inode, boolean checkQuota) throws QuotaExceededException {
-    final int pos = inodesInPath.getINodes().length - 1;
-    return addChild(inodesInPath, pos, inode, checkQuota);
-  }
+  @VisibleForTesting
+  public INodesInPath addLastINode(INodesInPath existing, INode inode,
+      boolean checkQuota) throws QuotaExceededException {
+    assert existing.getLastINode() != null &&
+        existing.getLastINode().isDirectory();
 
-  /** Add a node child to the inodes at index pos. 
-   * Its ancestors are stored at [0, pos-1].
-   * @return false if the child with this name already exists; 
-   *         otherwise return true;
-   * @throws QuotaExceededException is thrown if it violates quota limit
-   */
-  boolean addChild(INodesInPath iip, int pos, INode child, boolean checkQuota)
-      throws QuotaExceededException {
-    final INode[] inodes = iip.getINodes();
+    final int pos = existing.length();
     // Disallow creation of /.reserved. This may be created when loading
     // editlog/fsimage during upgrade since /.reserved was a valid name in older
     // release. This may also be called when a user tries to create a file
     // or directory /.reserved.
-    if (pos == 1 && inodes[0] == rootDir && isReservedName(child)) {
+    if (pos == 1 && existing.getINode(0) == rootDir && isReservedName(inode)) {
       throw new HadoopIllegalArgumentException(
-          "File name \"" + child.getLocalName() + "\" is reserved and cannot "
+          "File name \"" + inode.getLocalName() + "\" is reserved and cannot "
               + "be created. If this is during upgrade change the name of the "
               + "existing file or directory to another name before upgrading "
               + "to the new release.");
     }
+    final INodeDirectory parent = existing.getINode(pos - 1).asDirectory();
     // The filesystem limits are not really quotas, so this check may appear
     // odd. It's because a rename operation deletes the src, tries to add
     // to the dest, if that fails, re-adds the src from whence it came.
@@ -1240,76 +880,67 @@ boolean addChild(INodesInPath iip, int pos, INode child, boolean checkQuota)
     // original location becase a quota violation would cause the the item
     // to go "poof".  The fs limits must be bypassed for the same reason.
     if (checkQuota) {
-      verifyMaxComponentLength(child.getLocalNameBytes(), inodes, pos);
-      verifyMaxDirItems(inodes, pos);
+      final String parentPath = existing.getPath(pos - 1);
+      verifyMaxComponentLength(inode.getLocalNameBytes(), parentPath);
+      verifyMaxDirItems(parent, parentPath);
     }
     // always verify inode name
-    verifyINodeName(child.getLocalNameBytes());
-    
-    final Quota.Counts counts = child.computeQuotaUsage();
-    updateCount(iip, pos,
+    verifyINodeName(inode.getLocalNameBytes());
+
+    final Quota.Counts counts = inode.computeQuotaUsage();
+    updateCount(existing, pos,
         counts.get(Quota.NAMESPACE), counts.get(Quota.DISKSPACE), checkQuota);
-    boolean isRename = (child.getParent() != null);
-    final INodeDirectory parent = inodes[pos-1].asDirectory();
+    boolean isRename = (inode.getParent() != null);
     boolean added;
     try {
-      added = parent.addChild(child, true, iip.getLatestSnapshotId());
+      added = parent.addChild(inode, true, existing.getLatestSnapshotId());
     } catch (QuotaExceededException e) {
-      updateCountNoQuotaCheck(iip, pos,
+      updateCountNoQuotaCheck(existing, pos,
           -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
       throw e;
     }
     if (!added) {
-      updateCountNoQuotaCheck(iip, pos,
+      updateCountNoQuotaCheck(existing, pos,
           -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+      return null;
     } else {
-      iip.setINode(pos - 1, child.getParent());
       if (!isRename) {
-        AclStorage.copyINodeDefaultAcl(child);
+        AclStorage.copyINodeDefaultAcl(inode);
       }
-      addToInodeMap(child);
+      addToInodeMap(inode);
     }
-    return added;
+    return INodesInPath.append(existing, inode, inode.getLocalNameBytes());
   }
-  
-  boolean addLastINodeNoQuotaCheck(INodesInPath inodesInPath, INode i) {
+
+  INodesInPath addLastINodeNoQuotaCheck(INodesInPath existing, INode i) {
     try {
-      return addLastINode(inodesInPath, i, false);
+      return addLastINode(existing, i, false);
     } catch (QuotaExceededException e) {
       NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
     }
-    return false;
+    return null;
   }
-  
+
   /**
    * Remove the last inode in the path from the namespace.
-   * Count of each ancestor with quota is also updated.
+   * Note: the caller needs to update the ancestors' quota count.
+   *
    * @return -1 for failing to remove;
    *          0 for removing a reference whose referred inode has other 
    *            reference nodes;
-   *         >0 otherwise. 
+   *          1 otherwise.
    */
-  long removeLastINode(final INodesInPath iip)
-      throws QuotaExceededException {
+  @VisibleForTesting
+  public long removeLastINode(final INodesInPath iip) {
     final int latestSnapshot = iip.getLatestSnapshotId();
     final INode last = iip.getLastINode();
     final INodeDirectory parent = iip.getINode(-2).asDirectory();
     if (!parent.removeChild(last, latestSnapshot)) {
       return -1;
     }
-    
-    if (!last.isInLatestSnapshot(latestSnapshot)) {
-      final Quota.Counts counts = last.computeQuotaUsage();
-      updateCountNoQuotaCheck(iip, iip.getINodes().length - 1,
-          -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
 
-      if (INodeReference.tryRemoveReference(last) > 0) {
-        return 0;
-      } else {
-        return counts.get(Quota.NAMESPACE);
-      }
-    }
-    return 1;
+    return (!last.isInLatestSnapshot(latestSnapshot)
+        && INodeReference.tryRemoveReference(last) > 0) ? 0 : 1;
   }
 
   static String normalizePath(String src) {
@@ -1331,7 +962,81 @@ void addYieldCount(long value) {
   public INodeMap getINodeMap() {
     return inodeMap;
   }
-  
+
+  /**
+   * FSEditLogLoader implementation.
+   * Unlike FSNamesystem.truncate, this will not schedule block recovery.
+   */
+  void unprotectedTruncate(String src, String clientName, String clientMachine,
+                           long newLength, long mtime, Block truncateBlock)
+      throws UnresolvedLinkException, QuotaExceededException,
+      SnapshotAccessControlException, IOException {
+    INodesInPath iip = getINodesInPath(src, true);
+    INodeFile file = iip.getLastINode().asFile();
+    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+    boolean onBlockBoundary =
+        unprotectedTruncate(iip, newLength, collectedBlocks, mtime);
+
+    if(! onBlockBoundary) {
+      BlockInfoContiguous oldBlock = file.getLastBlock();
+      Block tBlk =
+      getFSNamesystem().prepareFileForTruncate(iip,
+          clientName, clientMachine, file.computeFileSize() - newLength,
+          truncateBlock);
+      assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
+          tBlk.getNumBytes() == truncateBlock.getNumBytes() :
+          "Should be the same block.";
+      if(oldBlock.getBlockId() != tBlk.getBlockId() &&
+         !file.isBlockInLatestSnapshot(oldBlock)) {
+        getBlockManager().removeBlockFromMap(oldBlock);
+      }
+    }
+    assert onBlockBoundary == (truncateBlock == null) :
+      "truncateBlock is null iff on block boundary: " + truncateBlock;
+    getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+  }
+
+  boolean truncate(INodesInPath iip, long newLength,
+                   BlocksMapUpdateInfo collectedBlocks,
+                   long mtime)
+      throws IOException {
+    writeLock();
+    try {
+      return unprotectedTruncate(iip, newLength, collectedBlocks, mtime);
+    } finally {
+      writeUnlock();
+    }
+  }
+
+  /**
+   * Truncate has the following properties:
+   * 1.) Any block deletions occur now.
+   * 2.) INode length is truncated now – new clients can only read up to
+   * the truncated length.
+   * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
+   * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
+   * 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
+   * 6.) Soft and hard Lease expiration require truncation recovery to complete.
+   *
+   * @return true if on the block boundary or false if recovery is need
+   */
+  boolean unprotectedTruncate(INodesInPath iip, long newLength,
+                              BlocksMapUpdateInfo collectedBlocks,
+                              long mtime) throws IOException {
+    assert hasWriteLock();
+    INodeFile file = iip.getLastINode().asFile();
+    int latestSnapshot = iip.getLatestSnapshotId();
+    file.recordModification(latestSnapshot, true);
+    long oldDiskspace = file.diskspaceConsumed();
+    long remainingLength =
+        file.collectBlocksBeyondMax(newLength, collectedBlocks);
+    file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
+    file.setModificationTime(mtime);
+    updateCount(iip, 0, file.diskspaceConsumed() - oldDiskspace, true);
+    // return whether on a block boundary
+    return (remainingLength - newLength) == 0;
+  }
+
   /**
    * This method is always called with writeLock of FSDirectory held.
    */
@@ -1396,77 +1101,7 @@ public INode getInode(long id) {
   int getInodeMapSize() {
     return inodeMap.size();
   }
-  
-  /**
-   * See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
-   * Sets quota for for a directory.
-   * @return INodeDirectory if any of the quotas have changed. null otherwise.
-   * @throws FileNotFoundException if the path does not exist.
-   * @throws PathIsNotDirectoryException if the path is not a directory.
-   * @throws QuotaExceededException if the directory tree size is 
-   *                                greater than the given quota
-   * @throws UnresolvedLinkException if a symlink is encountered in src.
-   * @throws SnapshotAccessControlException if path is in RO snapshot
-   */
-  INodeDirectory unprotectedSetQuota(String src, long nsQuota, long dsQuota)
-      throws FileNotFoundException, PathIsNotDirectoryException,
-      QuotaExceededException, UnresolvedLinkException,
-      SnapshotAccessControlException {
-    assert hasWriteLock();
-    // sanity check
-    if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET && 
-         nsQuota != HdfsConstants.QUOTA_RESET) || 
-        (dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET && 
-          dsQuota != HdfsConstants.QUOTA_RESET)) {
-      throw new IllegalArgumentException("Illegal value for nsQuota or " +
-                                         "dsQuota : " + nsQuota + " and " +
-                                         dsQuota);
-    }
-    
-    String srcs = normalizePath(src);
-    final INodesInPath iip = getINodesInPath4Write(srcs, true);
-    INodeDirectory dirNode = INodeDirectory.valueOf(iip.getLastINode(), srcs);
-    if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
-      throw new IllegalArgumentException("Cannot clear namespace quota on root.");
-    } else { // a directory inode
-      final Quota.Counts oldQuota = dirNode.getQuotaCounts();
-      final long oldNsQuota = oldQuota.get(Quota.NAMESPACE);
-      final long oldDsQuota = oldQuota.get(Quota.DISKSPACE);
-      if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
-        nsQuota = oldNsQuota;
-      }
-      if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
-        dsQuota = oldDsQuota;
-      }        
-      if (oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
-        return null;
-      }
 
-      final int latest = iip.getLatestSnapshotId();
-      dirNode.recordModification(latest);
-      dirNode.setQuota(nsQuota, dsQuota);
-      return dirNode;
-    }
-  }
-  
-  /**
-   * See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
-   * @return INodeDirectory if any of the quotas have changed. null otherwise.
-   * @throws SnapshotAccessControlException if path is in RO snapshot
-   * @see #unprotectedSetQuota(String, long, long)
-   */
-  INodeDirectory setQuota(String src, long nsQuota, long dsQuota)
-      throws FileNotFoundException, PathIsNotDirectoryException,
-      QuotaExceededException, UnresolvedLinkException,
-      SnapshotAccessControlException {
-    writeLock();
-    try {
-      return unprotectedSetQuota(src, nsQuota, dsQuota);
-    } finally {
-      writeUnlock();
-    }
-  }
-  
   long totalInodes() {
     readLock();
     try {
@@ -1477,50 +1112,6 @@ long totalInodes() {
     }
   }
 
-  /**
-   * Sets the access time on the file/directory. Logs it in the transaction log.
-   */
-  boolean setTimes(INode inode, long mtime, long atime, boolean force,
-                   int latestSnapshotId) throws QuotaExceededException {
-    writeLock();
-    try {
-      return unprotectedSetTimes(inode, mtime, atime, force, latestSnapshotId);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  boolean unprotectedSetTimes(String src, long mtime, long atime, boolean force) 
-      throws UnresolvedLinkException, QuotaExceededException {
-    assert hasWriteLock();
-    final INodesInPath i = getLastINodeInPath(src); 
-    return unprotectedSetTimes(i.getLastINode(), mtime, atime, force,
-        i.getLatestSnapshotId());
-  }
-
-  private boolean unprotectedSetTimes(INode inode, long mtime,
-      long atime, boolean force, int latest) throws QuotaExceededException {
-    assert hasWriteLock();
-    boolean status = false;
-    if (mtime != -1) {
-      inode = inode.setModificationTime(mtime, latest);
-      status = true;
-    }
-    if (atime != -1) {
-      long inodeTime = inode.getAccessTime();
-
-      // if the last access time update was within the last precision interval, then
-      // no need to store access time
-      if (atime <= inodeTime + getFSNamesystem().getAccessTimePrecision() && !force) {
-        status =  false;
-      } else {
-        inode.setAccessTime(atime, latest);
-        status = true;
-      }
-    } 
-    return status;
-  }
-
   /**
    * Reset the entire namespace tree.
    */
@@ -1537,113 +1128,6 @@ void reset() {
     }
   }
 
-  /**
-   * Add the specified path into the namespace.
-   */
-  INodeSymlink addSymlink(long id, String path, String target,
-                          long mtime, long atime, PermissionStatus perm)
-          throws UnresolvedLinkException, QuotaExceededException {
-    writeLock();
-    try {
-      return unprotectedAddSymlink(id, path, target, mtime, atime, perm);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  INodeSymlink unprotectedAddSymlink(long id, String path, String target,
-      long mtime, long atime, PermissionStatus perm)
-      throws UnresolvedLinkException, QuotaExceededException {
-    assert hasWriteLock();
-    final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
-        target);
-    return addINode(path, symlink) ? symlink : null;
-  }
-
-  /**
-   * Removes a list of XAttrs from an inode at a path.
-   *
-   * @param src path of inode
-   * @param toRemove XAttrs to be removed
-   * @return List of XAttrs that were removed
-   * @throws IOException if the inode does not exist, if quota is exceeded
-   */
-  List<XAttr> removeXAttrs(final String src, final List<XAttr> toRemove)
-      throws IOException {
-    writeLock();
-    try {
-      return unprotectedRemoveXAttrs(src, toRemove);
-    } finally {
-      writeUnlock();
-    }
-  }
-
-  List<XAttr> unprotectedRemoveXAttrs(final String src,
-      final List<XAttr> toRemove) throws IOException {
-    assert hasWriteLock();
-    INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
-    INode inode = resolveLastINode(src, iip);
-    int snapshotId = iip.getLatestSnapshotId();
-    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
-    List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
-    List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
-        removedXAttrs);
-    if (existingXAttrs.size() != newXAttrs.size()) {
-      XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
-      return removedXAttrs;
-    }
-    return null;
-  }
-
-  /**
-   * Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
-   * toFilter and puts them into filtered. Upon completion,
-   * toFilter contains the filter XAttrs that were not found, while
-   * fitleredXAttrs contains the XAttrs that were found.
-   *
-   * @param existingXAttrs Existing XAttrs to be filtered
-   * @param toFilter XAttrs to filter from the existing XAttrs
-   * @param filtered Return parameter, XAttrs that were filtered
-   * @return List of XAttrs that does not contain filtered XAttrs
-   */
-  @VisibleForTesting
-  List<XAttr> filterINodeXAttrs(final List<XAttr> existingXAttrs,
-      final List<XAttr> toFilter, final List<XAttr> filtered)
-    throws AccessControlException {
-    if (existingXAttrs == null || existingXAttrs.isEmpty() ||
-        toFilter == null || toFilter.isEmpty()) {
-      return existingXAttrs;
-    }
-
-    // Populate a new list with XAttrs that pass the filter
-    List<XAttr> newXAttrs =
-        Lists.newArrayListWithCapacity(existingXAttrs.size());
-    for (XAttr a : existingXAttrs) {
-      boolean add = true;
-      for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
-          ;) {
-        XAttr filter = it.next();
-        Preconditions.checkArgument(!KEYID_XATTR.equalsIgnoreValue(filter),
-            "The encryption zone xattr should never be deleted.");
-        if (UNREADABLE_BY_SUPERUSER_XATTR.equalsIgnoreValue(filter)) {
-          throw new AccessControlException("The xattr '" +
-              SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' can not be deleted.");
-        }
-        if (a.equalsIgnoreValue(filter)) {
-          add = false;
-          it.remove();
-          filtered.add(filter);
-          break;
-        }
-      }
-      if (add) {
-        newXAttrs.add(a);
-      }
-    }
-
-    return newXAttrs;
-  }
-
   boolean isInAnEZ(INodesInPath iip)
       throws UnresolvedLinkException, SnapshotAccessControlException {
     readLock();
@@ -1709,7 +1193,8 @@ void setFileEncryptionInfo(String src, FileEncryptionInfo info)
 
     writeLock();
     try {
-      unprotectedSetXAttrs(src, xAttrs, EnumSet.of(XAttrSetFlag.CREATE));
+      FSDirXAttrOp.unprotectedSetXAttrs(this, src, xAttrs,
+                                        EnumSet.of(XAttrSetFlag.CREATE));
     } finally {
       writeUnlock();
     }
@@ -1752,8 +1237,9 @@ FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId,
       final CipherSuite suite = encryptionZone.getSuite();
       final String keyName = encryptionZone.getKeyName();
 
-      XAttr fileXAttr = unprotectedGetXAttrByName(inode, snapshotId,
-          CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
+      XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByName(inode,
+                                                               snapshotId,
+                                                               CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
 
       if (fileXAttr == null) {
         NameNode.LOG.warn("Could not find encryption XAttr for file " +
@@ -1775,179 +1261,11 @@ FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId,
     }
   }
 
-  void setXAttrs(final String src, final List<XAttr> xAttrs,
-      final EnumSet<XAttrSetFlag> flag) throws IOException {
-    writeLock();
-    try {
-      unprotectedSetXAttrs(src, xAttrs, flag);
-    } finally {
-      writeUnlock();
-    }
-  }
-  
-  INode unprotectedSetXAttrs(final String src, final List<XAttr> xAttrs,
-      final EnumSet<XAttrSetFlag> flag)
-      throws QuotaExceededException, IOException {
-    assert hasWriteLock();
-    INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
-    INode inode = resolveLastINode(src, iip);
-    int snapshotId = iip.getLatestSnapshotId();
-    List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
-    List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, xAttrs, flag);
-    final boolean isFile = inode.isFile();
-
-    for (XAttr xattr : newXAttrs) {
-      final String xaName = XAttrHelper.getPrefixName(xattr);
-
-      /*
-       * If we're adding the encryption zone xattr, then add src to the list
-       * of encryption zones.
-       */
-      if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
-        final HdfsProtos.ZoneEncryptionInfoProto ezProto =
-            HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
-        ezManager.addEncryptionZone(inode.getId(),
-            PBHelper.convert(ezProto.getSuite()),
-            PBHelper.convert(ezProto.getCryptoProtocolVersion()),
-            ezProto.getKeyName());
-      }
-
-      if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
-        throw new IOException("Can only set '" +
-            SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file.");
-      }
-    }
-
-    XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
-    return inode;
-  }
-
-  List<XAttr> setINodeXAttrs(final List<XAttr> existingXAttrs,
-      final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
-      throws IOException {
-    // Check for duplicate XAttrs in toSet
-    // We need to use a custom comparator, so using a HashSet is not suitable
-    for (int i = 0; i < toSet.size(); i++) {
-      for (int j = i + 1; j < toSet.size(); j++) {
-        if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
-          throw new IOException("Cannot specify the same XAttr to be set " +
-              "more than once");
-        }
-      }
-    }
-
-    // Count the current number of user-visible XAttrs for limit checking
-    int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
-
-    // The XAttr list is copied to an exactly-sized array when it's stored,
-    // so there's no need to size it precisely here.
-    int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
-    newSize += toSet.size();
-    List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
-
-    // Check if the XAttr already exists to validate with the provided flag
-    for (XAttr xAttr: toSet) {
-      boolean exist = false;
-      if (existingXAttrs != null) {
-        for (XAttr a : existingXAttrs) {
-          if (a.equalsIgnoreValue(xAttr)) {
-            exist = true;
-            break;
-          }
-        }
-      }
-      XAttrSetFlag.validate(xAttr.getName(), exist, flag);
-      // add the new XAttr since it passed validation
-      xAttrs.add(xAttr);
-      if (isUserVisible(xAttr)) {
-        userVisibleXAttrsNum++;
-      }
-    }
-
-    // Add the existing xattrs back in, if they weren't already set
-    if (existingXAttrs != null) {
-      for (XAttr existing : existingXAttrs) {
-        boolean alreadySet = false;
-        for (XAttr set : toSet) {
-          if (set.equalsIgnoreValue(existing)) {
-            alreadySet = true;
-            break;
-          }
-        }
-        if (!alreadySet) {
-          xAttrs.add(existing);
-          if (isUserVisible(existing)) {
-            userVisibleXAttrsNum++;
-          }
-        }
-      }
-    }
-
-    if (userVisibleXAttrsNum > inodeXAttrsLimit) {
-      throw new IOException("Cannot add additional XAttr to inode, "
-          + "would exceed limit of " + inodeXAttrsLimit);
-    }
-
-    return xAttrs;
-  }
-  
-  private boolean isUserVisible(XAttr xAttr) {
-    if (xAttr.getNameSpace() == XAttr.NameSpace.USER || 
-        xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED) {
-      return true;
-    }
-    return false;
-  }
-  
-  List<XAttr> getXAttrs(String src) throws IOException {
-    String srcs = normalizePath(src);
-    readLock();
-    try {
-      INodesInPath iip = getLastINodeInPath(srcs, true);
-      INode inode = resolveLastINode(src, iip);
-      int snapshotId = iip.getPathSnapshotId();
-      return unprotectedGetXAttrs(inode, snapshotId);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  List<XAttr> getXAttrs(INode inode, int snapshotId) throws IOException {
-    readLock();
-    try {
-      return unprotectedGetXAttrs(inode, snapshotId);
-    } finally {
-      readUnlock();
-    }
-  }
-
-  private List<XAttr> unprotectedGetXAttrs(INode inode, int snapshotId)
-      throws IOException {
-    return XAttrStorage.readINodeXAttrs(inode, snapshotId);
-  }
-
-  private XAttr unprotectedGetXAttrByName(INode inode, int snapshotId,
-      String xAttrName)
-      throws IOException {
-    List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
-    if (xAttrs == null) {
-      return null;
-    }
-    for (XAttr x : xAttrs) {
-      if (XAttrHelper.getPrefixName(x)
-          .equals(xAttrName)) {
-        return x;
-      }
+  static INode resolveLastINode(INodesInPath iip) throws FileNotFoundException {
+    INode inode = iip.getLastINode();
+    if (inode == null) {
+      throw new FileNotFoundException("cannot find " + iip.getPath());
     }
-    return null;
-  }
-
-  static INode resolveLastINode(String src, INodesInPath iip)
-      throws FileNotFoundException {
-    INode[] inodes = iip.getINodes();
-    INode inode = inodes[inodes.length - 1];
-    if (inode == null)
-      throw new FileNotFoundException("cannot find " + src);
     return inode;
   }
 
@@ -2004,7 +1322,7 @@ public static boolean isReservedName(INode inode) {
 
   /** Check if a given path is reserved */
   public static boolean isReservedName(String src) {
-    return src.startsWith(DOT_RESERVED_PATH_PREFIX);
+    return src.startsWith(DOT_RESERVED_PATH_PREFIX + Path.SEPARATOR);
   }
 
   static boolean isReservedRawName(String src) {
@@ -2123,36 +1441,62 @@ private static String constructRemainingPath(String pathPrefix,
     return path.toString();
   }
 
-  /** @return the {@link INodesInPath} containing only the last inode. */
-  INodesInPath getLastINodeInPath(
-      String path, boolean resolveLink) throws UnresolvedLinkException {
-    return INodesInPath.resolve(rootDir, INode.getPathComponents(path), 1,
-            resolveLink);
+  INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
+    Preconditions.checkArgument(
+        src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
+        "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
+
+    final String dirPath = normalizePath(src.substring(0,
+        src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
+
+    final INode node = this.getINode(dirPath);
+    if (node != null && node.isDirectory()
+        && node.asDirectory().isSnapshottable()) {
+      return node;
+    }
+    return null;
+  }
+
+  INodesInPath getExistingPathINodes(byte[][] components)
+      throws UnresolvedLinkException {
+    return INodesInPath.resolve(rootDir, components, false);
+  }
+
+  /**
+   * Get {@link INode} associated with the file / directory.
+   */
+  public INodesInPath getINodesInPath4Write(String src)
+      throws UnresolvedLinkException, SnapshotAccessControlException {
+    return getINodesInPath4Write(src, true);
+  }
+
+  /**
+   * Get {@link INode} associated with the file / directory.
+   * @throws SnapshotAccessControlException if path is in RO snapshot
+   */
+  public INode getINode4Write(String src) throws UnresolvedLinkException,
+      SnapshotAccessControlException {
+    return getINodesInPath4Write(src, true).getLastINode();
   }
 
   /** @return the {@link INodesInPath} containing all inodes in the path. */
-  INodesInPath getINodesInPath(String path, boolean resolveLink
-  ) throws UnresolvedLinkException {
+  public INodesInPath getINodesInPath(String path, boolean resolveLink)
+      throws UnresolvedLinkException {
     final byte[][] components = INode.getPathComponents(path);
-    return INodesInPath.resolve(rootDir, components, components.length,
-            resolveLink);
+    return INodesInPath.resolve(rootDir, components, resolveLink);
   }
 
   /** @return the last inode in the path. */
-  INode getNode(String path, boolean resolveLink)
-          throws UnresolvedLinkException {
-    return getLastINodeInPath(path, resolveLink).getINode(0);
+  INode getINode(String path, boolean resolveLink)
+      throws UnresolvedLinkException {
+    return getINodesInPath(path, resolveLink).getLastINode();
   }
 
   /**
-   * @return the INode of the last component in src, or null if the last
-   * component does not exist.
-   * @throws UnresolvedLinkException if symlink can't be resolved
-   * @throws SnapshotAccessControlException if path is in RO snapshot
+   * Get {@link INode} associated with the file / directory.
    */
-  INode getINode4Write(String src, boolean resolveLink)
-          throws UnresolvedLinkException, SnapshotAccessControlException {
-    return getINodesInPath4Write(src, resolveLink).getLastINode();
+  public INode getINode(String src) throws UnresolvedLinkException {
+    return getINode(src, true);
   }
 
   /**
@@ -2164,7 +1508,7 @@ INodesInPath getINodesInPath4Write(String src, boolean resolveLink)
           throws UnresolvedLinkException, SnapshotAccessControlException {
     final byte[][] components = INode.getPathComponents(src);
     INodesInPath inodesInPath = INodesInPath.resolve(rootDir, components,
-            components.length, resolveLink);
+        resolveLink);
     if (inodesInPath.isSnapshot()) {
       throw new SnapshotAccessControlException(
               "Modification on a read-only snapshot is disallowed");
@@ -2239,11 +1583,10 @@ void checkPermission(FSPermissionChecker pc, INodesInPath iip,
     }
   }
 
-  HdfsFileStatus getAuditFileInfo(String path, boolean resolveSymlink)
-    throws IOException {
+  HdfsFileStatus getAuditFileInfo(INodesInPath iip)
+      throws IOException {
     return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
-      ? FSDirStatAndListingOp.getFileInfo(this, path, resolveSymlink, false,
-        false) : null;
+        ? FSDirStatAndListingOp.getFileInfo(this, iip, false, false) : null;
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 4a29b59957ddc..9ce3fa9e964e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -34,16 +34,17 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -54,6 +55,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllocateBlockIdOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllowSnapshotOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AppendOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CancelDelegationTokenOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CloseOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ConcatDeleteOp;
@@ -75,6 +77,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameSnapshotOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenewDelegationTokenOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetAclOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV1Op;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV2Op;
@@ -86,9 +89,9 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetXAttrOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TruncateOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateMasterKeyOp;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RollingUpgradeOp;
 import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -700,7 +703,19 @@ private void logRpcIds(FSEditLogOp op, boolean toLogRpcIds) {
       op.setRpcCallId(Server.getCallId());
     }
   }
-  
+
+  public void logAppendFile(String path, INodeFile file, boolean newBlock,
+      boolean toLogRpcIds) {
+    FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
+    assert uc != null;
+    AppendOp op = AppendOp.getInstance(cache.get()).setPath(path)
+        .setClientName(uc.getClientName())
+        .setClientMachine(uc.getClientMachine())
+        .setNewBlock(newBlock);
+    logRpcIds(op, toLogRpcIds);
+    logEdit(op);
+  }
+
   /** 
    * Add open lease record to edit log. 
    * Records the block locations of the last block.
@@ -756,10 +771,10 @@ public void logCloseFile(String path, INodeFile newNode) {
   
   public void logAddBlock(String path, INodeFile file) {
     Preconditions.checkArgument(file.isUnderConstruction());
-    BlockInfo[] blocks = file.getBlocks();
+    BlockInfoContiguous[] blocks = file.getBlocks();
     Preconditions.checkState(blocks != null && blocks.length > 0);
-    BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
-    BlockInfo lastBlock = blocks[blocks.length - 1];
+    BlockInfoContiguous pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
+    BlockInfoContiguous lastBlock = blocks[blocks.length - 1];
     AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path)
         .setPenultimateBlock(pBlock).setLastBlock(lastBlock);
     logEdit(op);
@@ -896,6 +911,21 @@ void logDelete(String src, long timestamp, boolean toLogRpcIds) {
     logRpcIds(op, toLogRpcIds);
     logEdit(op);
   }
+  
+  /**
+   * Add truncate file record to edit log
+   */
+  void logTruncate(String src, String clientName, String clientMachine,
+                   long size, long timestamp, Block truncateBlock) {
+    TruncateOp op = TruncateOp.getInstance(cache.get())
+      .setPath(src)
+      .setClientName(clientName)
+      .setClientMachine(clientMachine)
+      .setNewLength(size)
+      .setTimestamp(timestamp)
+      .setTruncateBlock(truncateBlock);
+    logEdit(op);
+  }
 
   /**
    * Add legacy block generation stamp record to edit log
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index d63545b5ce729..5fcad7428dd94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TruncateOp;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormat.renameReservedPathsOnUpgrade;
 import static org.apache.hadoop.util.Time.now;
 
@@ -33,6 +34,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -40,9 +42,8 @@
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.Storage;
@@ -52,6 +53,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AddCloseOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllocateBlockIdOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AllowSnapshotOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.AppendOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.BlockListUpdatingOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.CancelDelegationTokenOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ClearNSQuotaOp;
@@ -67,6 +69,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.ReassignLeaseOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveCacheDirectiveInfoOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveCachePoolOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveXAttrOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOldOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RenameSnapshotOp;
@@ -82,7 +85,6 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetReplicationOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetStoragePolicyOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetXAttrOp;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.RemoveXAttrOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SymlinkOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.TimesOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.UpdateBlocksOp;
@@ -95,8 +97,8 @@
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
 import org.apache.hadoop.hdfs.util.Holder;
+import org.apache.hadoop.util.ChunkedArrayList;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
@@ -324,31 +326,30 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       LOG.trace("replaying edit log: " + op);
     }
     final boolean toAddRetryCache = fsNamesys.hasRetryCache() && op.hasRpcIds();
-    
+
     switch (op.opCode) {
     case OP_ADD: {
       AddCloseOp addCloseOp = (AddCloseOp)op;
       final String path =
           renameReservedPathsOnUpgrade(addCloseOp.path, logVersion);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(op.opCode + ": " + path +
+      if (FSNamesystem.LOG.isDebugEnabled()) {
+        FSNamesystem.LOG.debug(op.opCode + ": " + path +
             " numblocks : " + addCloseOp.blocks.length +
             " clientHolder " + addCloseOp.clientName +
             " clientMachine " + addCloseOp.clientMachine);
       }
-      // There three cases here:
+      // There are 3 cases here:
       // 1. OP_ADD to create a new file
       // 2. OP_ADD to update file blocks
-      // 3. OP_ADD to open file for append
+      // 3. OP_ADD to open file for append (old append)
 
       // See if the file already exists (persistBlocks call)
-      final INodesInPath iip = fsDir.getINodesInPath(path, true);
-      final INode[] inodes = iip.getINodes();
-      INodeFile oldFile = INodeFile.valueOf(
-          inodes[inodes.length - 1], path, true);
+      INodesInPath iip = fsDir.getINodesInPath(path, true);
+      INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path, true);
       if (oldFile != null && addCloseOp.overwrite) {
         // This is OP_ADD with overwrite
-        fsDir.unprotectedDelete(path, addCloseOp.mtime);
+        FSDirDeleteOp.deleteForEditLog(fsDir, path, addCloseOp.mtime);
+        iip = INodesInPath.replace(iip, iip.length() - 1, null);
         oldFile = null;
       }
       INodeFile newFile = oldFile;
@@ -360,14 +361,18 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
         assert addCloseOp.blocks.length == 0;
 
         // add to the file tree
-        inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion,
-            lastInodeId);
-        newFile = fsDir.unprotectedAddFile(inodeId,
-            path, addCloseOp.permissions, addCloseOp.aclEntries,
-            addCloseOp.xAttrs,
-            replication, addCloseOp.mtime, addCloseOp.atime,
-            addCloseOp.blockSize, true, addCloseOp.clientName,
-            addCloseOp.clientMachine, addCloseOp.storagePolicyId);
+        inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId);
+        newFile = fsDir.addFileForEditLog(inodeId, iip.getExistingINodes(),
+            iip.getLastLocalName(),
+            addCloseOp.permissions,
+            addCloseOp.aclEntries,
+            addCloseOp.xAttrs, replication,
+            addCloseOp.mtime, addCloseOp.atime,
+            addCloseOp.blockSize, true,
+            addCloseOp.clientName,
+            addCloseOp.clientMachine,
+            addCloseOp.storagePolicyId);
+        iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
         fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
 
         // add the op into retry cache if necessary
@@ -379,19 +384,17 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
           fsNamesys.addCacheEntryWithPayload(addCloseOp.rpcClientId,
               addCloseOp.rpcCallId, stat);
         }
-      } else { // This is OP_ADD on an existing file
+      } else { // This is OP_ADD on an existing file (old append)
         if (!oldFile.isUnderConstruction()) {
           // This is case 3: a call to append() on an already-closed file.
           if (FSNamesystem.LOG.isDebugEnabled()) {
             FSNamesystem.LOG.debug("Reopening an already-closed file " +
                 "for append");
           }
-          LocatedBlock lb = fsNamesys.prepareFileForWrite(path,
-              oldFile, addCloseOp.clientName, addCloseOp.clientMachine, false, iip.getLatestSnapshotId(), false);
-          newFile = INodeFile.valueOf(fsDir.getINode(path),
-              path, true);
-          
-          // add the op into retry cache is necessary
+          LocatedBlock lb = fsNamesys.prepareFileForAppend(path, iip,
+              addCloseOp.clientName, addCloseOp.clientMachine, false, false,
+              false);
+          // add the op into retry cache if necessary
           if (toAddRetryCache) {
             HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus(
                 fsNamesys.dir,
@@ -410,7 +413,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       // Update the salient file attributes.
       newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
       newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
-      updateBlocks(fsDir, addCloseOp, newFile);
+      updateBlocks(fsDir, addCloseOp, iip, newFile);
       break;
     }
     case OP_CLOSE: {
@@ -424,13 +427,13 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
             " clientMachine " + addCloseOp.clientMachine);
       }
 
-      final INodesInPath iip = fsDir.getLastINodeInPath(path);
-      final INodeFile file = INodeFile.valueOf(iip.getINode(0), path);
+      final INodesInPath iip = fsDir.getINodesInPath(path, true);
+      final INodeFile file = INodeFile.valueOf(iip.getLastINode(), path);
 
       // Update the salient file attributes.
       file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
       file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
-      updateBlocks(fsDir, addCloseOp, file);
+      updateBlocks(fsDir, addCloseOp, iip, file);
 
       // Now close the file
       if (!file.isUnderConstruction() &&
@@ -449,6 +452,34 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       }
       break;
     }
+    case OP_APPEND: {
+      AppendOp appendOp = (AppendOp) op;
+      final String path = renameReservedPathsOnUpgrade(appendOp.path,
+          logVersion);
+      if (FSNamesystem.LOG.isDebugEnabled()) {
+        FSNamesystem.LOG.debug(op.opCode + ": " + path +
+            " clientName " + appendOp.clientName +
+            " clientMachine " + appendOp.clientMachine +
+            " newBlock " + appendOp.newBlock);
+      }
+      INodesInPath iip = fsDir.getINodesInPath4Write(path);
+      INodeFile file = INodeFile.valueOf(iip.getLastINode(), path);
+      if (!file.isUnderConstruction()) {
+        LocatedBlock lb = fsNamesys.prepareFileForAppend(path, iip,
+            appendOp.clientName, appendOp.clientMachine, appendOp.newBlock,
+            false, false);
+        // add the op into retry cache if necessary
+        if (toAddRetryCache) {
+          HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus(
+              fsNamesys.dir, HdfsFileStatus.EMPTY_NAME, file,
+              BlockStoragePolicySuite.ID_UNSPECIFIED,
+              Snapshot.CURRENT_STATE_ID, false, iip);
+          fsNamesys.addCacheEntryWithPayload(appendOp.rpcClientId,
+              appendOp.rpcCallId, new LastBlockWithStatus(lb, stat));
+        }
+      }
+      break;
+    }
     case OP_UPDATE_BLOCKS: {
       UpdateBlocksOp updateOp = (UpdateBlocksOp)op;
       final String path =
@@ -457,10 +488,10 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
         FSNamesystem.LOG.debug(op.opCode + ": " + path +
             " numblocks : " + updateOp.blocks.length);
       }
-      INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path),
-          path);
+      INodesInPath iip = fsDir.getINodesInPath(path, true);
+      INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
       // Update in-memory data structures
-      updateBlocks(fsDir, updateOp, oldFile);
+      updateBlocks(fsDir, updateOp, iip, oldFile);
       
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
@@ -483,9 +514,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       SetReplicationOp setReplicationOp = (SetReplicationOp)op;
       short replication = fsNamesys.getBlockManager().adjustReplication(
           setReplicationOp.replication);
-      fsDir.unprotectedSetReplication(
-          renameReservedPathsOnUpgrade(setReplicationOp.path, logVersion),
-                                      replication, null);
+      FSDirAttrOp.unprotectedSetReplication(fsDir, renameReservedPathsOnUpgrade(
+          setReplicationOp.path, logVersion), replication, null);
       break;
     }
     case OP_CONCAT_DELETE: {
@@ -496,7 +526,14 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
         srcs[i] =
             renameReservedPathsOnUpgrade(concatDeleteOp.srcs[i], logVersion);
       }
-      FSDirConcatOp.unprotectedConcat(fsDir, trg, srcs, concatDeleteOp.timestamp);
+      INodesInPath targetIIP = fsDir.getINodesInPath4Write(trg);
+      INodeFile[] srcFiles = new INodeFile[srcs.length];
+      for (int i = 0; i < srcs.length; i++) {
+        INodesInPath srcIIP = fsDir.getINodesInPath4Write(srcs[i]);
+        srcFiles[i] = srcIIP.getLastINode().asFile();
+      }
+      FSDirConcatOp.unprotectedConcat(fsDir, targetIIP, srcFiles,
+          concatDeleteOp.timestamp);
       
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(concatDeleteOp.rpcClientId,
@@ -508,7 +545,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       RenameOldOp renameOp = (RenameOldOp)op;
       final String src = renameReservedPathsOnUpgrade(renameOp.src, logVersion);
       final String dst = renameReservedPathsOnUpgrade(renameOp.dst, logVersion);
-      FSDirRenameOp.unprotectedRenameTo(fsDir, src, dst, renameOp.timestamp);
+      FSDirRenameOp.renameForEditLog(fsDir, src, dst, renameOp.timestamp);
       
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(renameOp.rpcClientId, renameOp.rpcCallId);
@@ -517,8 +554,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
     }
     case OP_DELETE: {
       DeleteOp deleteOp = (DeleteOp)op;
-      fsDir.unprotectedDelete(
-          renameReservedPathsOnUpgrade(deleteOp.path, logVersion),
+      FSDirDeleteOp.deleteForEditLog(
+          fsDir, renameReservedPathsOnUpgrade(deleteOp.path, logVersion),
           deleteOp.timestamp);
       
       if (toAddRetryCache) {
@@ -530,57 +567,56 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       MkdirOp mkdirOp = (MkdirOp)op;
       inodeId = getAndUpdateLastInodeId(mkdirOp.inodeId, logVersion,
           lastInodeId);
-      FSDirMkdirOp.unprotectedMkdir(fsDir, inodeId,
+      FSDirMkdirOp.mkdirForEditLog(fsDir, inodeId,
           renameReservedPathsOnUpgrade(mkdirOp.path, logVersion),
           mkdirOp.permissions, mkdirOp.aclEntries, mkdirOp.timestamp);
       break;
     }
     case OP_SET_GENSTAMP_V1: {
       SetGenstampV1Op setGenstampV1Op = (SetGenstampV1Op)op;
-      fsNamesys.getBlockIdManager().setGenerationStampV1(setGenstampV1Op.genStampV1);
+      fsNamesys.getBlockIdManager().setGenerationStampV1(
+          setGenstampV1Op.genStampV1);
       break;
     }
     case OP_SET_PERMISSIONS: {
       SetPermissionsOp setPermissionsOp = (SetPermissionsOp)op;
-      fsDir.unprotectedSetPermission(
-          renameReservedPathsOnUpgrade(setPermissionsOp.src, logVersion),
-          setPermissionsOp.permissions);
+      FSDirAttrOp.unprotectedSetPermission(fsDir, renameReservedPathsOnUpgrade(
+          setPermissionsOp.src, logVersion), setPermissionsOp.permissions);
       break;
     }
     case OP_SET_OWNER: {
       SetOwnerOp setOwnerOp = (SetOwnerOp)op;
-      fsDir.unprotectedSetOwner(
-          renameReservedPathsOnUpgrade(setOwnerOp.src, logVersion),
+      FSDirAttrOp.unprotectedSetOwner(
+          fsDir, renameReservedPathsOnUpgrade(setOwnerOp.src, logVersion),
           setOwnerOp.username, setOwnerOp.groupname);
       break;
     }
     case OP_SET_NS_QUOTA: {
       SetNSQuotaOp setNSQuotaOp = (SetNSQuotaOp)op;
-      fsDir.unprotectedSetQuota(
-          renameReservedPathsOnUpgrade(setNSQuotaOp.src, logVersion),
+      FSDirAttrOp.unprotectedSetQuota(
+          fsDir, renameReservedPathsOnUpgrade(setNSQuotaOp.src, logVersion),
           setNSQuotaOp.nsQuota, HdfsConstants.QUOTA_DONT_SET);
       break;
     }
     case OP_CLEAR_NS_QUOTA: {
       ClearNSQuotaOp clearNSQuotaOp = (ClearNSQuotaOp)op;
-      fsDir.unprotectedSetQuota(
-          renameReservedPathsOnUpgrade(clearNSQuotaOp.src, logVersion),
+      FSDirAttrOp.unprotectedSetQuota(
+          fsDir, renameReservedPathsOnUpgrade(clearNSQuotaOp.src, logVersion),
           HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET);
       break;
     }
 
     case OP_SET_QUOTA:
-      SetQuotaOp setQuotaOp = (SetQuotaOp)op;
-      fsDir.unprotectedSetQuota(
+      SetQuotaOp setQuotaOp = (SetQuotaOp) op;
+      FSDirAttrOp.unprotectedSetQuota(fsDir,
           renameReservedPathsOnUpgrade(setQuotaOp.src, logVersion),
           setQuotaOp.nsQuota, setQuotaOp.dsQuota);
       break;
 
     case OP_TIMES: {
       TimesOp timesOp = (TimesOp)op;
-
-      fsDir.unprotectedSetTimes(
-          renameReservedPathsOnUpgrade(timesOp.path, logVersion),
+      FSDirAttrOp.unprotectedSetTimes(
+          fsDir, renameReservedPathsOnUpgrade(timesOp.path, logVersion),
           timesOp.mtime, timesOp.atime, true);
       break;
     }
@@ -588,10 +624,12 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       SymlinkOp symlinkOp = (SymlinkOp)op;
       inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion,
           lastInodeId);
-      fsDir.unprotectedAddSymlink(inodeId,
-          renameReservedPathsOnUpgrade(symlinkOp.path, logVersion),
-          symlinkOp.value, symlinkOp.mtime, symlinkOp.atime,
-          symlinkOp.permissionStatus);
+      final String path = renameReservedPathsOnUpgrade(symlinkOp.path,
+          logVersion);
+      final INodesInPath iip = fsDir.getINodesInPath(path, false);
+      FSDirSymlinkOp.unprotectedAddSymlink(fsDir, iip.getExistingINodes(),
+          iip.getLastLocalName(), inodeId, symlinkOp.value, symlinkOp.mtime,
+          symlinkOp.atime, symlinkOp.permissionStatus);
       
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(symlinkOp.rpcClientId, symlinkOp.rpcCallId);
@@ -600,7 +638,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
     }
     case OP_RENAME: {
       RenameOp renameOp = (RenameOp)op;
-      FSDirRenameOp.unprotectedRenameTo(fsDir,
+      FSDirRenameOp.renameForEditLog(fsDir,
           renameReservedPathsOnUpgrade(renameOp.src, logVersion),
           renameReservedPathsOnUpgrade(renameOp.dst, logVersion),
           renameOp.timestamp, renameOp.options);
@@ -730,12 +768,14 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
     }
     case OP_SET_GENSTAMP_V2: {
       SetGenstampV2Op setGenstampV2Op = (SetGenstampV2Op) op;
-      fsNamesys.getBlockIdManager().setGenerationStampV2(setGenstampV2Op.genStampV2);
+      fsNamesys.getBlockIdManager().setGenerationStampV2(
+          setGenstampV2Op.genStampV2);
       break;
     }
     case OP_ALLOCATE_BLOCK_ID: {
       AllocateBlockIdOp allocateBlockIdOp = (AllocateBlockIdOp) op;
-      fsNamesys.getBlockIdManager().setLastAllocatedBlockId(allocateBlockIdOp.blockId);
+      fsNamesys.getBlockIdManager().setLastAllocatedBlockId(
+          allocateBlockIdOp.blockId);
       break;
     }
     case OP_ROLLING_UPGRADE_START: {
@@ -823,13 +863,16 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
     }
     case OP_SET_ACL: {
       SetAclOp setAclOp = (SetAclOp) op;
-      FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries);
+      FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries,
+          true);
       break;
     }
     case OP_SET_XATTR: {
       SetXAttrOp setXAttrOp = (SetXAttrOp) op;
-      fsDir.unprotectedSetXAttrs(setXAttrOp.src, setXAttrOp.xAttrs,
-          EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+      FSDirXAttrOp.unprotectedSetXAttrs(fsDir, setXAttrOp.src,
+                                        setXAttrOp.xAttrs,
+                                        EnumSet.of(XAttrSetFlag.CREATE,
+                                                   XAttrSetFlag.REPLACE));
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
       }
@@ -837,20 +880,29 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
     }
     case OP_REMOVE_XATTR: {
       RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
-      fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src,
-          removeXAttrOp.xAttrs);
+      FSDirXAttrOp.unprotectedRemoveXAttrs(fsDir, removeXAttrOp.src,
+                                           removeXAttrOp.xAttrs);
       if (toAddRetryCache) {
         fsNamesys.addCacheEntry(removeXAttrOp.rpcClientId,
             removeXAttrOp.rpcCallId);
       }
       break;
     }
+    case OP_TRUNCATE: {
+      TruncateOp truncateOp = (TruncateOp) op;
+      fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
+          truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp,
+          truncateOp.truncateBlock);
+      break;
+    }
     case OP_SET_STORAGE_POLICY: {
       SetStoragePolicyOp setStoragePolicyOp = (SetStoragePolicyOp) op;
       final String path = renameReservedPathsOnUpgrade(setStoragePolicyOp.path,
           logVersion);
       final INodesInPath iip = fsDir.getINodesInPath4Write(path);
-      fsDir.unprotectedSetStoragePolicy(iip, setStoragePolicyOp.policyId);
+      FSDirAttrOp.unprotectedSetStoragePolicy(
+          fsDir, fsNamesys.getBlockManager(), iip,
+          setStoragePolicyOp.policyId);
       break;
     }
     default:
@@ -881,7 +933,7 @@ private static String formatEditLogReplayError(EditLogInputStream in,
    */
   private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
       throws IOException {
-    BlockInfo[] oldBlocks = file.getBlocks();
+    BlockInfoContiguous[] oldBlocks = file.getBlocks();
     Block pBlock = op.getPenultimateBlock();
     Block newBlock= op.getLastBlock();
     
@@ -898,16 +950,16 @@ private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
       }
       
       oldLastBlock.setNumBytes(pBlock.getNumBytes());
-      if (oldLastBlock instanceof BlockInfoUnderConstruction) {
+      if (oldLastBlock instanceof BlockInfoContiguousUnderConstruction) {
         fsNamesys.getBlockManager().forceCompleteBlock(file,
-            (BlockInfoUnderConstruction) oldLastBlock);
+            (BlockInfoContiguousUnderConstruction) oldLastBlock);
         fsNamesys.getBlockManager().processQueuedMessagesForBlock(pBlock);
       }
     } else { // the penultimate block is null
       Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
     }
     // add the new block
-    BlockInfo newBI = new BlockInfoUnderConstruction(
+    BlockInfoContiguous newBI = new BlockInfoContiguousUnderConstruction(
           newBlock, file.getBlockReplication());
     fsNamesys.getBlockManager().addBlockCollection(newBI, file);
     file.addBlock(newBI);
@@ -919,9 +971,9 @@ private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
    * @throws IOException
    */
   private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
-      INodeFile file) throws IOException {
+      INodesInPath iip, INodeFile file) throws IOException {
     // Update its block list
-    BlockInfo[] oldBlocks = file.getBlocks();
+    BlockInfoContiguous[] oldBlocks = file.getBlocks();
     Block[] newBlocks = op.getBlocks();
     String path = op.getPath();
     
@@ -930,7 +982,7 @@ private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
     
     // First, update blocks in common
     for (int i = 0; i < oldBlocks.length && i < newBlocks.length; i++) {
-      BlockInfo oldBlock = oldBlocks[i];
+      BlockInfoContiguous oldBlock = oldBlocks[i];
       Block newBlock = newBlocks[i];
       
       boolean isLastBlock = i == newBlocks.length - 1;
@@ -948,11 +1000,11 @@ private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
         oldBlock.getGenerationStamp() != newBlock.getGenerationStamp();
       oldBlock.setGenerationStamp(newBlock.getGenerationStamp());
       
-      if (oldBlock instanceof BlockInfoUnderConstruction &&
+      if (oldBlock instanceof BlockInfoContiguousUnderConstruction &&
           (!isLastBlock || op.shouldCompleteLastBlock())) {
         changeMade = true;
         fsNamesys.getBlockManager().forceCompleteBlock(file,
-            (BlockInfoUnderConstruction) oldBlock);
+            (BlockInfoContiguousUnderConstruction) oldBlock);
       }
       if (changeMade) {
         // The state or gen-stamp of the block has changed. So, we may be
@@ -973,7 +1025,7 @@ private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
             + path);
       }
       Block oldBlock = oldBlocks[oldBlocks.length - 1];
-      boolean removed = fsDir.unprotectedRemoveBlock(path, file, oldBlock);
+      boolean removed = fsDir.unprotectedRemoveBlock(path, iip, file, oldBlock);
       if (!removed && !(op instanceof UpdateBlocksOp)) {
         throw new IOException("Trying to delete non-existant block " + oldBlock);
       }
@@ -981,19 +1033,19 @@ private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
       // We're adding blocks
       for (int i = oldBlocks.length; i < newBlocks.length; i++) {
         Block newBlock = newBlocks[i];
-        BlockInfo newBI;
+        BlockInfoContiguous newBI;
         if (!op.shouldCompleteLastBlock()) {
           // TODO: shouldn't this only be true for the last block?
           // what about an old-version fsync() where fsync isn't called
           // until several blocks in?
-          newBI = new BlockInfoUnderConstruction(
+          newBI = new BlockInfoContiguousUnderConstruction(
               newBlock, file.getBlockReplication());
         } else {
           // OP_CLOSE should add finalized blocks. This code path
           // is only executed when loading edits written by prior
           // versions of Hadoop. Current versions always log
           // OP_ADD operations as each block is allocated.
-          newBI = new BlockInfo(newBlock, file.getBlockReplication());
+          newBI = new BlockInfoContiguous(newBlock, file.getBlockReplication());
         }
         fsNamesys.getBlockManager().addBlockCollection(newBI, file);
         file.addBlock(newBI);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 11026fc80adfd..1629d808c983c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_ADD;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_APPEND;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_ADD_BLOCK;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_ADD_CACHE_DIRECTIVE;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_ADD_CACHE_POOL;
@@ -59,6 +60,7 @@
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_START_LOG_SEGMENT;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SYMLINK;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_TIMES;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_TRUNCATE;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_BLOCKS;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_MASTER_KEY;
 import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SET_STORAGE_POLICY;
@@ -180,6 +182,7 @@ public OpInstanceCache() {
       inst.put(OP_START_LOG_SEGMENT, new LogSegmentOp(OP_START_LOG_SEGMENT));
       inst.put(OP_END_LOG_SEGMENT, new LogSegmentOp(OP_END_LOG_SEGMENT));
       inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
+      inst.put(OP_TRUNCATE, new TruncateOp());
 
       inst.put(OP_ALLOW_SNAPSHOT, new AllowSnapshotOp());
       inst.put(OP_DISALLOW_SNAPSHOT, new DisallowSnapshotOp());
@@ -205,6 +208,7 @@ public OpInstanceCache() {
       inst.put(OP_SET_XATTR, new SetXAttrOp());
       inst.put(OP_REMOVE_XATTR, new RemoveXAttrOp());
       inst.put(OP_SET_STORAGE_POLICY, new SetStoragePolicyOp());
+      inst.put(OP_APPEND, new AppendOp());
     }
     
     public FSEditLogOp get(FSEditLogOpCodes opcode) {
@@ -426,7 +430,7 @@ static abstract class AddCloseOp extends FSEditLogOp implements BlockListUpdatin
     private AddCloseOp(FSEditLogOpCodes opCode) {
       super(opCode);
       storagePolicyId = BlockStoragePolicySuite.ID_UNSPECIFIED;
-      assert(opCode == OP_ADD || opCode == OP_CLOSE);
+      assert(opCode == OP_ADD || opCode == OP_CLOSE || opCode == OP_APPEND);
     }
 
     @Override
@@ -768,7 +772,7 @@ private AddOp() {
     }
 
     static AddOp getInstance(OpInstanceCache cache) {
-      return (AddOp)cache.get(OP_ADD);
+      return (AddOp) cache.get(OP_ADD);
     }
 
     @Override
@@ -786,7 +790,7 @@ public String toString() {
   }
 
   /**
-   * Although {@link ClientProtocol#appendFile} may also log a close op, we do
+   * Although {@link ClientProtocol#append} may also log a close op, we do
    * not need to record the rpc ids here since a successful appendFile op will
    * finally log an AddOp.
    */
@@ -812,6 +816,97 @@ public String toString() {
       return builder.toString();
     }
   }
+
+  static class AppendOp extends FSEditLogOp {
+    String path;
+    String clientName;
+    String clientMachine;
+    boolean newBlock;
+
+    private AppendOp() {
+      super(OP_APPEND);
+    }
+
+    static AppendOp getInstance(OpInstanceCache cache) {
+      return (AppendOp) cache.get(OP_APPEND);
+    }
+
+    AppendOp setPath(String path) {
+      this.path = path;
+      return this;
+    }
+
+    AppendOp setClientName(String clientName) {
+      this.clientName = clientName;
+      return this;
+    }
+
+    AppendOp setClientMachine(String clientMachine) {
+      this.clientMachine = clientMachine;
+      return this;
+    }
+
+    AppendOp setNewBlock(boolean newBlock) {
+      this.newBlock = newBlock;
+      return this;
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder builder = new StringBuilder();
+      builder.append("AppendOp ");
+      builder.append("[path=").append(path);
+      builder.append(", clientName=").append(clientName);
+      builder.append(", clientMachine=").append(clientMachine);
+      builder.append(", newBlock=").append(newBlock).append("]");
+      return builder.toString();
+    }
+
+    @Override
+    void resetSubFields() {
+      this.path = null;
+      this.clientName = null;
+      this.clientMachine = null;
+      this.newBlock = false;
+    }
+
+    @Override
+    void readFields(DataInputStream in, int logVersion) throws IOException {
+      this.path = FSImageSerialization.readString(in);
+      this.clientName = FSImageSerialization.readString(in);
+      this.clientMachine = FSImageSerialization.readString(in);
+      this.newBlock = FSImageSerialization.readBoolean(in);
+      readRpcIds(in, logVersion);
+    }
+
+    @Override
+    public void writeFields(DataOutputStream out) throws IOException {
+      FSImageSerialization.writeString(path, out);
+      FSImageSerialization.writeString(clientName, out);
+      FSImageSerialization.writeString(clientMachine, out);
+      FSImageSerialization.writeBoolean(newBlock, out);
+      writeRpcIds(rpcClientId, rpcCallId, out);
+    }
+
+    @Override
+    protected void toXml(ContentHandler contentHandler) throws SAXException {
+      XMLUtils.addSaxString(contentHandler, "PATH", path);
+      XMLUtils.addSaxString(contentHandler, "CLIENT_NAME", clientName);
+      XMLUtils.addSaxString(contentHandler, "CLIENT_MACHINE", clientMachine);
+      XMLUtils.addSaxString(contentHandler, "NEWBLOCK",
+          Boolean.toString(newBlock));
+      appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
+    }
+
+    @Override
+    void fromXml(Stanza st) throws InvalidXmlException {
+      this.path = st.getValue("PATH");
+      this.clientName = st.getValue("CLIENT_NAME");
+      this.clientMachine = st.getValue("CLIENT_MACHINE");
+      this.newBlock = Boolean.parseBoolean(st.getValue("NEWBLOCK"));
+      readRpcIdsFromXml(st);
+    }
+  }
   
   static class AddBlockOp extends FSEditLogOp {
     private String path;
@@ -1641,7 +1736,7 @@ protected void toXml(ContentHandler contentHandler) throws SAXException {
    * {@link ClientProtocol#updateBlockForPipeline},
    * {@link ClientProtocol#recoverLease}, {@link ClientProtocol#addBlock}) or
    * already bound with other editlog op which records rpc ids (
-   * {@link ClientProtocol#startFile}). Thus no need to record rpc ids here.
+   * {@link ClientProtocol#create}). Thus no need to record rpc ids here.
    */
   static class SetGenstampV1Op extends FSEditLogOp {
     long genStampV1;
@@ -2602,6 +2697,137 @@ protected void toXml(ContentHandler contentHandler) throws SAXException {
       readRpcIdsFromXml(st);
     }
   }
+
+  static class TruncateOp extends FSEditLogOp {
+    String src;
+    String clientName;
+    String clientMachine;
+    long newLength;
+    long timestamp;
+    Block truncateBlock;
+
+    private TruncateOp() {
+      super(OP_TRUNCATE);
+    }
+
+    static TruncateOp getInstance(OpInstanceCache cache) {
+      return (TruncateOp)cache.get(OP_TRUNCATE);
+    }
+
+    @Override
+    void resetSubFields() {
+      src = null;
+      clientName = null;
+      clientMachine = null;
+      newLength = 0L;
+      timestamp = 0L;
+    }
+
+    TruncateOp setPath(String src) {
+      this.src = src;
+      return this;
+    }
+
+    TruncateOp setClientName(String clientName) {
+      this.clientName = clientName;
+      return this;
+    }
+
+    TruncateOp setClientMachine(String clientMachine) {
+      this.clientMachine = clientMachine;
+      return this;
+    }
+
+    TruncateOp setNewLength(long newLength) {
+      this.newLength = newLength;
+      return this;
+    }
+
+    TruncateOp setTimestamp(long timestamp) {
+      this.timestamp = timestamp;
+      return this;
+    }
+
+    TruncateOp setTruncateBlock(Block truncateBlock) {
+      this.truncateBlock = truncateBlock;
+      return this;
+    }
+
+    @Override
+    void readFields(DataInputStream in, int logVersion) throws IOException {
+      src = FSImageSerialization.readString(in);
+      clientName = FSImageSerialization.readString(in);
+      clientMachine = FSImageSerialization.readString(in);
+      newLength = FSImageSerialization.readLong(in);
+      timestamp = FSImageSerialization.readLong(in);
+      Block[] blocks =
+          FSImageSerialization.readCompactBlockArray(in, logVersion);
+      assert blocks.length <= 1 : "Truncate op should have 1 or 0 blocks";
+      truncateBlock = (blocks.length == 0) ? null : blocks[0];
+    }
+
+    @Override
+    public void writeFields(DataOutputStream out) throws IOException {
+      FSImageSerialization.writeString(src, out);
+      FSImageSerialization.writeString(clientName, out);
+      FSImageSerialization.writeString(clientMachine, out);
+      FSImageSerialization.writeLong(newLength, out);
+      FSImageSerialization.writeLong(timestamp, out);
+      int size = truncateBlock != null ? 1 : 0;
+      Block[] blocks = new Block[size];
+      if (truncateBlock != null) {
+        blocks[0] = truncateBlock;
+      }
+      FSImageSerialization.writeCompactBlockArray(blocks, out);
+    }
+
+    @Override
+    protected void toXml(ContentHandler contentHandler) throws SAXException {
+      XMLUtils.addSaxString(contentHandler, "SRC", src);
+      XMLUtils.addSaxString(contentHandler, "CLIENTNAME", clientName);
+      XMLUtils.addSaxString(contentHandler, "CLIENTMACHINE", clientMachine);
+      XMLUtils.addSaxString(contentHandler, "NEWLENGTH",
+          Long.toString(newLength));
+      XMLUtils.addSaxString(contentHandler, "TIMESTAMP",
+          Long.toString(timestamp));
+      if(truncateBlock != null)
+        FSEditLogOp.blockToXml(contentHandler, truncateBlock);
+    }
+
+    @Override
+    void fromXml(Stanza st) throws InvalidXmlException {
+      this.src = st.getValue("SRC");
+      this.clientName = st.getValue("CLIENTNAME");
+      this.clientMachine = st.getValue("CLIENTMACHINE");
+      this.newLength = Long.parseLong(st.getValue("NEWLENGTH"));
+      this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
+      if (st.hasChildren("BLOCK"))
+        this.truncateBlock = FSEditLogOp.blockFromXml(st);
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder builder = new StringBuilder();
+      builder.append("TruncateOp [src=");
+      builder.append(src);
+      builder.append(", clientName=");
+      builder.append(clientName);
+      builder.append(", clientMachine=");
+      builder.append(clientMachine);
+      builder.append(", newLength=");
+      builder.append(newLength);
+      builder.append(", timestamp=");
+      builder.append(timestamp);
+      builder.append(", truncateBlock=");
+      builder.append(truncateBlock);
+      builder.append(", opCode=");
+      builder.append(opCode);
+      builder.append(", txid=");
+      builder.append(txid);
+      builder.append("]");
+      return builder.toString();
+    }
+  }
  
   /**
    * {@literal @Idempotent} for {@link ClientProtocol#recoverLease}. In the
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
index 86be54adb7f4f..6cd1617be8e50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
@@ -73,6 +73,8 @@ public enum FSEditLogOpCodes {
   OP_SET_XATTR                  ((byte) 43),
   OP_REMOVE_XATTR               ((byte) 44),
   OP_SET_STORAGE_POLICY         ((byte) 45),
+  OP_TRUNCATE                   ((byte) 46),
+  OP_APPEND                     ((byte) 47),
 
   // Note that the current range of the valid OP code is 0~127
   OP_INVALID                    ((byte) -1);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 8ac6926850efc..3b5d2c39cc61a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -29,9 +29,11 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -97,6 +99,15 @@ public class FSImage implements Closeable {
 
   protected NNStorageRetentionManager archivalManager;
 
+  /* Used to make sure there are no concurrent checkpoints for a given txid
+   * The checkpoint here could be one of the following operations.
+   * a. checkpoint when NN is in standby.
+   * b. admin saveNameSpace operation.
+   * c. download checkpoint file from any remote checkpointer.
+  */
+  private final Set<Long> currentlyCheckpointing =
+      Collections.<Long>synchronizedSet(new HashSet<Long>());
+
   /**
    * Construct an FSImage
    * @param conf Configuration
@@ -1058,18 +1069,26 @@ public synchronized void saveNamespace(FSNamesystem source, NameNodeFile nnf,
       editLog.endCurrentLogSegment(true);
     }
     long imageTxId = getLastAppliedOrWrittenTxId();
+    if (!addToCheckpointing(imageTxId)) {
+      throw new IOException(
+          "FS image is being downloaded from another NN at txid " + imageTxId);
+    }
     try {
-      saveFSImageInAllDirs(source, nnf, imageTxId, canceler);
-      storage.writeAll();
-    } finally {
-      if (editLogWasOpen) {
-        editLog.startLogSegmentAndWriteHeaderTxn(imageTxId + 1);
-        // Take this opportunity to note the current transaction.
-        // Even if the namespace save was cancelled, this marker
-        // is only used to determine what transaction ID is required
-        // for startup. So, it doesn't hurt to update it unnecessarily.
-        storage.writeTransactionIdFileToStorage(imageTxId + 1);
+      try {
+        saveFSImageInAllDirs(source, nnf, imageTxId, canceler);
+        storage.writeAll();
+      } finally {
+        if (editLogWasOpen) {
+          editLog.startLogSegmentAndWriteHeaderTxn(imageTxId + 1);
+          // Take this opportunity to note the current transaction.
+          // Even if the namespace save was cancelled, this marker
+          // is only used to determine what transaction ID is required
+          // for startup. So, it doesn't hurt to update it unnecessarily.
+          storage.writeTransactionIdFileToStorage(imageTxId + 1);
+        }
       }
+    } finally {
+      removeFromCheckpointing(imageTxId);
     }
   }
 
@@ -1078,7 +1097,22 @@ public synchronized void saveNamespace(FSNamesystem source, NameNodeFile nnf,
    */
   protected synchronized void saveFSImageInAllDirs(FSNamesystem source, long txid)
       throws IOException {
-    saveFSImageInAllDirs(source, NameNodeFile.IMAGE, txid, null);
+    if (!addToCheckpointing(txid)) {
+      throw new IOException(("FS image is being downloaded from another NN"));
+    }
+    try {
+      saveFSImageInAllDirs(source, NameNodeFile.IMAGE, txid, null);
+    } finally {
+      removeFromCheckpointing(txid);
+    }
+  }
+
+  public boolean addToCheckpointing(long txid) {
+    return currentlyCheckpointing.add(txid);
+  }
+
+  public void removeFromCheckpointing(long txid) {
+    currentlyCheckpointing.remove(txid);
   }
 
   private synchronized void saveFSImageInAllDirs(FSNamesystem source,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index e26f052c6c1c4..ed05d400cf9a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -51,8 +51,8 @@
 import org.apache.hadoop.hdfs.protocol.LayoutFlags;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
@@ -596,7 +596,7 @@ private int loadDirectory(DataInput in, Counter counter) throws IOException {
      // Rename .snapshot paths if we're doing an upgrade
      parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion());
      final INodeDirectory parent = INodeDirectory.valueOf(
-         namesystem.dir.getNode(parentPath, true), parentPath);
+         namesystem.dir.getINode(parentPath, true), parentPath);
      return loadChildren(parent, in, counter);
    }
 
@@ -683,7 +683,7 @@ private void addToParent(INodeDirectory parent, INode child) {
 
     public void updateBlocksMap(INodeFile file) {
       // Add file->block mapping
-      final BlockInfo[] blocks = file.getBlocks();
+      final BlockInfoContiguous[] blocks = file.getBlocks();
       if (blocks != null) {
         final BlockManager bm = namesystem.getBlockManager();
         for (int i = 0; i < blocks.length; i++) {
@@ -749,9 +749,9 @@ INode loadINode(final byte[] localName, boolean isSnapshotINode,
       // file
       
       // read blocks
-      BlockInfo[] blocks = new BlockInfo[numBlocks];
+      BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
       for (int j = 0; j < numBlocks; j++) {
-        blocks[j] = new BlockInfo(replication);
+        blocks[j] = new BlockInfoContiguous(replication);
         blocks[j].readFields(in);
       }
 
@@ -771,8 +771,8 @@ INode loadINode(final byte[] localName, boolean isSnapshotINode,
             clientMachine = FSImageSerialization.readString(in);
             // convert the last block to BlockUC
             if (blocks.length > 0) {
-              BlockInfo lastBlk = blocks[blocks.length - 1]; 
-              blocks[blocks.length - 1] = new BlockInfoUnderConstruction(
+              BlockInfoContiguous lastBlk = blocks[blocks.length - 1];
+              blocks[blocks.length - 1] = new BlockInfoContiguousUnderConstruction(
                   lastBlk, replication);
             }
           }
@@ -940,16 +940,16 @@ LayoutVersion.Feature.ADD_INODE_ID, getLayoutVersion())) {
           inSnapshot = true;
         } else {
           path = renameReservedPathsOnUpgrade(path, getLayoutVersion());
-          final INodesInPath iip = fsDir.getLastINodeInPath(path);
-          oldnode = INodeFile.valueOf(iip.getINode(0), path);
+          final INodesInPath iip = fsDir.getINodesInPath(path, true);
+          oldnode = INodeFile.valueOf(iip.getLastINode(), path);
         }
 
         FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
         oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
         if (oldnode.numBlocks() > 0) {
-          BlockInfo ucBlock = cons.getLastBlock();
+          BlockInfoContiguous ucBlock = cons.getLastBlock();
           // we do not replace the inode, just replace the last block of oldnode
-          BlockInfo info = namesystem.getBlockManager().addBlockCollection(
+          BlockInfoContiguous info = namesystem.getBlockManager().addBlockCollection(
               ucBlock, oldnode);
           oldnode.setBlock(oldnode.numBlocks() - 1, info);
         }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 26ca16a3edba6..f20add1a4e83c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -40,8 +40,8 @@
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext;
@@ -170,7 +170,7 @@ public static INodeDirectory loadINodeDirectory(INodeSection.INode n,
 
     public static void updateBlocksMap(INodeFile file, BlockManager bm) {
       // Add file->block mapping
-      final BlockInfo[] blocks = file.getBlocks();
+      final BlockInfoContiguous[] blocks = file.getBlocks();
       if (blocks != null) {
         for (int i = 0; i < blocks.length; i++) {
           file.setBlock(i, bm.addBlockCollection(blocks[i], file));
@@ -282,9 +282,9 @@ private INodeFile loadINodeFile(INodeSection.INode n) {
       short replication = (short) f.getReplication();
       LoaderContext state = parent.getLoaderContext();
 
-      BlockInfo[] blocks = new BlockInfo[bp.size()];
+      BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.size()];
       for (int i = 0, e = bp.size(); i < e; ++i) {
-        blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
+        blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
       }
       final PermissionStatus permissions = loadPermission(f.getPermission(),
           parent.getLoaderContext().getStringTable());
@@ -310,9 +310,9 @@ private INodeFile loadINodeFile(INodeSection.INode n) {
         INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
         file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
         if (blocks.length > 0) {
-          BlockInfo lastBlk = file.getLastBlock();
+          BlockInfoContiguous lastBlk = file.getLastBlock();
           // replace the last block of file
-          file.setBlock(file.numBlocks() - 1, new BlockInfoUnderConstruction(
+          file.setBlock(file.numBlocks() - 1, new BlockInfoContiguousUnderConstruction(
               lastBlk, replication));
         }
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 1c22ee9b75480..33f644d84189b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -33,8 +33,8 @@
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
@@ -126,17 +126,17 @@ static INodeFile readINodeUnderConstruction(
     long preferredBlockSize = in.readLong();
   
     int numBlocks = in.readInt();
-    BlockInfo[] blocks = new BlockInfo[numBlocks];
+    BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
     Block blk = new Block();
     int i = 0;
     for (; i < numBlocks-1; i++) {
       blk.readFields(in);
-      blocks[i] = new BlockInfo(blk, blockReplication);
+      blocks[i] = new BlockInfoContiguous(blk, blockReplication);
     }
     // last block is UNDER_CONSTRUCTION
     if(numBlocks > 0) {
       blk.readFields(in);
-      blocks[i] = new BlockInfoUnderConstruction(
+      blocks[i] = new BlockInfoContiguousUnderConstruction(
         blk, blockReplication, BlockUCState.UNDER_CONSTRUCTION, null);
     }
     PermissionStatus perm = PermissionStatus.read(in);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
index 931386cae9a55..388a1bf0cce2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageUtil.java
@@ -23,6 +23,7 @@
 import java.io.RandomAccessFile;
 import java.util.Arrays;
 
+import org.apache.commons.io.Charsets;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
@@ -32,7 +33,8 @@
 
 @InterfaceAudience.Private
 public final class FSImageUtil {
-  public static final byte[] MAGIC_HEADER = "HDFSIMG1".getBytes();
+  public static final byte[] MAGIC_HEADER =
+      "HDFSIMG1".getBytes(Charsets.UTF_8);
   public static final int FILE_VERSION = 1;
 
   public static boolean checkFileFormat(RandomAccessFile file)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e9ce78c06d9ec..a60f1eb8ed2f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -86,8 +86,6 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
 import static org.apache.hadoop.util.Time.now;
 
@@ -120,6 +118,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
@@ -151,7 +150,6 @@
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.XAttrSetFlag;
@@ -169,7 +167,6 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException;
 import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.AclException;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -204,8 +201,8 @@
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -241,6 +238,7 @@
 import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
 import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
 import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
+import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
 import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -252,7 +250,7 @@
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
 import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
+import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.RetriableException;
@@ -273,6 +271,7 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.Daemon;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.StringUtils;
@@ -281,6 +280,7 @@
 import org.apache.log4j.Appender;
 import org.apache.log4j.AsyncAppender;
 import org.apache.log4j.Logger;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.mortbay.util.ajax.JSON;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -337,11 +337,6 @@ public boolean isAuditEnabled() {
     return !isDefaultAuditLogger || auditLog.isInfoEnabled();
   }
 
-  private HdfsFileStatus getAuditFileInfo(String path, boolean resolveSymlink)
-      throws IOException {
-    return dir.getAuditFileInfo(path, resolveSymlink);
-  }
-  
   private void logAuditEvent(boolean succeeded, String cmd, String src)
       throws IOException {
     logAuditEvent(succeeded, cmd, src, null, null);
@@ -425,12 +420,9 @@ private void logAuditEvent(boolean succeeded,
   private final CacheManager cacheManager;
   private final DatanodeStatistics datanodeStatistics;
 
-  // whether setStoragePolicy is allowed.
-  private final boolean isStoragePolicyEnabled;
-
   private String nameserviceId;
 
-  private RollingUpgradeInfo rollingUpgradeInfo = null;
+  private volatile RollingUpgradeInfo rollingUpgradeInfo = null;
   /**
    * A flag that indicates whether the checkpointer should checkpoint a rollback
    * fsimage. The edit log tailer sets this flag. The checkpoint will create a
@@ -532,9 +524,6 @@ private void logAuditEvent(boolean succeeded,
 
   private final RetryCache retryCache;
 
-  private final boolean xattrsEnabled;
-  private final int xattrMaxSize;
-
   private KeyProviderCryptoExtension provider = null;
 
   private volatile boolean imageLoaded = false;
@@ -542,6 +531,9 @@ private void logAuditEvent(boolean succeeded,
 
   private final FSImage fsImage;
 
+  private final TopConf topConf;
+  private TopMetrics topMetrics;
+
   /**
    * Notify that loading of this FSDirectory is complete, and
    * it is imageLoaded for use
@@ -604,6 +596,7 @@ void clear() {
     snapshotManager.clearSnapshottableDirs();
     cacheManager.clear();
     setImageLoaded(false);
+    blockManager.clear();
   }
 
   @VisibleForTesting
@@ -750,10 +743,6 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
       this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics();
       this.blockIdManager = new BlockIdManager(blockManager);
 
-      this.isStoragePolicyEnabled =
-          conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
-                          DFS_STORAGE_POLICY_ENABLED_DEFAULT);
-
       this.fsOwner = UserGroupInformation.getCurrentUser();
       this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY, 
                                  DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
@@ -845,23 +834,11 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
       this.snapshotManager = new SnapshotManager(dir);
       this.cacheManager = new CacheManager(this, conf, blockManager);
       this.safeMode = new SafeModeInfo(conf);
+      this.topConf = new TopConf(conf);
       this.auditLoggers = initAuditLoggers(conf);
       this.isDefaultAuditLogger = auditLoggers.size() == 1 &&
         auditLoggers.get(0) instanceof DefaultAuditLogger;
       this.retryCache = ignoreRetryCache ? null : initRetryCache(conf);
-
-      this.xattrsEnabled = conf.getBoolean(
-          DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
-          DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
-      LOG.info("XAttrs enabled? " + xattrsEnabled);
-      this.xattrMaxSize = conf.getInt(
-          DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
-          DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
-      Preconditions.checkArgument(xattrMaxSize >= 0,
-          "Cannot set a negative value for the maximum size of an xattr (%s).",
-          DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
-      final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
-      LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);
     } catch(IOException e) {
       LOG.error(getClass().getSimpleName() + " initialization failed.", e);
       close();
@@ -968,13 +945,9 @@ private List<AuditLogger> initAuditLoggers(Configuration conf) {
     }
 
     // Add audit logger to calculate top users
-    if (conf.getBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY,
-        DFSConfigKeys.NNTOP_ENABLED_DEFAULT)) {
-      String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
-      TopConf nntopConf = new TopConf(conf);
-      TopMetrics.initSingleton(conf, NamenodeRole.NAMENODE.name(), sessionId,
-          nntopConf.nntopReportingPeriodsMs);
-      auditLoggers.add(new TopAuditLogger());
+    if (topConf.isEnabled) {
+      topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs);
+      auditLoggers.add(new TopAuditLogger(topMetrics));
     }
 
     return Collections.unmodifiableList(auditLoggers);
@@ -1492,47 +1465,20 @@ public void readLock() {
     this.fsLock.readLock().lock();
   }
   @Override
-  public void longReadLockInterruptibly() throws InterruptedException {
-    this.fsLock.longReadLock().lockInterruptibly();
-    try {
-      this.fsLock.readLock().lockInterruptibly();
-    } catch (InterruptedException ie) {
-      // In the event we're interrupted while getting the normal FSNS read lock,
-      // release the long read lock.
-      this.fsLock.longReadLock().unlock();
-      throw ie;
-    }
-  }
-  @Override
-  public void longReadUnlock() {
-    this.fsLock.readLock().unlock();
-    this.fsLock.longReadLock().unlock();
-  }
-  @Override
   public void readUnlock() {
     this.fsLock.readLock().unlock();
   }
   @Override
   public void writeLock() {
-    this.fsLock.longReadLock().lock();
     this.fsLock.writeLock().lock();
   }
   @Override
   public void writeLockInterruptibly() throws InterruptedException {
-    this.fsLock.longReadLock().lockInterruptibly();
-    try {
-      this.fsLock.writeLock().lockInterruptibly();
-    } catch (InterruptedException ie) {
-      // In the event we're interrupted while getting the normal FSNS write
-      // lock, release the long read lock.
-      this.fsLock.longReadLock().unlock();
-      throw ie;
-    }
+    this.fsLock.writeLock().lockInterruptibly();
   }
   @Override
   public void writeUnlock() {
     this.fsLock.writeLock().unlock();
-    this.fsLock.longReadLock().unlock();
   }
   @Override
   public boolean hasWriteLock() {
@@ -1687,36 +1633,21 @@ private boolean isAccessTimeSupported() {
    * @throws IOException
    */
   void setPermission(String src, FsPermission permission) throws IOException {
-    try {
-      setPermissionInt(src, permission);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "setPermission", src);
-      throw e;
-    }
-  }
-
-  private void setPermissionInt(final String srcArg, FsPermission permission)
-      throws IOException {
-    String src = srcArg;
-    HdfsFileStatus resultingStat = null;
-    FSPermissionChecker pc = getPermissionChecker();
+    HdfsFileStatus auditStat;
     checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot set permission for " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      dir.checkOwner(pc, iip);
-      dir.setPermission(src, permission);
-      getEditLog().logSetPermissions(src, permission);
-      resultingStat = getAuditFileInfo(src, false);
+      auditStat = FSDirAttrOp.setPermission(dir, src, permission);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "setPermission", src);
+      throw e;
     } finally {
       writeUnlock();
     }
     getEditLog().logSync();
-    logAuditEvent(true, "setPermission", srcArg, null, resultingStat);
+    logAuditEvent(true, "setPermission", src, null, auditStat);
   }
 
   /**
@@ -1725,44 +1656,33 @@ private void setPermissionInt(final String srcArg, FsPermission permission)
    */
   void setOwner(String src, String username, String group)
       throws IOException {
-    try {
-      setOwnerInt(src, username, group);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "setOwner", src);
-      throw e;
-    } 
-  }
-
-  private void setOwnerInt(final String srcArg, String username, String group)
-      throws IOException {
-    String src = srcArg;
-    HdfsFileStatus resultingStat = null;
-    FSPermissionChecker pc = getPermissionChecker();
+    HdfsFileStatus auditStat;
     checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot set owner for " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      dir.checkOwner(pc, iip);
-      if (!pc.isSuperUser()) {
-        if (username != null && !pc.getUser().equals(username)) {
-          throw new AccessControlException("Non-super user cannot change owner");
-        }
-        if (group != null && !pc.containsGroup(group)) {
-          throw new AccessControlException("User does not belong to " + group);
-        }
-      }
-      dir.setOwner(src, username, group);
-      getEditLog().logSetOwner(src, username, group);
-      resultingStat = getAuditFileInfo(src, false);
+      auditStat = FSDirAttrOp.setOwner(dir, src, username, group);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "setOwner", src);
+      throw e;
     } finally {
       writeUnlock();
     }
     getEditLog().logSync();
-    logAuditEvent(true, "setOwner", srcArg, null, resultingStat);
+    logAuditEvent(true, "setOwner", src, null, auditStat);
+  }
+
+  static class GetBlockLocationsResult {
+    final INodesInPath iip;
+    final LocatedBlocks blocks;
+    boolean updateAccessTime() {
+      return iip != null;
+    }
+    private GetBlockLocationsResult(INodesInPath iip, LocatedBlocks blocks) {
+      this.iip = iip;
+      this.blocks = blocks;
+    }
   }
 
   /**
@@ -1770,22 +1690,55 @@ private void setOwnerInt(final String srcArg, String username, String group)
    * @see ClientProtocol#getBlockLocations(String, long, long)
    */
   LocatedBlocks getBlockLocations(String clientMachine, String src,
-      long offset, long length) throws AccessControlException,
-      FileNotFoundException, UnresolvedLinkException, IOException {
-    LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true,
-        true);
+      long offset, long length) throws IOException {
+    checkOperation(OperationCategory.READ);
+    GetBlockLocationsResult res = null;
+    readLock();
+    try {
+      checkOperation(OperationCategory.READ);
+      res = getBlockLocations(src, offset, length, true, true);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "open", src);
+      throw e;
+    } finally {
+      readUnlock();
+    }
+
+    logAuditEvent(true, "open", src);
+
+    if (res.updateAccessTime()) {
+      writeLock();
+      final long now = now();
+      try {
+        checkOperation(OperationCategory.WRITE);
+        INode inode = res.iip.getLastINode();
+        boolean updateAccessTime = now > inode.getAccessTime() +
+            getAccessTimePrecision();
+        if (!isInSafeMode() && updateAccessTime) {
+          boolean changed = FSDirAttrOp.setTimes(dir,
+              inode, -1, now, false, res.iip.getLatestSnapshotId());
+          if (changed) {
+            getEditLog().logTimes(src, -1, now);
+          }
+        }
+      } catch (Throwable e) {
+        LOG.warn("Failed to update the access time of " + src, e);
+      } finally {
+        writeUnlock();
+      }
+    }
+
+    LocatedBlocks blocks = res.blocks;
     if (blocks != null) {
-      blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
-          blocks.getLocatedBlocks());
+      blockManager.getDatanodeManager().sortLocatedBlocks(
+          clientMachine, blocks.getLocatedBlocks());
 
       // lastBlock is not part of getLocatedBlocks(), might need to sort it too
       LocatedBlock lastBlock = blocks.getLastLocatedBlock();
       if (lastBlock != null) {
-        ArrayList<LocatedBlock> lastBlockList =
-            Lists.newArrayListWithCapacity(1);
-        lastBlockList.add(lastBlock);
-        blockManager.getDatanodeManager().sortLocatedBlocks(clientMachine,
-            lastBlockList);
+        ArrayList<LocatedBlock> lastBlockList = Lists.newArrayList(lastBlock);
+        blockManager.getDatanodeManager().sortLocatedBlocks(
+            clientMachine, lastBlockList);
       }
     }
     return blocks;
@@ -1794,24 +1747,11 @@ LocatedBlocks getBlockLocations(String clientMachine, String src,
   /**
    * Get block locations within the specified range.
    * @see ClientProtocol#getBlockLocations(String, long, long)
-   * @throws FileNotFoundException, UnresolvedLinkException, IOException
+   * @throws IOException
    */
-  LocatedBlocks getBlockLocations(String src, long offset, long length,
-      boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode)
-      throws FileNotFoundException, UnresolvedLinkException, IOException {
-    try {
-      return getBlockLocationsInt(src, offset, length, doAccessTime,
-                                  needBlockToken, checkSafeMode);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "open", src);
-      throw e;
-    }
-  }
-
-  private LocatedBlocks getBlockLocationsInt(String src, long offset,
-      long length, boolean doAccessTime, boolean needBlockToken,
-      boolean checkSafeMode)
-      throws FileNotFoundException, UnresolvedLinkException, IOException {
+  GetBlockLocationsResult getBlockLocations(
+      String src, long offset, long length, boolean needBlockToken,
+      boolean checkSafeMode) throws IOException {
     if (offset < 0) {
       throw new HadoopIllegalArgumentException(
           "Negative offset is not supported. File: " + src);
@@ -1820,16 +1760,16 @@ private LocatedBlocks getBlockLocationsInt(String src, long offset,
       throw new HadoopIllegalArgumentException(
           "Negative length is not supported. File: " + src);
     }
-    final LocatedBlocks ret = getBlockLocationsUpdateTimes(src,
-        offset, length, doAccessTime, needBlockToken);  
-    logAuditEvent(true, "open", src);
+    final GetBlockLocationsResult ret = getBlockLocationsInt(
+        src, offset, length, needBlockToken);
+
     if (checkSafeMode && isInSafeMode()) {
-      for (LocatedBlock b : ret.getLocatedBlocks()) {
+      for (LocatedBlock b : ret.blocks.getLocatedBlocks()) {
         // if safemode & no block locations yet then throw safemodeException
         if ((b.getLocations() == null) || (b.getLocations().length == 0)) {
           SafeModeException se = new SafeModeException(
               "Zero blocklocations for " + src, safeMode);
-          if (haEnabled && haContext != null && 
+          if (haEnabled && haContext != null &&
               haContext.getState().getServiceState() == HAServiceState.ACTIVE) {
             throw new RetriableException(se);
           } else {
@@ -1841,97 +1781,49 @@ private LocatedBlocks getBlockLocationsInt(String src, long offset,
     return ret;
   }
 
-  /*
-   * Get block locations within the specified range, updating the
-   * access times if necessary. 
-   */
-  private LocatedBlocks getBlockLocationsUpdateTimes(final String srcArg,
-      long offset, long length, boolean doAccessTime, boolean needBlockToken)
+  private GetBlockLocationsResult getBlockLocationsInt(
+      final String srcArg, long offset, long length, boolean needBlockToken)
       throws IOException {
     String src = srcArg;
     FSPermissionChecker pc = getPermissionChecker();
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
-    for (int attempt = 0; attempt < 2; attempt++) {
-      boolean isReadOp = (attempt == 0);
-      if (isReadOp) { // first attempt is with readlock
-        checkOperation(OperationCategory.READ);
-        readLock();
-      }  else { // second attempt is with  write lock
-        checkOperation(OperationCategory.WRITE);
-        writeLock(); // writelock is needed to set accesstime
-      }
-      try {
-        if (isReadOp) {
-          checkOperation(OperationCategory.READ);
-        } else {
-          checkOperation(OperationCategory.WRITE);
-        }
-        src = dir.resolvePath(pc, src, pathComponents);
-        final INodesInPath iip = dir.getINodesInPath(src, true);
-        if (isPermissionEnabled) {
-          dir.checkPathAccess(pc, iip, FsAction.READ);
-        }
+    src = dir.resolvePath(pc, src, pathComponents);
+    final INodesInPath iip = dir.getINodesInPath(src, true);
+    final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
+    if (isPermissionEnabled) {
+      dir.checkPathAccess(pc, iip, FsAction.READ);
+      checkUnreadableBySuperuser(pc, inode, iip.getPathSnapshotId());
+    }
 
-        // if the namenode is in safemode, then do not update access time
-        if (isInSafeMode()) {
-          doAccessTime = false;
-        }
+    final long fileSize = iip.isSnapshot()
+        ? inode.computeFileSize(iip.getPathSnapshotId())
+        : inode.computeFileSizeNotIncludingLastUcBlock();
+    boolean isUc = inode.isUnderConstruction();
+    if (iip.isSnapshot()) {
+      // if src indicates a snapshot file, we need to make sure the returned
+      // blocks do not exceed the size of the snapshot file.
+      length = Math.min(length, fileSize - offset);
+      isUc = false;
+    }
 
-        final INode[] inodes = iip.getINodes();
-        final INodeFile inode = INodeFile.valueOf(
-            inodes[inodes.length - 1], src);
-        if (isPermissionEnabled) {
-          checkUnreadableBySuperuser(pc, inode, iip.getPathSnapshotId());
-        }
-        if (!iip.isSnapshot() //snapshots are readonly, so don't update atime.
-            && doAccessTime && isAccessTimeSupported()) {
-          final long now = now();
-          if (now > inode.getAccessTime() + getAccessTimePrecision()) {
-            // if we have to set access time but we only have the readlock, then
-            // restart this entire operation with the writeLock.
-            if (isReadOp) {
-              continue;
-            }
-            boolean changed = dir.setTimes(inode, -1, now, false,
-                    iip.getLatestSnapshotId());
-            if (changed) {
-              getEditLog().logTimes(src, -1, now);
-            }
-          }
-        }
-        final long fileSize = iip.isSnapshot() ?
-            inode.computeFileSize(iip.getPathSnapshotId())
-            : inode.computeFileSizeNotIncludingLastUcBlock();
-        boolean isUc = inode.isUnderConstruction();
-        if (iip.isSnapshot()) {
-          // if src indicates a snapshot file, we need to make sure the returned
-          // blocks do not exceed the size of the snapshot file.
-          length = Math.min(length, fileSize - offset);
-          isUc = false;
-        }
+    final FileEncryptionInfo feInfo =
+        FSDirectory.isReservedRawName(srcArg) ? null
+            : dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId(), iip);
 
-        final FileEncryptionInfo feInfo =
-          FSDirectory.isReservedRawName(srcArg) ?
-          null : dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId(),
-              iip);
-
-        final LocatedBlocks blocks =
-          blockManager.createLocatedBlocks(inode.getBlocks(), fileSize,
-            isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo);
-        // Set caching information for the located blocks.
-        for (LocatedBlock lb: blocks.getLocatedBlocks()) {
-          cacheManager.setCachedLocations(lb);
-        }
-        return blocks;
-      } finally {
-        if (isReadOp) {
-          readUnlock();
-        } else {
-          writeUnlock();
-        }
-      }
+    final LocatedBlocks blocks = blockManager.createLocatedBlocks(
+        inode.getBlocks(iip.getPathSnapshotId()), fileSize,
+        isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo);
+
+    // Set caching information for the located blocks.
+    for (LocatedBlock lb : blocks.getLocatedBlocks()) {
+      cacheManager.setCachedLocations(lb);
     }
-    return null; // can never reach here
+
+    final long now = now();
+    boolean updateAccessTime = isAccessTimeSupported() && !isInSafeMode()
+        && !iip.isSnapshot()
+        && now > inode.getAccessTime() + getAccessTimePrecision();
+    return new GetBlockLocationsResult(updateAccessTime ? iip : null, blocks);
   }
 
   /**
@@ -1970,114 +1862,245 @@ void concat(String target, String [] srcs, boolean logRetryCache)
    * The access time is precise up to an hour. The transaction, if needed, is
    * written to the edits log but is not flushed.
    */
-  void setTimes(String src, long mtime, long atime) 
-      throws IOException, UnresolvedLinkException {
-    if (!isAccessTimeSupported() && atime != -1) {
-      throw new IOException("Access time for hdfs is not configured. " +
-                            " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY + " configuration parameter.");
-    }
+  void setTimes(String src, long mtime, long atime) throws IOException {
+    HdfsFileStatus auditStat;
+    checkOperation(OperationCategory.WRITE);
+    writeLock();
     try {
-      setTimesInt(src, mtime, atime);
+      checkOperation(OperationCategory.WRITE);
+      checkNameNodeSafeMode("Cannot set times " + src);
+      auditStat = FSDirAttrOp.setTimes(dir, src, mtime, atime);
     } catch (AccessControlException e) {
       logAuditEvent(false, "setTimes", src);
       throw e;
+    } finally {
+      writeUnlock();
+    }
+    getEditLog().logSync();
+    logAuditEvent(true, "setTimes", src, null, auditStat);
+  }
+
+  /**
+   * Truncate file to a lower length.
+   * Truncate cannot be reverted / recovered from as it causes data loss.
+   * Truncation at block boundary is atomic, otherwise it requires
+   * block recovery to truncate the last block of the file.
+   *
+   * @return true if client does not need to wait for block recovery,
+   * false if client needs to wait for block recovery.
+   */
+  boolean truncate(String src, long newLength,
+                   String clientName, String clientMachine,
+                   long mtime)
+      throws IOException, UnresolvedLinkException {
+    boolean ret;
+    try {
+      ret = truncateInt(src, newLength, clientName, clientMachine, mtime);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "truncate", src);
+      throw e;
     }
+    return ret;
   }
 
-  private void setTimesInt(final String srcArg, long mtime, long atime)
-    throws IOException {
+  boolean truncateInt(String srcArg, long newLength,
+                      String clientName, String clientMachine,
+                      long mtime)
+      throws IOException, UnresolvedLinkException {
     String src = srcArg;
-    HdfsFileStatus resultingStat = null;
+    if (NameNode.stateChangeLog.isDebugEnabled()) {
+      NameNode.stateChangeLog.debug("DIR* NameSystem.truncate: src="
+          + src + " newLength=" + newLength);
+    }
+    if (newLength < 0) {
+      throw new HadoopIllegalArgumentException(
+          "Cannot truncate to a negative file size: " + newLength + ".");
+    }
+    HdfsFileStatus stat = null;
     FSPermissionChecker pc = getPermissionChecker();
     checkOperation(OperationCategory.WRITE);
+    boolean res;
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     writeLock();
+    BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
     try {
       checkOperation(OperationCategory.WRITE);
-      checkNameNodeSafeMode("Cannot set times " + src);
+      checkNameNodeSafeMode("Cannot truncate for " + src);
       src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      // Write access is required to set access and modification times
-      if (isPermissionEnabled) {
-        dir.checkPathAccess(pc, iip, FsAction.WRITE);
-      }
-      final INode inode = iip.getLastINode();
-      if (inode != null) {
-        boolean changed = dir.setTimes(inode, mtime, atime, true,
-                iip.getLatestSnapshotId());
-        if (changed) {
-          getEditLog().logTimes(src, mtime, atime);
-        }
-        resultingStat = getAuditFileInfo(src, false);
-      } else {
-        throw new FileNotFoundException("File/Directory " + src + " does not exist.");
-      }
+      res = truncateInternal(src, newLength, clientName,
+          clientMachine, mtime, pc, toRemoveBlocks);
+      stat = dir.getAuditFileInfo(dir.getINodesInPath4Write(src, false));
     } finally {
       writeUnlock();
     }
-    logAuditEvent(true, "setTimes", srcArg, null, resultingStat);
+    getEditLog().logSync();
+    if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
+      removeBlocks(toRemoveBlocks);
+      toRemoveBlocks.clear();
+    }
+    logAuditEvent(true, "truncate", src, null, stat);
+    return res;
   }
 
   /**
-   * Create a symbolic link.
+   * Truncate a file to a given size
+   * Update the count at each ancestor directory with quota
    */
-  void createSymlink(String target, String link,
-      PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
-      throws IOException {
-    if (!DFSUtil.isValidName(link)) {
-      throw new InvalidPathException("Invalid link name: " + link);
+  boolean truncateInternal(String src, long newLength,
+                           String clientName, String clientMachine,
+                           long mtime, FSPermissionChecker pc,
+                           BlocksMapUpdateInfo toRemoveBlocks)
+      throws IOException, UnresolvedLinkException {
+    assert hasWriteLock();
+    INodesInPath iip = dir.getINodesInPath4Write(src, true);
+    if (isPermissionEnabled) {
+      dir.checkPathAccess(pc, iip, FsAction.WRITE);
     }
-    if (FSDirectory.isReservedName(target)) {
-      throw new InvalidPathException("Invalid target name: " + target);
+    INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
+    final BlockStoragePolicy lpPolicy =
+        blockManager.getStoragePolicy("LAZY_PERSIST");
+
+    if (lpPolicy != null &&
+        lpPolicy.getId() == file.getStoragePolicyID()) {
+      throw new UnsupportedOperationException(
+          "Cannot truncate lazy persist file " + src);
+    }
+    // Opening an existing file for truncate. May need lease recovery.
+    recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE,
+        iip, src, clientName, clientMachine, false);
+    // Truncate length check.
+    long oldLength = file.computeFileSize();
+    if(oldLength == newLength) {
+      return true;
     }
-
-    try {
-      createSymlinkInt(target, link, dirPerms, createParent, logRetryCache);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "createSymlink", link, target, null);
-      throw e;
+    if(oldLength < newLength) {
+      throw new HadoopIllegalArgumentException(
+          "Cannot truncate to a larger file size. Current size: " + oldLength +
+              ", truncate size: " + newLength + ".");
+    }
+    // Perform INodeFile truncation.
+    boolean onBlockBoundary = dir.truncate(iip, newLength,
+        toRemoveBlocks, mtime);
+    Block truncateBlock = null;
+    if(! onBlockBoundary) {
+      // Open file for write, but don't log into edits
+      long lastBlockDelta = file.computeFileSize() - newLength;
+      assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
+      truncateBlock = prepareFileForTruncate(iip, clientName, clientMachine,
+          lastBlockDelta, null);
+    }
+    getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime,
+        truncateBlock);
+    return onBlockBoundary;
+  }
+
+  /**
+   * Convert current INode to UnderConstruction.
+   * Recreate lease.
+   * Create new block for the truncated copy.
+   * Schedule truncation of the replicas.
+   *
+   * @return the returned block will be written to editLog and passed back into
+   * this method upon loading.
+   */
+  Block prepareFileForTruncate(INodesInPath iip,
+                               String leaseHolder,
+                               String clientMachine,
+                               long lastBlockDelta,
+                               Block newBlock)
+      throws IOException {
+    INodeFile file = iip.getLastINode().asFile();
+    String src = iip.getPath();
+    file.recordModification(iip.getLatestSnapshotId());
+    file.toUnderConstruction(leaseHolder, clientMachine);
+    assert file.isUnderConstruction() : "inode should be under construction.";
+    leaseManager.addLease(
+        file.getFileUnderConstructionFeature().getClientName(), src);
+    boolean shouldRecoverNow = (newBlock == null);
+    BlockInfoContiguous oldBlock = file.getLastBlock();
+    boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
+    if(newBlock == null) {
+      newBlock = (shouldCopyOnTruncate) ? createNewBlock(file.isStriped()) :
+          new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
+              nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
+    }
+
+    BlockInfoContiguousUnderConstruction truncatedBlockUC;
+    if(shouldCopyOnTruncate) {
+      // Add new truncateBlock into blocksMap and
+      // use oldBlock as a source for copy-on-truncate recovery
+      truncatedBlockUC = new BlockInfoContiguousUnderConstruction(newBlock,
+          file.getBlockReplication());
+      truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
+      truncatedBlockUC.setTruncateBlock(oldBlock);
+      file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
+      getBlockManager().addBlockCollection(truncatedBlockUC, file);
+
+      NameNode.stateChangeLog.info("BLOCK* prepareFileForTruncate: "
+          + "Scheduling copy-on-truncate to new size "
+          + truncatedBlockUC.getNumBytes() + " new block " + newBlock
+          + " old block " + truncatedBlockUC.getTruncateBlock());
+    } else {
+      // Use new generation stamp for in-place truncate recovery
+      blockManager.convertLastBlockToUnderConstruction(file, lastBlockDelta);
+      oldBlock = file.getLastBlock();
+      assert !oldBlock.isComplete() : "oldBlock should be under construction";
+      truncatedBlockUC = (BlockInfoContiguousUnderConstruction) oldBlock;
+      truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
+      truncatedBlockUC.getTruncateBlock().setNumBytes(
+          oldBlock.getNumBytes() - lastBlockDelta);
+      truncatedBlockUC.getTruncateBlock().setGenerationStamp(
+          newBlock.getGenerationStamp());
+
+      NameNode.stateChangeLog.debug("BLOCK* prepareFileForTruncate: "
+          + "Scheduling in-place block truncate to new size "
+          + truncatedBlockUC.getTruncateBlock().getNumBytes()
+          + " block=" + truncatedBlockUC);
+    }
+    if(shouldRecoverNow)
+      truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
+
+    // update the quota: use the preferred block size for UC block
+    final long diff =
+        file.getPreferredBlockSize() - truncatedBlockUC.getNumBytes();
+    dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
+    return newBlock;
+  }
+
+  /**
+   * Defines if a replica needs to be copied on truncate or
+   * can be truncated in place.
+   */
+  boolean shouldCopyOnTruncate(INodeFile file, BlockInfoContiguous blk) {
+    if(!isUpgradeFinalized()) {
+      return true;
     }
+    return file.isBlockInLatestSnapshot(blk);
   }
 
-  private void createSymlinkInt(String target, final String linkArg,
-      PermissionStatus dirPerms, boolean createParent, boolean logRetryCache) 
+  /**
+   * Create a symbolic link.
+   */
+  void createSymlink(String target, String link,
+      PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
       throws IOException {
-    String link = linkArg;
-    if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* NameSystem.createSymlink: target="
-          + target + " link=" + link);
-    }
-    HdfsFileStatus resultingStat = null;
-    FSPermissionChecker pc = getPermissionChecker();
+    waitForLoadingFSImage();
+    HdfsFileStatus auditStat = null;
     checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(link);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot create symlink " + link);
-      link = dir.resolvePath(pc, link, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(link, false);
-      if (!createParent) {
-        dir.verifyParentDir(iip, link);
-      }
-      if (!dir.isValidToCreate(link)) {
-        throw new IOException("failed to create link " + link 
-            +" either because the filename is invalid or the file exists");
-      }
-      if (isPermissionEnabled) {
-        dir.checkAncestorAccess(pc, iip, FsAction.WRITE);
-      }
-      // validate that we have enough inodes.
-      checkFsObjectLimit();
-
-      // add symbolic link to namespace
-      addSymlink(link, target, dirPerms, createParent, logRetryCache);
-      resultingStat = getAuditFileInfo(link, false);
+      auditStat = FSDirSymlinkOp.createSymlinkInt(this, target, link, dirPerms,
+                                                  createParent, logRetryCache);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "createSymlink", link, target, null);
+      throw e;
     } finally {
       writeUnlock();
     }
     getEditLog().logSync();
-    logAuditEvent(true, "createSymlink", linkArg, target, resultingStat);
+    logAuditEvent(true, "createSymlink", link, target, auditStat);
   }
 
   /**
@@ -2095,49 +2118,25 @@ private void createSymlinkInt(String target, final String linkArg,
    */
   boolean setReplication(final String src, final short replication)
       throws IOException {
-    try {
-      return setReplicationInt(src, replication);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "setReplication", src);
-      throw e;
-    }
-  }
-
-  private boolean setReplicationInt(final String srcArg,
-      final short replication) throws IOException {
-    String src = srcArg;
-    blockManager.verifyReplication(src, replication, null);
-    final boolean isFile;
-    FSPermissionChecker pc = getPermissionChecker();
-    checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    boolean success = false;
     waitForLoadingFSImage();
+    checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot set replication for " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      if (isPermissionEnabled) {
-        dir.checkPathAccess(pc, iip, FsAction.WRITE);
-      }
-
-      final short[] blockRepls = new short[2]; // 0: old, 1: new
-      final Block[] blocks = dir.setReplication(src, replication, blockRepls);
-      isFile = blocks != null;
-      if (isFile) {
-        getEditLog().logSetReplication(src, replication);
-        blockManager.setReplication(blockRepls[0], blockRepls[1], src, blocks);
-      }
+      success = FSDirAttrOp.setReplication(dir, blockManager, src, replication);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "setReplication", src);
+      throw e;
     } finally {
       writeUnlock();
     }
-
-    getEditLog().logSync();
-    if (isFile) {
-      logAuditEvent(true, "setReplication", srcArg);
+    if (success) {
+      getEditLog().logSync();
+      logAuditEvent(true, "setReplication", src);
     }
-    return isFile;
+    return success;
   }
 
   /**
@@ -2146,58 +2145,24 @@ private boolean setReplicationInt(final String srcArg,
    * @param src file/directory path
    * @param policyName storage policy name
    */
-  void setStoragePolicy(String src, final String policyName)
-      throws IOException {
-    try {
-      setStoragePolicyInt(src, policyName);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "setStoragePolicy", src);
-      throw e;
-    }
-  }
-
-  private void setStoragePolicyInt(String src, final String policyName)
-      throws IOException {
-    if (!isStoragePolicyEnabled) {
-      throw new IOException("Failed to set storage policy since "
-          + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
-    }
-    FSPermissionChecker pc = null;
-    if (isPermissionEnabled) {
-      pc = getPermissionChecker();
-    }
-
-    checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+  void setStoragePolicy(String src, String policyName) throws IOException {
+    HdfsFileStatus auditStat;
     waitForLoadingFSImage();
-    HdfsFileStatus fileStat;
+    checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot set storage policy for " + src);
-
-      src = FSDirectory.resolvePath(src, pathComponents, dir);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-
-      if (pc != null) {
-        dir.checkPermission(pc, iip, false, null, null, FsAction.WRITE, null, false);
-      }
-
-      // get the corresponding policy and make sure the policy name is valid
-      BlockStoragePolicy policy = blockManager.getStoragePolicy(policyName);
-      if (policy == null) {
-        throw new HadoopIllegalArgumentException(
-            "Cannot find a block policy with the name " + policyName);
-      }
-      dir.setStoragePolicy(iip, policy.getId());
-      getEditLog().logSetStoragePolicy(src, policy.getId());
-      fileStat = getAuditFileInfo(src, false);
+      auditStat = FSDirAttrOp.setStoragePolicy(
+          dir, blockManager, src, policyName);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "setStoragePolicy", src);
+      throw e;
     } finally {
       writeUnlock();
     }
-
     getEditLog().logSync();
-    logAuditEvent(true, "setStoragePolicy", src, null, fileStat);
+    logAuditEvent(true, "setStoragePolicy", src, null, auditStat);
   }
 
   /**
@@ -2209,25 +2174,18 @@ BlockStoragePolicy[] getStoragePolicies() throws IOException {
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      return blockManager.getStoragePolicies();
+      return FSDirAttrOp.getStoragePolicies(blockManager);
     } finally {
       readUnlock();
     }
   }
 
-  long getPreferredBlockSize(String filename) throws IOException {
-    FSPermissionChecker pc = getPermissionChecker();
+  long getPreferredBlockSize(String src) throws IOException {
     checkOperation(OperationCategory.READ);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(filename);
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      filename = dir.resolvePath(pc, filename, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath(filename, true);
-      if (isPermissionEnabled) {
-        dir.checkTraverse(pc, iip);
-      }
-      return dir.getPreferredBlockSize(filename);
+      return FSDirAttrOp.getPreferredBlockSize(dir, src);
     } finally {
       readUnlock();
     }
@@ -2425,13 +2383,21 @@ private HdfsFileStatus startFileInt(final String srcArg,
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot create file" + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      toRemoveBlocks = startFileInternal(pc, iip, permissions, holder,
-          clientMachine, create, overwrite, createParent, replication, 
-          blockSize, isLazyPersist, suite, protocolVersion, edek, logRetryCache);
-      stat = FSDirStatAndListingOp.getFileInfo(dir, src, false,
-          FSDirectory.isReservedRawName(srcArg), true);
+      dir.writeLock();
+      try {
+        src = dir.resolvePath(pc, src, pathComponents);
+        final INodesInPath iip = dir.getINodesInPath4Write(src);
+        toRemoveBlocks = startFileInternal(
+            pc, iip, permissions, holder,
+            clientMachine, create, overwrite,
+            createParent, replication, blockSize,
+            isLazyPersist, suite, protocolVersion, edek,
+            logRetryCache);
+        stat = FSDirStatAndListingOp.getFileInfo(
+            dir, src, false, FSDirectory.isReservedRawName(srcArg), true);
+      } finally {
+        dir.writeUnlock();
+      }
     } catch (StandbyException se) {
       skipSync = true;
       throw se;
@@ -2523,14 +2489,17 @@ private BlocksMapUpdateInfo startFileInternal(FSPermissionChecker pc,
         if (overwrite) {
           toRemoveBlocks = new BlocksMapUpdateInfo();
           List<INode> toRemoveINodes = new ChunkedArrayList<INode>();
-          long ret = dir.delete(src, toRemoveBlocks, toRemoveINodes, now());
+          long ret = FSDirDeleteOp.delete(dir, iip, toRemoveBlocks,
+                                          toRemoveINodes, now());
           if (ret >= 0) {
-            incrDeletedFileCount(ret);
-            removePathAndBlocks(src, null, toRemoveINodes, true);
+            iip = INodesInPath.replace(iip, iip.length() - 1, null);
+            FSDirDeleteOp.incrDeletedFileCount(ret);
+            removeLeasesAndINodes(src, toRemoveINodes, true);
           }
         } else {
           // If lease soft limit time is expired, recover the lease
-          recoverLeaseInternal(myFile, src, holder, clientMachine, false);
+          recoverLeaseInternal(RecoverLeaseOp.CREATE_FILE,
+              iip, src, holder, clientMachine, false);
           throw new FileAlreadyExistsException(src + " for client " +
               clientMachine + " already exists");
         }
@@ -2540,11 +2509,12 @@ private BlocksMapUpdateInfo startFileInternal(FSPermissionChecker pc,
       INodeFile newNode = null;
 
       // Always do an implicit mkdirs for parent directory tree.
-      Path parent = new Path(src).getParent();
-      if (parent != null && FSDirMkdirOp.mkdirsRecursively(dir,
-          parent.toString(), permissions, true, now())) {
-        newNode = dir.addFile(src, permissions, replication, blockSize,
-                              holder, clientMachine);
+      Map.Entry<INodesInPath, String> parent = FSDirMkdirOp
+          .createAncestorDirectories(dir, iip, permissions);
+      if (parent != null) {
+        iip = dir.addFile(parent.getKey(), parent.getValue(), permissions,
+            replication, blockSize, holder, clientMachine);
+        newNode = iip != null ? iip.getLastINode().asFile() : null;
       }
 
       if (newNode == null) {
@@ -2619,12 +2589,12 @@ private void setNewINodeStoragePolicy(INodeFile inode,
    * <p>
    * 
    * For description of parameters and exceptions thrown see
-   * {@link ClientProtocol#append(String, String)}
-   * 
+   * {@link ClientProtocol#append(String, String, EnumSetWritable)}
+   *
    * @return the last block locations if the block is partial or null otherwise
    */
   private LocatedBlock appendFileInternal(FSPermissionChecker pc,
-      INodesInPath iip, String holder, String clientMachine,
+      INodesInPath iip, String holder, String clientMachine, boolean newBlock,
       boolean logRetryCache) throws IOException {
     assert hasWriteLock();
     // Verify that the destination does not exist as a directory already.
@@ -2646,28 +2616,24 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc,
       INodeFile myFile = INodeFile.valueOf(inode, src, true);
       final BlockStoragePolicy lpPolicy =
           blockManager.getStoragePolicy("LAZY_PERSIST");
-
       if (lpPolicy != null &&
           lpPolicy.getId() == myFile.getStoragePolicyID()) {
         throw new UnsupportedOperationException(
             "Cannot append to lazy persist file " + src);
       }
-      // Opening an existing file for write - may need to recover lease.
-      recoverLeaseInternal(myFile, src, holder, clientMachine, false);
+      // Opening an existing file for append - may need to recover lease.
+      recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE,
+          iip, src, holder, clientMachine, false);
       
-      // recoverLeaseInternal may create a new InodeFile via 
-      // finalizeINodeFileUnderConstruction so we need to refresh 
-      // the referenced file.  
-      myFile = INodeFile.valueOf(dir.getINode(src), src, true);
-      final BlockInfo lastBlock = myFile.getLastBlock();
+      final BlockInfoContiguous lastBlock = myFile.getLastBlock();
       // Check that the block has at least minimum replication.
       if(lastBlock != null && lastBlock.isComplete() &&
           !getBlockManager().isSufficientlyReplicated(lastBlock)) {
         throw new IOException("append: lastBlock=" + lastBlock +
             " of src=" + src + " is not sufficiently replicated yet.");
       }
-      return prepareFileForWrite(src, myFile, holder, clientMachine, true,
-              iip.getLatestSnapshotId(), logRetryCache);
+      return prepareFileForAppend(src, iip, holder, clientMachine, newBlock,
+          true, logRetryCache);
     } catch (IOException ie) {
       NameNode.stateChangeLog.warn("DIR* NameSystem.append: " +ie.getMessage());
       throw ie;
@@ -2675,13 +2641,13 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc,
   }
   
   /**
-   * Replace current node with a INodeUnderConstruction.
+   * Convert current node to under construction.
    * Recreate in-memory lease record.
    * 
    * @param src path to the file
-   * @param file existing file object
    * @param leaseHolder identifier of the lease holder on this file
    * @param clientMachine identifier of the client machine
+   * @param newBlock if the data is appended to a new block
    * @param writeToEditLog whether to persist this change to the edit log
    * @param logRetryCache whether to record RPC ids in editlog for retry cache
    *                      rebuilding
@@ -2689,26 +2655,34 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc,
    * @throws UnresolvedLinkException
    * @throws IOException
    */
-  LocatedBlock prepareFileForWrite(String src, INodeFile file,
-                                   String leaseHolder, String clientMachine,
-                                   boolean writeToEditLog,
-                                   int latestSnapshot, boolean logRetryCache)
-      throws IOException {
-    file.recordModification(latestSnapshot);
-    final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine);
+  LocatedBlock prepareFileForAppend(String src, INodesInPath iip,
+      String leaseHolder, String clientMachine, boolean newBlock,
+      boolean writeToEditLog, boolean logRetryCache) throws IOException {
+    final INodeFile file = iip.getLastINode().asFile();
+    file.recordModification(iip.getLatestSnapshotId());
+    file.toUnderConstruction(leaseHolder, clientMachine);
 
-    leaseManager.addLease(cons.getFileUnderConstructionFeature()
-        .getClientName(), src);
-    
-    LocatedBlock ret = blockManager.convertLastBlockToUnderConstruction(cons);
-    if (ret != null) {
-      // update the quota: use the preferred block size for UC block
-      final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
-      dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication());
+    leaseManager.addLease(
+        file.getFileUnderConstructionFeature().getClientName(), src);
+
+    LocatedBlock ret = null;
+    if (!newBlock) {
+      ret = blockManager.convertLastBlockToUnderConstruction(file, 0);
+      if (ret != null) {
+        // update the quota: use the preferred block size for UC block
+        final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
+        dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
+      }
+    } else {
+      BlockInfoContiguous lastBlock = file.getLastBlock();
+      if (lastBlock != null) {
+        ExtendedBlock blk = new ExtendedBlock(this.getBlockPoolId(), lastBlock);
+        ret = new LocatedBlock(blk, new DatanodeInfo[0]);
+      }
     }
 
     if (writeToEditLog) {
-      getEditLog().logOpenFile(src, cons, false, logRetryCache);
+      getEditLog().logAppendFile(src, file, newBlock, logRetryCache);
     }
     return ret;
   }
@@ -2748,7 +2722,8 @@ boolean recoverLease(String src, String holder, String clientMachine)
         dir.checkPathAccess(pc, iip, FsAction.WRITE);
       }
   
-      recoverLeaseInternal(inode, src, holder, clientMachine, true);
+      recoverLeaseInternal(RecoverLeaseOp.RECOVER_LEASE,
+          iip, src, holder, clientMachine, true);
     } catch (StandbyException se) {
       skipSync = true;
       throw se;
@@ -2763,48 +2738,58 @@ boolean recoverLease(String src, String holder, String clientMachine)
     return false;
   }
 
-  private void recoverLeaseInternal(INodeFile fileInode, 
+  private enum RecoverLeaseOp {
+    CREATE_FILE,
+    APPEND_FILE,
+    TRUNCATE_FILE,
+    RECOVER_LEASE;
+    
+    private String getExceptionMessage(String src, String holder,
+        String clientMachine, String reason) {
+      return "Failed to " + this + " " + src + " for " + holder +
+          " on " + clientMachine + " because " + reason;
+    }
+  }
+
+  void recoverLeaseInternal(RecoverLeaseOp op, INodesInPath iip,
       String src, String holder, String clientMachine, boolean force)
       throws IOException {
     assert hasWriteLock();
-    if (fileInode != null && fileInode.isUnderConstruction()) {
+    INodeFile file = iip.getLastINode().asFile();
+    if (file != null && file.isUnderConstruction()) {
       //
       // If the file is under construction , then it must be in our
       // leases. Find the appropriate lease record.
       //
       Lease lease = leaseManager.getLease(holder);
-      //
-      // We found the lease for this file. And surprisingly the original
-      // holder is trying to recreate this file. This should never occur.
-      //
 
       if (!force && lease != null) {
         Lease leaseFile = leaseManager.getLeaseByPath(src);
         if (leaseFile != null && leaseFile.equals(lease)) {
+          // We found the lease for this file but the original
+          // holder is trying to obtain it again.
           throw new AlreadyBeingCreatedException(
-            "failed to create file " + src + " for " + holder +
-            " for client " + clientMachine +
-            " because current leaseholder is trying to recreate file.");
+              op.getExceptionMessage(src, holder, clientMachine,
+                  holder + " is already the current lease holder."));
         }
       }
       //
       // Find the original holder.
       //
-      FileUnderConstructionFeature uc = fileInode.getFileUnderConstructionFeature();
+      FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
       String clientName = uc.getClientName();
       lease = leaseManager.getLease(clientName);
       if (lease == null) {
         throw new AlreadyBeingCreatedException(
-          "failed to create file " + src + " for " + holder +
-          " for client " + clientMachine +
-          " because pendingCreates is non-null but no leases found.");
+            op.getExceptionMessage(src, holder, clientMachine,
+                "the file is under construction but no leases found."));
       }
       if (force) {
         // close now: no need to wait for soft lease expiration and 
         // close only the file src
         LOG.info("recoverLease: " + lease + ", src=" + src +
           " from client " + clientName);
-        internalReleaseLease(lease, src, holder);
+        internalReleaseLease(lease, src, iip, holder);
       } else {
         assert lease.getHolder().equals(clientName) :
           "Current lease holder " + lease.getHolder() +
@@ -2816,23 +2801,24 @@ private void recoverLeaseInternal(INodeFile fileInode,
         if (lease.expiredSoftLimit()) {
           LOG.info("startFile: recover " + lease + ", src=" + src + " client "
               + clientName);
-          boolean isClosed = internalReleaseLease(lease, src, null);
+          boolean isClosed = internalReleaseLease(lease, src, iip, null);
           if(!isClosed)
             throw new RecoveryInProgressException(
-                "Failed to close file " + src +
-                ". Lease recovery is in progress. Try again later.");
+                op.getExceptionMessage(src, holder, clientMachine,
+                    "lease recovery is in progress. Try again later."));
         } else {
-          final BlockInfo lastBlock = fileInode.getLastBlock();
+          final BlockInfoContiguous lastBlock = file.getLastBlock();
           if (lastBlock != null
               && lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
-            throw new RecoveryInProgressException("Recovery in progress, file ["
-                + src + "], " + "lease owner [" + lease.getHolder() + "]");
+            throw new RecoveryInProgressException(
+                op.getExceptionMessage(src, holder, clientMachine,
+                    "another recovery is in progress by "
+                        + clientName + " on " + uc.getClientMachine()));
           } else {
-            throw new AlreadyBeingCreatedException("Failed to create file ["
-                + src + "] for [" + holder + "] for client [" + clientMachine
-                + "], because this file is already being created by ["
-                + clientName + "] on ["
-                + uc.getClientMachine() + "]");
+            throw new AlreadyBeingCreatedException(
+                op.getExceptionMessage(src, holder, clientMachine,
+                    "this file lease is currently owned by "
+                        + clientName + " on " + uc.getClientMachine()));
           }
         }
       }
@@ -2842,11 +2828,12 @@ private void recoverLeaseInternal(INodeFile fileInode,
   /**
    * Append to an existing file in the namespace.
    */
-  LastBlockWithStatus appendFile(
-      String src, String holder, String clientMachine, boolean logRetryCache)
+  LastBlockWithStatus appendFile(String src, String holder,
+      String clientMachine, EnumSet<CreateFlag> flag, boolean logRetryCache)
       throws IOException {
     try {
-      return appendFileInt(src, holder, clientMachine, logRetryCache);
+      return appendFileInt(src, holder, clientMachine,
+          flag.contains(CreateFlag.NEW_BLOCK), logRetryCache);
     } catch (AccessControlException e) {
       logAuditEvent(false, "append", src);
       throw e;
@@ -2854,10 +2841,8 @@ LastBlockWithStatus appendFile(
   }
 
   private LastBlockWithStatus appendFileInt(final String srcArg, String holder,
-      String clientMachine, boolean logRetryCache)
-      throws AccessControlException, SafeModeException,
-      FileAlreadyExistsException, FileNotFoundException,
-      ParentNotDirectoryException, IOException {
+      String clientMachine, boolean newBlock, boolean logRetryCache)
+      throws IOException {
     String src = srcArg;
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: src=" + src
@@ -2876,7 +2861,8 @@ private LastBlockWithStatus appendFileInt(final String srcArg, String holder,
       checkNameNodeSafeMode("Cannot append to file" + src);
       src = dir.resolvePath(pc, src, pathComponents);
       final INodesInPath iip = dir.getINodesInPath4Write(src);
-      lb = appendFileInternal(pc, iip, holder, clientMachine, logRetryCache);
+      lb = appendFileInternal(pc, iip, holder, clientMachine, newBlock,
+          logRetryCache);
       stat = FSDirStatAndListingOp.getFileInfo(dir, src, false,
           FSDirectory.isReservedRawName(srcArg), true);
     } catch (StandbyException se) {
@@ -2924,13 +2910,11 @@ void setBlockPoolId(String bpid) {
    */
   LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
       ExtendedBlock previous, Set<Node> excludedNodes, 
-      List<String> favoredNodes)
-      throws LeaseExpiredException, NotReplicatedYetException,
-      QuotaExceededException, SafeModeException, UnresolvedLinkException,
-      IOException {
+      List<String> favoredNodes) throws IOException {
     final long blockSize;
-    final int replication;
+    final short numTargets;
     final byte storagePolicyID;
+    final boolean isStriped;
     Node clientNode = null;
     String clientMachine = null;
 
@@ -2968,7 +2952,11 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
           .getClientMachine();
       clientNode = blockManager.getDatanodeManager().getDatanodeByHost(
           clientMachine);
-      replication = pendingFile.getFileReplication();
+      // TODO: make block group size configurable (HDFS-7337)
+      isStriped = pendingFile.isStriped();
+      numTargets = isStriped ?
+          HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS :
+          pendingFile.getFileReplication();
       storagePolicyID = pendingFile.getStoragePolicyID();
     } finally {
       readUnlock();
@@ -2980,7 +2968,7 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
 
     // choose targets for the new block to be allocated.
     final DatanodeStorageInfo targets[] = getBlockManager().chooseTarget4NewBlock( 
-        src, replication, clientNode, excludedNodes, blockSize, favoredNodes,
+        src, numTargets, clientNode, excludedNodes, blockSize, favoredNodes,
         storagePolicyID);
 
     // Part II.
@@ -3006,8 +2994,8 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
           return onRetryBlock[0];
         } else {
           // add new chosen targets to already allocated block and return
-          BlockInfo lastBlockInFile = pendingFile.getLastBlock();
-          ((BlockInfoUnderConstruction) lastBlockInFile)
+          BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
+          ((BlockInfoContiguousUnderConstruction) lastBlockInFile)
               .setExpectedLocations(targets);
           offset = pendingFile.computeFileSize();
           return makeLocatedBlock(lastBlockInFile, targets, offset);
@@ -3015,13 +3003,13 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
       }
 
       // commit the last block and complete it if it has minimum replicas
-      commitOrCompleteLastBlock(pendingFile,
+      commitOrCompleteLastBlock(pendingFile, fileState.iip,
                                 ExtendedBlock.getLocalBlock(previous));
 
       // allocate new block, record block locations in INode.
-      newBlock = createNewBlock();
+      newBlock = createNewBlock(isStriped);
       INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
-      saveAllocatedBlock(src, inodesInPath, newBlock, targets);
+      saveAllocatedBlock(src, inodesInPath, newBlock, targets, isStriped);
 
       persistNewBlock(src, pendingFile);
       offset = pendingFile.computeFileSize();
@@ -3055,10 +3043,12 @@ private Node getClientNode(String clientMachine) {
   static class FileState {
     public final INodeFile inode;
     public final String path;
+    public final INodesInPath iip;
 
-    public FileState(INodeFile inode, String fullPath) {
+    public FileState(INodeFile inode, String fullPath, INodesInPath iip) {
       this.inode = inode;
       this.path = fullPath;
+      this.iip = iip;
     }
   }
 
@@ -3078,21 +3068,25 @@ FileState analyzeFileState(String src,
     checkFsObjectLimit();
 
     Block previousBlock = ExtendedBlock.getLocalBlock(previous);
-    INode inode;
+    final INode inode;
+    final INodesInPath iip;
     if (fileId == INodeId.GRANDFATHER_INODE_ID) {
       // Older clients may not have given us an inode ID to work with.
       // In this case, we have to try to resolve the path and hope it
       // hasn't changed or been deleted since the file was opened for write.
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
+      iip = dir.getINodesInPath4Write(src);
       inode = iip.getLastINode();
     } else {
       // Newer clients pass the inode ID, so we can just get the inode
       // directly.
       inode = dir.getInode(fileId);
-      if (inode != null) src = inode.getFullPathName();
+      iip = INodesInPath.fromINode(inode);
+      if (inode != null) {
+        src = iip.getPath();
+      }
     }
     final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
-    BlockInfo lastBlockInFile = pendingFile.getLastBlock();
+    BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
     if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
       // The block that the client claims is the current last block
       // doesn't match up with what we think is the last block. There are
@@ -3120,7 +3114,7 @@ FileState analyzeFileState(String src,
       //    changed the namesystem state yet.
       //    We run this analysis again in Part II where case 4 is impossible.
 
-      BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
+      BlockInfoContiguous penultimateBlock = pendingFile.getPenultimateBlock();
       if (previous == null &&
           lastBlockInFile != null &&
           lastBlockInFile.getNumBytes() == pendingFile.getPreferredBlockSize() &&
@@ -3147,9 +3141,9 @@ FileState analyzeFileState(String src,
             src + ". Returning previously allocated block " + lastBlockInFile);
         long offset = pendingFile.computeFileSize();
         onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
-            ((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
+            ((BlockInfoContiguousUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
             offset);
-        return new FileState(pendingFile, src);
+        return new FileState(pendingFile, src, iip);
       } else {
         // Case 3
         throw new IOException("Cannot allocate block in " + src + ": " +
@@ -3162,7 +3156,7 @@ FileState analyzeFileState(String src,
     if (!checkFileProgress(src, pendingFile, false)) {
       throw new NotReplicatedYetException("Not replicated yet: " + src);
     }
-    return new FileState(pendingFile, src);
+    return new FileState(pendingFile, src, iip);
   }
 
   LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
@@ -3240,8 +3234,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId,
    * The client would like to let go of the given block
    */
   boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
-      throws LeaseExpiredException, FileNotFoundException,
-      UnresolvedLinkException, IOException {
+      throws IOException {
     if(NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: " + b
           + "of file " + src);
@@ -3257,21 +3250,24 @@ boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
       src = dir.resolvePath(pc, src, pathComponents);
 
       final INode inode;
+      final INodesInPath iip;
       if (fileId == INodeId.GRANDFATHER_INODE_ID) {
         // Older clients may not have given us an inode ID to work with.
         // In this case, we have to try to resolve the path and hope it
         // hasn't changed or been deleted since the file was opened for write.
-        inode = dir.getINode(src);
+        iip = dir.getINodesInPath(src, true);
+        inode = iip.getLastINode();
       } else {
         inode = dir.getInode(fileId);
-        if (inode != null) src = inode.getFullPathName();
+        iip = INodesInPath.fromINode(inode);
+        if (inode != null) {
+          src = iip.getPath();
+        }
       }
       final INodeFile file = checkLease(src, holder, inode, fileId);
 
-      //
       // Remove the block from the pending creates list
-      //
-      boolean removed = dir.removeBlock(src, file,
+      boolean removed = dir.removeBlock(src, iip, file,
           ExtendedBlock.getLocalBlock(b));
       if (!removed) {
         return true;
@@ -3290,8 +3286,7 @@ boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
   }
 
   private INodeFile checkLease(String src, String holder, INode inode,
-                               long fileId)
-      throws LeaseExpiredException, FileNotFoundException {
+      long fileId) throws LeaseExpiredException, FileNotFoundException {
     assert hasReadLock();
     final String ident = src + " (inode " + fileId + ")";
     if (inode == null) {
@@ -3368,29 +3363,30 @@ boolean completeFile(final String srcArg, String holder,
     return success;
   }
 
-  private boolean completeFileInternal(String src, 
-      String holder, Block last, long fileId) throws SafeModeException,
-      UnresolvedLinkException, IOException {
+  private boolean completeFileInternal(String src, String holder, Block last,
+      long fileId) throws IOException {
     assert hasWriteLock();
     final INodeFile pendingFile;
+    final INodesInPath iip;
+    INode inode = null;
     try {
-      final INode inode;
       if (fileId == INodeId.GRANDFATHER_INODE_ID) {
         // Older clients may not have given us an inode ID to work with.
         // In this case, we have to try to resolve the path and hope it
         // hasn't changed or been deleted since the file was opened for write.
-        final INodesInPath iip = dir.getLastINodeInPath(src);
-        inode = iip.getINode(0);
+        iip = dir.getINodesInPath(src, true);
+        inode = iip.getLastINode();
       } else {
         inode = dir.getInode(fileId);
-        if (inode != null) src = inode.getFullPathName();
+        iip = INodesInPath.fromINode(inode);
+        if (inode != null) {
+          src = iip.getPath();
+        }
       }
       pendingFile = checkLease(src, holder, inode, fileId);
     } catch (LeaseExpiredException lee) {
-      final INode inode = dir.getINode(src);
-      if (inode != null
-          && inode.isFile()
-          && !inode.asFile().isUnderConstruction()) {
+      if (inode != null && inode.isFile() &&
+          !inode.asFile().isUnderConstruction()) {
         // This could be a retry RPC - i.e the client tried to close
         // the file, but missed the RPC response. Thus, it is trying
         // again to close the file. If the file still exists and
@@ -3415,7 +3411,7 @@ private boolean completeFileInternal(String src,
     }
 
     // commit the last block and complete it if it has minimum replicas
-    commitOrCompleteLastBlock(pendingFile, last);
+    commitOrCompleteLastBlock(pendingFile, iip, last);
 
     if (!checkFileProgress(src, pendingFile, true)) {
       return false;
@@ -3434,13 +3430,16 @@ private boolean completeFileInternal(String src,
    *                     The last INode is the INode for {@code src} file.
    * @param newBlock newly allocated block to be save
    * @param targets target datanodes where replicas of the new block is placed
+   * @param isStriped is the file under striping or contigunous layout?
    * @throws QuotaExceededException If addition of block exceeds space quota
    */
-  BlockInfo saveAllocatedBlock(String src, INodesInPath inodesInPath,
-      Block newBlock, DatanodeStorageInfo[] targets)
+  BlockInfoContiguous saveAllocatedBlock(String src, INodesInPath inodesInPath,
+                               Block newBlock, DatanodeStorageInfo[] targets,
+                               boolean isStriped)
           throws IOException {
     assert hasWriteLock();
-    BlockInfo b = dir.addBlock(src, inodesInPath, newBlock, targets);
+    BlockInfoContiguous b = dir.addBlock(src, inodesInPath,
+        newBlock, targets, isStriped);
     NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
     DatanodeStorageInfo.incrementBlocksScheduled(targets);
     return b;
@@ -3448,10 +3447,11 @@ BlockInfo saveAllocatedBlock(String src, INodesInPath inodesInPath,
 
   /**
    * Create new block with a unique block id and a new generation stamp.
+   * @param isStriped is the file under striping or contiguous layout?
    */
-  Block createNewBlock() throws IOException {
+  Block createNewBlock(boolean isStriped) throws IOException {
     assert hasWriteLock();
-    Block b = new Block(nextBlockId(), 0, 0);
+    Block b = new Block(nextBlockId(isStriped), 0, 0);
     // Increment the generation stamp for every new block.
     b.setGenerationStamp(nextGenerationStamp(false));
     return b;
@@ -3467,14 +3467,14 @@ private boolean checkFileProgress(String src, INodeFile v, boolean checkall) {
     try {
       if (checkall) {
         // check all blocks of the file.
-        for (BlockInfo block: v.getBlocks()) {
+        for (BlockInfoContiguous block: v.getBlocks()) {
           if (!isCompleteBlock(src, block, blockManager.minReplication)) {
             return false;
           }
         }
       } else {
         // check the penultimate block of this file
-        BlockInfo b = v.getPenultimateBlock();
+        BlockInfoContiguous b = v.getPenultimateBlock();
         if (b != null
             && !isCompleteBlock(src, b, blockManager.minReplication)) {
           return false;
@@ -3486,9 +3486,9 @@ private boolean checkFileProgress(String src, INodeFile v, boolean checkall) {
     }
   }
 
-  private static boolean isCompleteBlock(String src, BlockInfo b, int minRepl) {
+  private static boolean isCompleteBlock(String src, BlockInfoContiguous b, int minRepl) {
     if (!b.isComplete()) {
-      final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)b;
+      final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)b;
       final int numNodes = b.numNodes();
       LOG.info("BLOCK* " + b + " is not COMPLETE (ucState = "
           + uc.getBlockUCState() + ", replication# = " + numNodes
@@ -3580,99 +3580,35 @@ void renameTo(final String src, final String dst,
    * description of exceptions
    */
   boolean delete(String src, boolean recursive, boolean logRetryCache)
-      throws AccessControlException, SafeModeException,
-      UnresolvedLinkException, IOException {
-
+      throws IOException {
+    waitForLoadingFSImage();
+    checkOperation(OperationCategory.WRITE);
+    BlocksMapUpdateInfo toRemovedBlocks = null;
+    writeLock();
     boolean ret = false;
     try {
-      ret = deleteInt(src, recursive, logRetryCache);
+      checkOperation(OperationCategory.WRITE);
+      checkNameNodeSafeMode("Cannot delete " + src);
+      toRemovedBlocks = FSDirDeleteOp.delete(
+          this, src, recursive, logRetryCache);
+      ret = toRemovedBlocks != null;
     } catch (AccessControlException e) {
       logAuditEvent(false, "delete", src);
       throw e;
+    } finally {
+      writeUnlock();
     }
-    return ret;
-  }
-      
-  private boolean deleteInt(String src, boolean recursive, boolean logRetryCache)
-      throws AccessControlException, SafeModeException,
-      UnresolvedLinkException, IOException {
-    if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
-    }
-    boolean status = deleteInternal(src, recursive, true, logRetryCache);
-    if (status) {
-      logAuditEvent(true, "delete", src);
+    if (toRemovedBlocks != null) {
+      removeBlocks(toRemovedBlocks); // Incremental deletion of blocks
     }
-    return status;
+    logAuditEvent(true, "delete", src);
+    return ret;
   }
-    
+
   FSPermissionChecker getPermissionChecker()
       throws AccessControlException {
     return dir.getPermissionChecker();
   }
-  
-  /**
-   * Remove a file/directory from the namespace.
-   * <p>
-   * For large directories, deletion is incremental. The blocks under
-   * the directory are collected and deleted a small number at a time holding
-   * the {@link FSNamesystem} lock.
-   * <p>
-   * For small directory or file the deletion is done in one shot.
-   * 
-   * @see ClientProtocol#delete(String, boolean) for description of exceptions
-   */
-  private boolean deleteInternal(String src, boolean recursive,
-      boolean enforcePermission, boolean logRetryCache)
-      throws AccessControlException, SafeModeException, UnresolvedLinkException,
-             IOException {
-    BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
-    List<INode> removedINodes = new ChunkedArrayList<INode>();
-    FSPermissionChecker pc = getPermissionChecker();
-    checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
-    boolean ret = false;
-
-    waitForLoadingFSImage();
-    writeLock();
-    try {
-      checkOperation(OperationCategory.WRITE);
-      checkNameNodeSafeMode("Cannot delete " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src, false);
-      if (!recursive && dir.isNonEmptyDirectory(iip)) {
-        throw new PathIsNotEmptyDirectoryException(src + " is non empty");
-      }
-      if (enforcePermission && isPermissionEnabled) {
-        dir.checkPermission(pc, iip, false, null, FsAction.WRITE, null,
-            FsAction.ALL, true);
-      }
-
-      long mtime = now();
-      // Unlink the target directory from directory tree
-      long filesRemoved = dir.delete(src, collectedBlocks, removedINodes,
-              mtime);
-      if (filesRemoved < 0) {
-        return false;
-      }
-      getEditLog().logDelete(src, mtime, logRetryCache);
-      incrDeletedFileCount(filesRemoved);
-      // Blocks/INodes will be handled later
-      removePathAndBlocks(src, null, removedINodes, true);
-      ret = true;
-    } finally {
-      writeUnlock();
-    }
-    getEditLog().logSync(); 
-    removeBlocks(collectedBlocks); // Incremental deletion of blocks
-    collectedBlocks.clear();
-
-    if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "
-        + src +" is removed");
-    }
-    return ret;
-  }
 
   /**
    * From the given list, incrementally remove the blocks from blockManager
@@ -3699,15 +3635,14 @@ void removeBlocks(BlocksMapUpdateInfo blocks) {
   }
   
   /**
-   * Remove leases, inodes and blocks related to a given path
+   * Remove leases and inodes related to a given path
    * @param src The given path
-   * @param blocks Containing the list of blocks to be deleted from blocksMap
-   * @param removedINodes Containing the list of inodes to be removed from 
+   * @param removedINodes Containing the list of inodes to be removed from
    *                      inodesMap
    * @param acquireINodeMapLock Whether to acquire the lock for inode removal
    */
-  void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks,
-      List<INode> removedINodes, final boolean acquireINodeMapLock) {
+  void removeLeasesAndINodes(String src, List<INode> removedINodes,
+      final boolean acquireINodeMapLock) {
     assert hasWriteLock();
     leaseManager.removeLeaseWithPrefixPath(src);
     // remove inodes from inodesMap
@@ -3724,11 +3659,6 @@ void removePathAndBlocks(String src, BlocksMapUpdateInfo blocks,
       }
       removedINodes.clear();
     }
-    if (blocks == null) {
-      return;
-    }
-    
-    removeBlocksAndUpdateSafemodeTotal(blocks);
   }
 
   /**
@@ -3748,7 +3678,7 @@ void removeBlocksAndUpdateSafemodeTotal(BlocksMapUpdateInfo blocks) {
 
     for (Block b : blocks.getToDeleteList()) {
       if (trackBlockCounts) {
-        BlockInfo bi = getStoredBlock(b);
+        BlockInfoContiguous bi = getStoredBlock(b);
         if (bi.isComplete()) {
           numRemovedComplete++;
           if (bi.numNodes() >= blockManager.minReplication) {
@@ -3887,20 +3817,14 @@ ContentSummary getContentSummary(final String src) throws IOException {
    * 
    * Note: This does not support ".inodes" relative path.
    */
-  void setQuota(String path, long nsQuota, long dsQuota)
-      throws IOException, UnresolvedLinkException {
-    checkSuperuserPrivilege();
+  void setQuota(String src, long nsQuota, long dsQuota)
+      throws IOException {
     checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
-      checkNameNodeSafeMode("Cannot set quota on " + path);
-      INodeDirectory changed = dir.setQuota(path, nsQuota, dsQuota);
-      if (changed != null) {
-        final Quota.Counts q = changed.getQuotaCounts();
-        getEditLog().logSetQuota(path,
-                q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE));
-      }
+      checkNameNodeSafeMode("Cannot set quota on " + src);
+      FSDirAttrOp.setQuota(dir, src, nsQuota, dsQuota);
     } finally {
       writeUnlock();
     }
@@ -3917,7 +3841,7 @@ void setQuota(String path, long nsQuota, long dsQuota)
    * @throws IOException if path does not exist
    */
   void fsync(String src, long fileId, String clientName, long lastBlockLength)
-      throws IOException, UnresolvedLinkException {
+      throws IOException {
     NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
     checkOperation(OperationCategory.WRITE);
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
@@ -3965,20 +3889,18 @@ void fsync(String src, long fileId, String clientName, long lastBlockLength)
    *         false if block recovery has been initiated. Since the lease owner
    *         has been changed and logged, caller should call logSync().
    */
-  boolean internalReleaseLease(Lease lease, String src, 
-      String recoveryLeaseHolder) throws AlreadyBeingCreatedException, 
-      IOException, UnresolvedLinkException {
+  boolean internalReleaseLease(Lease lease, String src, INodesInPath iip,
+      String recoveryLeaseHolder) throws IOException {
     LOG.info("Recovering " + lease + ", src=" + src);
     assert !isInSafeMode();
     assert hasWriteLock();
 
-    final INodesInPath iip = dir.getLastINodeInPath(src);
-    final INodeFile pendingFile = iip.getINode(0).asFile();
+    final INodeFile pendingFile = iip.getLastINode().asFile();
     int nrBlocks = pendingFile.numBlocks();
-    BlockInfo[] blocks = pendingFile.getBlocks();
+    BlockInfoContiguous[] blocks = pendingFile.getBlocks();
 
     int nrCompleteBlocks;
-    BlockInfo curBlock = null;
+    BlockInfoContiguous curBlock = null;
     for(nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
       curBlock = blocks[nrCompleteBlocks];
       if(!curBlock.isComplete())
@@ -4013,9 +3935,9 @@ boolean internalReleaseLease(Lease lease, String src,
 
     // The last block is not COMPLETE, and
     // that the penultimate block if exists is either COMPLETE or COMMITTED
-    final BlockInfo lastBlock = pendingFile.getLastBlock();
+    final BlockInfoContiguous lastBlock = pendingFile.getLastBlock();
     BlockUCState lastBlockState = lastBlock.getBlockUCState();
-    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
+    BlockInfoContiguous penultimateBlock = pendingFile.getPenultimateBlock();
 
     // If penultimate block doesn't exist then its minReplication is met
     boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
@@ -4048,7 +3970,18 @@ boolean internalReleaseLease(Lease lease, String src,
       throw new AlreadyBeingCreatedException(message);
     case UNDER_CONSTRUCTION:
     case UNDER_RECOVERY:
-      final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock;
+      final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)lastBlock;
+      // determine if last block was intended to be truncated
+      Block recoveryBlock = uc.getTruncateBlock();
+      boolean truncateRecovery = recoveryBlock != null;
+      boolean copyOnTruncate = truncateRecovery &&
+          recoveryBlock.getBlockId() != uc.getBlockId();
+      assert !copyOnTruncate ||
+          recoveryBlock.getBlockId() < uc.getBlockId() &&
+          recoveryBlock.getGenerationStamp() < uc.getGenerationStamp() &&
+          recoveryBlock.getNumBytes() > uc.getNumBytes() :
+            "wrong recoveryBlock";
+
       // setup the last block locations from the blockManager if not known
       if (uc.getNumExpectedLocations() == 0) {
         uc.setExpectedLocations(blockManager.getStorages(lastBlock));
@@ -4069,6 +4002,11 @@ boolean internalReleaseLease(Lease lease, String src,
       // start recovery of the last block for this file
       long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
       lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
+      if(copyOnTruncate) {
+        uc.setGenerationStamp(blockRecoveryId);
+      } else if(truncateRecovery) {
+        recoveryBlock.setGenerationStamp(blockRecoveryId);
+      }
       uc.initializeBlockRecovery(blockRecoveryId);
       leaseManager.renewLease(lease);
       // Cannot close file right now, since the last block requires recovery.
@@ -4102,7 +4040,7 @@ Lease reassignLeaseInternal(Lease lease, String src, String newHolder,
   }
 
   private void commitOrCompleteLastBlock(final INodeFile fileINode,
-      final Block commitBlock) throws IOException {
+      final INodesInPath iip, final Block commitBlock) throws IOException {
     assert hasWriteLock();
     Preconditions.checkArgument(fileINode.isUnderConstruction());
     if (!blockManager.commitOrCompleteLastBlock(fileINode, commitBlock)) {
@@ -4113,8 +4051,7 @@ private void commitOrCompleteLastBlock(final INodeFile fileINode,
     final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();    
     if (diff > 0) {
       try {
-        String path = fileINode.getFullPathName();
-        dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
+        dir.updateSpaceConsumed(iip, 0, -diff*fileINode.getFileReplication());
       } catch (IOException e) {
         LOG.warn("Unexpected exception while updating disk space.", e);
       }
@@ -4122,8 +4059,7 @@ private void commitOrCompleteLastBlock(final INodeFile fileINode,
   }
 
   private void finalizeINodeFileUnderConstruction(String src,
-      INodeFile pendingFile, int latestSnapshot) throws IOException,
-      UnresolvedLinkException {
+      INodeFile pendingFile, int latestSnapshot) throws IOException {
     assert hasWriteLock();
 
     FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
@@ -4135,22 +4071,22 @@ private void finalizeINodeFileUnderConstruction(String src,
     // The file is no longer pending.
     // Create permanent INode, update blocks. No need to replace the inode here
     // since we just remove the uc feature from pendingFile
-    final INodeFile newFile = pendingFile.toCompleteFile(now());
+    pendingFile.toCompleteFile(now());
 
     waitForLoadingFSImage();
     // close file and persist block allocations for this file
-    closeFile(src, newFile);
+    closeFile(src, pendingFile);
 
-    blockManager.checkReplication(newFile);
+    blockManager.checkReplication(pendingFile);
   }
 
   @VisibleForTesting
-  BlockInfo getStoredBlock(Block block) {
+  BlockInfoContiguous getStoredBlock(Block block) {
     return blockManager.getStoredBlock(block);
   }
   
   @Override
-  public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
+  public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC) {
     assert hasReadLock();
     final BlockCollection bc = blockUC.getBlockCollection();
     if (bc == null || !(bc instanceof INodeFile)
@@ -4158,11 +4094,10 @@ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
       return false;
     }
 
-    INodeFile inodeUC = (INodeFile) bc;
-    String fullName = inodeUC.getName();
+    String fullName = bc.getName();
     try {
       if (fullName != null && fullName.startsWith(Path.SEPARATOR)
-          && dir.getINode(fullName) == inodeUC) {
+          && dir.getINode(fullName) == bc) {
         // If file exists in normal path then no need to look in snapshot
         return false;
       }
@@ -4171,7 +4106,7 @@ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
       return false;
     }
     /*
-     * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and
+     * 1. if bc is under construction and also with snapshot, and
      * bc is not in the current fsdirectory tree, bc must represent a snapshot
      * file. 
      * 2. if fullName is not an absolute path, bc cannot be existent in the 
@@ -4182,12 +4117,11 @@ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
     return true;
   }
 
-  void commitBlockSynchronization(ExtendedBlock lastblock,
+  void commitBlockSynchronization(ExtendedBlock oldBlock,
       long newgenerationstamp, long newlength,
       boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
-      String[] newtargetstorages)
-      throws IOException, UnresolvedLinkException {
-    LOG.info("commitBlockSynchronization(lastblock=" + lastblock
+      String[] newtargetstorages) throws IOException {
+    LOG.info("commitBlockSynchronization(oldBlock=" + oldBlock
              + ", newgenerationstamp=" + newgenerationstamp
              + ", newlength=" + newlength
              + ", newtargets=" + Arrays.asList(newtargets)
@@ -4205,18 +4139,18 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
   
       checkNameNodeSafeMode(
           "Cannot commitBlockSynchronization while in safe mode");
-      final BlockInfo storedBlock = getStoredBlock(
-          ExtendedBlock.getLocalBlock(lastblock));
+      final BlockInfoContiguous storedBlock = getStoredBlock(
+          ExtendedBlock.getLocalBlock(oldBlock));
       if (storedBlock == null) {
         if (deleteblock) {
           // This may be a retry attempt so ignore the failure
           // to locate the block.
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Block (=" + lastblock + ") not found");
+            LOG.debug("Block (=" + oldBlock + ") not found");
           }
           return;
         } else {
-          throw new IOException("Block (=" + lastblock + ") not found");
+          throw new IOException("Block (=" + oldBlock + ") not found");
         }
       }
       //
@@ -4243,34 +4177,40 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
             + iFile.getFullPathName() + ", likely due to delayed block"
             + " removal");
       }
-      if (!iFile.isUnderConstruction() || storedBlock.isComplete()) {
+      if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) &&
+          iFile.getLastBlock().isComplete()) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Unexpected block (=" + lastblock
+          LOG.debug("Unexpected block (=" + oldBlock
                     + ") since the file (=" + iFile.getLocalName()
                     + ") is not under construction");
         }
         return;
       }
 
-      long recoveryId =
-        ((BlockInfoUnderConstruction)storedBlock).getBlockRecoveryId();
+      BlockInfoContiguousUnderConstruction truncatedBlock =
+          (BlockInfoContiguousUnderConstruction) iFile.getLastBlock();
+      long recoveryId = truncatedBlock.getBlockRecoveryId();
+      boolean copyTruncate =
+          truncatedBlock.getBlockId() != storedBlock.getBlockId();
       if(recoveryId != newgenerationstamp) {
         throw new IOException("The recovery id " + newgenerationstamp
                               + " does not match current recovery id "
-                              + recoveryId + " for block " + lastblock); 
+                              + recoveryId + " for block " + oldBlock);
       }
 
       if (deleteblock) {
-        Block blockToDel = ExtendedBlock.getLocalBlock(lastblock);
+        Block blockToDel = ExtendedBlock.getLocalBlock(oldBlock);
         boolean remove = iFile.removeLastBlock(blockToDel);
         if (remove) {
-          blockManager.removeBlockFromMap(storedBlock);
+          blockManager.removeBlock(storedBlock);
         }
       }
       else {
         // update last block
-        storedBlock.setGenerationStamp(newgenerationstamp);
-        storedBlock.setNumBytes(newlength);
+        if(!copyTruncate) {
+          storedBlock.setGenerationStamp(newgenerationstamp);
+          storedBlock.setNumBytes(newlength);
+        }
 
         // find the DatanodeDescriptor objects
         // There should be no locations in the blockManager till now because the
@@ -4300,7 +4240,11 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
             DatanodeStorageInfo storageInfo =
                 trimmedTargets.get(i).getStorageInfo(trimmedStorages.get(i));
             if (storageInfo != null) {
-              storageInfo.addBlock(storedBlock);
+              if(copyTruncate) {
+                storageInfo.addBlock(truncatedBlock);
+              } else {
+                storageInfo.addBlock(storedBlock);
+              }
             }
           }
         }
@@ -4310,11 +4254,22 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
             blockManager.getDatanodeManager().getDatanodeStorageInfos(
                 trimmedTargets.toArray(new DatanodeID[trimmedTargets.size()]),
                 trimmedStorages.toArray(new String[trimmedStorages.size()]));
-        iFile.setLastBlock(storedBlock, trimmedStorageInfos);
+        if(copyTruncate) {
+          iFile.setLastBlock(truncatedBlock, trimmedStorageInfos);
+        } else {
+          iFile.setLastBlock(storedBlock, trimmedStorageInfos);
+        }
       }
 
       if (closeFile) {
-        src = closeFileCommitBlocks(iFile, storedBlock);
+        if(copyTruncate) {
+          src = closeFileCommitBlocks(iFile, truncatedBlock);
+          if(!iFile.isBlockInLatestSnapshot(storedBlock)) {
+            blockManager.removeBlock(storedBlock);
+          }
+        } else {
+          src = closeFileCommitBlocks(iFile, storedBlock);
+        }
       } else {
         // If this commit does not want to close the file, persist blocks
         src = iFile.getFullPathName();
@@ -4325,13 +4280,13 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
     }
     getEditLog().logSync();
     if (closeFile) {
-      LOG.info("commitBlockSynchronization(newblock=" + lastblock
+      LOG.info("commitBlockSynchronization(oldBlock=" + oldBlock
           + ", file=" + src
           + ", newgenerationstamp=" + newgenerationstamp
           + ", newlength=" + newlength
           + ", newtargets=" + Arrays.asList(newtargets) + ") successful");
     } else {
-      LOG.info("commitBlockSynchronization(" + lastblock + ") successful");
+      LOG.info("commitBlockSynchronization(" + oldBlock + ") successful");
     }
   }
 
@@ -4342,12 +4297,13 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
    * @throws IOException on error
    */
   @VisibleForTesting
-  String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
+  String closeFileCommitBlocks(INodeFile pendingFile, BlockInfoContiguous storedBlock)
       throws IOException {
-    String src = pendingFile.getFullPathName();
+    final INodesInPath iip = INodesInPath.fromINode(pendingFile);
+    final String src = iip.getPath();
 
     // commit the last block and complete it if it has minimum replicas
-    commitOrCompleteLastBlock(pendingFile, storedBlock);
+    commitOrCompleteLastBlock(pendingFile, iip, storedBlock);
 
     //remove lease, close file
     finalizeINodeFileUnderConstruction(src, pendingFile,
@@ -4523,10 +4479,6 @@ private void persistBlocks(String path, INodeFile file,
     }
   }
 
-  void incrDeletedFileCount(long count) {
-    NameNode.getNameNodeMetrics().incrFilesDeleted(count);
-  }
-
   /**
    * Close file.
    * @param path
@@ -4544,41 +4496,6 @@ private void closeFile(String path, INodeFile file) {
     }
   }
 
-  /**
-   * Add the given symbolic link to the fs. Record it in the edits log.
-   */
-  private INodeSymlink addSymlink(String path, String target,
-                                  PermissionStatus dirPerms,
-                                  boolean createParent, boolean logRetryCache)
-      throws UnresolvedLinkException, FileAlreadyExistsException,
-      QuotaExceededException, SnapshotAccessControlException, AclException {
-    waitForLoadingFSImage();
-
-    final long modTime = now();
-    if (createParent) {
-      final String parent = new Path(path).getParent().toString();
-      if (!FSDirMkdirOp.mkdirsRecursively(dir, parent, dirPerms, true,
-          modTime)) {
-        return null;
-      }
-    }
-    final String userName = dirPerms.getUserName();
-    long id = dir.allocateNewInodeId();
-    INodeSymlink newNode = dir.addSymlink(id, path, target, modTime, modTime,
-            new PermissionStatus(userName, null, FsPermission.getDefault()));
-    if (newNode == null) {
-      NameNode.stateChangeLog.info("addSymlink: failed to add " + path);
-      return null;
-    }
-    getEditLog().logSymlink(path, target, modTime, modTime, newNode,
-        logRetryCache);
-
-    if(NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("addSymlink: " + path + " is added");
-    }
-    return newNode;
-  }
-
   /**
    * Periodically calls hasAvailableResources of NameNodeResourceChecker, and if
    * there are found to be insufficient resources available, causes the NN to
@@ -4688,7 +4605,7 @@ private void clearCorruptLazyPersistFiles()
 
         while (it.hasNext()) {
           Block b = it.next();
-          BlockInfo blockInfo = blockManager.getStoredBlock(b);
+          BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
           if (blockInfo.getBlockCollection().getStoragePolicyID() == lpPolicy.getId()) {
             filesToDelete.add(blockInfo.getBlockCollection());
           }
@@ -4696,7 +4613,13 @@ private void clearCorruptLazyPersistFiles()
 
         for (BlockCollection bc : filesToDelete) {
           LOG.warn("Removing lazyPersist file " + bc.getName() + " with no replicas.");
-          deleteInternal(bc.getName(), false, false, false);
+          BlocksMapUpdateInfo toRemoveBlocks =
+          FSDirDeleteOp.deleteInternal(
+              FSNamesystem.this, bc.getName(),
+              INodesInPath.fromINode((INodeFile) bc), false);
+          if (toRemoveBlocks != null) {
+            removeBlocks(toRemoveBlocks); // Incremental deletion of blocks
+          }
         }
       } finally {
         writeUnlock();
@@ -5629,7 +5552,7 @@ public void decrementSafeBlockCount(Block b) {
     SafeModeInfo safeMode = this.safeMode;
     if (safeMode == null) // mostly true
       return;
-    BlockInfo storedBlock = getStoredBlock(b);
+    BlockInfoContiguous storedBlock = getStoredBlock(b);
     if (storedBlock.isComplete()) {
       safeMode.decrementSafeBlockCount((short)blockManager.countNodes(b).liveReplicas());
     }
@@ -5827,7 +5750,7 @@ private void checkUnreadableBySuperuser(FSPermissionChecker pc,
       INode inode, int snapshotId)
       throws IOException {
     if (pc.isSuperUser()) {
-      for (XAttr xattr : dir.getXAttrs(inode, snapshotId)) {
+      for (XAttr xattr : FSDirXAttrOp.getXAttrs(dir, inode, snapshotId)) {
         if (XAttrHelper.getPrefixName(xattr).
             equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) {
           throw new AccessControlException("Access is denied for " +
@@ -6049,6 +5972,27 @@ public int getNumStaleStorages() {
     return getBlockManager().getDatanodeManager().getNumStaleStorages();
   }
 
+  @Override // FSNamesystemMBean
+  public String getTopUserOpCounts() {
+    if (!topConf.isEnabled) {
+      return null;
+    }
+
+    Date now = new Date();
+    final List<RollingWindowManager.TopWindow> topWindows =
+        topMetrics.getTopWindows();
+    Map<String, Object> topMap = new TreeMap<String, Object>();
+    topMap.put("windows", topWindows);
+    topMap.put("timestamp", DFSUtil.dateToIso8601String(now));
+    ObjectMapper mapper = new ObjectMapper();
+    try {
+      return mapper.writeValueAsString(topMap);
+    } catch (IOException e) {
+      LOG.warn("Failed to fetch TopUser metrics", e);
+    }
+    return null;
+  }
+
   /**
    * Increments, logs and then returns the stamp
    */
@@ -6070,11 +6014,13 @@ long nextGenerationStamp(boolean legacyBlock)
 
   /**
    * Increments, logs and then returns the block ID
+   * @param isStriped is the file under striping or contiguous layout?
    */
-  private long nextBlockId() throws IOException {
+  private long nextBlockId(boolean isStriped) throws IOException {
     assert hasWriteLock();
     checkNameNodeSafeMode("Cannot get next block ID");
-    final long blockId = blockIdManager.nextBlockId();
+    final long blockId = isStriped ?
+        blockIdManager.nextBlockGroupId() : blockIdManager.nextBlockId();
     getEditLog().logAllocateBlockId(blockId);
     // NB: callers sync the log
     return blockId;
@@ -6091,13 +6037,22 @@ private boolean isFileDeleted(INodeFile file) {
     INode tmpChild = file;
     INodeDirectory tmpParent = file.getParent();
     while (true) {
-      if (tmpParent == null ||
-          tmpParent.searchChildren(tmpChild.getLocalNameBytes()) < 0) {
+      if (tmpParent == null) {
+        return true;
+      }
+
+      INode childINode = tmpParent.getChild(tmpChild.getLocalNameBytes(),
+          Snapshot.CURRENT_STATE_ID);
+      if (childINode == null || !childINode.equals(tmpChild)) {
+        // a newly created INode with the same name as an already deleted one
+        // would be a different INode than the deleted one
         return true;
       }
+
       if (tmpParent.isRoot()) {
         break;
       }
+
       tmpChild = tmpParent;
       tmpParent = tmpParent.getParent();
     }
@@ -6116,7 +6071,7 @@ private INodeFile checkUCBlock(ExtendedBlock block,
         + "access token for block " + block);
     
     // check stored block state
-    BlockInfo storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
+    BlockInfoContiguous storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
     if (storedBlock == null || 
         storedBlock.getBlockUCState() != BlockUCState.UNDER_CONSTRUCTION) {
         throw new IOException(block + 
@@ -6245,8 +6200,8 @@ private void updatePipelineInternal(String clientName, ExtendedBlock oldBlock,
     assert hasWriteLock();
     // check the vadility of the block and lease holder name
     final INodeFile pendingFile = checkUCBlock(oldBlock, clientName);
-    final BlockInfoUnderConstruction blockinfo
-        = (BlockInfoUnderConstruction)pendingFile.getLastBlock();
+    final BlockInfoContiguousUnderConstruction blockinfo
+        = (BlockInfoContiguousUnderConstruction)pendingFile.getLastBlock();
 
     // check new GS & length: this is not expected
     if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
@@ -7147,11 +7102,6 @@ public ReentrantReadWriteLock getFsLockForTests() {
     return fsLock.coarseLock;
   }
   
-  @VisibleForTesting
-  public ReentrantLock getLongReadLockForTests() {
-    return fsLock.longReadLock;
-  }
-
   @VisibleForTesting
   public ReentrantLock getCpLockForTests() {
     return cpLock;
@@ -7468,16 +7418,11 @@ public void setNeedRollbackFsImage(boolean needRollbackFsImage) {
 
   @Override  // NameNodeMXBean
   public RollingUpgradeInfo.Bean getRollingUpgradeStatus() {
-    readLock();
-    try {
-      RollingUpgradeInfo upgradeInfo = getRollingUpgradeInfo();
-      if (upgradeInfo != null) {
-        return new RollingUpgradeInfo.Bean(upgradeInfo);
-      }
-      return null;
-    } finally {
-      readUnlock();
+    RollingUpgradeInfo upgradeInfo = getRollingUpgradeInfo();
+    if (upgradeInfo != null) {
+      return new RollingUpgradeInfo.Bean(upgradeInfo);
     }
+    return null;
   }
 
   /** Is rolling upgrade in progress? */
@@ -7905,7 +7850,8 @@ private void createEncryptionZoneInt(final String srcArg, String cipher,
       List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
       xAttrs.add(ezXAttr);
       getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
-      resultingStat = getAuditFileInfo(src, false);
+      final INodesInPath iip = dir.getINodesInPath4Write(src, false);
+      resultingStat = dir.getAuditFileInfo(iip);
     } finally {
       writeUnlock();
     }
@@ -7939,7 +7885,7 @@ EncryptionZone getEZForPath(final String srcArg)
         dir.checkPathAccess(pc, iip, FsAction.READ);
       }
       final EncryptionZone ret = dir.getEZForPath(iip);
-      resultingStat = getAuditFileInfo(src, false);
+      resultingStat = dir.getAuditFileInfo(iip);
       success = true;
       return ret;
     } finally {
@@ -7967,136 +7913,35 @@ BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
     }
   }
 
-  /**
-   * Set xattr for a file or directory.
-   * 
-   * @param src
-   *          - path on which it sets the xattr
-   * @param xAttr
-   *          - xAttr details to set
-   * @param flag
-   *          - xAttrs flags
-   * @throws AccessControlException
-   * @throws SafeModeException
-   * @throws UnresolvedLinkException
-   * @throws IOException
-   */
   void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
                 boolean logRetryCache)
-      throws AccessControlException, SafeModeException,
-      UnresolvedLinkException, IOException {
-    try {
-      setXAttrInt(src, xAttr, flag, logRetryCache);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "setXAttr", src);
-      throw e;
-    }
-  }
-  
-  private void setXAttrInt(final String srcArg, XAttr xAttr,
-      EnumSet<XAttrSetFlag> flag, boolean logRetryCache) throws IOException {
-    String src = srcArg;
-    checkXAttrsConfigFlag();
-    checkXAttrSize(xAttr);
-    HdfsFileStatus resultingStat = null;
-    FSPermissionChecker pc = getPermissionChecker();
-    XAttrPermissionFilter.checkPermissionForApi(pc, xAttr,
-        FSDirectory.isReservedRawName(src));
+      throws IOException {
     checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    HdfsFileStatus auditStat = null;
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot set XAttr on " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      checkXAttrChangeAccess(iip, xAttr, pc);
-      List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
-      xAttrs.add(xAttr);
-      dir.setXAttrs(src, xAttrs, flag);
-      getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
-      resultingStat = getAuditFileInfo(src, false);
+      auditStat = FSDirXAttrOp.setXAttr(dir, src, xAttr, flag, logRetryCache);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "setXAttr", src);
+      throw e;
     } finally {
       writeUnlock();
     }
     getEditLog().logSync();
-    logAuditEvent(true, "setXAttr", srcArg, null, resultingStat);
+    logAuditEvent(true, "setXAttr", src, null, auditStat);
   }
 
-  /**
-   * Verifies that the combined size of the name and value of an xattr is within
-   * the configured limit. Setting a limit of zero disables this check.
-   */
-  private void checkXAttrSize(XAttr xAttr) {
-    if (xattrMaxSize == 0) {
-      return;
-    }
-    int size = xAttr.getName().getBytes(Charsets.UTF_8).length;
-    if (xAttr.getValue() != null) {
-      size += xAttr.getValue().length;
-    }
-    if (size > xattrMaxSize) {
-      throw new HadoopIllegalArgumentException(
-          "The XAttr is too big. The maximum combined size of the"
-          + " name and value is " + xattrMaxSize
-          + ", but the total size is " + size);
-    }
-  }
-  
-  List<XAttr> getXAttrs(final String srcArg, List<XAttr> xAttrs)
+  List<XAttr> getXAttrs(final String src, List<XAttr> xAttrs)
       throws IOException {
-    String src = srcArg;
-    checkXAttrsConfigFlag();
-    FSPermissionChecker pc = getPermissionChecker();
-    final boolean isRawPath = FSDirectory.isReservedRawName(src);
-    boolean getAll = xAttrs == null || xAttrs.isEmpty();
-    if (!getAll) {
-      try {
-        XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
-      } catch (AccessControlException e) {
-        logAuditEvent(false, "getXAttrs", srcArg);
-        throw e;
-      }
-    }
     checkOperation(OperationCategory.READ);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath(src, true);
-      if (isPermissionEnabled) {
-        dir.checkPathAccess(pc, iip, FsAction.READ);
-      }
-      List<XAttr> all = dir.getXAttrs(src);
-      List<XAttr> filteredAll = XAttrPermissionFilter.
-          filterXAttrsForApi(pc, all, isRawPath);
-      if (getAll) {
-        return filteredAll;
-      } else {
-        if (filteredAll == null || filteredAll.isEmpty()) {
-          return null;
-        }
-        List<XAttr> toGet = Lists.newArrayListWithCapacity(xAttrs.size());
-        for (XAttr xAttr : xAttrs) {
-          boolean foundIt = false;
-          for (XAttr a : filteredAll) {
-            if (xAttr.getNameSpace() == a.getNameSpace()
-                && xAttr.getName().equals(a.getName())) {
-              toGet.add(a);
-              foundIt = true;
-              break;
-            }
-          }
-          if (!foundIt) {
-            throw new IOException(
-                "At least one of the attributes provided was not found.");
-          }
-        }
-        return toGet;
-      }
+      return FSDirXAttrOp.getXAttrs(dir, src, xAttrs);
     } catch (AccessControlException e) {
-      logAuditEvent(false, "getXAttrs", srcArg);
+      logAuditEvent(false, "getXAttrs", src);
       throw e;
     } finally {
       readUnlock();
@@ -8104,23 +7949,11 @@ List<XAttr> getXAttrs(final String srcArg, List<XAttr> xAttrs)
   }
 
   List<XAttr> listXAttrs(String src) throws IOException {
-    checkXAttrsConfigFlag();
-    final FSPermissionChecker pc = getPermissionChecker();
-    final boolean isRawPath = FSDirectory.isReservedRawName(src);
     checkOperation(OperationCategory.READ);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath(src, true);
-      if (isPermissionEnabled) {
-        /* To access xattr names, you need EXECUTE in the owning directory. */
-        dir.checkParentAccess(pc, iip, FsAction.EXECUTE);
-      }
-      final List<XAttr> all = dir.getXAttrs(src);
-      return XAttrPermissionFilter.
-        filterXAttrsForApi(pc, all, isRawPath);
+      return FSDirXAttrOp.listXAttrs(dir, src);
     } catch (AccessControlException e) {
       logAuditEvent(false, "listXAttrs", src);
       throw e;
@@ -8129,77 +7962,23 @@ List<XAttr> listXAttrs(String src) throws IOException {
     }
   }
 
-  /**
-   * Remove an xattr for a file or directory.
-   *
-   * @param src
-   *          - path to remove the xattr from
-   * @param xAttr
-   *          - xAttr to remove
-   * @throws AccessControlException
-   * @throws SafeModeException
-   * @throws UnresolvedLinkException
-   * @throws IOException
-   */
   void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
       throws IOException {
-    try {
-      removeXAttrInt(src, xAttr, logRetryCache);
-    } catch (AccessControlException e) {
-      logAuditEvent(false, "removeXAttr", src);
-      throw e;
-    }
-  }
-
-  void removeXAttrInt(final String srcArg, XAttr xAttr, boolean logRetryCache)
-      throws IOException {
-    String src = srcArg;
-    checkXAttrsConfigFlag();
-    HdfsFileStatus resultingStat = null;
-    FSPermissionChecker pc = getPermissionChecker();
-    XAttrPermissionFilter.checkPermissionForApi(pc, xAttr,
-        FSDirectory.isReservedRawName(src));
     checkOperation(OperationCategory.WRITE);
-    byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+    HdfsFileStatus auditStat = null;
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot remove XAttr entry on " + src);
-      src = dir.resolvePath(pc, src, pathComponents);
-      final INodesInPath iip = dir.getINodesInPath4Write(src);
-      checkXAttrChangeAccess(iip, xAttr, pc);
-
-      List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
-      xAttrs.add(xAttr);
-      List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
-      if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
-        getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache);
-      } else {
-        throw new IOException(
-            "No matching attributes found for remove operation");
-      }
-      resultingStat = getAuditFileInfo(src, false);
+      auditStat = FSDirXAttrOp.removeXAttr(dir, src, xAttr, logRetryCache);
+    } catch (AccessControlException e) {
+      logAuditEvent(false, "removeXAttr", src);
+      throw e;
     } finally {
       writeUnlock();
     }
     getEditLog().logSync();
-    logAuditEvent(true, "removeXAttr", srcArg, null, resultingStat);
-  }
-
-  private void checkXAttrChangeAccess(INodesInPath iip, XAttr xAttr,
-      FSPermissionChecker pc) throws AccessControlException {
-    if (isPermissionEnabled && xAttr.getNameSpace() == XAttr.NameSpace.USER) {
-      final INode inode = iip.getLastINode();
-      if (inode != null &&
-          inode.isDirectory() &&
-          inode.getFsPermission().getStickyBit()) {
-        if (!pc.isSuperUser()) {
-          dir.checkOwner(pc, iip);
-        }
-      } else {
-        dir.checkPathAccess(pc, iip, FsAction.WRITE);
-      }
-    }
+    logAuditEvent(true, "removeXAttr", src, null, auditStat);
   }
 
   void checkAccess(String src, FsAction mode) throws IOException {
@@ -8210,8 +7989,8 @@ void checkAccess(String src, FsAction mode) throws IOException {
       checkOperation(OperationCategory.READ);
       src = FSDirectory.resolvePath(src, pathComponents, dir);
       final INodesInPath iip = dir.getINodesInPath(src, true);
-      INode[] inodes = iip.getINodes();
-      if (inodes[inodes.length - 1] == null) {
+      INode inode = iip.getLastINode();
+      if (inode == null) {
         throw new FileNotFoundException("Path not found");
       }
       if (isPermissionEnabled) {
@@ -8311,13 +8090,5 @@ private static void enableAsyncAuditLog() {
     }
   }
 
-  private void checkXAttrsConfigFlag() throws IOException {
-    if (!xattrsEnabled) {
-      throw new IOException(String.format(
-          "The XAttr operation has been rejected.  "
-              + "Support for XAttrs has been disabled by setting %s to false.",
-          DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY));
-    }
-  }
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index f0312849b7bb0..7e820d8502b35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -34,24 +34,6 @@ class FSNamesystemLock implements ReadWriteLock {
   @VisibleForTesting
   protected ReentrantReadWriteLock coarseLock;
   
-  /**
-   * When locking the FSNS for a read that may take a long time, we take this
-   * lock before taking the regular FSNS read lock. All writers also take this
-   * lock before taking the FSNS write lock. Regular (short) readers do not
-   * take this lock at all, instead relying solely on the synchronization of the
-   * regular FSNS lock.
-   * 
-   * This scheme ensures that:
-   * 1) In the case of normal (fast) ops, readers proceed concurrently and
-   *    writers are not starved.
-   * 2) In the case of long read ops, short reads are allowed to proceed
-   *    concurrently during the duration of the long read.
-   * 
-   * See HDFS-5064 for more context.
-   */
-  @VisibleForTesting
-  protected final ReentrantLock longReadLock = new ReentrantLock(true);
-  
   FSNamesystemLock(boolean fair) {
     this.coarseLock = new ReentrantReadWriteLock(fair);
   }
@@ -66,10 +48,6 @@ public Lock writeLock() {
     return coarseLock.writeLock();
   }
 
-  public Lock longReadLock() {
-    return longReadLock;
-  }
-  
   public int getReadHoldCount() {
     return coarseLock.getReadHoldCount();
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index a0455dc6b824e..050848492d4d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -20,6 +20,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.Stack;
 
@@ -46,6 +47,12 @@ class FSPermissionChecker {
   /** @return a string for throwing {@link AccessControlException} */
   private String toAccessControlString(INode inode, int snapshotId,
       FsAction access, FsPermission mode) {
+    return toAccessControlString(inode, snapshotId, access, mode, false);
+  }
+
+  /** @return a string for throwing {@link AccessControlException} */
+  private String toAccessControlString(INode inode, int snapshotId, FsAction access,
+      FsPermission mode, boolean deniedFromAcl) {
     StringBuilder sb = new StringBuilder("Permission denied: ")
       .append("user=").append(user).append(", ")
       .append("access=").append(access).append(", ")
@@ -54,6 +61,9 @@ private String toAccessControlString(INode inode, int snapshotId,
       .append(inode.getGroupName(snapshotId)).append(':')
       .append(inode.isDirectory() ? 'd' : '-')
       .append(mode);
+    if (deniedFromAcl) {
+      sb.append("+");
+    }
     return sb.toString();
   }
 
@@ -144,22 +154,25 @@ void checkPermission(INodesInPath inodesInPath, boolean doCheckOwner,
     // check if (parentAccess != null) && file exists, then check sb
     // If resolveLink, the check is performed on the link target.
     final int snapshotId = inodesInPath.getPathSnapshotId();
-    final INode[] inodes = inodesInPath.getINodes();
-    int ancestorIndex = inodes.length - 2;
-    for(; ancestorIndex >= 0 && inodes[ancestorIndex] == null;
-        ancestorIndex--);
-    checkTraverse(inodes, ancestorIndex, snapshotId);
+    final int length = inodesInPath.length();
+    final INode last = length > 0 ? inodesInPath.getLastINode() : null;
+    final INode parent = length > 1 ? inodesInPath.getINode(-2) : null;
+
+    checkTraverse(inodesInPath, snapshotId);
 
-    final INode last = inodes[inodes.length - 1];
     if (parentAccess != null && parentAccess.implies(FsAction.WRITE)
-        && inodes.length > 1 && last != null) {
-      checkStickyBit(inodes[inodes.length - 2], last, snapshotId);
+        && length > 1 && last != null) {
+      checkStickyBit(parent, last, snapshotId);
     }
-    if (ancestorAccess != null && inodes.length > 1) {
-      check(inodes, ancestorIndex, snapshotId, ancestorAccess);
+    if (ancestorAccess != null && length > 1) {
+      List<INode> inodes = inodesInPath.getReadOnlyINodes();
+      INode ancestor = null;
+      for (int i = inodes.size() - 2; i >= 0 && (ancestor = inodes.get(i)) ==
+          null; i--);
+      check(ancestor, snapshotId, ancestorAccess);
     }
-    if (parentAccess != null && inodes.length > 1) {
-      check(inodes, inodes.length - 2, snapshotId, parentAccess);
+    if (parentAccess != null && length > 1 && parent != null) {
+      check(parent, snapshotId, parentAccess);
     }
     if (access != null) {
       check(last, snapshotId, access);
@@ -184,10 +197,15 @@ private void checkOwner(INode inode, int snapshotId
   }
 
   /** Guarded by {@link FSNamesystem#readLock()} */
-  private void checkTraverse(INode[] inodes, int last, int snapshotId
-      ) throws AccessControlException {
-    for(int j = 0; j <= last; j++) {
-      check(inodes[j], snapshotId, FsAction.EXECUTE);
+  private void checkTraverse(INodesInPath iip, int snapshotId)
+      throws AccessControlException {
+    List<INode> inodes = iip.getReadOnlyINodes();
+    for (int i = 0; i < inodes.size() - 1; i++) {
+      INode inode = inodes.get(i);
+      if (inode == null) {
+        break;
+      }
+      check(inode, snapshotId, FsAction.EXECUTE);
     }
   }
 
@@ -215,14 +233,8 @@ private void checkSubAccess(INode inode, int snapshotId, FsAction access,
   }
 
   /** Guarded by {@link FSNamesystem#readLock()} */
-  private void check(INode[] inodes, int i, int snapshotId, FsAction access
-      ) throws AccessControlException {
-    check(i >= 0? inodes[i]: null, snapshotId, access);
-  }
-
-  /** Guarded by {@link FSNamesystem#readLock()} */
-  private void check(INode inode, int snapshotId, FsAction access
-      ) throws AccessControlException {
+  private void check(INode inode, int snapshotId, FsAction access)
+      throws AccessControlException {
     if (inode == null) {
       return;
     }
@@ -335,7 +347,7 @@ private void checkAccessAcl(INode inode, int snapshotId, FsAction access,
     }
 
     throw new AccessControlException(
-      toAccessControlString(inode, snapshotId, access, mode));
+      toAccessControlString(inode, snapshotId, access, mode, true));
   }
 
   /** Guarded by {@link FSNamesystem#readLock()} */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index 6001db5ccea6a..921803c96c4b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -300,7 +300,7 @@ private static List<EditLogFile> matchEditLogs(File[] filesInStorage,
             .matcher(name);
         if (staleInprogressEditsMatch.matches()) {
           try {
-            long startTxId = Long.valueOf(staleInprogressEditsMatch.group(1));
+            long startTxId = Long.parseLong(staleInprogressEditsMatch.group(1));
             ret.add(new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID,
                 true));
             continue;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
index 896bedb08a5b0..1ebdde645d707 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
@@ -20,9 +20,8 @@
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 
 /**
@@ -59,10 +58,10 @@ public String getClientMachine() {
    */
   void updateLengthOfLastBlock(INodeFile f, long lastBlockLength)
       throws IOException {
-    BlockInfo lastBlock = f.getLastBlock();
+    BlockInfoContiguous lastBlock = f.getLastBlock();
     assert (lastBlock != null) : "The last block for path "
         + f.getFullPathName() + " is null when updating its length";
-    assert (lastBlock instanceof BlockInfoUnderConstruction)
+    assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
         : "The last block for path " + f.getFullPathName()
             + " is not a BlockInfoUnderConstruction when updating its length";
     lastBlock.setNumBytes(lastBlockLength);
@@ -75,11 +74,11 @@ void updateLengthOfLastBlock(INodeFile f, long lastBlockLength)
    */
   void cleanZeroSizeBlock(final INodeFile f,
       final BlocksMapUpdateInfo collectedBlocks) {
-    final BlockInfo[] blocks = f.getBlocks();
+    final BlockInfoContiguous[] blocks = f.getBlocks();
     if (blocks != null && blocks.length > 0
-        && blocks[blocks.length - 1] instanceof BlockInfoUnderConstruction) {
-      BlockInfoUnderConstruction lastUC =
-          (BlockInfoUnderConstruction) blocks[blocks.length - 1];
+        && blocks[blocks.length - 1] instanceof BlockInfoContiguousUnderConstruction) {
+      BlockInfoContiguousUnderConstruction lastUC =
+          (BlockInfoContiguousUnderConstruction) blocks[blocks.length - 1];
       if (lastUC.getNumBytes() == 0) {
         // this is a 0-sized block. do not need check its UC state here
         collectedBlocks.addDeleteBlock(lastUC);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 44549303f925a..bafde456d5c99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -36,8 +36,8 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
 import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
-import org.apache.hadoop.hdfs.util.ChunkedArrayList;
 import org.apache.hadoop.hdfs.util.Diff;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.StringUtils;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -96,8 +96,7 @@ public final String getUserName() {
   abstract void setUser(String user);
 
   /** Set user */
-  final INode setUser(String user, int latestSnapshotId)
-      throws QuotaExceededException {
+  final INode setUser(String user, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     setUser(user);
     return this;
@@ -121,8 +120,7 @@ public final String getGroupName() {
   abstract void setGroup(String group);
 
   /** Set group */
-  final INode setGroup(String group, int latestSnapshotId)
-      throws QuotaExceededException {
+  final INode setGroup(String group, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     setGroup(group);
     return this;
@@ -147,8 +145,7 @@ public final FsPermission getFsPermission() {
   abstract void setPermission(FsPermission permission);
 
   /** Set the {@link FsPermission} of this {@link INode} */
-  INode setPermission(FsPermission permission, int latestSnapshotId) 
-      throws QuotaExceededException {
+  INode setPermission(FsPermission permission, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     setPermission(permission);
     return this;
@@ -163,8 +160,7 @@ public final AclFeature getAclFeature() {
 
   abstract void addAclFeature(AclFeature aclFeature);
 
-  final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId)
-      throws QuotaExceededException {
+  final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     addAclFeature(aclFeature);
     return this;
@@ -172,8 +168,7 @@ final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId)
 
   abstract void removeAclFeature();
 
-  final INode removeAclFeature(int latestSnapshotId)
-      throws QuotaExceededException {
+  final INode removeAclFeature(int latestSnapshotId) {
     recordModification(latestSnapshotId);
     removeAclFeature();
     return this;
@@ -198,8 +193,7 @@ public final XAttrFeature getXAttrFeature() {
    */
   abstract void addXAttrFeature(XAttrFeature xAttrFeature);
   
-  final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId) 
-      throws QuotaExceededException {
+  final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     addXAttrFeature(xAttrFeature);
     return this;
@@ -210,8 +204,7 @@ final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId)
    */
   abstract void removeXAttrFeature();
   
-  final INode removeXAttrFeature(int lastestSnapshotId)
-      throws QuotaExceededException {
+  final INode removeXAttrFeature(int lastestSnapshotId) {
     recordModification(lastestSnapshotId);
     removeXAttrFeature();
     return this;
@@ -227,7 +220,7 @@ public INodeAttributes getSnapshotINode(final int snapshotId) {
 
   /** Is this inode in the latest snapshot? */
   public final boolean isInLatestSnapshot(final int latestSnapshotId) {
-    if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) {
+    if (latestSnapshotId == Snapshot.CURRENT_STATE_ID || latestSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
       return false;
     }
     // if parent is a reference node, parent must be a renamed node. We can 
@@ -242,15 +235,12 @@ public final boolean isInLatestSnapshot(final int latestSnapshotId) {
     if (!parentDir.isInLatestSnapshot(latestSnapshotId)) {
       return false;
     }
-    final INode child = parentDir.getChild(getLocalNameBytes(),
-        latestSnapshotId);
+    final INode child = parentDir.getChild(getLocalNameBytes(), latestSnapshotId);
     if (this == child) {
       return true;
     }
-    if (child == null || !(child.isReference())) {
-      return false;
-    }
-    return this == child.asReference().getReferredINode();
+    return child != null && child.isReference() &&
+        this == child.asReference().getReferredINode();
   }
   
   /** @return true if the given inode is an ancestor directory of this inode. */
@@ -300,8 +290,7 @@ public final boolean shouldRecordInSrcSnapshot(final int latestInDst) {
    *                         Note that it is {@link Snapshot#CURRENT_STATE_ID} 
    *                         if no snapshots have been taken.
    */
-  abstract void recordModification(final int latestSnapshotId)
-      throws QuotaExceededException;
+  abstract void recordModification(final int latestSnapshotId);
 
   /** Check whether it's a reference. */
   public boolean isReference() {
@@ -414,8 +403,7 @@ public INodeSymlink asSymlink() {
    */
   public abstract Quota.Counts cleanSubtree(final int snapshotId,
       int priorSnapshotId, BlocksMapUpdateInfo collectedBlocks,
-      List<INode> removedINodes, boolean countDiffChange)
-      throws QuotaExceededException;
+      List<INode> removedINodes);
   
   /**
    * Destroy self and clear everything! If the INode is a file, this method
@@ -641,15 +629,14 @@ public final long getModificationTime() {
   }
 
   /** Update modification time if it is larger than the current value. */
-  public abstract INode updateModificationTime(long mtime, int latestSnapshotId) 
-      throws QuotaExceededException;
+  public abstract INode updateModificationTime(long mtime, int latestSnapshotId);
 
   /** Set the last modification time of inode. */
   public abstract void setModificationTime(long modificationTime);
 
   /** Set the last modification time of inode. */
   public final INode setModificationTime(long modificationTime,
-      int latestSnapshotId) throws QuotaExceededException {
+      int latestSnapshotId) {
     recordModification(latestSnapshotId);
     setModificationTime(modificationTime);
     return this;
@@ -678,8 +665,7 @@ public final long getAccessTime() {
   /**
    * Set last access time of inode.
    */
-  public final INode setAccessTime(long accessTime, int latestSnapshotId)
-      throws QuotaExceededException {
+  public final INode setAccessTime(long accessTime, int latestSnapshotId) {
     recordModification(latestSnapshotId);
     setAccessTime(accessTime);
     return this;
@@ -769,8 +755,7 @@ public final StringBuffer dumpTreeRecursively() {
 
   @VisibleForTesting
   public final void dumpTreeRecursively(PrintStream out) {
-    dumpTreeRecursively(new PrintWriter(out, true), new StringBuilder(),
-        Snapshot.CURRENT_STATE_ID);
+    out.println(dumpTreeRecursively().toString());
   }
 
   /**
@@ -817,11 +802,15 @@ public List<Block> getToDeleteList() {
      * @param toDelete the to-be-deleted block
      */
     public void addDeleteBlock(Block toDelete) {
-      if (toDelete != null) {
-        toDeleteList.add(toDelete);
-      }
+      assert toDelete != null : "toDelete is null";
+      toDeleteList.add(toDelete);
     }
-    
+
+    public void removeDeleteBlock(Block block) {
+      assert block != null : "block is null";
+      toDeleteList.remove(block);
+    }
+
     /**
      * Clear {@link BlocksMapUpdateInfo#toDeleteList}
      */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java
index 8b0a5f018b113..0f76b682ebff6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeAttributes.java
@@ -75,6 +75,9 @@ public static abstract class SnapshotCopy implements INodeAttributes {
         XAttrFeature xAttrFeature) {
       this.name = name;
       this.permission = PermissionStatusFormat.toLong(permissions);
+      if (aclFeature != null) {
+        aclFeature = AclStorage.addAclFeature(aclFeature);
+      }
       this.aclFeature = aclFeature;
       this.modificationTime = modificationTime;
       this.accessTime = accessTime;
@@ -84,7 +87,11 @@ public static abstract class SnapshotCopy implements INodeAttributes {
     SnapshotCopy(INode inode) {
       this.name = inode.getLocalNameBytes();
       this.permission = inode.getPermissionLong();
-      this.aclFeature = inode.getAclFeature();
+      if (inode.getAclFeature() != null) {
+        aclFeature = AclStorage.addAclFeature(inode.getAclFeature());
+      } else {
+        aclFeature = null;
+      }
       this.modificationTime = inode.getModificationTime();
       this.accessTime = inode.getAccessTime();
       this.xAttrFeature = inode.getXAttrFeature();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index 797a62cc10714..6bf57f0dd615f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.PathIsNotDirectoryException;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
@@ -93,6 +92,12 @@ public INodeDirectory(INodeDirectory other, boolean adopt,
       }
     }
     this.features = featuresToCopy;
+    AclFeature aclFeature = getFeature(AclFeature.class);
+    if (aclFeature != null) {
+      // for the de-duplication of AclFeature
+      removeFeature(aclFeature);
+      addFeature(AclStorage.addAclFeature(aclFeature));
+    }
   }
 
   /** @return true unconditionally. */
@@ -346,8 +351,7 @@ INodeReference.WithName replaceChild4ReferenceWithName(INode oldChild,
   }
 
   @Override
-  public void recordModification(int latestSnapshotId)
-      throws QuotaExceededException {
+  public void recordModification(int latestSnapshotId) {
     if (isInLatestSnapshot(latestSnapshotId)
         && !shouldRecordInSrcSnapshot(latestSnapshotId)) {
       // add snapshot feature if necessary
@@ -366,7 +370,7 @@ public void recordModification(int latestSnapshotId)
    * @return the child inode, which may be replaced.
    */
   public INode saveChild2Snapshot(final INode child, final int latestSnapshotId,
-      final INode snapshotCopy) throws QuotaExceededException {
+      final INode snapshotCopy) {
     if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) {
       return child;
     }
@@ -465,8 +469,7 @@ static int nextChild(ReadOnlyList<INode> children, byte[] name) {
   /**
    * Remove the specified child from this directory.
    */
-  public boolean removeChild(INode child, int latestSnapshotId)
-      throws QuotaExceededException {
+  public boolean removeChild(INode child, int latestSnapshotId) {
     if (isInLatestSnapshot(latestSnapshotId)) {
       // create snapshot feature if necessary
       DirectoryWithSnapshotFeature sf = this.getDirectoryWithSnapshotFeature();
@@ -731,8 +734,7 @@ public void clear() {
   /** Call cleanSubtree(..) recursively down the subtree. */
   public Quota.Counts cleanSubtreeRecursively(final int snapshot,
       int prior, final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final Map<INode, INode> excludedNodes, 
-      final boolean countDiffChange) throws QuotaExceededException {
+      final List<INode> removedINodes, final Map<INode, INode> excludedNodes) {
     Quota.Counts counts = Quota.Counts.newInstance();
     // in case of deletion snapshot, since this call happens after we modify
     // the diff list, the snapshot to be deleted has been combined or renamed
@@ -747,7 +749,7 @@ public Quota.Counts cleanSubtreeRecursively(final int snapshot,
         continue;
       } else {
         Quota.Counts childCounts = child.cleanSubtree(snapshot, prior,
-            collectedBlocks, removedINodes, countDiffChange);
+            collectedBlocks, removedINodes);
         counts.add(childCounts);
       }
     }
@@ -764,6 +766,9 @@ public void destroyAndCollectBlocks(final BlocksMapUpdateInfo collectedBlocks,
     for (INode child : getChildrenList(Snapshot.CURRENT_STATE_ID)) {
       child.destroyAndCollectBlocks(collectedBlocks, removedINodes);
     }
+    if (getAclFeature() != null) {
+      AclStorage.removeAclFeature(getAclFeature());
+    }
     clear();
     removedINodes.add(this);
   }
@@ -771,13 +776,12 @@ public void destroyAndCollectBlocks(final BlocksMapUpdateInfo collectedBlocks,
   @Override
   public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange)
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
     // there is snapshot data
     if (sf != null) {
       return sf.cleanDirectory(this, snapshotId, priorSnapshotId,
-          collectedBlocks, removedINodes, countDiffChange);
+          collectedBlocks, removedINodes);
     }
     // there is no snapshot data
     if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID
@@ -790,7 +794,7 @@ public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
     } else {
       // process recursively down the subtree
       Quota.Counts counts = cleanSubtreeRecursively(snapshotId, priorSnapshotId,
-          collectedBlocks, removedINodes, null, countDiffChange);
+          collectedBlocks, removedINodes, null);
       if (isQuotaSet()) {
         getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
             -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index ccf3df1e72632..0f3fcd5a20848 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -24,17 +24,21 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
@@ -113,17 +117,17 @@ static long toLong(long preferredBlockSize, short replication,
 
   private long header = 0L;
 
-  private BlockInfo[] blocks;
+  private BlockInfoContiguous[] blocks;
 
   INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
-            long atime, BlockInfo[] blklist, short replication,
+            long atime, BlockInfoContiguous[] blklist, short replication,
             long preferredBlockSize) {
     this(id, name, permissions, mtime, atime, blklist, replication,
          preferredBlockSize, (byte) 0);
   }
 
   INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
-      long atime, BlockInfo[] blklist, short replication,
+      long atime, BlockInfoContiguous[] blklist, short replication,
       long preferredBlockSize, byte storagePolicyID) {
     super(id, name, permissions, mtime, atime);
     header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
@@ -218,20 +222,21 @@ private void assertAllBlocksComplete() {
   }
 
   @Override // BlockCollection
-  public void setBlock(int index, BlockInfo blk) {
+  public void setBlock(int index, BlockInfoContiguous blk) {
     this.blocks[index] = blk;
   }
 
   @Override // BlockCollection, the file should be under construction
-  public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock,
-      DatanodeStorageInfo[] locations) throws IOException {
+  public BlockInfoContiguousUnderConstruction setLastBlock(
+      BlockInfoContiguous lastBlock, DatanodeStorageInfo[] locations)
+      throws IOException {
     Preconditions.checkState(isUnderConstruction(),
         "file is no longer under construction");
 
     if (numBlocks() == 0) {
       throw new IOException("Failed to set last block: File is empty.");
     }
-    BlockInfoUnderConstruction ucBlock =
+    BlockInfoContiguousUnderConstruction ucBlock =
       lastBlock.convertToBlockUnderConstruction(
           BlockUCState.UNDER_CONSTRUCTION, locations);
     setBlock(numBlocks() - 1, ucBlock);
@@ -254,7 +259,7 @@ boolean removeLastBlock(Block oldblock) {
     }
 
     //copy to a new list
-    BlockInfo[] newlist = new BlockInfo[size_1];
+    BlockInfoContiguous[] newlist = new BlockInfoContiguous[size_1];
     System.arraycopy(blocks, 0, newlist, 0, size_1);
     setBlocks(newlist);
     return true;
@@ -302,8 +307,11 @@ public INodeFileAttributes getSnapshotINode(final int snapshotId) {
   }
 
   @Override
-  public void recordModification(final int latestSnapshotId)
-      throws QuotaExceededException {
+  public void recordModification(final int latestSnapshotId) {
+    recordModification(latestSnapshotId, false);
+  }
+
+  public void recordModification(final int latestSnapshotId, boolean withBlocks) {
     if (isInLatestSnapshot(latestSnapshotId)
         && !shouldRecordInSrcSnapshot(latestSnapshotId)) {
       // the file is in snapshot, create a snapshot feature if it does not have
@@ -312,10 +320,10 @@ public void recordModification(final int latestSnapshotId)
         sf = addSnapshotFeature(null);
       }
       // record self in the diff list if necessary
-      sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null);
+      sf.getDiffs().saveSelf2Snapshot(latestSnapshotId, this, null, withBlocks);
     }
   }
-  
+
   public FileDiffList getDiffs() {
     FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
     if (sf != null) {
@@ -405,19 +413,34 @@ public long getHeaderLong() {
   }
 
   /** @return the diskspace required for a full block. */
-  final long getBlockDiskspace() {
+  final long getPreferredBlockDiskspace() {
     return getPreferredBlockSize() * getBlockReplication();
   }
 
   /** @return the blocks of the file. */
   @Override
-  public BlockInfo[] getBlocks() {
+  public BlockInfoContiguous[] getBlocks() {
     return this.blocks;
   }
 
+  /** @return blocks of the file corresponding to the snapshot. */
+  public BlockInfoContiguous[] getBlocks(int snapshot) {
+    if(snapshot == CURRENT_STATE_ID || getDiffs() == null)
+      return getBlocks();
+    FileDiff diff = getDiffs().getDiffById(snapshot);
+    BlockInfoContiguous[] snapshotBlocks =
+        diff == null ? getBlocks() : diff.getBlocks();
+    if(snapshotBlocks != null)
+      return snapshotBlocks;
+    // Blocks are not in the current snapshot
+    // Find next snapshot with blocks present or return current file blocks
+    snapshotBlocks = getDiffs().findLaterSnapshotBlocks(snapshot);
+    return (snapshotBlocks == null) ? getBlocks() : snapshotBlocks;
+  }
+
   void updateBlockCollection() {
     if (blocks != null) {
-      for(BlockInfo b : blocks) {
+      for(BlockInfoContiguous b : blocks) {
         b.setBlockCollection(this);
       }
     }
@@ -433,7 +456,8 @@ void concatBlocks(INodeFile[] inodes) {
       totalAddedBlocks += f.blocks.length;
     }
     
-    BlockInfo[] newlist = new BlockInfo[size + totalAddedBlocks];
+    BlockInfoContiguous[] newlist =
+        new BlockInfoContiguous[size + totalAddedBlocks];
     System.arraycopy(this.blocks, 0, newlist, 0, size);
     
     for(INodeFile in: inodes) {
@@ -448,12 +472,12 @@ void concatBlocks(INodeFile[] inodes) {
   /**
    * add a block to the block list
    */
-  void addBlock(BlockInfo newblock) {
+  void addBlock(BlockInfoContiguous newblock) {
     if (this.blocks == null) {
-      this.setBlocks(new BlockInfo[]{newblock});
+      this.setBlocks(new BlockInfoContiguous[]{newblock});
     } else {
       int size = this.blocks.length;
-      BlockInfo[] newlist = new BlockInfo[size + 1];
+      BlockInfoContiguous[] newlist = new BlockInfoContiguous[size + 1];
       System.arraycopy(this.blocks, 0, newlist, 0, size);
       newlist[size] = newblock;
       this.setBlocks(newlist);
@@ -461,19 +485,18 @@ void addBlock(BlockInfo newblock) {
   }
 
   /** Set the blocks. */
-  public void setBlocks(BlockInfo[] blocks) {
+  public void setBlocks(BlockInfoContiguous[] blocks) {
     this.blocks = blocks;
   }
 
   @Override
   public Quota.Counts cleanSubtree(final int snapshot, int priorSnapshotId,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange)
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if (sf != null) {
       return sf.cleanFile(this, snapshot, priorSnapshotId, collectedBlocks,
-          removedINodes, countDiffChange);
+          removedINodes);
     }
     Quota.Counts counts = Quota.Counts.newInstance();
     if (snapshot == CURRENT_STATE_ID) {
@@ -498,21 +521,24 @@ public Quota.Counts cleanSubtree(final int snapshot, int priorSnapshotId,
   public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
       final List<INode> removedINodes) {
     if (blocks != null && collectedBlocks != null) {
-      for (BlockInfo blk : blocks) {
+      for (BlockInfoContiguous blk : blocks) {
         collectedBlocks.addDeleteBlock(blk);
         blk.setBlockCollection(null);
       }
     }
     setBlocks(null);
+    if (getAclFeature() != null) {
+      AclStorage.removeAclFeature(getAclFeature());
+    }
     clear();
     removedINodes.add(this);
-    
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if (sf != null) {
+      sf.getDiffs().destroyAndCollectSnapshotBlocks(collectedBlocks);
       sf.clearDiffs();
     }
   }
-  
+
   @Override
   public String getName() {
     // Get the full path name of this inode.
@@ -528,11 +554,9 @@ public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
     if (sf != null) {
       FileDiffList fileDiffList = sf.getDiffs();
       int last = fileDiffList.getLastSnapshotId();
-      List<FileDiff> diffs = fileDiffList.asList();
 
       if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
           || last == Snapshot.CURRENT_STATE_ID) {
-        nsDelta += diffs.size();
         dsDelta = diskspaceConsumed();
       } else if (last < lastSnapshotId) {
         dsDelta = computeFileSize(true, false) * getFileReplication();
@@ -551,39 +575,23 @@ public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
   @Override
   public final ContentSummaryComputationContext computeContentSummary(
       final ContentSummaryComputationContext summary) {
-    computeContentSummary4Snapshot(summary.getCounts());
-    computeContentSummary4Current(summary.getCounts());
-    return summary;
-  }
-
-  private void computeContentSummary4Snapshot(final Content.Counts counts) {
-    // file length and diskspace only counted for the latest state of the file
-    // i.e. either the current state or the last snapshot
+    final Content.Counts counts = summary.getCounts();
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
-    if (sf != null) {
+    if (sf == null) {
+      counts.add(Content.LENGTH, computeFileSize());
+      counts.add(Content.FILE, 1);
+    } else {
       final FileDiffList diffs = sf.getDiffs();
       final int n = diffs.asList().size();
       counts.add(Content.FILE, n);
       if (n > 0 && sf.isCurrentFileDeleted()) {
         counts.add(Content.LENGTH, diffs.getLast().getFileSize());
-      }
-
-      if (sf.isCurrentFileDeleted()) {
-        final long lastFileSize = diffs.getLast().getFileSize();
-        counts.add(Content.DISKSPACE, lastFileSize * getBlockReplication());
+      } else {
+        counts.add(Content.LENGTH, computeFileSize());
       }
     }
-  }
-
-  private void computeContentSummary4Current(final Content.Counts counts) {
-    FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
-    if (sf != null && sf.isCurrentFileDeleted()) {
-      return;
-    }
-
-    counts.add(Content.LENGTH, computeFileSize());
-    counts.add(Content.FILE, 1);
     counts.add(Content.DISKSPACE, diskspaceConsumed());
+    return summary;
   }
 
   /** The same as computeFileSize(null). */
@@ -634,7 +642,7 @@ public final long computeFileSize(boolean includesLastUcBlock,
     final int last = blocks.length - 1;
     //check if the last block is BlockInfoUnderConstruction
     long size = blocks[last].getNumBytes();
-    if (blocks[last] instanceof BlockInfoUnderConstruction) {
+    if (blocks[last] instanceof BlockInfoContiguousUnderConstruction) {
        if (!includesLastUcBlock) {
          size = 0;
        } else if (usePreferredBlockSize4LastUcBlock) {
@@ -648,9 +656,37 @@ public final long computeFileSize(boolean includesLastUcBlock,
     return size;
   }
 
+  /**
+   * Compute size consumed by all blocks of the current file,
+   * including blocks in its snapshots.
+   * Use preferred block size for the last block if it is under construction.
+   */
   public final long diskspaceConsumed() {
-    // use preferred block size for the last block if it is under construction
-    return computeFileSize(true, true) * getBlockReplication();
+    FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
+    if(sf == null) {
+      return computeFileSize(true, true) * getBlockReplication();
+    }
+
+    // Collect all distinct blocks
+    long size = 0;
+    Set<Block> allBlocks = new HashSet<Block>(Arrays.asList(getBlocks()));
+    List<FileDiff> diffs = sf.getDiffs().asList();
+    for(FileDiff diff : diffs) {
+      BlockInfoContiguous[] diffBlocks = diff.getBlocks();
+      if (diffBlocks != null) {
+        allBlocks.addAll(Arrays.asList(diffBlocks));
+      }
+    }
+    for(Block block : allBlocks) {
+      size += block.getNumBytes();
+    }
+    // check if the last block is under construction
+    BlockInfoContiguous lastBlock = getLastBlock();
+    if(lastBlock != null &&
+        lastBlock instanceof BlockInfoContiguousUnderConstruction) {
+      size += getPreferredBlockSize() - lastBlock.getNumBytes();
+    }
+    return size * getBlockReplication();
   }
 
   public final long diskspaceConsumed(int lastSnapshotId) {
@@ -665,7 +701,7 @@ public final long diskspaceConsumed(int lastSnapshotId) {
   /**
    * Return the penultimate allocated block for this file.
    */
-  BlockInfo getPenultimateBlock() {
+  BlockInfoContiguous getPenultimateBlock() {
     if (blocks == null || blocks.length <= 1) {
       return null;
     }
@@ -673,7 +709,7 @@ BlockInfo getPenultimateBlock() {
   }
 
   @Override
-  public BlockInfo getLastBlock() {
+  public BlockInfoContiguous getLastBlock() {
     return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
   }
 
@@ -693,4 +729,107 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
     out.print(blocks == null || blocks.length == 0? null: blocks[0]);
     out.println();
   }
+
+  /**
+   * Remove full blocks at the end file up to newLength
+   * @return sum of sizes of the remained blocks
+   */
+  public long collectBlocksBeyondMax(final long max,
+      final BlocksMapUpdateInfo collectedBlocks) {
+    final BlockInfoContiguous[] oldBlocks = getBlocks();
+    if (oldBlocks == null)
+      return 0;
+    // find the minimum n such that the size of the first n blocks > max
+    int n = 0;
+    long size = 0;
+    for(; n < oldBlocks.length && max > size; n++) {
+      size += oldBlocks[n].getNumBytes();
+    }
+    if (n >= oldBlocks.length)
+      return size;
+
+    // starting from block n, the data is beyond max.
+    // resize the array.
+    truncateBlocksTo(n);
+
+    // collect the blocks beyond max
+    if (collectedBlocks != null) {
+      for(; n < oldBlocks.length; n++) {
+        collectedBlocks.addDeleteBlock(oldBlocks[n]);
+      }
+    }
+    return size;
+  }
+
+  void truncateBlocksTo(int n) {
+    final BlockInfoContiguous[] newBlocks;
+    if (n == 0) {
+      newBlocks = BlockInfoContiguous.EMPTY_ARRAY;
+    } else {
+      newBlocks = new BlockInfoContiguous[n];
+      System.arraycopy(getBlocks(), 0, newBlocks, 0, n);
+    }
+    // set new blocks
+    setBlocks(newBlocks);
+  }
+
+  public void collectBlocksBeyondSnapshot(BlockInfoContiguous[] snapshotBlocks,
+                                          BlocksMapUpdateInfo collectedBlocks) {
+    BlockInfoContiguous[] oldBlocks = getBlocks();
+    if(snapshotBlocks == null || oldBlocks == null)
+      return;
+    // Skip blocks in common between the file and the snapshot
+    int n = 0;
+    while(n < oldBlocks.length && n < snapshotBlocks.length &&
+          oldBlocks[n] == snapshotBlocks[n]) {
+      n++;
+    }
+    truncateBlocksTo(n);
+    // Collect the remaining blocks of the file
+    while(n < oldBlocks.length) {
+      collectedBlocks.addDeleteBlock(oldBlocks[n++]);
+    }
+  }
+
+  /** Exclude blocks collected for deletion that belong to a snapshot. */
+  void excludeSnapshotBlocks(int snapshotId,
+                             BlocksMapUpdateInfo collectedBlocks) {
+    if(collectedBlocks == null || collectedBlocks.getToDeleteList().isEmpty())
+      return;
+    FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
+    if(sf == null)
+      return;
+    BlockInfoContiguous[] snapshotBlocks =
+        getDiffs().findEarlierSnapshotBlocks(snapshotId);
+    if(snapshotBlocks == null)
+      return;
+    List<Block> toDelete = collectedBlocks.getToDeleteList();
+    for(Block blk : snapshotBlocks) {
+      if(toDelete.contains(blk))
+        collectedBlocks.removeDeleteBlock(blk);
+    }
+  }
+
+  /**
+   * @return true if the block is contained in a snapshot or false otherwise.
+   */
+  boolean isBlockInLatestSnapshot(BlockInfoContiguous block) {
+    FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
+    if (sf == null || sf.getDiffs() == null) {
+      return false;
+    }
+    BlockInfoContiguous[] snapshotBlocks = getDiffs()
+        .findEarlierSnapshotBlocks(getDiffs().getLastSnapshotId());
+    return snapshotBlocks != null &&
+        Arrays.asList(snapshotBlocks).contains(block);
+  }
+
+  @VisibleForTesting
+  /**
+   * @return true if the file is in the striping layout.
+   */
+  // TODO: move erasure coding policy to file XAttr (HDFS-7337)
+  public boolean isStriped() {
+    return getStoragePolicyID() == HdfsConstants.EC_STORAGE_POLICY_ID;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index 5009c58dc10a9..8629bf82aabed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -22,8 +22,6 @@
 
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.namenode.Quota.Counts;
 import org.apache.hadoop.util.GSet;
@@ -95,8 +93,7 @@ public INode get(long id) {
         "", "", new FsPermission((short) 0)), 0, 0) {
       
       @Override
-      void recordModification(int latestSnapshotId)
-          throws QuotaExceededException {
+      void recordModification(int latestSnapshotId) {
       }
       
       @Override
@@ -119,8 +116,7 @@ public ContentSummaryComputationContext computeContentSummary(
       
       @Override
       public Counts cleanSubtree(int snapshotId, int priorSnapshotId,
-          BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes,
-          boolean countDiffChange) throws QuotaExceededException {
+          BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
         return null;
       }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
index a4766d1156142..b39151a083c49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 
 import com.google.common.base.Preconditions;
+import org.mortbay.log.Log;
 
 /**
  * An anonymous reference to an inode.
@@ -265,8 +266,7 @@ public final long getModificationTime(int snapshotId) {
   }
   
   @Override
-  public final INode updateModificationTime(long mtime, int latestSnapshotId) 
-      throws QuotaExceededException {
+  public final INode updateModificationTime(long mtime, int latestSnapshotId) {
     return referred.updateModificationTime(mtime, latestSnapshotId);
   }
   
@@ -296,17 +296,15 @@ public final byte getLocalStoragePolicyID() {
   }
 
   @Override
-  final void recordModification(int latestSnapshotId)
-      throws QuotaExceededException {
+  final void recordModification(int latestSnapshotId) {
     referred.recordModification(latestSnapshotId);
   }
 
   @Override // used by WithCount
   public Quota.Counts cleanSubtree(int snapshot, int prior,
-      BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes,
-      final boolean countDiffChange) throws QuotaExceededException {
+      BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
     return referred.cleanSubtree(snapshot, prior, collectedBlocks,
-        removedINodes, countDiffChange);
+        removedINodes);
   }
 
   @Override // used by WithCount
@@ -537,8 +535,7 @@ public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
     @Override
     public Quota.Counts cleanSubtree(final int snapshot, int prior,
         final BlocksMapUpdateInfo collectedBlocks,
-        final List<INode> removedINodes, final boolean countDiffChange)
-        throws QuotaExceededException {
+        final List<INode> removedINodes) {
       // since WithName node resides in deleted list acting as a snapshot copy,
       // the parameter snapshot must be non-null
       Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
@@ -554,11 +551,15 @@ public Quota.Counts cleanSubtree(final int snapshot, int prior,
       }
 
       Quota.Counts counts = getReferredINode().cleanSubtree(snapshot, prior,
-          collectedBlocks, removedINodes, false);
+          collectedBlocks, removedINodes);
       INodeReference ref = getReferredINode().getParentReference();
       if (ref != null) {
-        ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
-            -counts.get(Quota.DISKSPACE), true);
+        try {
+          ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
+              -counts.get(Quota.DISKSPACE), true);
+        } catch (QuotaExceededException e) {
+          Log.warn("Should not have QuotaExceededException");
+        }
       }
       
       if (snapshot < lastSnapshotId) {
@@ -597,7 +598,7 @@ public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
           }
           try {
             Quota.Counts counts = referred.cleanSubtree(snapshot, prior,
-                collectedBlocks, removedINodes, false);
+                collectedBlocks, removedINodes);
             INodeReference ref = getReferredINode().getParentReference();
             if (ref != null) {
               ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
@@ -653,8 +654,7 @@ public DstReference(INodeDirectory parent, WithCount referred,
     
     @Override
     public Quota.Counts cleanSubtree(int snapshot, int prior,
-        BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes,
-        final boolean countDiffChange) throws QuotaExceededException {
+        BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
       if (snapshot == Snapshot.CURRENT_STATE_ID
           && prior == Snapshot.NO_SNAPSHOT_ID) {
         Quota.Counts counts = Quota.Counts.newInstance();
@@ -676,7 +676,7 @@ public Quota.Counts cleanSubtree(int snapshot, int prior,
           return Quota.Counts.newInstance();
         }
         return getReferredINode().cleanSubtree(snapshot, prior,
-            collectedBlocks, removedINodes, countDiffChange);
+            collectedBlocks, removedINodes);
       }
     }
     
@@ -714,15 +714,11 @@ public void destroyAndCollectBlocks(
           Preconditions.checkState(file.isWithSnapshot());
           // make sure we mark the file as deleted
           file.getFileWithSnapshotFeature().deleteCurrentFile();
-          try {
-            // when calling cleanSubtree of the referred node, since we 
-            // compute quota usage updates before calling this destroy 
-            // function, we use true for countDiffChange
-            referred.cleanSubtree(snapshot, prior, collectedBlocks,
-                removedINodes, true);
-          } catch (QuotaExceededException e) {
-            LOG.error("should not exceed quota while snapshot deletion", e);
-          }
+          // when calling cleanSubtree of the referred node, since we
+          // compute quota usage updates before calling this destroy
+          // function, we use true for countDiffChange
+          referred.cleanSubtree(snapshot, prior, collectedBlocks,
+              removedINodes);
         } else if (referred.isDirectory()) {
           // similarly, if referred is a directory, it must be an
           // INodeDirectory with snapshot
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
index 45a4bc82a978e..617c99afbbe8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
@@ -23,10 +23,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
-import org.apache.hadoop.hdfs.server.namenode.AclFeature;
-import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
 
 /**
  * An {@link INode} representing a symbolic link.
@@ -47,7 +44,7 @@ public class INodeSymlink extends INodeWithAdditionalFields {
   }
 
   @Override
-  void recordModification(int latestSnapshotId) throws QuotaExceededException {
+  void recordModification(int latestSnapshotId) {
     if (isInLatestSnapshot(latestSnapshotId)) {
       INodeDirectory parent = getParent();
       parent.saveChild2Snapshot(this, latestSnapshotId, new INodeSymlink(this));
@@ -77,7 +74,7 @@ public byte[] getSymlink() {
   @Override
   public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange) {
+      final List<INode> removedINodes) {
     if (snapshotId == Snapshot.CURRENT_STATE_ID
         && priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
       destroyAndCollectBlocks(collectedBlocks, removedINodes);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
index 93da052a278f7..8ff15a8277e6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.util.LongBitFormat;
 import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
@@ -219,7 +218,7 @@ public long getPermissionLong() {
   }
 
   @Override
-  final AclFeature getAclFeature(int snapshotId) {
+  public final AclFeature getAclFeature(int snapshotId) {
     if (snapshotId != Snapshot.CURRENT_STATE_ID) {
       return getSnapshotINode(snapshotId).getAclFeature();
     }
@@ -239,8 +238,7 @@ final long getModificationTime(int snapshotId) {
 
   /** Update modification time if it is larger than the current value. */
   @Override
-  public final INode updateModificationTime(long mtime, int latestSnapshotId) 
-      throws QuotaExceededException {
+  public final INode updateModificationTime(long mtime, int latestSnapshotId) {
     Preconditions.checkState(isDirectory());
     if (mtime <= modificationTime) {
       return this;
@@ -330,6 +328,7 @@ public void removeAclFeature() {
     AclFeature f = getAclFeature();
     Preconditions.checkNotNull(f);
     removeFeature(f);
+    AclStorage.removeAclFeature(f);
   }
 
   public void addAclFeature(AclFeature f) {
@@ -337,7 +336,7 @@ public void addAclFeature(AclFeature f) {
     if (f1 != null)
       throw new IllegalStateException("Duplicated ACLFeature");
 
-    addFeature(f);
+    addFeature(AclStorage.addAclFeature(f));
   }
   
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 58f5f3d029fe3..389b62b2cbf60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -18,7 +18,11 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
@@ -31,6 +35,9 @@
 
 import com.google.common.base.Preconditions;
 
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.ID_INTEGER_COMPARATOR;
+
 /**
  * Contains INodes information resolved from a given path.
  */
@@ -54,7 +61,6 @@ static INodesInPath fromINode(INode inode) {
     }
     final byte[][] path = new byte[depth][];
     final INode[] inodes = new INode[depth];
-    final INodesInPath iip = new INodesInPath(path, depth);
     tmp = inode;
     index = depth;
     while (tmp != null) {
@@ -63,8 +69,7 @@ static INodesInPath fromINode(INode inode) {
       inodes[index] = tmp;
       tmp = tmp.getParent();
     }
-    iip.setINodes(inodes);
-    return iip;
+    return new INodesInPath(inodes, path);
   }
 
   /**
@@ -85,18 +90,11 @@ private static String constructPath(byte[][] components, int start, int end) {
     return buf.toString();
   }
 
-  static INodesInPath resolve(final INodeDirectory startingDir,
-      final byte[][] components) throws UnresolvedLinkException {
-    return resolve(startingDir, components, components.length, false);
-  }
-
   /**
-   * Retrieve existing INodes from a path. If existing is big enough to store
-   * all path components (existing and non-existing), then existing INodes
-   * will be stored starting from the root INode into existing[0]; if
-   * existing is not big enough to store all path components, then only the
-   * last existing and non existing INodes will be stored so that
-   * existing[existing.length-1] refers to the INode of the final component.
+   * Retrieve existing INodes from a path. For non-snapshot path,
+   * the number of INodes is equal to the number of path components. For
+   * snapshot path (e.g., /foo/.snapshot/s1/bar), the number of INodes is
+   * (number_of_path_components - 1).
    * 
    * An UnresolvedPathException is always thrown when an intermediate path 
    * component refers to a symbolic link. If the final path component refers 
@@ -106,58 +104,44 @@ static INodesInPath resolve(final INodeDirectory startingDir,
    * <p>
    * Example: <br>
    * Given the path /c1/c2/c3 where only /c1/c2 exists, resulting in the
-   * following path components: ["","c1","c2","c3"],
-   * 
-   * <p>
-   * <code>getExistingPathINodes(["","c1","c2"], [?])</code> should fill the
-   * array with [c2] <br>
-   * <code>getExistingPathINodes(["","c1","c2","c3"], [?])</code> should fill the
-   * array with [null]
-   * 
-   * <p>
-   * <code>getExistingPathINodes(["","c1","c2"], [?,?])</code> should fill the
-   * array with [c1,c2] <br>
-   * <code>getExistingPathINodes(["","c1","c2","c3"], [?,?])</code> should fill
-   * the array with [c2,null]
+   * following path components: ["","c1","c2","c3"]
    * 
    * <p>
-   * <code>getExistingPathINodes(["","c1","c2"], [?,?,?,?])</code> should fill
-   * the array with [rootINode,c1,c2,null], <br>
-   * <code>getExistingPathINodes(["","c1","c2","c3"], [?,?,?,?])</code> should
+   * <code>getExistingPathINodes(["","c1","c2"])</code> should fill
+   * the array with [rootINode,c1,c2], <br>
+   * <code>getExistingPathINodes(["","c1","c2","c3"])</code> should
    * fill the array with [rootINode,c1,c2,null]
    * 
    * @param startingDir the starting directory
    * @param components array of path component name
-   * @param numOfINodes number of INodes to return
    * @param resolveLink indicates whether UnresolvedLinkException should
    *        be thrown when the path refers to a symbolic link.
    * @return the specified number of existing INodes in the path
    */
   static INodesInPath resolve(final INodeDirectory startingDir,
-      final byte[][] components, final int numOfINodes, 
-      final boolean resolveLink) throws UnresolvedLinkException {
+      final byte[][] components, final boolean resolveLink)
+      throws UnresolvedLinkException {
     Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
 
     INode curNode = startingDir;
-    final INodesInPath existing = new INodesInPath(components, numOfINodes);
     int count = 0;
-    int index = numOfINodes - components.length;
-    if (index > 0) {
-      index = 0;
-    }
+    int inodeNum = 0;
+    INode[] inodes = new INode[components.length];
+    boolean isSnapshot = false;
+    int snapshotId = CURRENT_STATE_ID;
+
     while (count < components.length && curNode != null) {
-      final boolean lastComp = (count == components.length - 1);      
-      if (index >= 0) {
-        existing.addNode(curNode);
-      }
+      final boolean lastComp = (count == components.length - 1);
+      inodes[inodeNum++] = curNode;
       final boolean isRef = curNode.isReference();
       final boolean isDir = curNode.isDirectory();
-      final INodeDirectory dir = isDir? curNode.asDirectory(): null;  
+      final INodeDirectory dir = isDir? curNode.asDirectory(): null;
       if (!isRef && isDir && dir.isWithSnapshot()) {
         //if the path is a non-snapshot path, update the latest snapshot.
-        if (!existing.isSnapshot()) {
-          existing.updateLatestSnapshotId(dir.getDirectoryWithSnapshotFeature()
-              .getLastSnapshotId());
+        if (!isSnapshot && shouldUpdateLatestId(
+            dir.getDirectoryWithSnapshotFeature().getLastSnapshotId(),
+            snapshotId)) {
+          snapshotId = dir.getDirectoryWithSnapshotFeature().getLastSnapshotId();
         }
       } else if (isRef && isDir && !lastComp) {
         // If the curNode is a reference node, need to check its dstSnapshot:
@@ -170,19 +154,18 @@ static INodesInPath resolve(final INodeDirectory startingDir,
         // the latest snapshot if lastComp is true. In case of the operation is
         // a modification operation, we do a similar check in corresponding 
         // recordModification method.
-        if (!existing.isSnapshot()) {
+        if (!isSnapshot) {
           int dstSnapshotId = curNode.asReference().getDstSnapshotId();
-          int latest = existing.getLatestSnapshotId();
-          if (latest == Snapshot.CURRENT_STATE_ID || // no snapshot in dst tree of rename
-              (dstSnapshotId != Snapshot.CURRENT_STATE_ID && 
-                dstSnapshotId >= latest)) { // the above scenario 
-            int lastSnapshot = Snapshot.CURRENT_STATE_ID;
+          if (snapshotId == CURRENT_STATE_ID || // no snapshot in dst tree of rename
+              (dstSnapshotId != CURRENT_STATE_ID &&
+               dstSnapshotId >= snapshotId)) { // the above scenario
+            int lastSnapshot = CURRENT_STATE_ID;
             DirectoryWithSnapshotFeature sf;
             if (curNode.isDirectory() && 
                 (sf = curNode.asDirectory().getDirectoryWithSnapshotFeature()) != null) {
               lastSnapshot = sf.getLastSnapshotId();
             }
-            existing.setSnapshotId(lastSnapshot);
+            snapshotId = lastSnapshot;
           }
         }
       }
@@ -210,11 +193,7 @@ static INodesInPath resolve(final INodeDirectory startingDir,
       if (isDotSnapshotDir(childName) && dir.isSnapshottable()) {
         // skip the ".snapshot" in components
         count++;
-        index++;
-        existing.isSnapshot = true;
-        if (index >= 0) { // decrease the capacity by 1 to account for .snapshot
-          existing.capacity--;
-        }
+        isSnapshot = true;
         // check if ".snapshot" is the last element of components
         if (count == components.length - 1) {
           break;
@@ -222,65 +201,98 @@ static INodesInPath resolve(final INodeDirectory startingDir,
         // Resolve snapshot root
         final Snapshot s = dir.getSnapshot(components[count + 1]);
         if (s == null) {
-          //snapshot not found
-          curNode = null;
+          curNode = null; // snapshot not found
         } else {
           curNode = s.getRoot();
-          existing.setSnapshotId(s.getId());
-        }
-        if (index >= -1) {
-          existing.snapshotRootIndex = existing.numNonNull;
+          snapshotId = s.getId();
         }
       } else {
         // normal case, and also for resolving file/dir under snapshot root
-        curNode = dir.getChild(childName, existing.getPathSnapshotId());
+        curNode = dir.getChild(childName,
+            isSnapshot ? snapshotId : CURRENT_STATE_ID);
       }
       count++;
-      index++;
     }
-    return existing;
+    if (isSnapshot && !isDotSnapshotDir(components[components.length - 1])) {
+      // for snapshot path shrink the inode array. however, for path ending with
+      // .snapshot, still keep last the null inode in the array
+      INode[] newNodes = new INode[components.length - 1];
+      System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
+      inodes = newNodes;
+    }
+    return new INodesInPath(inodes, components, isSnapshot, snapshotId);
+  }
+
+  private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
+    return snapshotId == CURRENT_STATE_ID || (sid != CURRENT_STATE_ID &&
+        ID_INTEGER_COMPARATOR.compare(snapshotId, sid) < 0);
   }
 
-  private final byte[][] path;
   /**
-   * Array with the specified number of INodes resolved for a given path.
+   * Replace an inode of the given INodesInPath in the given position. We do a
+   * deep copy of the INode array.
+   * @param pos the position of the replacement
+   * @param inode the new inode
+   * @return a new INodesInPath instance
    */
-  private INode[] inodes;
+  public static INodesInPath replace(INodesInPath iip, int pos, INode inode) {
+    Preconditions.checkArgument(iip.length() > 0 && pos > 0 // no for root
+        && pos < iip.length());
+    if (iip.getINode(pos) == null) {
+      Preconditions.checkState(iip.getINode(pos - 1) != null);
+    }
+    INode[] inodes = new INode[iip.inodes.length];
+    System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
+    inodes[pos] = inode;
+    return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
+  }
+
   /**
-   * Indicate the number of non-null elements in {@link #inodes}
+   * Extend a given INodesInPath with a child INode. The child INode will be
+   * appended to the end of the new INodesInPath.
    */
-  private int numNonNull;
+  public static INodesInPath append(INodesInPath iip, INode child,
+      byte[] childName) {
+    Preconditions.checkArgument(!iip.isSnapshot && iip.length() > 0);
+    Preconditions.checkArgument(iip.getLastINode() != null && iip
+        .getLastINode().isDirectory());
+    INode[] inodes = new INode[iip.length() + 1];
+    System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length - 1);
+    inodes[inodes.length - 1] = child;
+    byte[][] path = new byte[iip.path.length + 1][];
+    System.arraycopy(iip.path, 0, path, 0, path.length - 1);
+    path[path.length - 1] = childName;
+    return new INodesInPath(inodes, path, false, iip.snapshotId);
+  }
+
+  private final byte[][] path;
   /**
-   * The path for a snapshot file/dir contains the .snapshot thus makes the
-   * length of the path components larger the number of inodes. We use
-   * the capacity to control this special case.
+   * Array with the specified number of INodes resolved for a given path.
    */
-  private int capacity;
+  private final INode[] inodes;
   /**
    * true if this path corresponds to a snapshot
    */
-  private boolean isSnapshot;
-  /**
-   * index of the {@link Snapshot.Root} node in the inodes array,
-   * -1 for non-snapshot paths.
-   */
-  private int snapshotRootIndex;
+  private final boolean isSnapshot;
   /**
    * For snapshot paths, it is the id of the snapshot; or 
    * {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For 
    * non-snapshot paths, it is the id of the latest snapshot found in the path;
    * or {@link Snapshot#CURRENT_STATE_ID} if no snapshot is found.
    */
-  private int snapshotId = Snapshot.CURRENT_STATE_ID; 
+  private final int snapshotId;
 
-  private INodesInPath(byte[][] path, int number) {
+  private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
+      int snapshotId) {
+    Preconditions.checkArgument(inodes != null && path != null);
+    this.inodes = inodes;
     this.path = path;
-    assert (number >= 0);
-    inodes = new INode[number];
-    capacity = number;
-    numNonNull = 0;
-    isSnapshot = false;
-    snapshotRootIndex = -1;
+    this.isSnapshot = isSnapshot;
+    this.snapshotId = snapshotId;
+  }
+
+  private INodesInPath(INode[] inodes, byte[][] path) {
+    this(inodes, path, false, CURRENT_STATE_ID);
   }
 
   /**
@@ -296,102 +308,125 @@ public int getLatestSnapshotId() {
    * For non-snapshot paths, return {@link Snapshot#CURRENT_STATE_ID}.
    */
   public int getPathSnapshotId() {
-    return isSnapshot ? snapshotId : Snapshot.CURRENT_STATE_ID;
+    return isSnapshot ? snapshotId : CURRENT_STATE_ID;
   }
 
-  private void setSnapshotId(int sid) {
-    snapshotId = sid;
-  }
-  
-  private void updateLatestSnapshotId(int sid) {
-    if (snapshotId == Snapshot.CURRENT_STATE_ID
-        || (sid != Snapshot.CURRENT_STATE_ID && Snapshot.ID_INTEGER_COMPARATOR
-            .compare(snapshotId, sid) < 0)) {
-      snapshotId = sid;
-    }
-  }
-
-  /**
-   * @return a new array of inodes excluding the null elements introduced by
-   * snapshot path elements. E.g., after resolving path "/dir/.snapshot",
-   * {@link #inodes} is {/, dir, null}, while the returned array only contains
-   * inodes of "/" and "dir". Note the length of the returned array is always
-   * equal to {@link #capacity}.
-   */
-  INode[] getINodes() {
-    if (capacity == inodes.length) {
-      return inodes;
-    }
-
-    INode[] newNodes = new INode[capacity];
-    System.arraycopy(inodes, 0, newNodes, 0, capacity);
-    return newNodes;
-  }
-  
   /**
    * @return the i-th inode if i >= 0;
    *         otherwise, i < 0, return the (length + i)-th inode.
    */
   public INode getINode(int i) {
-    return inodes[i >= 0? i: inodes.length + i];
+    if (inodes == null || inodes.length == 0) {
+      throw new NoSuchElementException("inodes is null or empty");
+    }
+    int index = i >= 0 ? i : inodes.length + i;
+    if (index < inodes.length && index >= 0) {
+      return inodes[index];
+    } else {
+      throw new NoSuchElementException("inodes.length == " + inodes.length);
+    }
   }
   
   /** @return the last inode. */
   public INode getLastINode() {
-    return inodes[inodes.length - 1];
+    return getINode(-1);
   }
 
   byte[] getLastLocalName() {
     return path[path.length - 1];
   }
 
+  public byte[][] getPathComponents() {
+    return path;
+  }
+
   /** @return the full path in string form */
   public String getPath() {
     return DFSUtil.byteArray2PathString(path);
   }
 
+  public String getParentPath() {
+    return getPath(path.length - 1);
+  }
+
+  public String getPath(int pos) {
+    return DFSUtil.byteArray2PathString(path, 0, pos);
+  }
+
   /**
-   * @return index of the {@link Snapshot.Root} node in the inodes array,
-   * -1 for non-snapshot paths.
+   * @param offset start endpoint (inclusive)
+   * @param length number of path components
+   * @return sub-list of the path
    */
-  int getSnapshotRootIndex() {
-    return this.snapshotRootIndex;
+  public List<String> getPath(int offset, int length) {
+    Preconditions.checkArgument(offset >= 0 && length >= 0 && offset + length
+        <= path.length);
+    ImmutableList.Builder<String> components = ImmutableList.builder();
+    for (int i = offset; i < offset + length; i++) {
+      components.add(DFSUtil.bytes2String(path[i]));
+    }
+    return components.build();
   }
-  
+
+  public int length() {
+    return inodes.length;
+  }
+
+  public List<INode> getReadOnlyINodes() {
+    return Collections.unmodifiableList(Arrays.asList(inodes));
+  }
+
   /**
-   * @return isSnapshot true for a snapshot path
+   * @param length number of ancestral INodes in the returned INodesInPath
+   *               instance
+   * @return the INodesInPath instance containing ancestral INodes. Note that
+   * this method only handles non-snapshot paths.
    */
-  boolean isSnapshot() {
-    return this.isSnapshot;
+  private INodesInPath getAncestorINodesInPath(int length) {
+    Preconditions.checkArgument(length >= 0 && length < inodes.length);
+    Preconditions.checkState(!isSnapshot());
+    final INode[] anodes = new INode[length];
+    final byte[][] apath = new byte[length][];
+    System.arraycopy(this.inodes, 0, anodes, 0, length);
+    System.arraycopy(this.path, 0, apath, 0, length);
+    return new INodesInPath(anodes, apath, false, snapshotId);
   }
-  
+
   /**
-   * Add an INode at the end of the array
+   * @return an INodesInPath instance containing all the INodes in the parent
+   *         path. We do a deep copy here.
    */
-  private void addNode(INode node) {
-    inodes[numNonNull++] = node;
+  public INodesInPath getParentINodesInPath() {
+    return inodes.length > 1 ? getAncestorINodesInPath(inodes.length - 1) :
+        null;
   }
 
-  private void setINodes(INode inodes[]) {
-    this.inodes = inodes;
-    this.numNonNull = this.inodes.length;
-  }
-  
-  void setINode(int i, INode inode) {
-    inodes[i >= 0? i: inodes.length + i] = inode;
-  }
-  
-  void setLastINode(INode last) {
-    inodes[inodes.length - 1] = last;
+  /**
+   * @return a new INodesInPath instance that only contains exisitng INodes.
+   * Note that this method only handles non-snapshot paths.
+   */
+  public INodesInPath getExistingINodes() {
+    Preconditions.checkState(!isSnapshot());
+    int i = 0;
+    for (; i < inodes.length; i++) {
+      if (inodes[i] == null) {
+        break;
+      }
+    }
+    INode[] existing = new INode[i];
+    byte[][] existingPath = new byte[i][];
+    System.arraycopy(inodes, 0, existing, 0, i);
+    System.arraycopy(path, 0, existingPath, 0, i);
+    return new INodesInPath(existing, existingPath, false, snapshotId);
   }
-  
+
   /**
-   * @return The number of non-null elements
+   * @return isSnapshot true for a snapshot path
    */
-  int getNumNonNull() {
-    return numNonNull;
+  boolean isSnapshot() {
+    return this.isSnapshot;
   }
-  
+
   private static String toString(INode inode) {
     return inode == null? null: inode.getLocalName();
   }
@@ -420,20 +455,16 @@ private String toString(boolean vaildateObject) {
       }
       b.append("], length=").append(inodes.length);
     }
-    b.append("\n  numNonNull = ").append(numNonNull)
-     .append("\n  capacity   = ").append(capacity)
-     .append("\n  isSnapshot        = ").append(isSnapshot)
-     .append("\n  snapshotRootIndex = ").append(snapshotRootIndex)
+    b.append("\n  isSnapshot        = ").append(isSnapshot)
      .append("\n  snapshotId        = ").append(snapshotId);
     return b.toString();
   }
 
   void validate() {
-    // check parent up to snapshotRootIndex or numNonNull
-    final int n = snapshotRootIndex >= 0? snapshotRootIndex + 1: numNonNull;  
+    // check parent up to snapshotRootIndex if this is a snapshot path
     int i = 0;
     if (inodes[i] != null) {
-      for(i++; i < n && inodes[i] != null; i++) {
+      for(i++; i < inodes.length && inodes[i] != null; i++) {
         final INodeDirectory parent_i = inodes[i].getParent();
         final INodeDirectory parent_i_1 = inodes[i-1].getParent();
         if (parent_i != inodes[i-1] &&
@@ -447,8 +478,8 @@ void validate() {
         }
       }
     }
-    if (i != n) {
-      throw new AssertionError("i = " + i + " != " + n
+    if (i != inodes.length) {
+      throw new AssertionError("i = " + i + " != " + inodes.length
           + ", this=" + toString(false));
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
index d10aacc77685f..702c8f14db1d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
@@ -81,9 +81,6 @@ public class ImageServlet extends HttpServlet {
   private static final String LATEST_FSIMAGE_VALUE = "latest";
   private static final String IMAGE_FILE_TYPE = "imageFile";
 
-  private static final Set<Long> currentlyDownloadingCheckpoints =
-    Collections.synchronizedSet(new HashSet<Long>());
-  
   @Override
   public void doGet(final HttpServletRequest request,
       final HttpServletResponse response) throws ServletException, IOException {
@@ -467,17 +464,20 @@ public Void run() throws Exception {
 
               final NameNodeFile nnf = parsedParams.getNameNodeFile();
 
-              if (!currentlyDownloadingCheckpoints.add(txid)) {
+              if (!nnImage.addToCheckpointing(txid)) {
                 response.sendError(HttpServletResponse.SC_CONFLICT,
-                    "Another checkpointer is already in the process of uploading a"
-                        + " checkpoint made at transaction ID " + txid);
+                    "Either current namenode is checkpointing or another"
+                        + " checkpointer is already in the process of "
+                        + "uploading a checkpoint made at transaction ID "
+                        + txid);
                 return null;
               }
               try {
                 if (nnImage.getStorage().findImageFile(nnf, txid) != null) {
                   response.sendError(HttpServletResponse.SC_CONFLICT,
-                      "Another checkpointer already uploaded an checkpoint "
-                          + "for txid " + txid);
+                      "Either current namenode has checkpointed or "
+                          + "another checkpointer already uploaded an "
+                          + "checkpoint for txid " + txid);
                   return null;
                 }
 
@@ -502,7 +502,7 @@ public Void run() throws Exception {
                   stream.close();
                 }
               } finally {
-                currentlyDownloadingCheckpoints.remove(txid);
+                nnImage.removeFromCheckpointing(txid);
               }
               return null;
             }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index cd3fc23b2aae6..5345b46de5c15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -53,15 +53,22 @@ public static EventBatch translate(FSEditLogOp op) {
             .groupName(addOp.permissions.getGroupName())
             .perms(addOp.permissions.getPermission())
             .overwrite(addOp.overwrite)
+            .defaultBlockSize(addOp.blockSize)
             .iNodeType(Event.CreateEvent.INodeType.FILE).build() });
-      } else {
+      } else { // append
         return new EventBatch(op.txid,
-            new Event[] { new Event.AppendEvent(addOp.path) });
+            new Event[]{new Event.AppendEvent.Builder()
+                .path(addOp.path)
+                .build()});
       }
     case OP_CLOSE:
       FSEditLogOp.CloseOp cOp = (FSEditLogOp.CloseOp) op;
       return new EventBatch(op.txid, new Event[] {
           new Event.CloseEvent(cOp.path, getSize(cOp), cOp.mtime) });
+    case OP_APPEND:
+      FSEditLogOp.AppendOp appendOp = (FSEditLogOp.AppendOp) op;
+      return new EventBatch(op.txid, new Event[] {new Event.AppendEvent
+          .Builder().path(appendOp.path).newBlock(appendOp.newBlock).build()});
     case OP_SET_REPLICATION:
       FSEditLogOp.SetReplicationOp setRepOp = (FSEditLogOp.SetReplicationOp) op;
       return new EventBatch(op.txid,
@@ -72,25 +79,40 @@ public static EventBatch translate(FSEditLogOp op) {
     case OP_CONCAT_DELETE:
       FSEditLogOp.ConcatDeleteOp cdOp = (FSEditLogOp.ConcatDeleteOp) op;
       List<Event> events = Lists.newArrayList();
-      events.add(new Event.AppendEvent(cdOp.trg));
+      events.add(new Event.AppendEvent.Builder()
+          .path(cdOp.trg)
+          .build());
       for (String src : cdOp.srcs) {
-        events.add(new Event.UnlinkEvent(src, cdOp.timestamp));
+        events.add(new Event.UnlinkEvent.Builder()
+          .path(src)
+          .timestamp(cdOp.timestamp)
+          .build());
       }
       events.add(new Event.CloseEvent(cdOp.trg, -1, cdOp.timestamp));
       return new EventBatch(op.txid, events.toArray(new Event[0]));
     case OP_RENAME_OLD:
       FSEditLogOp.RenameOldOp rnOpOld = (FSEditLogOp.RenameOldOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.RenameEvent(rnOpOld.src,
-              rnOpOld.dst, rnOpOld.timestamp) });
+          new Event.RenameEvent.Builder()
+              .srcPath(rnOpOld.src)
+              .dstPath(rnOpOld.dst)
+              .timestamp(rnOpOld.timestamp)
+              .build() });
     case OP_RENAME:
       FSEditLogOp.RenameOp rnOp = (FSEditLogOp.RenameOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) });
+          new Event.RenameEvent.Builder()
+            .srcPath(rnOp.src)
+            .dstPath(rnOp.dst)
+            .timestamp(rnOp.timestamp)
+            .build() });
     case OP_DELETE:
       FSEditLogOp.DeleteOp delOp = (FSEditLogOp.DeleteOp) op;
       return new EventBatch(op.txid, new Event[] {
-          new Event.UnlinkEvent(delOp.path, delOp.timestamp) });
+          new Event.UnlinkEvent.Builder()
+            .path(delOp.path)
+            .timestamp(delOp.timestamp)
+            .build() });
     case OP_MKDIR:
       FSEditLogOp.MkdirOp mkOp = (FSEditLogOp.MkdirOp) op;
       return new EventBatch(op.txid,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index e13a5c67e9e11..0dafaae7fc7d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -37,7 +37,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.util.Daemon;
 
@@ -116,14 +116,14 @@ synchronized long getNumUnderConstructionBlocks() {
         final INodeFile cons;
         try {
           cons = this.fsnamesystem.getFSDirectory().getINode(path).asFile();
-            Preconditions.checkState(cons.isUnderConstruction());
+          Preconditions.checkState(cons.isUnderConstruction());
         } catch (UnresolvedLinkException e) {
           throw new AssertionError("Lease files should reside on this FS");
         }
-        BlockInfo[] blocks = cons.getBlocks();
+        BlockInfoContiguous[] blocks = cons.getBlocks();
         if(blocks == null)
           continue;
-        for(BlockInfo b : blocks) {
+        for(BlockInfoContiguous b : blocks) {
           if(!b.isComplete())
             numUCBlocks++;
         }
@@ -481,8 +481,10 @@ synchronized boolean checkLeases() {
       leaseToCheck.getPaths().toArray(leasePaths);
       for(String p : leasePaths) {
         try {
+          INodesInPath iip = fsnamesystem.getFSDirectory().getINodesInPath(p,
+              true);
           boolean completed = fsnamesystem.internalReleaseLease(leaseToCheck, p,
-              HdfsServerConstants.NAMENODE_LEASE_HOLDER);
+              iip, HdfsServerConstants.NAMENODE_LEASE_HOLDER);
           if (LOG.isDebugEnabled()) {
             if (completed) {
               LOG.debug("Lease recovery for " + p + " is complete. File closed.");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index a71d158b11617..9dcf25b41a8fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -21,9 +21,6 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -43,11 +40,21 @@
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.hdfs.server.namenode.ha.*;
+import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
+import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
+import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
+import org.apache.hadoop.hdfs.server.namenode.ha.StandbyState;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressMetrics;
-import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -58,7 +65,6 @@
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
-import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
 import org.apache.hadoop.tools.GetUserMappingsProtocol;
 import org.apache.hadoop.tracing.SpanReceiverHost;
 import org.apache.hadoop.tracing.TraceAdminProtocol;
@@ -67,6 +73,9 @@
 import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.ServicePlugin;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.LogManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.management.ObjectName;
 
@@ -79,11 +88,46 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_SERVICE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
 
@@ -252,9 +296,12 @@ public long getProtocolVersion(String protocol,
   }
     
   public static final int DEFAULT_PORT = 8020;
-  public static final Log LOG = LogFactory.getLog(NameNode.class.getName());
-  public static final Log stateChangeLog = LogFactory.getLog("org.apache.hadoop.hdfs.StateChange");
-  public static final Log blockStateChangeLog = LogFactory.getLog("BlockStateChange");
+  public static final Logger LOG =
+      LoggerFactory.getLogger(NameNode.class.getName());
+  public static final Logger stateChangeLog =
+      LoggerFactory.getLogger("org.apache.hadoop.hdfs.StateChange");
+  public static final Logger blockStateChangeLog =
+      LoggerFactory.getLogger("BlockStateChange");
   public static final HAState ACTIVE_STATE = new ActiveState();
   public static final HAState STANDBY_STATE = new StandbyState();
   
@@ -265,6 +312,7 @@ public long getProtocolVersion(String protocol,
   private final boolean haEnabled;
   private final HAContext haContext;
   protected final boolean allowStaleStandbyReads;
+  private AtomicBoolean started = new AtomicBoolean(false); 
 
   
   /** httpServer */
@@ -347,7 +395,7 @@ public void setClientNamenodeAddress(Configuration conf) {
       return;
     }
 
-    LOG.info(FS_DEFAULT_NAME_KEY + " is " + nnAddr);
+    LOG.info("{} is {}", FS_DEFAULT_NAME_KEY, nnAddr);
     URI nnUri = URI.create(nnAddr);
 
     String nnHost = nnUri.getHost();
@@ -367,8 +415,8 @@ public void setClientNamenodeAddress(Configuration conf) {
       clientNamenodeAddress = null;
       return;
     }
-    LOG.info("Clients are to use " + clientNamenodeAddress + " to access"
-        + " this namenode/service.");
+    LOG.info("Clients are to use {} to access"
+        + " this namenode/service.", clientNamenodeAddress );
   }
 
   /**
@@ -389,7 +437,7 @@ public static InetSocketAddress getAddress(String address) {
    */
   public static void setServiceAddress(Configuration conf,
                                            String address) {
-    LOG.info("Setting ADDRESS " + address);
+    LOG.info("Setting ADDRESS {}", address);
     conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, address);
   }
   
@@ -775,6 +823,7 @@ protected NameNode(Configuration conf, NamenodeRole role)
       this.stop();
       throw e;
     }
+    this.started.set(true);
   }
 
   protected HAState createHAState(StartupOption startOpt) {
@@ -1006,7 +1055,7 @@ private static boolean initializeSharedEdits(Configuration conf,
     initializeGenericKeys(conf, nsId, namenodeId);
     
     if (conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY) == null) {
-      LOG.fatal("No shared edits directory configured for namespace " +
+      LOG.error("No shared edits directory configured for namespace " +
           nsId + " namenode " + namenodeId);
       return false;
     }
@@ -1180,7 +1229,7 @@ static StartupOption parseArguments(String args[]) {
             i++;
             if (i >= argsLen) {
               // if no cluster id specified, return null
-              LOG.fatal("Must specify a valid cluster ID after the "
+              LOG.error("Must specify a valid cluster ID after the "
                   + StartupOption.CLUSTERID.getName() + " flag");
               return null;
             }
@@ -1190,7 +1239,7 @@ static StartupOption parseArguments(String args[]) {
                 clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) ||
                 clusterId.equalsIgnoreCase(
                     StartupOption.NONINTERACTIVE.getName())) {
-              LOG.fatal("Must specify a valid cluster ID after the "
+              LOG.error("Must specify a valid cluster ID after the "
                   + StartupOption.CLUSTERID.getName() + " flag");
               return null;
             }
@@ -1227,7 +1276,7 @@ static StartupOption parseArguments(String args[]) {
               i += 2;
               startOpt.setClusterId(args[i]);
             } else {
-              LOG.fatal("Must specify a valid cluster ID after the "
+              LOG.error("Must specify a valid cluster ID after the "
                   + StartupOption.CLUSTERID.getName() + " flag");
               return null;
             }
@@ -1241,7 +1290,7 @@ static StartupOption parseArguments(String args[]) {
               i += 1;
             }
           } else {
-            LOG.fatal("Unknown upgrade flag " + flag);
+            LOG.error("Unknown upgrade flag " + flag);
             return null;
           }
         }
@@ -1249,7 +1298,7 @@ static StartupOption parseArguments(String args[]) {
         startOpt = StartupOption.ROLLINGUPGRADE;
         ++i;
         if (i >= argsLen) {
-          LOG.fatal("Must specify a rolling upgrade startup option "
+          LOG.error("Must specify a rolling upgrade startup option "
               + RollingUpgradeStartupOption.getAllOptionString());
           return null;
         }
@@ -1271,7 +1320,7 @@ static StartupOption parseArguments(String args[]) {
           } else if (StartupOption.FORCE.getName().equals(args[i])) {
             startOpt.setForceFormat(true);
           } else {
-            LOG.fatal("Invalid argument: " + args[i]);
+            LOG.error("Invalid argument: " + args[i]);
             return null;
           }
         }
@@ -1511,7 +1560,7 @@ public static void main(String argv[]) throws Exception {
         namenode.join();
       }
     } catch (Throwable e) {
-      LOG.fatal("Failed to start namenode.", e);
+      LOG.error("Failed to start namenode.", e);
       terminate(1, e);
     }
   }
@@ -1638,7 +1687,7 @@ protected synchronized void doImmediateShutdown(Throwable t)
     String message = "Error encountered requiring NN shutdown. " +
         "Shutting down immediately.";
     try {
-      LOG.fatal(message, t);
+      LOG.error(message, t);
     } catch (Throwable ignored) {
       // This is unlikely to happen, but there's nothing we can do if it does.
     }
@@ -1743,7 +1792,14 @@ public boolean isStandbyState() {
   public boolean isActiveState() {
     return (state.equals(ACTIVE_STATE));
   }
-  
+
+  /**
+   * Returns whether the NameNode is completely started
+   */
+  boolean isStarted() {
+    return this.started.get();
+  }
+
   /**
    * Check that a request to change this node's HA state is valid.
    * In particular, verifies that, if auto failover is enabled, non-forced
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 512913b3accb1..848fa33e48a6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -69,8 +69,10 @@ public static enum Feature implements LayoutFeature {
     CREATE_OVERWRITE(-58, "Use single editlog record for " +
       "creating file with overwrite"),
     XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"),
-    BLOCK_STORAGE_POLICY(-60, "Block Storage policy");
-    
+    BLOCK_STORAGE_POLICY(-60, "Block Storage policy"),
+    TRUNCATE(-61, "Truncate"),
+    APPEND_NEW_BLOCK(-62, "Support appending to new block");
+
     private final FeatureInfo info;
 
     /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 876afbaa2df67..1c434e82d5ddb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -23,6 +23,7 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_PATH_DEPTH;
 import static org.apache.hadoop.hdfs.protocol.HdfsConstants.MAX_PATH_LENGTH;
+import static org.apache.hadoop.util.Time.now;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -36,7 +37,6 @@
 
 import com.google.common.collect.Lists;
 
-import org.apache.commons.logging.Log;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
@@ -68,7 +68,7 @@
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
-import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.inotify.EventBatch;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
 import org.apache.hadoop.hdfs.protocol.AclException;
@@ -177,6 +177,7 @@
 import org.apache.hadoop.tracing.TraceAdminProtocolServerSideTranslatorPB;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.util.VersionUtil;
+import org.slf4j.Logger;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingService;
@@ -187,9 +188,10 @@
  */
 class NameNodeRpcServer implements NamenodeProtocols {
   
-  private static final Log LOG = NameNode.LOG;
-  private static final Log stateChangeLog = NameNode.stateChangeLog;
-  private static final Log blockStateChangeLog = NameNode.blockStateChangeLog;
+  private static final Logger LOG = NameNode.LOG;
+  private static final Logger stateChangeLog = NameNode.stateChangeLog;
+  private static final Logger blockStateChangeLog = NameNode
+      .blockStateChangeLog;
   
   // Dependencies from other parts of NN.
   protected final FSNamesystem namesystem;
@@ -478,12 +480,14 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
       throw new IllegalArgumentException(
         "Unexpected not positive size: "+size);
     }
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return namesystem.getBlockManager().getBlocks(datanode, size); 
   }
 
   @Override // NamenodeProtocol
   public ExportedBlockKeys getBlockKeys() throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return namesystem.getBlockManager().getBlockKeys();
   }
@@ -492,6 +496,7 @@ public ExportedBlockKeys getBlockKeys() throws IOException {
   public void errorReport(NamenodeRegistration registration,
                           int errorCode, 
                           String msg) throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
     namesystem.checkSuperuserPrivilege();
     verifyRequest(registration);
@@ -504,6 +509,7 @@ public void errorReport(NamenodeRegistration registration,
   @Override // NamenodeProtocol
   public NamenodeRegistration registerSubordinateNamenode(
       NamenodeRegistration registration) throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     verifyLayoutVersion(registration.getVersion());
     NamenodeRegistration myRegistration = nn.setRegistration();
@@ -513,7 +519,8 @@ public NamenodeRegistration registerSubordinateNamenode(
 
   @Override // NamenodeProtocol
   public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
-  throws IOException {
+      throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     verifyRequest(registration);
     if(!nn.isRole(NamenodeRole.NAMENODE))
@@ -536,6 +543,7 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
   @Override // NamenodeProtocol
   public void endCheckpoint(NamenodeRegistration registration,
                             CheckpointSignature sig) throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -553,18 +561,21 @@ public void endCheckpoint(NamenodeRegistration registration,
   @Override // ClientProtocol
   public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
       throws IOException {
+    checkNNStartup();
     return namesystem.getDelegationToken(renewer);
   }
 
   @Override // ClientProtocol
   public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
       throws InvalidToken, IOException {
+    checkNNStartup();
     return namesystem.renewDelegationToken(token);
   }
 
   @Override // ClientProtocol
   public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
       throws IOException {
+    checkNNStartup();
     namesystem.cancelDelegationToken(token);
   }
   
@@ -573,6 +584,7 @@ public LocatedBlocks getBlockLocations(String src,
                                           long offset, 
                                           long length) 
       throws IOException {
+    checkNNStartup();
     metrics.incrGetBlockLocations();
     return namesystem.getBlockLocations(getClientMachine(), 
                                         src, offset, length);
@@ -580,6 +592,7 @@ public LocatedBlocks getBlockLocations(String src,
   
   @Override // ClientProtocol
   public FsServerDefaults getServerDefaults() throws IOException {
+    checkNNStartup();
     return namesystem.getServerDefaults();
   }
 
@@ -589,6 +602,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
       boolean createParent, short replication, long blockSize, 
       CryptoProtocolVersion[] supportedVersions)
       throws IOException {
+    checkNNStartup();
     String clientMachine = getClientMachine();
     if (stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.create: file "
@@ -621,14 +635,16 @@ public HdfsFileStatus create(String src, FsPermission masked,
   }
 
   @Override // ClientProtocol
-  public LastBlockWithStatus append(String src, String clientName) 
-      throws IOException {
+  public LastBlockWithStatus append(String src, String clientName,
+      EnumSetWritable<CreateFlag> flag) throws IOException {
+    checkNNStartup();
     String clientMachine = getClientMachine();
     if (stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.append: file "
           +src+" for "+clientName+" at "+clientMachine);
     }
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null);
+    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
+        null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (LastBlockWithStatus) cacheEntry.getPayload();
     }
@@ -636,7 +652,7 @@ public LastBlockWithStatus append(String src, String clientName)
     LastBlockWithStatus info = null;
     boolean success = false;
     try {
-      info = namesystem.appendFile(src, clientName, clientMachine,
+      info = namesystem.appendFile(src, clientName, clientMachine, flag.get(),
           cacheEntry != null);
       success = true;
     } finally {
@@ -648,36 +664,42 @@ public LastBlockWithStatus append(String src, String clientName)
 
   @Override // ClientProtocol
   public boolean recoverLease(String src, String clientName) throws IOException {
+    checkNNStartup();
     String clientMachine = getClientMachine();
     return namesystem.recoverLease(src, clientName, clientMachine);
   }
 
   @Override // ClientProtocol
   public boolean setReplication(String src, short replication) 
-    throws IOException {  
+    throws IOException {
+    checkNNStartup();
     return namesystem.setReplication(src, replication);
   }
 
   @Override
   public void setStoragePolicy(String src, String policyName)
       throws IOException {
+    checkNNStartup();
     namesystem.setStoragePolicy(src, policyName);
   }
 
   @Override
   public BlockStoragePolicy[] getStoragePolicies() throws IOException {
+    checkNNStartup();
     return namesystem.getStoragePolicies();
   }
 
   @Override // ClientProtocol
   public void setPermission(String src, FsPermission permissions)
       throws IOException {
+    checkNNStartup();
     namesystem.setPermission(src, permissions);
   }
 
   @Override // ClientProtocol
   public void setOwner(String src, String username, String groupname)
       throws IOException {
+    checkNNStartup();
     namesystem.setOwner(src, username, groupname);
   }
   
@@ -686,6 +708,7 @@ public LocatedBlock addBlock(String src, String clientName,
       ExtendedBlock previous, DatanodeInfo[] excludedNodes, long fileId,
       String[] favoredNodes)
       throws IOException {
+    checkNNStartup();
     if (stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*BLOCK* NameNode.addBlock: file " + src
           + " fileId=" + fileId + " for " + clientName);
@@ -713,6 +736,7 @@ public LocatedBlock getAdditionalDatanode(final String src,
       final DatanodeInfo[] excludes,
       final int numAdditionalNodes, final String clientName
       ) throws IOException {
+    checkNNStartup();
     if (LOG.isDebugEnabled()) {
       LOG.debug("getAdditionalDatanode: src=" + src
           + ", fileId=" + fileId
@@ -741,6 +765,7 @@ public LocatedBlock getAdditionalDatanode(final String src,
   @Override // ClientProtocol
   public void abandonBlock(ExtendedBlock b, long fileId, String src,
         String holder) throws IOException {
+    checkNNStartup();
     if(stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*BLOCK* NameNode.abandonBlock: "
           +b+" of file "+src);
@@ -754,6 +779,7 @@ public void abandonBlock(ExtendedBlock b, long fileId, String src,
   public boolean complete(String src, String clientName,
                           ExtendedBlock last,  long fileId)
       throws IOException {
+    checkNNStartup();
     if(stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.complete: "
           + src + " fileId=" + fileId +" for " + clientName);
@@ -769,12 +795,14 @@ public boolean complete(String src, String clientName,
    */
   @Override // ClientProtocol, DatanodeProtocol
   public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
+    checkNNStartup();
     namesystem.reportBadBlocks(blocks);
   }
 
   @Override // ClientProtocol
   public LocatedBlock updateBlockForPipeline(ExtendedBlock block, String clientName)
       throws IOException {
+    checkNNStartup();
     return namesystem.updateBlockForPipeline(block, clientName);
   }
 
@@ -783,6 +811,7 @@ public LocatedBlock updateBlockForPipeline(ExtendedBlock block, String clientNam
   public void updatePipeline(String clientName, ExtendedBlock oldBlock,
       ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
       throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -804,6 +833,7 @@ public void commitBlockSynchronization(ExtendedBlock block,
       boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
       String[] newtargetstorages)
       throws IOException {
+    checkNNStartup();
     namesystem.commitBlockSynchronization(block, newgenerationstamp,
         newlength, closeFile, deleteblock, newtargets, newtargetstorages);
   }
@@ -811,12 +841,14 @@ public void commitBlockSynchronization(ExtendedBlock block,
   @Override // ClientProtocol
   public long getPreferredBlockSize(String filename) 
       throws IOException {
+    checkNNStartup();
     return namesystem.getPreferredBlockSize(filename);
   }
     
   @Deprecated
   @Override // ClientProtocol
   public boolean rename(String src, String dst) throws IOException {
+    checkNNStartup();
     if(stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
     }
@@ -844,6 +876,7 @@ public boolean rename(String src, String dst) throws IOException {
   
   @Override // ClientProtocol
   public void concat(String trg, String[] src) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -861,6 +894,7 @@ public void concat(String trg, String[] src) throws IOException {
   @Override // ClientProtocol
   public void rename2(String src, String dst, Options.Rename... options)
       throws IOException {
+    checkNNStartup();
     if(stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
     }
@@ -882,8 +916,26 @@ public void rename2(String src, String dst, Options.Rename... options)
     metrics.incrFilesRenamed();
   }
 
+  @Override // ClientProtocol
+  public boolean truncate(String src, long newLength, String clientName)
+      throws IOException {
+    checkNNStartup();
+    if(stateChangeLog.isDebugEnabled()) {
+      stateChangeLog.debug("*DIR* NameNode.truncate: " + src + " to " +
+          newLength);
+    }
+    String clientMachine = getClientMachine();
+    try {
+      return namesystem.truncate(
+          src, newLength, clientName, clientMachine, now());
+    } finally {
+      metrics.incrFilesTruncated();
+    }
+  }
+
   @Override // ClientProtocol
   public boolean delete(String src, boolean recursive) throws IOException {
+    checkNNStartup();
     if (stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
           + ", recursive=" + recursive);
@@ -918,6 +970,7 @@ private boolean checkPathLength(String src) {
   @Override // ClientProtocol
   public boolean mkdirs(String src, FsPermission masked, boolean createParent)
       throws IOException {
+    checkNNStartup();
     if(stateChangeLog.isDebugEnabled()) {
       stateChangeLog.debug("*DIR* NameNode.mkdirs: " + src);
     }
@@ -932,12 +985,14 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
 
   @Override // ClientProtocol
   public void renewLease(String clientName) throws IOException {
+    checkNNStartup();
     namesystem.renewLease(clientName);        
   }
 
   @Override // ClientProtocol
   public DirectoryListing getListing(String src, byte[] startAfter,
       boolean needLocation) throws IOException {
+    checkNNStartup();
     DirectoryListing files = namesystem.getListing(
         src, startAfter, needLocation);
     if (files != null) {
@@ -949,23 +1004,27 @@ public DirectoryListing getListing(String src, byte[] startAfter,
 
   @Override // ClientProtocol
   public HdfsFileStatus getFileInfo(String src)  throws IOException {
+    checkNNStartup();
     metrics.incrFileInfoOps();
     return namesystem.getFileInfo(src, true);
   }
   
   @Override // ClientProtocol
   public boolean isFileClosed(String src) throws IOException{
+    checkNNStartup();
     return namesystem.isFileClosed(src);
   }
   
   @Override // ClientProtocol
-  public HdfsFileStatus getFileLinkInfo(String src) throws IOException { 
+  public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
+    checkNNStartup();
     metrics.incrFileInfoOps();
     return namesystem.getFileInfo(src, false);
   }
   
   @Override // ClientProtocol
   public long[] getStats() throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ);
     return namesystem.getStats();
   }
@@ -973,28 +1032,23 @@ public long[] getStats() throws IOException {
   @Override // ClientProtocol
   public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
   throws IOException {
+    checkNNStartup();
     DatanodeInfo results[] = namesystem.datanodeReport(type);
-    if (results == null ) {
-      throw new IOException("Failed to get datanode report for " + type
-          + " datanodes.");
-    }
     return results;
   }
     
   @Override // ClientProtocol
   public DatanodeStorageReport[] getDatanodeStorageReport(
       DatanodeReportType type) throws IOException {
+    checkNNStartup();
     final DatanodeStorageReport[] reports = namesystem.getDatanodeStorageReport(type);
-    if (reports == null ) {
-      throw new IOException("Failed to get datanode storage report for " + type
-          + " datanodes.");
-    }
     return reports;
   }
 
   @Override // ClientProtocol
   public boolean setSafeMode(SafeModeAction action, boolean isChecked)
       throws IOException {
+    checkNNStartup();
     OperationCategory opCategory = OperationCategory.UNCHECKED;
     if (isChecked) {
       if (action == SafeModeAction.SAFEMODE_GET) {
@@ -1009,11 +1063,13 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
 
   @Override // ClientProtocol
   public boolean restoreFailedStorage(String arg) throws IOException { 
+    checkNNStartup();
     return namesystem.restoreFailedStorage(arg);
   }
 
   @Override // ClientProtocol
   public void saveNamespace() throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1029,17 +1085,20 @@ public void saveNamespace() throws IOException {
   
   @Override // ClientProtocol
   public long rollEdits() throws AccessControlException, IOException {
+    checkNNStartup();
     CheckpointSignature sig = namesystem.rollEditLog();
     return sig.getCurSegmentTxId();
   }
 
   @Override // ClientProtocol
   public void refreshNodes() throws IOException {
+    checkNNStartup();
     namesystem.refreshNodes();
   }
 
   @Override // NamenodeProtocol
   public long getTransactionID() throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
     namesystem.checkSuperuserPrivilege();
     return namesystem.getFSImage().getLastAppliedOrWrittenTxId();
@@ -1047,6 +1106,7 @@ public long getTransactionID() throws IOException {
   
   @Override // NamenodeProtocol
   public long getMostRecentCheckpointTxId() throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.UNCHECKED);
     namesystem.checkSuperuserPrivilege();
     return namesystem.getFSImage().getMostRecentCheckpointTxId();
@@ -1054,13 +1114,15 @@ public long getMostRecentCheckpointTxId() throws IOException {
   
   @Override // NamenodeProtocol
   public CheckpointSignature rollEditLog() throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return namesystem.rollEditLog();
   }
   
   @Override // NamenodeProtocol
   public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
-  throws IOException {
+      throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ);
     namesystem.checkSuperuserPrivilege();
     return namesystem.getEditLog().getEditLogManifest(sinceTxId);
@@ -1068,11 +1130,13 @@ public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
     
   @Override // ClientProtocol
   public void finalizeUpgrade() throws IOException {
+    checkNNStartup();
     namesystem.finalizeUpgrade();
   }
 
   @Override // ClientProtocol
   public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOException {
+    checkNNStartup();
     LOG.info("rollingUpgrade " + action);
     switch(action) {
     case QUERY:
@@ -1089,12 +1153,14 @@ public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action) throws IOE
 
   @Override // ClientProtocol
   public void metaSave(String filename) throws IOException {
+    checkNNStartup();
     namesystem.metaSave(filename);
   }
 
   @Override // ClientProtocol
   public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
       throws IOException {
+    checkNNStartup();
     String[] cookieTab = new String[] { cookie };
     Collection<FSNamesystem.CorruptFileBlockInfo> fbs =
       namesystem.listCorruptFileBlocks(path, cookieTab);
@@ -1115,17 +1181,25 @@ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
    */
   @Override // ClientProtocol
   public void setBalancerBandwidth(long bandwidth) throws IOException {
+    checkNNStartup();
     namesystem.setBalancerBandwidth(bandwidth);
   }
   
   @Override // ClientProtocol
   public ContentSummary getContentSummary(String path) throws IOException {
+    checkNNStartup();
     return namesystem.getContentSummary(path);
   }
 
   @Override // ClientProtocol
-  public void setQuota(String path, long namespaceQuota, long diskspaceQuota) 
+  public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
+                       StorageType type)
       throws IOException {
+    checkNNStartup();
+    if (type != null) {
+      throw new UnsupportedActionException(
+          "Quota by storage type support is not fully supported by namenode yet.");
+    }
     namesystem.setQuota(path, namespaceQuota, diskspaceQuota);
   }
   
@@ -1133,25 +1207,27 @@ public void setQuota(String path, long namespaceQuota, long diskspaceQuota)
   public void fsync(String src, long fileId, String clientName,
                     long lastBlockLength)
       throws IOException {
+    checkNNStartup();
     namesystem.fsync(src, fileId, clientName, lastBlockLength);
   }
 
   @Override // ClientProtocol
   public void setTimes(String src, long mtime, long atime) 
       throws IOException {
+    checkNNStartup();
     namesystem.setTimes(src, mtime, atime);
   }
 
   @Override // ClientProtocol
   public void createSymlink(String target, String link, FsPermission dirPerms,
       boolean createParent) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
 
-    metrics.incrCreateSymlinkOps();
-    /* We enforce the MAX_PATH_LENGTH limit even though a symlink target 
+    /* We enforce the MAX_PATH_LENGTH limit even though a symlink target
      * URI may refer to a non-HDFS file system. 
      */
     if (!checkPathLength(link)) {
@@ -1159,9 +1235,7 @@ public void createSymlink(String target, String link, FsPermission dirPerms,
                             " character limit");
                             
     }
-    if ("".equals(target)) {
-      throw new IOException("Invalid symlink target");
-    }
+
     final UserGroupInformation ugi = getRemoteUser();
 
     boolean success = false;
@@ -1178,6 +1252,7 @@ public void createSymlink(String target, String link, FsPermission dirPerms,
 
   @Override // ClientProtocol
   public String getLinkTarget(String path) throws IOException {
+    checkNNStartup();
     metrics.incrGetLinkTargetOps();
     HdfsFileStatus stat = null;
     try {
@@ -1200,6 +1275,7 @@ public String getLinkTarget(String path) throws IOException {
   @Override // DatanodeProtocol
   public DatanodeRegistration registerDatanode(DatanodeRegistration nodeReg)
       throws IOException {
+    checkNNStartup();
     verifySoftwareVersion(nodeReg);
     namesystem.registerDatanode(nodeReg);
     return nodeReg;
@@ -1210,6 +1286,7 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration nodeReg,
       StorageReport[] report, long dnCacheCapacity, long dnCacheUsed,
       int xmitsInProgress, int xceiverCount,
       int failedVolumes) throws IOException {
+    checkNNStartup();
     verifyRequest(nodeReg);
     return namesystem.handleHeartbeat(nodeReg, report,
         dnCacheCapacity, dnCacheUsed, xceiverCount, xmitsInProgress,
@@ -1219,6 +1296,7 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration nodeReg,
   @Override // DatanodeProtocol
   public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
       String poolId, StorageBlockReport[] reports) throws IOException {
+    checkNNStartup();
     verifyRequest(nodeReg);
     if(blockStateChangeLog.isDebugEnabled()) {
       blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: "
@@ -1250,6 +1328,7 @@ public DatanodeCommand blockReport(DatanodeRegistration nodeReg,
   @Override
   public DatanodeCommand cacheReport(DatanodeRegistration nodeReg,
       String poolId, List<Long> blockIds) throws IOException {
+    checkNNStartup();
     verifyRequest(nodeReg);
     if (blockStateChangeLog.isDebugEnabled()) {
       blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: "
@@ -1262,6 +1341,7 @@ public DatanodeCommand cacheReport(DatanodeRegistration nodeReg,
   @Override // DatanodeProtocol
   public void blockReceivedAndDeleted(DatanodeRegistration nodeReg, String poolId,
       StorageReceivedDeletedBlocks[] receivedAndDeletedBlocks) throws IOException {
+    checkNNStartup();
     verifyRequest(nodeReg);
     metrics.incrBlockReceivedAndDeletedOps();
     if(blockStateChangeLog.isDebugEnabled()) {
@@ -1277,6 +1357,7 @@ public void blockReceivedAndDeleted(DatanodeRegistration nodeReg, String poolId,
   @Override // DatanodeProtocol
   public void errorReport(DatanodeRegistration nodeReg,
                           int errorCode, String msg) throws IOException { 
+    checkNNStartup();
     String dnName = 
        (nodeReg == null) ? "Unknown DataNode" : nodeReg.toString();
 
@@ -1298,6 +1379,7 @@ public void errorReport(DatanodeRegistration nodeReg,
     
   @Override // DatanodeProtocol, NamenodeProtocol
   public NamespaceInfo versionRequest() throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return namesystem.getNamespaceInfo();
   }
@@ -1322,6 +1404,7 @@ private void verifyRequest(NodeRegistration nodeReg) throws IOException {
 
   @Override // RefreshAuthorizationPolicyProtocol
   public void refreshServiceAcl() throws IOException {
+    checkNNStartup();
     if (!serviceAuthEnabled) {
       throw new AuthorizationException("Service Level Authorization not enabled!");
     }
@@ -1372,28 +1455,32 @@ public String[] getGroupsForUser(String user) throws IOException {
   }
 
   @Override // HAServiceProtocol
-  public synchronized void monitorHealth() 
-      throws HealthCheckFailedException, AccessControlException {
+  public synchronized void monitorHealth() throws HealthCheckFailedException,
+      AccessControlException, IOException {
+    checkNNStartup();
     nn.monitorHealth();
   }
   
   @Override // HAServiceProtocol
   public synchronized void transitionToActive(StateChangeRequestInfo req) 
-      throws ServiceFailedException, AccessControlException {
+      throws ServiceFailedException, AccessControlException, IOException {
+    checkNNStartup();
     nn.checkHaStateChange(req);
     nn.transitionToActive();
   }
   
   @Override // HAServiceProtocol
   public synchronized void transitionToStandby(StateChangeRequestInfo req) 
-      throws ServiceFailedException, AccessControlException {
+      throws ServiceFailedException, AccessControlException, IOException {
+    checkNNStartup();
     nn.checkHaStateChange(req);
     nn.transitionToStandby();
   }
 
   @Override // HAServiceProtocol
   public synchronized HAServiceStatus getServiceStatus() 
-      throws AccessControlException, ServiceFailedException {
+      throws AccessControlException, ServiceFailedException, IOException {
+    checkNNStartup();
     return nn.getServiceStatus();
   }
 
@@ -1428,7 +1515,7 @@ private void verifySoftwareVersion(DatanodeRegistration dnReg)
         IncorrectVersionException ive = new IncorrectVersionException(
             messagePrefix + " and CTime of DN ('" + dnCTime +
             "') does not match CTime of NN ('" + nnCTime + "')");
-        LOG.warn(ive);
+        LOG.warn(ive.toString(), ive);
         throw ive;
       } else {
         LOG.info(messagePrefix +
@@ -1450,12 +1537,14 @@ private static String getClientMachine() {
 
   @Override
   public DataEncryptionKey getDataEncryptionKey() throws IOException {
+    checkNNStartup();
     return namesystem.getBlockManager().generateDataEncryptionKey();
   }
 
   @Override
   public String createSnapshot(String snapshotRoot, String snapshotName)
       throws IOException {
+    checkNNStartup();
     if (!checkPathLength(snapshotRoot)) {
       throw new IOException("createSnapshot: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
@@ -1480,6 +1569,7 @@ public String createSnapshot(String snapshotRoot, String snapshotName)
   @Override
   public void deleteSnapshot(String snapshotRoot, String snapshotName)
       throws IOException {
+    checkNNStartup();
     metrics.incrDeleteSnapshotOps();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1497,6 +1587,7 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName)
   @Override
   // Client Protocol
   public void allowSnapshot(String snapshotRoot) throws IOException {
+    checkNNStartup();
     metrics.incrAllowSnapshotOps();
     namesystem.allowSnapshot(snapshotRoot);
   }
@@ -1504,13 +1595,16 @@ public void allowSnapshot(String snapshotRoot) throws IOException {
   @Override
   // Client Protocol
   public void disallowSnapshot(String snapshot) throws IOException {
+    checkNNStartup();
     metrics.incrDisAllowSnapshotOps();
     namesystem.disallowSnapshot(snapshot);
   }
 
   @Override
+  // ClientProtocol
   public void renameSnapshot(String snapshotRoot, String snapshotOldName,
       String snapshotNewName) throws IOException {
+    checkNNStartup();
     if (snapshotNewName == null || snapshotNewName.isEmpty()) {
       throw new IOException("The new snapshot name is null or empty.");
     }
@@ -1532,24 +1626,27 @@ public void renameSnapshot(String snapshotRoot, String snapshotOldName,
   @Override // Client Protocol
   public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
       throws IOException {
+    checkNNStartup();
     SnapshottableDirectoryStatus[] status = namesystem
         .getSnapshottableDirListing();
     metrics.incrListSnapshottableDirOps();
     return status;
   }
 
-  @Override
+  @Override // ClientProtocol
   public SnapshotDiffReport getSnapshotDiffReport(String snapshotRoot,
       String earlierSnapshotName, String laterSnapshotName) throws IOException {
+    checkNNStartup();
     SnapshotDiffReport report = namesystem.getSnapshotDiffReport(snapshotRoot,
         earlierSnapshotName, laterSnapshotName);
     metrics.incrSnapshotDiffReportOps();
     return report;
   }
 
-  @Override
+  @Override // ClientProtocol
   public long addCacheDirective(
       CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
+    checkNNStartup();
     CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
       (retryCache, null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1567,9 +1664,10 @@ public long addCacheDirective(
     return ret;
   }
 
-  @Override
+  @Override // ClientProtocol
   public void modifyCacheDirective(
       CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1584,8 +1682,9 @@ public void modifyCacheDirective(
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public void removeCacheDirective(long id) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1599,17 +1698,19 @@ public void removeCacheDirective(long id) throws IOException {
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(long prevId,
       CacheDirectiveInfo filter) throws IOException {
+    checkNNStartup();
     if (filter == null) {
       filter = new CacheDirectiveInfo.Builder().build();
     }
     return namesystem.listCacheDirectives(prevId, filter);
   }
 
-  @Override
+  @Override //ClientProtocol
   public void addCachePool(CachePoolInfo info) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1623,8 +1724,9 @@ public void addCachePool(CachePoolInfo info) throws IOException {
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public void modifyCachePool(CachePoolInfo info) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1638,8 +1740,9 @@ public void modifyCachePool(CachePoolInfo info) throws IOException {
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public void removeCachePool(String cachePoolName) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1653,47 +1756,55 @@ public void removeCachePool(String cachePoolName) throws IOException {
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public BatchedEntries<CachePoolEntry> listCachePools(String prevKey)
       throws IOException {
+    checkNNStartup();
     return namesystem.listCachePools(prevKey != null ? prevKey : "");
   }
 
-  @Override
+  @Override // ClientProtocol
   public void modifyAclEntries(String src, List<AclEntry> aclSpec)
       throws IOException {
+    checkNNStartup();
     namesystem.modifyAclEntries(src, aclSpec);
   }
 
-  @Override
+  @Override // ClienProtocol
   public void removeAclEntries(String src, List<AclEntry> aclSpec)
       throws IOException {
+    checkNNStartup();
     namesystem.removeAclEntries(src, aclSpec);
   }
 
-  @Override
+  @Override // ClientProtocol
   public void removeDefaultAcl(String src) throws IOException {
+    checkNNStartup();
     namesystem.removeDefaultAcl(src);
   }
 
-  @Override
+  @Override // ClientProtocol
   public void removeAcl(String src) throws IOException {
+    checkNNStartup();
     namesystem.removeAcl(src);
   }
 
-  @Override
+  @Override // ClientProtocol
   public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
+    checkNNStartup();
     namesystem.setAcl(src, aclSpec);
   }
 
-  @Override
+  @Override // ClientProtocol
   public AclStatus getAclStatus(String src) throws IOException {
+    checkNNStartup();
     return namesystem.getAclStatus(src);
   }
   
-  @Override
+  @Override // ClientProtocol
   public void createEncryptionZone(String src, String keyName)
     throws IOException {
+    checkNNStartup();
     final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1707,21 +1818,24 @@ public void createEncryptionZone(String src, String keyName)
     }
   }
 
-  @Override
+  @Override // ClientProtocol
   public EncryptionZone getEZForPath(String src)
     throws IOException {
+    checkNNStartup();
     return namesystem.getEZForPath(src);
   }
 
-  @Override
+  @Override // ClientProtocol
   public BatchedEntries<EncryptionZone> listEncryptionZones(
       long prevId) throws IOException {
+    checkNNStartup();
     return namesystem.listEncryptionZones(prevId);
   }
 
-  @Override
+  @Override // ClientProtocol
   public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
       throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1735,19 +1849,22 @@ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
     }
   }
   
-  @Override
+  @Override // ClientProtocol
   public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs) 
       throws IOException {
+    checkNNStartup();
     return namesystem.getXAttrs(src, xAttrs);
   }
 
-  @Override
+  @Override // ClientProtocol
   public List<XAttr> listXAttrs(String src) throws IOException {
+    checkNNStartup();
     return namesystem.listXAttrs(src);
   }
   
-  @Override
+  @Override // ClientProtocol
   public void removeXAttr(String src, XAttr xAttr) throws IOException {
+    checkNNStartup();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1761,13 +1878,21 @@ public void removeXAttr(String src, XAttr xAttr) throws IOException {
     }
   }
 
-  @Override
+  private void checkNNStartup() throws IOException {
+    if (!this.nn.isStarted()) {
+      throw new IOException(this.nn.getRole() + " still not started");
+    }
+  }
+
+  @Override // ClientProtocol
   public void checkAccess(String path, FsAction mode) throws IOException {
+    checkNNStartup();
     namesystem.checkAccess(path, mode);
   }
 
   @Override // ClientProtocol
   public long getCurrentEditLogTxid() throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ); // only active
     namesystem.checkSuperuserPrivilege();
     // if it's not yet open for write, we may be in the process of transitioning
@@ -1796,6 +1921,7 @@ private static FSEditLogOp readOp(EditLogInputStream elis)
 
   @Override // ClientProtocol
   public EventBatchList getEditsFromTxid(long txid) throws IOException {
+    checkNNStartup();
     namesystem.checkOperation(OperationCategory.READ); // only active
     namesystem.checkSuperuserPrivilege();
     int maxEventsPerRPC = nn.conf.getInt(
@@ -1879,20 +2005,23 @@ public EventBatchList getEditsFromTxid(long txid) throws IOException {
     return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
   }
 
-  @Override
+  @Override // TraceAdminProtocol
   public SpanReceiverInfo[] listSpanReceivers() throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return nn.spanReceiverHost.listSpanReceivers();
   }
 
-  @Override
+  @Override // TraceAdminProtocol
   public long addSpanReceiver(SpanReceiverInfo info) throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     return nn.spanReceiverHost.addSpanReceiver(info);
   }
 
-  @Override
+  @Override // TraceAdminProtocol
   public void removeSpanReceiver(long id) throws IOException {
+    checkNNStartup();
     namesystem.checkSuperuserPrivilege();
     nn.spanReceiverHost.removeSpanReceiver(id);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index f82f0ea06a3e7..dc9494dac98d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -61,7 +61,7 @@
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
@@ -230,7 +230,7 @@ public void blockIdCK(String blockId) {
       //get blockInfo
       Block block = new Block(Block.getBlockId(blockId));
       //find which file this block belongs to
-      BlockInfo blockInfo = bm.getStoredBlock(block);
+      BlockInfoContiguous blockInfo = bm.getStoredBlock(block);
       if(blockInfo == null) {
         out.println("Block "+ blockId +" " + NONEXISTENT_STATUS);
         LOG.warn("Block "+ blockId + " " + NONEXISTENT_STATUS);
@@ -443,12 +443,15 @@ void check(String parent, HdfsFileStatus file, Result res) throws IOException {
     long fileLen = file.getLen();
     // Get block locations without updating the file access time 
     // and without block access tokens
-    LocatedBlocks blocks;
+    LocatedBlocks blocks = null;
+    FSNamesystem fsn = namenode.getNamesystem();
+    fsn.readLock();
     try {
-      blocks = namenode.getNamesystem().getBlockLocations(path, 0,
-          fileLen, false, false, false);
+      blocks = fsn.getBlockLocations(path, 0, fileLen, false, false).blocks;
     } catch (FileNotFoundException fnfe) {
       blocks = null;
+    } finally {
+      fsn.readUnlock();
     }
     if (blocks == null) { // the file is deleted
       return;
@@ -640,10 +643,6 @@ private void copyBlocksToLostFound(String parent, HdfsFileStatus file,
         }
         if (fos == null) {
           fos = dfs.create(target + "/" + chain, true);
-          if (fos == null) {
-            throw new IOException("Failed to copy " + fullName +
-                " to /lost+found: could not store chain " + chain);
-          }
           chain++;
         }
         
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index 40c4765f91c10..3442e7be6b7cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
 import org.apache.hadoop.hdfs.util.RwLock;
 import org.apache.hadoop.ipc.StandbyException;
@@ -45,5 +45,5 @@ public interface Namesystem extends RwLock, SafeMode {
 
   public void checkOperation(OperationCategory read) throws StandbyException;
 
-  public boolean isInSnapshot(BlockInfoUnderConstruction blockUC);
+  public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC);
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 160371a646e3e..c1e9d7f51482f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -528,10 +528,18 @@ private static MD5Hash receiveFile(String url, List<File> localPaths,
         fos.getChannel().force(true);
         fos.close();
       }
+
+      // Something went wrong and did not finish reading.
+      // Remove the temporary files.
+      if (!finishedReceiving) {
+        deleteTmpFiles(localPaths);
+      }
+
       if (finishedReceiving && received != advertisedSize) {
         // only throw this exception if we think we read all of it on our end
         // -- otherwise a client-side IOException would be masked by this
         // exception that makes it look like a server-side problem!
+        deleteTmpFiles(localPaths);
         throw new IOException("File " + url + " received length " + received +
                               " is not of the advertised size " +
                               advertisedSize);
@@ -548,6 +556,7 @@ private static MD5Hash receiveFile(String url, List<File> localPaths,
       
       if (advertisedDigest != null &&
           !computedDigest.equals(advertisedDigest)) {
+        deleteTmpFiles(localPaths);
         throw new IOException("File " + url + " computed digest " +
             computedDigest + " does not match advertised digest " + 
             advertisedDigest);
@@ -558,6 +567,19 @@ private static MD5Hash receiveFile(String url, List<File> localPaths,
     }    
   }
 
+  private static void deleteTmpFiles(List<File> files) {
+    if (files == null) {
+      return;
+    }
+
+    LOG.info("Deleting temporary files: " + files);
+    for (File file : files) {
+      if (!file.delete()) {
+        LOG.warn("Deleting " + file + " has failed");
+      }
+    }
+  }
+
   private static MD5Hash parseMD5Header(HttpURLConnection connection) {
     String header = connection.getHeaderField(MD5_HEADER);
     return (header != null) ? new MD5Hash(header) : null;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
index 79dabb33899e8..95f943da9289c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
@@ -100,7 +100,7 @@ static void checkPermissionForApi(FSPermissionChecker pc,
   static List<XAttr> filterXAttrsForApi(FSPermissionChecker pc,
       List<XAttr> xAttrs, boolean isRawPath) {
     assert xAttrs != null : "xAttrs can not be null";
-    if (xAttrs == null || xAttrs.isEmpty()) {
+    if (xAttrs.isEmpty()) {
       return xAttrs;
     }
     
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
index 708591b45eadf..86f4bd624ec92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
@@ -164,4 +164,11 @@ public interface FSNamesystemMBean {
    */
   public int getNumStaleStorages();
 
+  /**
+   * Returns a nested JSON object listing the top users for different RPC 
+   * operations over tracked time windows.
+   * 
+   * @return JSON string
+   */
+  public String getTopUserOpCounts();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
index 42942dc51bcf8..94e845ba3c641 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
@@ -47,6 +47,7 @@ public class NameNodeMetrics {
   @Metric MutableCounterLong filesAppended;
   @Metric MutableCounterLong getBlockLocations;
   @Metric MutableCounterLong filesRenamed;
+  @Metric MutableCounterLong filesTruncated;
   @Metric MutableCounterLong getListingOps;
   @Metric MutableCounterLong deleteFileOps;
   @Metric("Number of files/dirs deleted by delete or rename operations")
@@ -173,6 +174,10 @@ public void incrFilesRenamed() {
     filesRenamed.incr();
   }
 
+  public void incrFilesTruncated() {
+    filesTruncated.incr();
+  }
+
   public void incrFilesDeleted(long delta) {
     filesDeleted.incr(delta);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index d918495765d1f..8187d0b6b7616 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -22,8 +22,6 @@
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
@@ -71,8 +69,7 @@ public void clear() {
   public final Quota.Counts deleteSnapshotDiff(final int snapshot,
       final int prior, final N currentINode,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, boolean countDiffChange) 
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     int snapshotIndex = Collections.binarySearch(diffs, snapshot);
     
     Quota.Counts counts = Quota.Counts.newInstance();
@@ -83,14 +80,6 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
         diffs.get(snapshotIndex).setSnapshotId(prior);
       } else { // there is no snapshot before
         removed = diffs.remove(0);
-        if (countDiffChange) {
-          counts.add(Quota.NAMESPACE, 1);
-        } else {
-          // the currentINode must be a descendant of a WithName node, which set
-          // countDiffChange to false. In that case we should count in the diff
-          // change when updating the quota usage in the current tree
-          currentINode.addSpaceConsumed(-1, 0, false);
-        }
         counts.add(removed.destroyDiffAndCollectBlocks(currentINode,
             collectedBlocks, removedINodes));
       }
@@ -101,11 +90,6 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
       } else {
         // combine the to-be-removed diff with its previous diff
         removed = diffs.remove(snapshotIndex);
-        if (countDiffChange) {
-          counts.add(Quota.NAMESPACE, 1);
-        } else {
-          currentINode.addSpaceConsumed(-1, 0, false);
-        }
         if (previous.snapshotINode == null) {
           previous.snapshotINode = removed.snapshotINode;
         }
@@ -120,9 +104,7 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
   }
 
   /** Add an {@link AbstractINodeDiff} for the given snapshot. */
-  final D addDiff(int latestSnapshotId, N currentINode)
-      throws QuotaExceededException {
-    currentINode.addSpaceConsumed(1, 0, true);
+  final D addDiff(int latestSnapshotId, N currentINode) {
     return addLast(createDiff(latestSnapshotId, currentINode));
   }
 
@@ -163,9 +145,12 @@ public final int getLastSnapshotId() {
    *                  id, otherwise <=.
    * @return The id of the latest snapshot before the given snapshot.
    */
-  private final int getPrior(int anchorId, boolean exclusive) {
+  public final int getPrior(int anchorId, boolean exclusive) {
     if (anchorId == Snapshot.CURRENT_STATE_ID) {
-      return getLastSnapshotId();
+      int last = getLastSnapshotId();
+      if(exclusive && last == anchorId)
+        return Snapshot.NO_SNAPSHOT_ID;
+      return last;
     }
     final int i = Collections.binarySearch(diffs, anchorId);
     if (exclusive) { // must be the one before
@@ -272,28 +257,19 @@ public A getSnapshotINode(final int snapshotId, final A currentINode) {
    * Check if the latest snapshot diff exists.  If not, add it.
    * @return the latest snapshot diff, which is never null.
    */
-  final D checkAndAddLatestSnapshotDiff(int latestSnapshotId, N currentINode)
-      throws QuotaExceededException {
+  final D checkAndAddLatestSnapshotDiff(int latestSnapshotId, N currentINode) {
     final D last = getLast();
-    if (last != null
-        && Snapshot.ID_INTEGER_COMPARATOR.compare(last.getSnapshotId(),
-            latestSnapshotId) >= 0) {
-      return last;
-    } else {
-      try {
-        return addDiff(latestSnapshotId, currentINode);
-      } catch(NSQuotaExceededException e) {
-        e.setMessagePrefix("Failed to record modification for snapshot");
-        throw e;
-      }
-    }
+    return (last != null && Snapshot.ID_INTEGER_COMPARATOR
+        .compare(last.getSnapshotId(), latestSnapshotId) >= 0) ?
+        last : addDiff(latestSnapshotId, currentINode);
   }
 
   /** Save the snapshot copy to the latest snapshot. */
-  public void saveSelf2Snapshot(int latestSnapshotId, N currentINode,
-      A snapshotCopy) throws QuotaExceededException {
+  public D saveSelf2Snapshot(int latestSnapshotId, N currentINode,
+      A snapshotCopy) {
+    D diff = null;
     if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) {
-      D diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode);
+      diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode);
       if (diff.snapshotINode == null) {
         if (snapshotCopy == null) {
           snapshotCopy = createSnapshotCopy(currentINode);
@@ -301,6 +277,7 @@ public void saveSelf2Snapshot(int latestSnapshotId, N currentINode,
         diff.saveSnapshotCopy(snapshotCopy);
       }
     }
+    return diff;
   }
   
   @Override
@@ -312,4 +289,4 @@ public Iterator<D> iterator() {
   public String toString() {
     return getClass().getSimpleName() + ": " + diffs;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 42dc5e7c2321b..0ed99e642dc86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -215,7 +215,7 @@ public Snapshot removeSnapshot(INodeDirectory snapshotRoot,
       int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId());
       try {
         Quota.Counts counts = snapshotRoot.cleanSubtree(snapshot.getId(),
-            prior, collectedBlocks, removedINodes, true);
+            prior, collectedBlocks, removedINodes);
         INodeDirectory parent = snapshotRoot.getParent();
         if (parent != null) {
           // there will not be any WithName node corresponding to the deleted
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 9c9d435a34ebf..a0008dec92f4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -28,6 +28,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.namenode.AclStorage;
 import org.apache.hadoop.hdfs.server.namenode.Content;
 import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
@@ -69,7 +70,7 @@ private ChildrenDiff(final List<INode> created, final List<INode> deleted) {
      * Replace the given child from the created/deleted list.
      * @return true if the child is replaced; false if the child is not found.
      */
-    private final boolean replace(final ListType type,
+    private boolean replace(final ListType type,
         final INode oldChild, final INode newChild) {
       final List<INode> list = getList(type);
       final int i = search(list, oldChild.getLocalNameBytes());
@@ -82,7 +83,7 @@ private final boolean replace(final ListType type,
       return true;
     }
 
-    private final boolean removeChild(ListType type, final INode child) {
+    private boolean removeChild(ListType type, final INode child) {
       final List<INode> list = getList(type);
       final int i = searchIndex(type, child.getLocalNameBytes());
       if (i >= 0 && list.get(i) == child) {
@@ -317,6 +318,10 @@ Quota.Counts destroyDiffAndCollectBlocks(INodeDirectory currentINode,
       // this diff has been deleted
       Quota.Counts counts = Quota.Counts.newInstance();
       counts.add(diff.destroyDeletedList(collectedBlocks, removedINodes));
+      INodeDirectoryAttributes snapshotINode = getSnapshotINode();
+      if (snapshotINode != null && snapshotINode.getAclFeature() != null) {
+        AclStorage.removeAclFeature(snapshotINode.getAclFeature());
+      }
       return counts;
     }
   }
@@ -405,15 +410,14 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
           && snapshot != Snapshot.CURRENT_STATE_ID) {
         // this inode has been renamed before the deletion of the DstReference
         // subtree
-        inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes,
-            true);
+        inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
       } else { 
         // for DstReference node, continue this process to its subtree
         destroyDstSubtree(inode.asReference().getReferredINode(), snapshot,
             prior, collectedBlocks, removedINodes);
       }
     } else if (inode.isFile()) {
-      inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes, true);
+      inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
     } else if (inode.isDirectory()) {
       Map<INode, INode> excludedNodes = null;
       INodeDirectory dir = inode.asDirectory();
@@ -428,7 +432,7 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
         
         if (snapshot != Snapshot.CURRENT_STATE_ID) {
           diffList.deleteSnapshotDiff(snapshot, prior, dir, collectedBlocks,
-              removedINodes, true);
+              removedINodes);
         }
         priorDiff = diffList.getDiffById(prior);
         if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
@@ -458,8 +462,7 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
   private static Quota.Counts cleanDeletedINode(INode inode,
       final int post, final int prior,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange)
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     Quota.Counts counts = Quota.Counts.newInstance();
     Deque<INode> queue = new ArrayDeque<INode>();
     queue.addLast(inode);
@@ -468,15 +471,14 @@ private static Quota.Counts cleanDeletedINode(INode inode,
       if (topNode instanceof INodeReference.WithName) {
         INodeReference.WithName wn = (INodeReference.WithName) topNode;
         if (wn.getLastSnapshotId() >= post) {
-          wn.cleanSubtree(post, prior, collectedBlocks, removedINodes,
-              countDiffChange);
+          wn.cleanSubtree(post, prior, collectedBlocks, removedINodes);
         }
         // For DstReference node, since the node is not in the created list of
         // prior, we should treat it as regular file/dir
       } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
         INodeFile file = topNode.asFile();
         counts.add(file.getDiffs().deleteSnapshotDiff(post, prior, file,
-            collectedBlocks, removedINodes, countDiffChange));
+            collectedBlocks, removedINodes));
       } else if (topNode.isDirectory()) {
         INodeDirectory dir = topNode.asDirectory();
         ChildrenDiff priorChildrenDiff = null;
@@ -556,7 +558,7 @@ public boolean addChild(INodeDirectory parent, INode inode,
    * needs to make sure that parent is in the given snapshot "latest".
    */
   public boolean removeChild(INodeDirectory parent, INode child,
-      int latestSnapshotId) throws QuotaExceededException {
+      int latestSnapshotId) {
     // For a directory that is not a renamed node, if isInLatestSnapshot returns
     // false, the directory is not in the latest snapshot, thus we do not need
     // to record the removed child in any snapshot.
@@ -602,8 +604,7 @@ public INode getChild(INodeDirectory currentINode, byte[] name,
   
   /** Used to record the modification of a symlink node */
   public INode saveChild2Snapshot(INodeDirectory currentINode,
-      final INode child, final int latestSnapshotId, final INode snapshotCopy)
-      throws QuotaExceededException {
+      final INode child, final int latestSnapshotId, final INode snapshotCopy) {
     Preconditions.checkArgument(!child.isDirectory(),
         "child is a directory, child=%s", child);
     Preconditions.checkArgument(latestSnapshotId != Snapshot.CURRENT_STATE_ID);
@@ -635,7 +636,6 @@ public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
         deleted.computeQuotaUsage(counts, false, Snapshot.CURRENT_STATE_ID);
       }
     }
-    counts.add(Quota.NAMESPACE, diffs.asList().size());
     return counts;
   }
   
@@ -709,8 +709,7 @@ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
   public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
       final int snapshot, int prior,
       final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange)
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     Quota.Counts counts = Quota.Counts.newInstance();
     Map<INode, INode> priorCreated = null;
     Map<INode, INode> priorDeleted = null;
@@ -723,7 +722,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
             collectedBlocks, removedINodes));
       }
       counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
-          collectedBlocks, removedINodes, priorDeleted, countDiffChange));
+          collectedBlocks, removedINodes, priorDeleted));
     } else {
       // update prior
       prior = getDiffs().updatePrior(snapshot, prior);
@@ -740,9 +739,9 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
       }
       
       counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior,
-          currentINode, collectedBlocks, removedINodes, countDiffChange));
+          currentINode, collectedBlocks, removedINodes));
       counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
-          collectedBlocks, removedINodes, priorDeleted, countDiffChange));
+          collectedBlocks, removedINodes, priorDeleted));
 
       // check priorDiff again since it may be created during the diff deletion
       if (prior != Snapshot.NO_SNAPSHOT_ID) {
@@ -759,7 +758,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
                 ListType.CREATED)) {
               if (priorCreated.containsKey(cNode)) {
                 counts.add(cNode.cleanSubtree(snapshot, Snapshot.NO_SNAPSHOT_ID,
-                    collectedBlocks, removedINodes, countDiffChange));
+                    collectedBlocks, removedINodes));
               }
             }
           }
@@ -776,7 +775,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
               ListType.DELETED)) {
             if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
               counts.add(cleanDeletedINode(dNode, snapshot, prior,
-                  collectedBlocks, removedINodes, countDiffChange));
+                  collectedBlocks, removedINodes));
             }
           }
         }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 86ba03d6af25d..ebe4603309504 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -36,6 +36,10 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -229,6 +233,20 @@ private void loadFileDiffList(InputStream in, INodeFile file, int size)
 
         FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
             pbf.getFileSize());
+        List<BlockProto> bpl = pbf.getBlocksList();
+        BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.size()];
+        for(int j = 0, e = bpl.size(); j < e; ++j) {
+          Block blk = PBHelper.convert(bpl.get(j));
+          BlockInfoContiguous storedBlock =  fsn.getBlockManager().getStoredBlock(blk);
+          if(storedBlock == null) {
+            storedBlock = fsn.getBlockManager().addBlockCollection(
+                new BlockInfoContiguous(blk, copy.getFileReplication()), file);
+          }
+          blocks[j] = storedBlock;
+        }
+        if(blocks.length > 0) {
+          diff.setBlocks(blocks);
+        }
         diffs.addFirst(diff);
       }
       file.addSnapshotFeature(diffs);
@@ -472,6 +490,11 @@ private void serializeFileDiffList(INodeFile file, OutputStream out)
           SnapshotDiffSection.FileDiff.Builder fb = SnapshotDiffSection.FileDiff
               .newBuilder().setSnapshotId(diff.getSnapshotId())
               .setFileSize(diff.getFileSize());
+          if(diff.getBlocks() != null) {
+            for(Block block : diff.getBlocks()) {
+              fb.addBlocks(PBHelper.convert(block));
+            }
+          }
           INodeFileAttributes copy = diff.snapshotINode;
           if (copy != null) {
             fb.setName(ByteString.copyFrom(copy.getLocalNameBytes()))
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
index 919ab564c6669..10bba67ffc6da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
@@ -19,8 +19,10 @@
 
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -37,10 +39,13 @@ public class FileDiff extends
 
   /** The file size at snapshot creation time. */
   private final long fileSize;
+  /** A copy of the INodeFile block list. Used in truncate. */
+  private BlockInfoContiguous[] blocks;
 
   FileDiff(int snapshotId, INodeFile file) {
     super(snapshotId, null, null);
     fileSize = file.computeFileSize();
+    blocks = null;
   }
 
   /** Constructor used by FSImage loading */
@@ -48,20 +53,40 @@ public class FileDiff extends
       FileDiff posteriorDiff, long fileSize) {
     super(snapshotId, snapshotINode, posteriorDiff);
     this.fileSize = fileSize;
+    blocks = null;
   }
 
   /** @return the file size in the snapshot. */
   public long getFileSize() {
     return fileSize;
   }
-  
+
+  /**
+   * Copy block references into the snapshot
+   * up to the current {@link #fileSize}.
+   * Should be done only once.
+   */
+  public void setBlocks(BlockInfoContiguous[] blocks) {
+    if(this.blocks != null)
+      return;
+    int numBlocks = 0;
+    for(long s = 0; numBlocks < blocks.length && s < fileSize; numBlocks++)
+      s += blocks[numBlocks].getNumBytes();
+    this.blocks = Arrays.copyOf(blocks, numBlocks);
+  }
+
+  public BlockInfoContiguous[] getBlocks() {
+    return blocks;
+  }
+
   @Override
   Quota.Counts combinePosteriorAndCollectBlocks(INodeFile currentINode,
       FileDiff posterior, BlocksMapUpdateInfo collectedBlocks,
       final List<INode> removedINodes) {
-    return currentINode.getFileWithSnapshotFeature()
-        .updateQuotaAndCollectBlocks(currentINode, posterior, collectedBlocks,
-            removedINodes);
+    FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature();
+    assert sf != null : "FileWithSnapshotFeature is null";
+    return sf.updateQuotaAndCollectBlocks(
+        currentINode, posterior, collectedBlocks, removedINodes);
   }
   
   @Override
@@ -91,4 +116,13 @@ Quota.Counts destroyDiffAndCollectBlocks(INodeFile currentINode,
         .updateQuotaAndCollectBlocks(currentINode, this, collectedBlocks,
             removedINodes);
   }
+
+  public void destroyAndCollectSnapshotBlocks(
+      BlocksMapUpdateInfo collectedBlocks) {
+    if(blocks == null || collectedBlocks == null)
+      return;
+    for(BlockInfoContiguous blk : blocks)
+      collectedBlocks.addDeleteBlock(blk);
+    blocks = null;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index b0a973d28caca..5ea23dc3bea9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -17,6 +17,12 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.snapshot;
 
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
 
@@ -33,4 +39,94 @@ FileDiff createDiff(int snapshotId, INodeFile file) {
   INodeFileAttributes createSnapshotCopy(INodeFile currentINode) {
     return new INodeFileAttributes.SnapshotCopy(currentINode);
   }
+
+  public void destroyAndCollectSnapshotBlocks(
+      BlocksMapUpdateInfo collectedBlocks) {
+    for(FileDiff d : asList())
+      d.destroyAndCollectSnapshotBlocks(collectedBlocks);
+  }
+
+  public void saveSelf2Snapshot(int latestSnapshotId, INodeFile iNodeFile,
+      INodeFileAttributes snapshotCopy, boolean withBlocks) {
+    final FileDiff diff =
+        super.saveSelf2Snapshot(latestSnapshotId, iNodeFile, snapshotCopy);
+    if(withBlocks)  // Store blocks if this is the first update
+      diff.setBlocks(iNodeFile.getBlocks());
+  }
+
+  public BlockInfoContiguous[] findEarlierSnapshotBlocks(int snapshotId) {
+    assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
+    if(snapshotId == Snapshot.CURRENT_STATE_ID) {
+      return null;
+    }
+    List<FileDiff> diffs = this.asList();
+    int i = Collections.binarySearch(diffs, snapshotId);
+    BlockInfoContiguous[] blocks = null;
+    for(i = i >= 0 ? i : -i; i < diffs.size(); i--) {
+      blocks = diffs.get(i).getBlocks();
+      if(blocks != null) {
+        break;
+      }
+    }
+    return blocks;
+  }
+
+  public BlockInfoContiguous[] findLaterSnapshotBlocks(int snapshotId) {
+    assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
+    if(snapshotId == Snapshot.CURRENT_STATE_ID) {
+      return null;
+    }
+    List<FileDiff> diffs = this.asList();
+    int i = Collections.binarySearch(diffs, snapshotId);
+    BlockInfoContiguous[] blocks = null;
+    for(i = i >= 0 ? i+1 : -i-1; i < diffs.size(); i++) {
+      blocks = diffs.get(i).getBlocks();
+      if(blocks != null) {
+        break;
+      }
+    }
+    return blocks;
+  }
+
+  /**
+   * Copy blocks from the removed snapshot into the previous snapshot
+   * up to the file length of the latter.
+   * Collect unused blocks of the removed snapshot.
+   */
+  void combineAndCollectSnapshotBlocks(INodeFile file,
+                                       FileDiff removed,
+                                       BlocksMapUpdateInfo collectedBlocks,
+                                       List<INode> removedINodes) {
+    BlockInfoContiguous[] removedBlocks = removed.getBlocks();
+    if(removedBlocks == null) {
+      FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
+      assert sf != null : "FileWithSnapshotFeature is null";
+      if(sf.isCurrentFileDeleted())
+        sf.collectBlocksAndClear(file, collectedBlocks, removedINodes);
+      return;
+    }
+    int p = getPrior(removed.getSnapshotId(), true);
+    FileDiff earlierDiff = p == Snapshot.NO_SNAPSHOT_ID ? null : getDiffById(p);
+    // Copy blocks to the previous snapshot if not set already
+    if(earlierDiff != null)
+      earlierDiff.setBlocks(removedBlocks);
+    BlockInfoContiguous[] earlierBlocks =
+        (earlierDiff == null ? new BlockInfoContiguous[]{} : earlierDiff.getBlocks());
+    // Find later snapshot (or file itself) with blocks
+    BlockInfoContiguous[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
+    laterBlocks = (laterBlocks==null) ? file.getBlocks() : laterBlocks;
+    // Skip blocks, which belong to either the earlier or the later lists
+    int i = 0;
+    for(; i < removedBlocks.length; i++) {
+      if(i < earlierBlocks.length && removedBlocks[i] == earlierBlocks[i])
+        continue;
+      if(i < laterBlocks.length && removedBlocks[i] == laterBlocks[i])
+        continue;
+      break;
+    }
+    // Collect the remaining blocks of the file
+    while(i < removedBlocks.length) {
+      collectedBlocks.addDeleteBlock(removedBlocks[i++]);
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 9e57b151954d1..419557bd6dcc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -20,10 +20,11 @@
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+import org.apache.hadoop.hdfs.server.namenode.AclStorage;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
 import org.apache.hadoop.hdfs.server.namenode.Quota;
@@ -115,8 +116,7 @@ public String getDetailedString() {
   
   public Quota.Counts cleanFile(final INodeFile file, final int snapshotId,
       int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks,
-      final List<INode> removedINodes, final boolean countDiffChange)
-      throws QuotaExceededException {
+      final List<INode> removedINodes) {
     if (snapshotId == Snapshot.CURRENT_STATE_ID) {
       // delete the current file while the file has snapshot feature
       if (!isCurrentFileDeleted()) {
@@ -128,7 +128,7 @@ public Quota.Counts cleanFile(final INodeFile file, final int snapshotId,
     } else { // delete the snapshot
       priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId);
       return diffs.deleteSnapshotDiff(snapshotId, priorSnapshotId, file,
-          collectedBlocks, removedINodes, countDiffChange);
+          collectedBlocks, removedINodes);
     }
   }
   
@@ -148,19 +148,24 @@ public Quota.Counts updateQuotaAndCollectBlocks(INodeFile file,
       } else if (replication > currentRepl) {  
         oldDiskspace = oldDiskspace / file.getBlockReplication() * replication;
       }
+      AclFeature aclFeature = removed.getSnapshotINode().getAclFeature();
+      if (aclFeature != null) {
+        AclStorage.removeAclFeature(aclFeature);
+      }
     }
-    
-    collectBlocksAndClear(file, collectedBlocks, removedINodes);
-    
+
+    getDiffs().combineAndCollectSnapshotBlocks(
+        file, removed, collectedBlocks, removedINodes);
+
     long dsDelta = oldDiskspace - file.diskspaceConsumed();
     return Quota.Counts.newInstance(0, dsDelta);
   }
-  
+
   /**
    * If some blocks at the end of the block list no longer belongs to
    * any inode, collect them and update the block list.
    */
-  private void collectBlocksAndClear(final INodeFile file,
+  public void collectBlocksAndClear(final INodeFile file,
       final BlocksMapUpdateInfo info, final List<INode> removedINodes) {
     // check if everything is deleted.
     if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) {
@@ -169,47 +174,19 @@ private void collectBlocksAndClear(final INodeFile file,
     }
     // find max file size.
     final long max;
+    FileDiff diff = getDiffs().getLast();
     if (isCurrentFileDeleted()) {
-      final FileDiff last = getDiffs().getLast();
-      max = last == null? 0: last.getFileSize();
+      max = diff == null? 0: diff.getFileSize();
     } else { 
       max = file.computeFileSize();
     }
 
-    collectBlocksBeyondMax(file, max, info);
-  }
-
-  private void collectBlocksBeyondMax(final INodeFile file, final long max,
-      final BlocksMapUpdateInfo collectedBlocks) {
-    final BlockInfo[] oldBlocks = file.getBlocks();
-    if (oldBlocks != null) {
-      //find the minimum n such that the size of the first n blocks > max
-      int n = 0;
-      for(long size = 0; n < oldBlocks.length && max > size; n++) {
-        size += oldBlocks[n].getNumBytes();
-      }
-      
-      // starting from block n, the data is beyond max.
-      if (n < oldBlocks.length) {
-        // resize the array.  
-        final BlockInfo[] newBlocks;
-        if (n == 0) {
-          newBlocks = BlockInfo.EMPTY_ARRAY;
-        } else {
-          newBlocks = new BlockInfo[n];
-          System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
-        }
-        
-        // set new blocks
-        file.setBlocks(newBlocks);
-
-        // collect the blocks beyond max.  
-        if (collectedBlocks != null) {
-          for(; n < oldBlocks.length; n++) {
-            collectedBlocks.addDeleteBlock(oldBlocks[n]);
-          }
-        }
-      }
-    }
+    // Collect blocks that should be deleted
+    FileDiff last = diffs.getLast();
+    BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks();
+    if(snapshotBlocks == null)
+      file.collectBlocksBeyondMax(max, info);
+    else
+      file.collectBlocksBeyondSnapshot(snapshotBlocks, info);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
index 4f26b171c83aa..49c91536215d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
@@ -19,6 +19,7 @@
 
 import java.net.InetAddress;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -36,6 +37,14 @@
 public class TopAuditLogger implements AuditLogger {
   public static final Logger LOG = LoggerFactory.getLogger(TopAuditLogger.class);
 
+  private final TopMetrics topMetrics;
+
+  public TopAuditLogger(TopMetrics topMetrics) {
+    Preconditions.checkNotNull(topMetrics, "Cannot init with a null " +
+        "TopMetrics");
+    this.topMetrics = topMetrics;
+  }
+
   @Override
   public void initialize(Configuration conf) {
   }
@@ -43,12 +52,11 @@ public void initialize(Configuration conf) {
   @Override
   public void logAuditEvent(boolean succeeded, String userName,
       InetAddress addr, String cmd, String src, String dst, FileStatus status) {
-
-    TopMetrics instance = TopMetrics.getInstance();
-    if (instance != null) {
-      instance.report(succeeded, userName, addr, cmd, src, dst, status);
-    } else {
-      LOG.error("TopMetrics is not initialized yet!");
+    try {
+      topMetrics.report(succeeded, userName, addr, cmd, src, dst, status);
+    } catch (Throwable t) {
+      LOG.error("An error occurred while reflecting the event in top service, "
+          + "event: (cmd={},userName={})", cmd, userName);
     }
 
     if (LOG.isDebugEnabled()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
index 0f4ebac5fd1d2..ba820323b20b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.top;
 
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.primitives.Ints;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -27,34 +30,34 @@
  */
 @InterfaceAudience.Private
 public final class TopConf {
-
-  public static final String TOP_METRICS_REGISTRATION_NAME = "topusers";
-  public static final String TOP_METRICS_RECORD_NAME = "topparam";
   /**
-   * A meta command representing the total number of commands
+   * Whether TopMetrics are enabled
    */
-  public static final String CMD_TOTAL = "total";
+  public final boolean isEnabled;
+
   /**
-   * A meta user representing all users
+   * A meta command representing the total number of calls to all commands
    */
-  public static String ALL_USERS = "ALL";
+  public static final String ALL_CMDS = "*";
 
   /**
    * nntop reporting periods in milliseconds
    */
-  public final long[] nntopReportingPeriodsMs;
+  public final int[] nntopReportingPeriodsMs;
 
   public TopConf(Configuration conf) {
+    isEnabled = conf.getBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY,
+        DFSConfigKeys.NNTOP_ENABLED_DEFAULT);
     String[] periodsStr = conf.getTrimmedStrings(
         DFSConfigKeys.NNTOP_WINDOWS_MINUTES_KEY,
         DFSConfigKeys.NNTOP_WINDOWS_MINUTES_DEFAULT);
-    nntopReportingPeriodsMs = new long[periodsStr.length];
+    nntopReportingPeriodsMs = new int[periodsStr.length];
     for (int i = 0; i < periodsStr.length; i++) {
-      nntopReportingPeriodsMs[i] = Integer.parseInt(periodsStr[i]) *
-          60L * 1000L; //min to ms
+      nntopReportingPeriodsMs[i] = Ints.checkedCast(
+          TimeUnit.MINUTES.toMillis(Integer.parseInt(periodsStr[i])));
     }
-    for (long aPeriodMs: nntopReportingPeriodsMs) {
-      Preconditions.checkArgument(aPeriodMs >= 60L * 1000L,
+    for (int aPeriodMs: nntopReportingPeriodsMs) {
+      Preconditions.checkArgument(aPeriodMs >= TimeUnit.MINUTES.toMillis(1),
           "minimum reporting period is 1 min!");
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
index e8a4e23fdf085..ab55392886278 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
@@ -17,67 +17,50 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.top.metrics;
 
-import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
-import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
 import java.net.InetAddress;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
 import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.MetricsSource;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
-import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.MetricValueMap;
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow;
 
-/***
- * The interface to the top metrics
+/**
+ * The interface to the top metrics.
+ * <p/>
+ * Metrics are collected by a custom audit logger, {@link org.apache.hadoop
+ * .hdfs.server.namenode.top.TopAuditLogger}, which calls TopMetrics to
+ * increment per-operation, per-user counts on every audit log call. These
+ * counts are used to show the top users by NameNode operation as well as
+ * across all operations.
+ * <p/>
+ * TopMetrics maintains these counts for a configurable number of time
+ * intervals, e.g. 1min, 5min, 25min. Each interval is tracked by a
+ * RollingWindowManager.
  * <p/>
- * The producers use the {@link #report} method to report events and the
- * consumers use {@link #getMetrics(MetricsCollector, boolean)} to retrieve the
- * current top metrics. The default consumer is JMX but it could be any other
- * user interface.
+ * These metrics are published as a JSON string via {@link org.apache.hadoop
+ * .hdfs.server .namenode.metrics.FSNamesystemMBean#getTopWindows}. This is
+ * done by calling {@link org.apache.hadoop.hdfs.server.namenode.top.window
+ * .RollingWindowManager#snapshot} on each RollingWindowManager.
  * <p/>
  * Thread-safe: relies on thread-safety of RollingWindowManager
  */
 @InterfaceAudience.Private
-public class TopMetrics implements MetricsSource {
+public class TopMetrics {
   public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
 
-  enum Singleton {
-    INSTANCE;
-
-    volatile TopMetrics impl = null;
-
-    synchronized TopMetrics init(Configuration conf, String processName,
-        String sessionId, long[] reportingPeriods) {
-      if (impl == null) {
-        impl =
-            create(conf, processName, sessionId, reportingPeriods,
-                DefaultMetricsSystem.instance());
-      }
-      logConf(conf);
-      return impl;
-    }
-  }
-
   private static void logConf(Configuration conf) {
     LOG.info("NNTop conf: " + DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_KEY +
         " = " +  conf.get(DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_KEY));
@@ -87,128 +70,35 @@ private static void logConf(Configuration conf) {
         " = " +  conf.get(DFSConfigKeys.NNTOP_WINDOWS_MINUTES_KEY));
   }
 
-  /**
-   * Return only the shortest periods for default
-   * TODO: make it configurable
-   */
-  final boolean smallestOnlyDefault = true;
-
-  /**
-   * The smallest of reporting periods
-   */
-  long smallestPeriod = Long.MAX_VALUE;
-
-  /**
-   * processName and sessionId might later be leveraged later when we aggregate
-   * report from multiple federated name nodes
-   */
-  final String processName, sessionId;
-
   /**
    * A map from reporting periods to WindowManager. Thread-safety is provided by
    * the fact that the mapping is not changed after construction.
    */
-  final Map<Long, RollingWindowManager> rollingWindowManagers =
-      new HashMap<Long, RollingWindowManager>();
+  final Map<Integer, RollingWindowManager> rollingWindowManagers =
+      new HashMap<Integer, RollingWindowManager>();
 
-  TopMetrics(Configuration conf, String processName, String sessionId,
-      long[] reportingPeriods) {
-    this.processName = processName;
-    this.sessionId = sessionId;
+  public TopMetrics(Configuration conf, int[] reportingPeriods) {
+    logConf(conf);
     for (int i = 0; i < reportingPeriods.length; i++) {
-      smallestPeriod = Math.min(smallestPeriod, reportingPeriods[i]);
       rollingWindowManagers.put(reportingPeriods[i], new RollingWindowManager(
           conf, reportingPeriods[i]));
     }
   }
 
-  public static TopMetrics create(Configuration conf, String processName,
-      String sessionId, long[] reportingPeriods, MetricsSystem ms) {
-    return ms.register(TopConf.TOP_METRICS_REGISTRATION_NAME,
-        "top metrics of the namenode in a last period of time", new TopMetrics(
-            conf, processName, sessionId, reportingPeriods));
-  }
-
-  public static TopMetrics initSingleton(Configuration conf,
-      String processName, String sessionId, long[] reportingPeriods) {
-    return Singleton.INSTANCE.init(conf, processName, sessionId,
-        reportingPeriods);
-  }
-
-  public static TopMetrics getInstance() {
-    TopMetrics topMetrics = Singleton.INSTANCE.impl;
-    Preconditions.checkArgument(topMetrics != null,
-          "The TopMetric singleton instance is not initialized."
-              + " Have you called initSingleton first?");
-    return topMetrics;
-  }
-
   /**
-   * In testing, the previous initialization should be reset if the entire
-   * metric system is reinitialized
+   * Get a list of the current TopWindow statistics, one TopWindow per tracked
+   * time interval.
    */
-  @VisibleForTesting
-  public static void reset() {
-    Singleton.INSTANCE.impl = null;
-  }
-
-  @Override
-  public void getMetrics(MetricsCollector collector, boolean all) {
-    long realTime = Time.monotonicNow();
-    getMetrics(smallestOnlyDefault, realTime, collector, all);
-  }
-
-  public void getMetrics(boolean smallestOnly, long currTime,
-      MetricsCollector collector, boolean all) {
-    for (Entry<Long, RollingWindowManager> entry : rollingWindowManagers
+  public List<TopWindow> getTopWindows() {
+    long monoTime = Time.monotonicNow();
+    List<TopWindow> windows = Lists.newArrayListWithCapacity
+        (rollingWindowManagers.size());
+    for (Entry<Integer, RollingWindowManager> entry : rollingWindowManagers
         .entrySet()) {
-      if (!smallestOnly || smallestPeriod == entry.getKey()) {
-        getMetrics(currTime, collector, entry.getKey(), entry.getValue(), all);
-      }
-    }
-  }
-
-  /**
-   * Get metrics for a particular recording period and its corresponding
-   * {@link RollingWindowManager}
-   * <p/>
-   *
-   * @param collector the metric collector
-   * @param period the reporting period
-   * @param rollingWindowManager the window manager corresponding to the
-   *          reporting period
-   * @param all currently ignored
-   */
-  void getMetrics(long currTime, MetricsCollector collector, Long period,
-      RollingWindowManager rollingWindowManager, boolean all) {
-    MetricsRecordBuilder rb =
-        collector.addRecord(createTopMetricsRecordName(period))
-            .setContext("namenode").tag(ProcessName, processName)
-            .tag(SessionId, sessionId);
-
-    MetricValueMap snapshotMetrics = rollingWindowManager.snapshot(currTime);
-    LOG.debug("calling snapshot, result size is: " + snapshotMetrics.size());
-    for (Map.Entry<String, Number> entry : snapshotMetrics.entrySet()) {
-      String key = entry.getKey();
-      Number value = entry.getValue();
-      LOG.debug("checking an entry: key: {} value: {}", key, value);
-      long min = period / 1000L / 60L; //ms -> min
-      String desc = "top user of name node in the past " + min + " minutes";
-
-      if (value instanceof Integer) {
-        rb.addGauge(info(key, desc), (Integer) value);
-      } else if (value instanceof Long) {
-        rb.addGauge(info(key, desc), (Long) value);
-      } else if (value instanceof Float) {
-        rb.addGauge(info(key, desc), (Float) value);
-      } else if (value instanceof Double) {
-        rb.addGauge(info(key, desc), (Double) value);
-      } else {
-        LOG.warn("Unsupported metric type: " + value.getClass());
-      }
+      TopWindow window = entry.getValue().snapshot(monoTime);
+      windows.add(window);
     }
-    LOG.debug("END iterating over metrics, result size is: {}",
-        snapshotMetrics.size());
+    return windows;
   }
 
   /**
@@ -216,18 +106,10 @@ void getMetrics(long currTime, MetricsCollector collector, Long period,
    * log file. This is to be consistent when {@link TopMetrics} is charged with
    * data read back from log files instead of being invoked directly by the
    * FsNamesystem
-   *
-   * @param succeeded
-   * @param userName
-   * @param addr
-   * @param cmd
-   * @param src
-   * @param dst
-   * @param status
    */
   public void report(boolean succeeded, String userName, InetAddress addr,
       String cmd, String src, String dst, FileStatus status) {
-    //currently we nntop makes use of only the username and the command
+    // currently nntop only makes use of the username and the command
     report(userName, cmd);
   }
 
@@ -239,27 +121,11 @@ public void report(String userName, String cmd) {
   public void report(long currTime, String userName, String cmd) {
     LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
     userName = UserGroupInformation.trimLoginMethod(userName);
-    try {
-      for (RollingWindowManager rollingWindowManager : rollingWindowManagers
-          .values()) {
-        rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
-        rollingWindowManager.recordMetric(currTime,
-            TopConf.CMD_TOTAL, userName, 1);
-      }
-    } catch (Throwable t) {
-      LOG.error("An error occurred while reflecting the event in top service, "
-          + "event: (time,cmd,userName)=(" + currTime + "," + cmd + ","
-          + userName);
+    for (RollingWindowManager rollingWindowManager : rollingWindowManagers
+        .values()) {
+      rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
+      rollingWindowManager.recordMetric(currTime,
+          TopConf.ALL_CMDS, userName, 1);
     }
   }
-
-  /***
-   *
-   * @param period the reporting period length in ms
-   * @return
-   */
-  public static String createTopMetricsRecordName(Long period) {
-    return TopConf.TOP_METRICS_RECORD_NAME + "-" + period;
-  }
-
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
index d818cce26efd3..00e708766e0d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
@@ -17,21 +17,22 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.top.window;
 
-import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
+import java.util.Stack;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
 
 /**
  * A class to manage the set of {@link RollingWindow}s. This class is the
@@ -46,25 +47,93 @@ public class RollingWindowManager {
   public static final Logger LOG = LoggerFactory.getLogger(
       RollingWindowManager.class);
 
-  private int windowLenMs;
-  private int bucketsPerWindow; // e.g., 10 buckets per minute
-  private int topUsersCnt; // e.g., report top 10 metrics
+  private final int windowLenMs;
+  private final int bucketsPerWindow; // e.g., 10 buckets per minute
+  private final int topUsersCnt; // e.g., report top 10 metrics
+
+  static private class RollingWindowMap extends
+      ConcurrentHashMap<String, RollingWindow> {
+    private static final long serialVersionUID = -6785807073237052051L;
+  }
 
   /**
-   * Create a metric name composed of the command and user
-   *
-   * @param command the command executed
-   * @param user    the user
-   * @return a composed metric name
+   * Represents a snapshot of the rolling window. It contains one Op per 
+   * operation in the window, with ranked users for each Op.
    */
-  @VisibleForTesting
-  public static String createMetricName(String command, String user) {
-    return command + "." + user;
+  public static class TopWindow {
+    private final int windowMillis;
+    private final List<Op> top;
+
+    public TopWindow(int windowMillis) {
+      this.windowMillis = windowMillis;
+      this.top = Lists.newArrayList();
+    }
+
+    public void addOp(Op op) {
+      top.add(op);
+    }
+
+    public int getWindowLenMs() {
+      return windowMillis;
+    }
+
+    public List<Op> getOps() {
+      return top;
+    }
   }
 
-  static private class RollingWindowMap extends
-      ConcurrentHashMap<String, RollingWindow> {
-    private static final long serialVersionUID = -6785807073237052051L;
+  /**
+   * Represents an operation within a TopWindow. It contains a ranked 
+   * set of the top users for the operation.
+   */
+  public static class Op {
+    private final String opType;
+    private final List<User> topUsers;
+    private final long totalCount;
+
+    public Op(String opType, long totalCount) {
+      this.opType = opType;
+      this.topUsers = Lists.newArrayList();
+      this.totalCount = totalCount;
+    }
+
+    public void addUser(User u) {
+      topUsers.add(u);
+    }
+
+    public String getOpType() {
+      return opType;
+    }
+
+    public List<User> getTopUsers() {
+      return topUsers;
+    }
+
+    public long getTotalCount() {
+      return totalCount;
+    }
+  }
+
+  /**
+   * Represents a user who called an Op within a TopWindow. Specifies the 
+   * user and the number of times the user called the operation.
+   */
+  public static class User {
+    private final String user;
+    private final long count;
+
+    public User(String user, long count) {
+      this.user = user;
+      this.count = count;
+    }
+
+    public String getUser() {
+      return user;
+    }
+
+    public long getCount() {
+      return count;
+    }
   }
 
   /**
@@ -75,8 +144,9 @@ static private class RollingWindowMap extends
   public ConcurrentHashMap<String, RollingWindowMap> metricMap =
       new ConcurrentHashMap<String, RollingWindowMap>();
 
-  public RollingWindowManager(Configuration conf, long reportingPeriodMs) {
-    windowLenMs = (int) reportingPeriodMs;
+  public RollingWindowManager(Configuration conf, int reportingPeriodMs) {
+    
+    windowLenMs = reportingPeriodMs;
     bucketsPerWindow =
         conf.getInt(DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_KEY,
             DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_DEFAULT);
@@ -112,53 +182,71 @@ public void recordMetric(long time, String command, String user, long delta) {
    * Take a snapshot of current top users in the past period.
    *
    * @param time the current time
-   * @return a map between the top metrics and their values. The user is encoded
-   * in the metric name. Refer to {@link RollingWindowManager#createMetricName} for
-   * the actual format.
+   * @return a TopWindow describing the top users for each metric in the 
+   * window.
    */
-  public MetricValueMap snapshot(long time) {
-    MetricValueMap map = new MetricValueMap();
-    Set<String> metricNames = metricMap.keySet();
-    LOG.debug("iterating in reported metrics, size={} values={}",
-        metricNames.size(), metricNames);
-    for (Map.Entry<String,RollingWindowMap> rwEntry: metricMap.entrySet()) {
-      String metricName = rwEntry.getKey();
-      RollingWindowMap rollingWindows = rwEntry.getValue();
-      TopN topN = new TopN(topUsersCnt);
-      Iterator<Map.Entry<String, RollingWindow>> iterator =
-          rollingWindows.entrySet().iterator();
-      while (iterator.hasNext()) {
-        Map.Entry<String, RollingWindow> entry = iterator.next();
-        String userName = entry.getKey();
-        RollingWindow aWindow = entry.getValue();
-        long windowSum = aWindow.getSum(time);
-        // do the gc here
-        if (windowSum == 0) {
-          LOG.debug("gc window of metric: {} userName: {}",
-              metricName, userName);
-          iterator.remove();
-          continue;
-        }
-        LOG.debug("offer window of metric: {} userName: {} sum: {}",
-            metricName, userName, windowSum);
-        topN.offer(new NameValuePair(userName, windowSum));
-      }
-      int n = topN.size();
-      LOG.info("topN size for command " + metricName + " is: " + n);
-      if (n == 0) {
+  public TopWindow snapshot(long time) {
+    TopWindow window = new TopWindow(windowLenMs);
+    if (LOG.isDebugEnabled()) {
+      Set<String> metricNames = metricMap.keySet();
+      LOG.debug("iterating in reported metrics, size={} values={}",
+          metricNames.size(), metricNames);
+    }
+    for (Map.Entry<String, RollingWindowMap> entry : metricMap.entrySet()) {
+      String metricName = entry.getKey();
+      RollingWindowMap rollingWindows = entry.getValue();
+      TopN topN = getTopUsersForMetric(time, metricName, rollingWindows);
+      final int size = topN.size();
+      if (size == 0) {
         continue;
       }
-      String allMetricName =
-          createMetricName(metricName, TopConf.ALL_USERS);
-      map.put(allMetricName, Long.valueOf(topN.total));
-      for (int i = 0; i < n; i++) {
-        NameValuePair userEntry = topN.poll();
-        String userMetricName =
-            createMetricName(metricName, userEntry.name);
-        map.put(userMetricName, Long.valueOf(userEntry.value));
+      Op op = new Op(metricName, topN.getTotal());
+      window.addOp(op);
+      // Reverse the users from the TopUsers using a stack, 
+      // since we'd like them sorted in descending rather than ascending order
+      Stack<NameValuePair> reverse = new Stack<NameValuePair>();
+      for (int i = 0; i < size; i++) {
+        reverse.push(topN.poll());
       }
+      for (int i = 0; i < size; i++) {
+        NameValuePair userEntry = reverse.pop();
+        User user = new User(userEntry.name, Long.valueOf(userEntry.value));
+        op.addUser(user);
+      }
+    }
+    return window;
+  }
+
+  /**
+   * Calculates the top N users over a time interval.
+   * 
+   * @param time the current time
+   * @param metricName Name of metric
+   * @return
+   */
+  private TopN getTopUsersForMetric(long time, String metricName, 
+      RollingWindowMap rollingWindows) {
+    TopN topN = new TopN(topUsersCnt);
+    Iterator<Map.Entry<String, RollingWindow>> iterator =
+        rollingWindows.entrySet().iterator();
+    while (iterator.hasNext()) {
+      Map.Entry<String, RollingWindow> entry = iterator.next();
+      String userName = entry.getKey();
+      RollingWindow aWindow = entry.getValue();
+      long windowSum = aWindow.getSum(time);
+      // do the gc here
+      if (windowSum == 0) {
+        LOG.debug("gc window of metric: {} userName: {}",
+            metricName, userName);
+        iterator.remove();
+        continue;
+      }
+      LOG.debug("offer window of metric: {} userName: {} sum: {}",
+          metricName, userName, windowSum);
+      topN.offer(new NameValuePair(userName, windowSum));
     }
-    return map;
+    LOG.info("topN size for command {} is: {}", metricName, topN.size());
+    return topN;
   }
 
   /**
@@ -190,7 +278,8 @@ private RollingWindow getRollingWindow(String metric, String user) {
   }
 
   /**
-   * A pair of a name and its corresponding value
+   * A pair of a name and its corresponding value. Defines a custom 
+   * comparator so the TopN PriorityQueue sorts based on the count.
    */
   static private class NameValuePair implements Comparable<NameValuePair> {
     String name;
@@ -254,12 +343,4 @@ public long getTotal() {
       return total;
     }
   }
-
-  /**
-   * A mapping from metric names to their absolute values and their percentage
-   */
-  @InterfaceAudience.Private
-  public static class MetricValueMap extends HashMap<String, Number> {
-    private static final long serialVersionUID = 8936732010242400171L;
-  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index e688bb624b414..0a6f133099262 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -57,7 +57,7 @@
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.XAttrHelper;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -614,10 +614,12 @@ public Response postRoot(
       @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
           final BufferSizeParam bufferSize,
       @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
-          final ExcludeDatanodesParam excludeDatanodes
+          final ExcludeDatanodesParam excludeDatanodes,
+      @QueryParam(NewLengthParam.NAME) @DefaultValue(NewLengthParam.DEFAULT)
+          final NewLengthParam newLength
       ) throws IOException, InterruptedException {
     return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs,
-        bufferSize, excludeDatanodes);
+        bufferSize, excludeDatanodes, newLength);
   }
 
   /** Handle HTTP POST request. */
@@ -641,11 +643,13 @@ public Response post(
       @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
           final BufferSizeParam bufferSize,
       @QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
-          final ExcludeDatanodesParam excludeDatanodes
+          final ExcludeDatanodesParam excludeDatanodes,
+      @QueryParam(NewLengthParam.NAME) @DefaultValue(NewLengthParam.DEFAULT)
+          final NewLengthParam newLength
       ) throws IOException, InterruptedException {
 
     init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize,
-        excludeDatanodes);
+        excludeDatanodes, newLength);
 
     return ugi.doAs(new PrivilegedExceptionAction<Response>() {
       @Override
@@ -653,7 +657,7 @@ public Response run() throws IOException, URISyntaxException {
         try {
           return post(ugi, delegation, username, doAsUser,
               path.getAbsolutePath(), op, concatSrcs, bufferSize,
-              excludeDatanodes);
+              excludeDatanodes, newLength);
         } finally {
           reset();
         }
@@ -670,9 +674,11 @@ private Response post(
       final PostOpParam op,
       final ConcatSourcesParam concatSrcs,
       final BufferSizeParam bufferSize,
-      final ExcludeDatanodesParam excludeDatanodes
+      final ExcludeDatanodesParam excludeDatanodes,
+      final NewLengthParam newLength
       ) throws IOException, URISyntaxException {
     final NameNode namenode = (NameNode)context.getAttribute("name.node");
+    final NamenodeProtocols np = getRPCServer(namenode);
 
     switch(op.getValue()) {
     case APPEND:
@@ -684,9 +690,17 @@ private Response post(
     }
     case CONCAT:
     {
-      getRPCServer(namenode).concat(fullpath, concatSrcs.getAbsolutePaths());
+      np.concat(fullpath, concatSrcs.getAbsolutePaths());
       return Response.ok().build();
     }
+    case TRUNCATE:
+    {
+      // We treat each rest request as a separate client.
+      final boolean b = np.truncate(fullpath, newLength.getValue(), 
+          "DFSClient_" + DFSUtil.getSecureRandom().nextLong());
+      final String js = JsonUtil.toJsonString("boolean", b);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
     default:
       throw new UnsupportedOperationException(op + " is not supported");
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
index b7199ba3803dd..ced3296f48637 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
@@ -22,6 +22,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -54,6 +55,7 @@ public class BlockRecoveryCommand extends DatanodeCommand {
   @InterfaceStability.Evolving
   public static class RecoveringBlock extends LocatedBlock {
     private final long newGenerationStamp;
+    private final Block recoveryBlock;
 
     /**
      * Create RecoveringBlock.
@@ -61,6 +63,17 @@ public static class RecoveringBlock extends LocatedBlock {
     public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) {
       super(b, locs, -1, false); // startOffset is unknown
       this.newGenerationStamp = newGS;
+      this.recoveryBlock = null;
+    }
+
+    /**
+     * Create RecoveringBlock with copy-on-truncate option.
+     */
+    public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs,
+        Block recoveryBlock) {
+      super(b, locs, -1, false); // startOffset is unknown
+      this.newGenerationStamp = recoveryBlock.getGenerationStamp();
+      this.recoveryBlock = recoveryBlock;
     }
 
     /**
@@ -70,6 +83,13 @@ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) {
     public long getNewGenerationStamp() {
       return newGenerationStamp;
     }
+
+    /**
+     * Return the new block.
+     */
+    public Block getNewBlock() {
+      return recoveryBlock;
+    }
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
index 4fe07b9b4b37d..4d224d5fd19ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java
@@ -47,6 +47,7 @@ public enum State {
   private final String storageID;
   private final State state;
   private final StorageType storageType;
+  private static final String STORAGE_ID_PREFIX = "DS-";
 
   /**
    * Create a storage with {@link State#NORMAL} and {@link StorageType#DEFAULT}.
@@ -80,7 +81,23 @@ public StorageType getStorageType() {
    * @return unique storage ID
    */
   public static String generateUuid() {
-    return "DS-" + UUID.randomUUID();
+    return STORAGE_ID_PREFIX + UUID.randomUUID();
+  }
+
+  /**
+   * Verify that a given string is a storage ID in the "DS-..uuid.." format.
+   */
+  public static boolean isValidStorageId(final String storageID) {
+    try {
+      // Attempt to parse the UUID.
+      if (storageID != null && storageID.indexOf(STORAGE_ID_PREFIX) == 0) {
+        UUID.fromString(storageID.substring(STORAGE_ID_PREFIX.length()));
+        return true;
+      }
+    } catch (IllegalArgumentException iae) {
+    }
+
+    return false;
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
index 62915b4cbbd46..72cb0c161233e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
@@ -67,5 +67,6 @@ ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
    * Update replica with the new generation stamp and length.  
    */
   String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId,
-      long newLength) throws IOException;
+                                    long newBlockId, long newLength)
+      throws IOException;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
index 8d7544d768b8b..eed9a6e380119 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
@@ -52,4 +52,12 @@ public ServerCommand(int action) {
   public int getAction() {
     return this.action;
   }
+
+  public String toString() {
+    final StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName());
+    sb.append("/");
+    sb.append(action);
+    return sb.toString();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
new file mode 100644
index 0000000000000..153fb36a10766
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.tools.TableListing;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Helper methods for CacheAdmin/CryptoAdmin/StoragePolicyAdmin
+ */
+public class AdminHelper {
+  /**
+   * Maximum length for printed lines
+   */
+  static final int MAX_LINE_WIDTH = 80;
+  static final String HELP_COMMAND_NAME = "-help";
+
+  static DistributedFileSystem getDFS(Configuration conf)
+      throws IOException {
+    FileSystem fs = FileSystem.get(conf);
+    if (!(fs instanceof DistributedFileSystem)) {
+      throw new IllegalArgumentException("FileSystem " + fs.getUri() +
+          " is not an HDFS file system");
+    }
+    return (DistributedFileSystem)fs;
+  }
+
+  /**
+   * NN exceptions contain the stack trace as part of the exception message.
+   * When it's a known error, pretty-print the error and squish the stack trace.
+   */
+  static String prettifyException(Exception e) {
+    return e.getClass().getSimpleName() + ": "
+        + e.getLocalizedMessage().split("\n")[0];
+  }
+
+  static TableListing getOptionDescriptionListing() {
+    return new TableListing.Builder()
+        .addField("").addField("", true)
+        .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
+  }
+
+  /**
+   * Parses a time-to-live value from a string
+   * @return The ttl in milliseconds
+   * @throws IOException if it could not be parsed
+   */
+  static Long parseTtlString(String maxTtlString) throws IOException {
+    Long maxTtl = null;
+    if (maxTtlString != null) {
+      if (maxTtlString.equalsIgnoreCase("never")) {
+        maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER;
+      } else {
+        maxTtl = DFSUtil.parseRelativeTime(maxTtlString);
+      }
+    }
+    return maxTtl;
+  }
+
+  static Long parseLimitString(String limitString) {
+    Long limit = null;
+    if (limitString != null) {
+      if (limitString.equalsIgnoreCase("unlimited")) {
+        limit = CachePoolInfo.LIMIT_UNLIMITED;
+      } else {
+        limit = Long.parseLong(limitString);
+      }
+    }
+    return limit;
+  }
+
+  static Command determineCommand(String commandName, Command[] commands) {
+    Preconditions.checkNotNull(commands);
+    if (HELP_COMMAND_NAME.equals(commandName)) {
+      return new HelpCommand(commands);
+    }
+    for (Command command : commands) {
+      if (command.getName().equals(commandName)) {
+        return command;
+      }
+    }
+    return null;
+  }
+
+  static void printUsage(boolean longUsage, String toolName,
+      Command[] commands) {
+    Preconditions.checkNotNull(commands);
+    System.err.println("Usage: bin/hdfs " + toolName + " [COMMAND]");
+    final HelpCommand helpCommand = new HelpCommand(commands);
+    for (AdminHelper.Command command : commands) {
+      if (longUsage) {
+        System.err.print(command.getLongUsage());
+      } else {
+        System.err.print("          " + command.getShortUsage());
+      }
+    }
+    System.err.print(longUsage ? helpCommand.getLongUsage() :
+        ("          " + helpCommand.getShortUsage()));
+    System.err.println();
+  }
+
+  interface Command {
+    String getName();
+    String getShortUsage();
+    String getLongUsage();
+    int run(Configuration conf, List<String> args) throws IOException;
+  }
+
+  static class HelpCommand implements Command {
+    private final Command[] commands;
+
+    public HelpCommand(Command[] commands) {
+      Preconditions.checkNotNull(commands != null);
+      this.commands = commands;
+    }
+
+    @Override
+    public String getName() {
+      return HELP_COMMAND_NAME;
+    }
+
+    @Override
+    public String getShortUsage() {
+      return "[-help <command-name>]\n";
+    }
+
+    @Override
+    public String getLongUsage() {
+      final TableListing listing = AdminHelper.getOptionDescriptionListing();
+      listing.addRow("<command-name>", "The command for which to get " +
+          "detailed help. If no command is specified, print detailed help for " +
+          "all commands");
+      return getShortUsage() + "\n" +
+          "Get detailed help about a command.\n\n" +
+          listing.toString();
+    }
+
+    @Override
+    public int run(Configuration conf, List<String> args) throws IOException {
+      if (args.size() == 0) {
+        for (AdminHelper.Command command : commands) {
+          System.err.println(command.getLongUsage());
+        }
+        return 0;
+      }
+      if (args.size() != 1) {
+        System.out.println("You must give exactly one argument to -help.");
+        return 0;
+      }
+      final String commandName = args.get(0);
+      // prepend a dash to match against the command names
+      final AdminHelper.Command command = AdminHelper
+          .determineCommand("-" + commandName, commands);
+      if (command == null) {
+        System.err.print("Unknown command '" + commandName + "'.\n");
+        System.err.print("Valid help command names are:\n");
+        String separator = "";
+        for (AdminHelper.Command c : commands) {
+          System.err.print(separator + c.getName().substring(1));
+          separator = ", ";
+        }
+        System.err.print("\n");
+        return 1;
+      }
+      System.err.print(command.getLongUsage());
+      return 0;
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index dbe228452636c..6888ea8e85e31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.CacheFlag;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -53,11 +52,6 @@
 @InterfaceAudience.Private
 public class CacheAdmin extends Configured implements Tool {
 
-  /**
-   * Maximum length for printed lines
-   */
-  private static final int MAX_LINE_WIDTH = 80;
-
   public CacheAdmin() {
     this(null);
   }
@@ -69,16 +63,17 @@ public CacheAdmin(Configuration conf) {
   @Override
   public int run(String[] args) throws IOException {
     if (args.length == 0) {
-      printUsage(false);
+      AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
       return 1;
     }
-    Command command = determineCommand(args[0]);
+    AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+        COMMANDS);
     if (command == null) {
       System.err.println("Can't understand command '" + args[0] + "'");
       if (!args[0].startsWith("-")) {
         System.err.println("Command names must start with dashes.");
       }
-      printUsage(false);
+      AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
       return 1;
     }
     List<String> argsList = new LinkedList<String>();
@@ -88,7 +83,7 @@ public int run(String[] args) throws IOException {
     try {
       return command.run(getConf(), argsList);
     } catch (IllegalArgumentException e) {
-      System.err.println(prettifyException(e));
+      System.err.println(AdminHelper.prettifyException(e));
       return -1;
     }
   }
@@ -98,64 +93,9 @@ public static void main(String[] argsArray) throws IOException {
     System.exit(cacheAdmin.run(argsArray));
   }
 
-  private static DistributedFileSystem getDFS(Configuration conf)
+  private static CacheDirectiveInfo.Expiration parseExpirationString(String ttlString)
       throws IOException {
-    FileSystem fs = FileSystem.get(conf);
-    if (!(fs instanceof DistributedFileSystem)) {
-      throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
-      " is not an HDFS file system");
-    }
-    return (DistributedFileSystem)fs;
-  }
-
-  /**
-   * NN exceptions contain the stack trace as part of the exception message.
-   * When it's a known error, pretty-print the error and squish the stack trace.
-   */
-  private static String prettifyException(Exception e) {
-    return e.getClass().getSimpleName() + ": "
-        + e.getLocalizedMessage().split("\n")[0];
-  }
-
-  private static TableListing getOptionDescriptionListing() {
-    TableListing listing = new TableListing.Builder()
-    .addField("").addField("", true)
-    .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
-    return listing;
-  }
-
-  /**
-   * Parses a time-to-live value from a string
-   * @return The ttl in milliseconds
-   * @throws IOException if it could not be parsed
-   */
-  private static Long parseTtlString(String maxTtlString) throws IOException {
-    Long maxTtl = null;
-    if (maxTtlString != null) {
-      if (maxTtlString.equalsIgnoreCase("never")) {
-        maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER;
-      } else {
-        maxTtl = DFSUtil.parseRelativeTime(maxTtlString);
-      }
-    }
-    return maxTtl;
-  }
-
-  private static Long parseLimitString(String limitString) {
-    Long limit = null;
-    if (limitString != null) {
-      if (limitString.equalsIgnoreCase("unlimited")) {
-        limit = CachePoolInfo.LIMIT_UNLIMITED;
-      } else {
-        limit = Long.parseLong(limitString);
-      }
-    }
-    return limit;
-  }
-
-  private static Expiration parseExpirationString(String ttlString)
-      throws IOException {
-    Expiration ex = null;
+    CacheDirectiveInfo.Expiration ex = null;
     if (ttlString != null) {
       if (ttlString.equalsIgnoreCase("never")) {
         ex = CacheDirectiveInfo.Expiration.NEVER;
@@ -167,14 +107,8 @@ private static Expiration parseExpirationString(String ttlString)
     return ex;
   }
 
-  interface Command {
-    String getName();
-    String getShortUsage();
-    String getLongUsage();
-    int run(Configuration conf, List<String> args) throws IOException;
-  }
-
-  private static class AddCacheDirectiveInfoCommand implements Command {
+  private static class AddCacheDirectiveInfoCommand
+      implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-addDirective";
@@ -190,7 +124,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("<path>", "A path to cache. The path can be " +
           "a directory or a file.");
       listing.addRow("<pool-name>", "The pool to which the directive will be " +
@@ -252,7 +186,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
         return 1;
       }
         
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       CacheDirectiveInfo directive = builder.build();
       EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
       if (force) {
@@ -262,7 +196,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
         long id = dfs.addCacheDirective(directive, flags);
         System.out.println("Added cache directive " + id);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
 
@@ -270,7 +204,8 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class RemoveCacheDirectiveInfoCommand implements Command {
+  private static class RemoveCacheDirectiveInfoCommand
+      implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-removeDirective";
@@ -283,7 +218,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("<id>", "The id of the cache directive to remove.  " + 
         "You must have write permission on the pool of the " +
         "directive in order to remove it.  To see a list " +
@@ -318,19 +253,20 @@ public int run(Configuration conf, List<String> args) throws IOException {
         System.err.println("Usage is " + getShortUsage());
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         dfs.getClient().removeCacheDirective(id);
         System.out.println("Removed cached directive " + id);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       return 0;
     }
   }
 
-  private static class ModifyCacheDirectiveInfoCommand implements Command {
+  private static class ModifyCacheDirectiveInfoCommand
+      implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-modifyDirective";
@@ -345,7 +281,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("<id>", "The ID of the directive to modify (required)");
       listing.addRow("<path>", "A path to cache. The path can be " +
           "a directory or a file. (optional)");
@@ -415,7 +351,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
         System.err.println("No modifications were specified.");
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
       if (force) {
         flags.add(CacheFlag.FORCE);
@@ -424,14 +360,15 @@ public int run(Configuration conf, List<String> args) throws IOException {
         dfs.modifyCacheDirective(builder.build(), flags);
         System.out.println("Modified cache directive " + idString);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       return 0;
     }
   }
 
-  private static class RemoveCacheDirectiveInfosCommand implements Command {
+  private static class RemoveCacheDirectiveInfosCommand
+      implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-removeDirectives";
@@ -444,7 +381,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("-path <path>", "The path of the cache directives to remove.  " +
         "You must have write permission on the pool of the directive in order " +
         "to remove it.  To see a list of cache directives, use the " +
@@ -468,7 +405,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
       }
       int exitCode = 0;
       try {
-        DistributedFileSystem dfs = getDFS(conf);
+        DistributedFileSystem dfs = AdminHelper.getDFS(conf);
         RemoteIterator<CacheDirectiveEntry> iter =
             dfs.listCacheDirectives(
                 new CacheDirectiveInfo.Builder().
@@ -480,12 +417,12 @@ public int run(Configuration conf, List<String> args) throws IOException {
             System.out.println("Removed cache directive " +
                 entry.getInfo().getId());
           } catch (IOException e) {
-            System.err.println(prettifyException(e));
+            System.err.println(AdminHelper.prettifyException(e));
             exitCode = 2;
           }
         }
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         exitCode = 2;
       }
       if (exitCode == 0) {
@@ -496,7 +433,8 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class ListCacheDirectiveInfoCommand implements Command {
+  private static class ListCacheDirectiveInfoCommand
+      implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-listDirectives";
@@ -510,7 +448,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("-stats", "List path-based cache directive statistics.");
       listing.addRow("<path>", "List only " +
           "cache directives with this path. " +
@@ -559,7 +497,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
       }
       TableListing tableListing = tableBuilder.build();
       try {
-        DistributedFileSystem dfs = getDFS(conf);
+        DistributedFileSystem dfs = AdminHelper.getDFS(conf);
         RemoteIterator<CacheDirectiveEntry> iter =
             dfs.listCacheDirectives(builder.build());
         int numEntries = 0;
@@ -587,7 +525,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
             row.add("" + stats.getFilesNeeded());
             row.add("" + stats.getFilesCached());
           }
-          tableListing.addRow(row.toArray(new String[0]));
+          tableListing.addRow(row.toArray(new String[row.size()]));
           numEntries++;
         }
         System.out.print(String.format("Found %d entr%s%n",
@@ -596,14 +534,14 @@ public int run(Configuration conf, List<String> args) throws IOException {
           System.out.print(tableListing);
         }
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       return 0;
     }
   }
 
-  private static class AddCachePoolCommand implements Command {
+  private static class AddCachePoolCommand implements AdminHelper.Command {
 
     private static final String NAME = "-addPool";
 
@@ -621,7 +559,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
 
       listing.addRow("<name>", "Name of the new pool.");
       listing.addRow("<owner>", "Username of the owner of the pool. " +
@@ -669,13 +607,13 @@ public int run(Configuration conf, List<String> args) throws IOException {
         info.setMode(new FsPermission(mode));
       }
       String limitString = StringUtils.popOptionWithArgument("-limit", args);
-      Long limit = parseLimitString(limitString);
+      Long limit = AdminHelper.parseLimitString(limitString);
       if (limit != null) {
         info.setLimit(limit);
       }
       String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
       try {
-        Long maxTtl = parseTtlString(maxTtlString);
+        Long maxTtl = AdminHelper.parseTtlString(maxTtlString);
         if (maxTtl != null) {
           info.setMaxRelativeExpiryMs(maxTtl);
         }
@@ -691,11 +629,11 @@ public int run(Configuration conf, List<String> args) throws IOException {
         System.err.println("Usage is " + getShortUsage());
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         dfs.addCachePool(info);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       System.out.println("Successfully added cache pool " + name + ".");
@@ -703,7 +641,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class ModifyCachePoolCommand implements Command {
+  private static class ModifyCachePoolCommand implements AdminHelper.Command {
 
     @Override
     public String getName() {
@@ -719,7 +657,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
 
       listing.addRow("<name>", "Name of the pool to modify.");
       listing.addRow("<owner>", "Username of the owner of the pool");
@@ -733,7 +671,7 @@ public String getLongUsage() {
       return getShortUsage() + "\n" +
           WordUtils.wrap("Modifies the metadata of an existing cache pool. " +
           "See usage of " + AddCachePoolCommand.NAME + " for more details.",
-          MAX_LINE_WIDTH) + "\n\n" +
+          AdminHelper.MAX_LINE_WIDTH) + "\n\n" +
           listing.toString();
     }
 
@@ -745,11 +683,11 @@ public int run(Configuration conf, List<String> args) throws IOException {
       Integer mode = (modeString == null) ?
           null : Integer.parseInt(modeString, 8);
       String limitString = StringUtils.popOptionWithArgument("-limit", args);
-      Long limit = parseLimitString(limitString);
+      Long limit = AdminHelper.parseLimitString(limitString);
       String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
-      Long maxTtl = null;
+      Long maxTtl;
       try {
-        maxTtl = parseTtlString(maxTtlString);
+        maxTtl = AdminHelper.parseTtlString(maxTtlString);
       } catch (IOException e) {
         System.err.println(
             "Error while parsing maxTtl value: " + e.getMessage());
@@ -794,11 +732,11 @@ public int run(Configuration conf, List<String> args) throws IOException {
             "change in the cache pool.");
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         dfs.modifyCachePool(info);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       System.out.print("Successfully modified cache pool " + name);
@@ -827,7 +765,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class RemoveCachePoolCommand implements Command {
+  private static class RemoveCachePoolCommand implements AdminHelper.Command {
 
     @Override
     public String getName() {
@@ -843,7 +781,7 @@ public String getShortUsage() {
     public String getLongUsage() {
       return getShortUsage() + "\n" +
           WordUtils.wrap("Remove a cache pool. This also uncaches paths " +
-              "associated with the pool.\n\n", MAX_LINE_WIDTH) +
+              "associated with the pool.\n\n", AdminHelper.MAX_LINE_WIDTH) +
           "<name>  Name of the cache pool to remove.\n";
     }
 
@@ -861,11 +799,11 @@ public int run(Configuration conf, List<String> args) throws IOException {
         System.err.println("Usage is " + getShortUsage());
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         dfs.removeCachePool(name);
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       System.out.println("Successfully removed cache pool " + name + ".");
@@ -873,7 +811,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class ListCachePoolsCommand implements Command {
+  private static class ListCachePoolsCommand implements AdminHelper.Command {
 
     @Override
     public String getName() {
@@ -887,15 +825,14 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("-stats", "Display additional cache pool statistics.");
       listing.addRow("<name>", "If specified, list only the named cache pool.");
 
       return getShortUsage() + "\n" +
           WordUtils.wrap("Display information about one or more cache pools, " +
-              "e.g. name, owner, group, permissions, etc.", MAX_LINE_WIDTH) +
-          "\n\n" +
-          listing.toString();
+              "e.g. name, owner, group, permissions, etc.",
+              AdminHelper.MAX_LINE_WIDTH) + "\n\n" + listing.toString();
     }
 
     @Override
@@ -908,7 +845,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
         System.err.println("Usage is " + getShortUsage());
         return 1;
       }
-      DistributedFileSystem dfs = getDFS(conf);
+      DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       TableListing.Builder builder = new TableListing.Builder().
           addField("NAME", Justification.LEFT).
           addField("OWNER", Justification.LEFT).
@@ -949,7 +886,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
             String maxTtlString = null;
 
             if (maxTtl != null) {
-              if (maxTtl.longValue() == CachePoolInfo.RELATIVE_EXPIRY_NEVER) {
+              if (maxTtl == CachePoolInfo.RELATIVE_EXPIRY_NEVER) {
                 maxTtlString  = "never";
               } else {
                 maxTtlString = DFSUtil.durationToString(maxTtl);
@@ -964,7 +901,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
               row.add(Long.toString(stats.getFilesNeeded()));
               row.add(Long.toString(stats.getFilesCached()));
             }
-            listing.addRow(row.toArray(new String[] {}));
+            listing.addRow(row.toArray(new String[row.size()]));
             ++numResults;
             if (name != null) {
               break;
@@ -972,7 +909,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
           }
         }
       } catch (IOException e) {
-        System.err.println(prettifyException(e));
+        System.err.println(AdminHelper.prettifyException(e));
         return 2;
       }
       System.out.print(String.format("Found %d result%s.%n", numResults,
@@ -985,61 +922,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class HelpCommand implements Command {
-    @Override
-    public String getName() {
-      return "-help";
-    }
-
-    @Override
-    public String getShortUsage() {
-      return "[-help <command-name>]\n";
-    }
-
-    @Override
-    public String getLongUsage() {
-      TableListing listing = getOptionDescriptionListing();
-      listing.addRow("<command-name>", "The command for which to get " +
-          "detailed help. If no command is specified, print detailed help for " +
-          "all commands");
-      return getShortUsage() + "\n" +
-        "Get detailed help about a command.\n\n" +
-        listing.toString();
-    }
-
-    @Override
-    public int run(Configuration conf, List<String> args) throws IOException {
-      if (args.size() == 0) {
-        for (Command command : COMMANDS) {
-          System.err.println(command.getLongUsage());
-        }
-        return 0;
-      }
-      if (args.size() != 1) {
-        System.out.println("You must give exactly one argument to -help.");
-        return 0;
-      }
-      String commandName = args.get(0);
-      // prepend a dash to match against the command names
-      Command command = determineCommand("-"+commandName);
-      if (command == null) {
-        System.err.print("Sorry, I don't know the command '" +
-          commandName + "'.\n");
-        System.err.print("Valid help command names are:\n");
-        String separator = "";
-        for (Command c : COMMANDS) {
-          System.err.print(separator + c.getName().substring(1));
-          separator = ", ";
-        }
-        System.err.print("\n");
-        return 1;
-      }
-      System.err.print(command.getLongUsage());
-      return 0;
-    }
-  }
-
-  private static final Command[] COMMANDS = {
+  private static final AdminHelper.Command[] COMMANDS = {
     new AddCacheDirectiveInfoCommand(),
     new ModifyCacheDirectiveInfoCommand(),
     new ListCacheDirectiveInfoCommand(),
@@ -1048,29 +931,6 @@ public int run(Configuration conf, List<String> args) throws IOException {
     new AddCachePoolCommand(),
     new ModifyCachePoolCommand(),
     new RemoveCachePoolCommand(),
-    new ListCachePoolsCommand(),
-    new HelpCommand(),
+    new ListCachePoolsCommand()
   };
-
-  private static void printUsage(boolean longUsage) {
-    System.err.println(
-        "Usage: bin/hdfs cacheadmin [COMMAND]");
-    for (Command command : COMMANDS) {
-      if (longUsage) {
-        System.err.print(command.getLongUsage());
-      } else {
-        System.err.print("          " + command.getShortUsage());
-      }
-    }
-    System.err.println();
-  }
-
-  private static Command determineCommand(String commandName) {
-    for (int i = 0; i < COMMANDS.length; i++) {
-      if (COMMANDS[i].getName().equals(commandName)) {
-        return COMMANDS[i];
-      }
-    }
-    return null;
-  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index bb52ddd1531a5..09825b0fed2b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -39,11 +38,6 @@
 @InterfaceAudience.Private
 public class CryptoAdmin extends Configured implements Tool {
 
-  /**
-   * Maximum length for printed lines
-   */
-  private static final int MAX_LINE_WIDTH = 80;
-
   public CryptoAdmin() {
     this(null);
   }
@@ -55,16 +49,17 @@ public CryptoAdmin(Configuration conf) {
   @Override
   public int run(String[] args) throws IOException {
     if (args.length == 0) {
-      printUsage(false);
+      AdminHelper.printUsage(false, "crypto", COMMANDS);
       return 1;
     }
-    final Command command = determineCommand(args[0]);
+    final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+        COMMANDS);
     if (command == null) {
       System.err.println("Can't understand command '" + args[0] + "'");
       if (!args[0].startsWith("-")) {
         System.err.println("Command names must start with dashes.");
       }
-      printUsage(false);
+      AdminHelper.printUsage(false, "crypto", COMMANDS);
       return 1;
     }
     final List<String> argsList = new LinkedList<String>();
@@ -84,16 +79,6 @@ public static void main(String[] argsArray) throws IOException {
     System.exit(cryptoAdmin.run(argsArray));
   }
 
-  private static DistributedFileSystem getDFS(Configuration conf)
-      throws IOException {
-    final FileSystem fs = FileSystem.get(conf);
-    if (!(fs instanceof DistributedFileSystem)) {
-      throw new IllegalArgumentException("FileSystem " + fs.getUri() +
-      " is not an HDFS file system");
-    }
-    return (DistributedFileSystem) fs;
-  }
-
   /**
    * NN exceptions contain the stack trace as part of the exception message.
    * When it's a known error, pretty-print the error and squish the stack trace.
@@ -103,21 +88,7 @@ private static String prettifyException(Exception e) {
       e.getLocalizedMessage().split("\n")[0];
   }
 
-  private static TableListing getOptionDescriptionListing() {
-    final TableListing listing = new TableListing.Builder()
-      .addField("").addField("", true)
-      .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
-    return listing;
-  }
-
-  interface Command {
-    String getName();
-    String getShortUsage();
-    String getLongUsage();
-    int run(Configuration conf, List<String> args) throws IOException;
-  }
-
-  private static class CreateZoneCommand implements Command {
+  private static class CreateZoneCommand implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-createZone";
@@ -130,7 +101,7 @@ public String getShortUsage() {
 
     @Override
     public String getLongUsage() {
-      final TableListing listing = getOptionDescriptionListing();
+      final TableListing listing = AdminHelper.getOptionDescriptionListing();
       listing.addRow("<path>", "The path of the encryption zone to create. " +
         "It must be an empty directory.");
       listing.addRow("<keyName>", "Name of the key to use for the " +
@@ -160,7 +131,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
         return 1;
       }
 
-      final DistributedFileSystem dfs = getDFS(conf);
+      final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         dfs.createEncryptionZone(new Path(path), keyName);
         System.out.println("Added encryption zone " + path);
@@ -173,7 +144,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class ListZonesCommand implements Command {
+  private static class ListZonesCommand implements AdminHelper.Command {
     @Override
     public String getName() {
       return "-listZones";
@@ -197,11 +168,11 @@ public int run(Configuration conf, List<String> args) throws IOException {
         return 1;
       }
 
-      final DistributedFileSystem dfs = getDFS(conf);
+      final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
       try {
         final TableListing listing = new TableListing.Builder()
           .addField("").addField("", true)
-          .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build();
+          .wrapWidth(AdminHelper.MAX_LINE_WIDTH).hideHeaders().build();
         final RemoteIterator<EncryptionZone> it = dfs.listEncryptionZones();
         while (it.hasNext()) {
           EncryptionZone ez = it.next();
@@ -217,85 +188,8 @@ public int run(Configuration conf, List<String> args) throws IOException {
     }
   }
 
-  private static class HelpCommand implements Command {
-    @Override
-    public String getName() {
-      return "-help";
-    }
-
-    @Override
-    public String getShortUsage() {
-      return "[-help <command-name>]\n";
-    }
-
-    @Override
-    public String getLongUsage() {
-      final TableListing listing = getOptionDescriptionListing();
-      listing.addRow("<command-name>", "The command for which to get " +
-          "detailed help. If no command is specified, print detailed help for " +
-          "all commands");
-      return getShortUsage() + "\n" +
-        "Get detailed help about a command.\n\n" +
-        listing.toString();
-    }
-
-    @Override
-    public int run(Configuration conf, List<String> args) throws IOException {
-      if (args.size() == 0) {
-        for (Command command : COMMANDS) {
-          System.err.println(command.getLongUsage());
-        }
-        return 0;
-      }
-      if (args.size() != 1) {
-        System.out.println("You must give exactly one argument to -help.");
-        return 0;
-      }
-      final String commandName = args.get(0);
-      // prepend a dash to match against the command names
-      final Command command = determineCommand("-"+commandName);
-      if (command == null) {
-        System.err.print("Sorry, I don't know the command '" +
-          commandName + "'.\n");
-        System.err.print("Valid help command names are:\n");
-        String separator = "";
-        for (Command c : COMMANDS) {
-          System.err.print(separator + c.getName().substring(1));
-          separator = ", ";
-        }
-        System.err.print("\n");
-        return 1;
-      }
-      System.err.print(command.getLongUsage());
-      return 0;
-    }
-  }
-
-  private static final Command[] COMMANDS = {
+  private static final AdminHelper.Command[] COMMANDS = {
     new CreateZoneCommand(),
-    new ListZonesCommand(),
-    new HelpCommand(),
+    new ListZonesCommand()
   };
-
-  private static void printUsage(boolean longUsage) {
-    System.err.println(
-        "Usage: bin/hdfs crypto [COMMAND]");
-    for (Command command : COMMANDS) {
-      if (longUsage) {
-        System.err.print(command.getLongUsage());
-      } else {
-        System.err.print("          " + command.getShortUsage());
-      }
-    }
-    System.err.println();
-  }
-
-  private static Command determineCommand(String commandName) {
-    for (int i = 0; i < COMMANDS.length; i++) {
-      if (COMMANDS[i].getName().equals(commandName)) {
-        return COMMANDS[i];
-      }
-    }
-    return null;
-  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 484ac1238be07..77c77fe53e711 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -52,7 +52,6 @@
 import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFormat;
 import org.apache.hadoop.hdfs.client.BlockReportOptions;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -68,12 +67,11 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
+import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.ipc.GenericRefreshProtocol;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
@@ -176,7 +174,7 @@ private static class SetQuotaCommand extends DFSAdminCommand {
       "\t\tNote: A quota of 1 would force the directory to remain empty.\n";
 
     private final long quota; // the quota to be set
-    
+
     /** Constructor */
     SetQuotaCommand(String[] args, int pos, FileSystem fs) {
       super(fs);
@@ -209,19 +207,27 @@ public void run(Path path) throws IOException {
   /** A class that supports command clearSpaceQuota */
   private static class ClearSpaceQuotaCommand extends DFSAdminCommand {
     private static final String NAME = "clrSpaceQuota";
-    private static final String USAGE = "-"+NAME+" <dirname>...<dirname>";
+    private static final String USAGE = "-"+NAME+" <dirname>...<dirname> -storageType <storagetype>";
     private static final String DESCRIPTION = USAGE + ": " +
     "Clear the disk space quota for each directory <dirName>.\n" +
     "\t\tFor each directory, attempt to clear the quota. An error will be reported if\n" +
     "\t\t1. the directory does not exist or is a file, or\n" +
     "\t\t2. user is not an administrator.\n" +
-    "\t\tIt does not fault if the directory has no quota.";
-    
+    "\t\tIt does not fault if the directory has no quota.\n" +
+    "\t\tThe storage type specific quota is cleared when -storageType option is specified.";
+
+    private StorageType type;
+
     /** Constructor */
     ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
       super(fs);
       CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
       List<String> parameters = c.parse(args, pos);
+      String storageTypeString =
+          StringUtils.popOptionWithArgument("-storageType", parameters);
+      if (storageTypeString != null) {
+        this.type = StorageType.parseStorageType(storageTypeString);
+      }
       this.args = parameters.toArray(new String[parameters.size()]);
     }
     
@@ -241,7 +247,11 @@ public String getCommandName() {
 
     @Override
     public void run(Path path) throws IOException {
-      dfs.setQuota(path, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_RESET);
+      if (type != null) {
+        dfs.setQuotaByStorageType(path, type, HdfsConstants.QUOTA_RESET);
+      } else {
+        dfs.setQuota(path, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_RESET);
+      }
     }
   }
   
@@ -249,7 +259,7 @@ public void run(Path path) throws IOException {
   private static class SetSpaceQuotaCommand extends DFSAdminCommand {
     private static final String NAME = "setSpaceQuota";
     private static final String USAGE =
-      "-"+NAME+" <quota> <dirname>...<dirname>";
+      "-"+NAME+" <quota> <dirname>...<dirname> -storageType <storagetype>";
     private static final String DESCRIPTION = USAGE + ": " +
       "Set the disk space quota <quota> for each directory <dirName>.\n" + 
       "\t\tThe space quota is a long integer that puts a hard limit\n" +
@@ -261,9 +271,11 @@ private static class SetSpaceQuotaCommand extends DFSAdminCommand {
       "\t\tFor each directory, attempt to set the quota. An error will be reported if\n" +
       "\t\t1. N is not a positive integer, or\n" +
       "\t\t2. user is not an administrator, or\n" +
-      "\t\t3. the directory does not exist or is a file, or\n";
+      "\t\t3. the directory does not exist or is a file.\n" +
+      "\t\tThe storage type specific quota is set when -storageType option is specified.\n";
 
     private long quota; // the quota to be set
+    private StorageType type;
     
     /** Constructor */
     SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
@@ -276,6 +288,11 @@ private static class SetSpaceQuotaCommand extends DFSAdminCommand {
       } catch (NumberFormatException nfe) {
         throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
       }
+      String storageTypeString =
+          StringUtils.popOptionWithArgument("-storageType", parameters);
+      if (storageTypeString != null) {
+        this.type = StorageType.parseStorageType(storageTypeString);
+      }
       
       this.args = parameters.toArray(new String[parameters.size()]);
     }
@@ -296,7 +313,11 @@ public String getCommandName() {
 
     @Override
     public void run(Path path) throws IOException {
-      dfs.setQuota(path, HdfsConstants.QUOTA_DONT_SET, quota);
+      if (type != null) {
+        dfs.setQuotaByStorageType(path, type, quota);
+      } else {
+        dfs.setQuota(path, HdfsConstants.QUOTA_DONT_SET, quota);
+      }
     }
   }
 
@@ -398,8 +419,6 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep
     "\t[-shutdownDatanode <datanode_host:ipc_port> [upgrade]]\n" +
     "\t[-getDatanodeInfo <datanode_host:ipc_port>]\n" +
     "\t[-metasave filename]\n" +
-    "\t[-setStoragePolicy path policyName]\n" +
-    "\t[-getStoragePolicy path]\n" +
     "\t[-triggerBlockReport [-incremental] <datanode_host:ipc_port>]\n" +
     "\t[-help [cmd]]\n";
 
@@ -608,35 +627,6 @@ private boolean waitExitSafeMode(ClientProtocol nn, boolean inSafeMode)
     return inSafeMode;
   }
 
-  public int setStoragePolicy(String[] argv) throws IOException {
-    DistributedFileSystem dfs = getDFS();
-    dfs.setStoragePolicy(new Path(argv[1]), argv[2]);
-    System.out.println("Set storage policy " + argv[2] + " on " + argv[1]);
-    return 0;
-  }
-
-  public int getStoragePolicy(String[] argv) throws IOException {
-    DistributedFileSystem dfs = getDFS();
-    HdfsFileStatus status = dfs.getClient().getFileInfo(argv[1]);
-    if (status == null) {
-      throw new FileNotFoundException("File/Directory does not exist: "
-          + argv[1]);
-    }
-    byte storagePolicyId = status.getStoragePolicy();
-    if (storagePolicyId == BlockStoragePolicySuite.ID_UNSPECIFIED) {
-      System.out.println("The storage policy of " + argv[1] + " is unspecified");
-      return 0;
-    }
-    BlockStoragePolicy[] policies = dfs.getStoragePolicies();
-    for (BlockStoragePolicy p : policies) {
-      if (p.getId() == storagePolicyId) {
-        System.out.println("The storage policy of " + argv[1] + ":\n" + p);
-        return 0;
-      }
-    }
-    throw new IOException("Cannot identify the storage policy for " + argv[1]);
-  }
-
   public int triggerBlockReport(String[] argv) throws IOException {
     List<String> args = new LinkedList<String>();
     for (int j = 1; j < argv.length; j++) {
@@ -1016,12 +1006,6 @@ private void printHelp(String cmd) {
         + "\tGet the information about the given datanode. This command can\n"
         + "\tbe used for checking if a datanode is alive.\n";
 
-    String setStoragePolicy = "-setStoragePolicy path policyName\n"
-        + "\tSet the storage policy for a file/directory.\n";
-
-    String getStoragePolicy = "-getStoragePolicy path\n"
-        + "\tGet the storage policy for a file/directory.\n";
-
     String triggerBlockReport =
       "-triggerBlockReport [-incremental] <datanode_host:ipc_port>\n"
         + "\tTrigger a block report for the datanode.\n"
@@ -1087,10 +1071,6 @@ private void printHelp(String cmd) {
       System.out.println(shutdownDatanode);
     } else if ("getDatanodeInfo".equalsIgnoreCase(cmd)) {
       System.out.println(getDatanodeInfo);
-    } else if ("setStoragePolicy".equalsIgnoreCase(cmd))  {
-      System.out.println(setStoragePolicy);
-    } else if ("getStoragePolicy".equalsIgnoreCase(cmd))  {
-      System.out.println(getStoragePolicy);
     } else if ("help".equals(cmd)) {
       System.out.println(help);
     } else {
@@ -1123,8 +1103,6 @@ private void printHelp(String cmd) {
       System.out.println(disallowSnapshot);
       System.out.println(shutdownDatanode);
       System.out.println(getDatanodeInfo);
-      System.out.println(setStoragePolicy);
-      System.out.println(getStoragePolicy);
       System.out.println(triggerBlockReport);
       System.out.println(help);
       System.out.println();
@@ -1476,7 +1454,7 @@ int getReconfigurationStatus(String nodeType, String address,
           } else {
             out.print("FAILED: ");
           }
-          out.printf("Change property %s\n\tFrom: \"%s\"\n\tTo: \"%s\"\n",
+          out.printf("Change property %s%n\tFrom: \"%s\"%n\tTo: \"%s\"%n",
               result.getKey().prop, result.getKey().oldVal,
               result.getKey().newVal);
           if (result.getValue().isPresent()) {
@@ -1555,12 +1533,6 @@ private static void printUsage(String cmd) {
     } else if ("-safemode".equals(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
           + " [-safemode enter | leave | get | wait]");
-    } else if ("-setStoragePolicy".equals(cmd)) {
-      System.err.println("Usage: java DFSAdmin"
-          + " [-setStoragePolicy path policyName]");
-    } else if ("-getStoragePolicy".equals(cmd)) {
-      System.err.println("Usage: java DFSAdmin"
-          + " [-getStoragePolicy path]");
     } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
           + " [-allowSnapshot <snapshotDir>]");
@@ -1780,21 +1752,11 @@ public int run(String[] argv) throws Exception {
         printUsage(cmd);
         return exitCode;
       }
-    } else if ("-setStoragePolicy".equals(cmd)) {
-      if (argv.length != 3) {
-        printUsage(cmd);
-        return exitCode;
-      }
     } else if ("-triggerBlockReport".equals(cmd)) {
       if (argv.length < 1) {
         printUsage(cmd);
         return exitCode;
       }
-    } else if ("-getStoragePolicy".equals(cmd)) {
-      if (argv.length != 2) {
-        printUsage(cmd);
-        return exitCode;
-      }
     }
     
     // initialize DFSAdmin
@@ -1868,10 +1830,6 @@ public int run(String[] argv) throws Exception {
         exitCode = getDatanodeInfo(argv, i);
       } else if ("-reconfig".equals(cmd)) {
         exitCode = reconfig(argv, i);
-      } else if ("-setStoragePolicy".equals(cmd)) {
-        exitCode = setStoragePolicy(argv);
-      } else if ("-getStoragePolicy".equals(cmd)) {
-        exitCode = getStoragePolicy(argv);
       } else if ("-triggerBlockReport".equals(cmd)) {
         exitCode = triggerBlockReport(argv);
       } else if ("-help".equals(cmd)) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 0d73b4329911c..98f2030760149 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HAUtil;
@@ -229,14 +230,14 @@ private Integer listCorruptFileBlocks(String dir, String baseUrl)
    * @return Returns http address or null if failure.
    * @throws IOException if we can't determine the active NN address
    */
-  private URI getCurrentNamenodeAddress() throws IOException {
+  private URI getCurrentNamenodeAddress(Path target) throws IOException {
     //String nnAddress = null;
     Configuration conf = getConf();
 
     //get the filesystem object to verify it is an HDFS system
-    FileSystem fs;
+    final FileSystem fs;
     try {
-      fs = FileSystem.get(conf);
+      fs = target.getFileSystem(conf);
     } catch (IOException ioe) {
       System.err.println("FileSystem is inaccessible due to:\n"
           + StringUtils.stringifyException(ioe));
@@ -254,16 +255,6 @@ private URI getCurrentNamenodeAddress() throws IOException {
   private int doWork(final String[] args) throws IOException {
     final StringBuilder url = new StringBuilder();
     
-    URI namenodeAddress = getCurrentNamenodeAddress();
-    if (namenodeAddress == null) {
-      //Error message already output in {@link #getCurrentNamenodeAddress()}
-      System.err.println("DFSck exiting.");
-      return 0;
-    }
-
-    url.append(namenodeAddress.toString());
-    System.err.println("Connecting to namenode via " + url.toString());
-    
     url.append("/fsck?ugi=").append(ugi.getShortUserName());
     String dir = null;
     boolean doListCorruptFileBlocks = false;
@@ -309,7 +300,20 @@ else if (args[idx].equals("-list-corruptfileblocks")) {
     if (null == dir) {
       dir = "/";
     }
-    url.append("&path=").append(URLEncoder.encode(dir, "UTF-8"));
+
+    final Path dirpath = new Path(dir);
+    final URI namenodeAddress = getCurrentNamenodeAddress(dirpath);
+    if (namenodeAddress == null) {
+      //Error message already output in {@link #getCurrentNamenodeAddress()}
+      System.err.println("DFSck exiting.");
+      return 0;
+    }
+
+    url.insert(0, namenodeAddress.toString());
+    url.append("&path=").append(URLEncoder.encode(
+        Path.getPathWithoutSchemeAndAuthority(dirpath).toString(), "UTF-8"));
+    System.err.println("Connecting to namenode via " + url.toString());
+
     if (doListCorruptFileBlocks) {
       return listCorruptFileBlocks(dir, url.toString());
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetStoragePolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetStoragePolicies.java
deleted file mode 100644
index d2793ebb73951..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetStoragePolicies.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.tools;
-
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-
-/**
- * A tool listing all the existing block storage policies. No argument is
- * required when using this tool.
- */
-public class GetStoragePolicies extends Configured implements Tool {
-
-  @Override
-  public int run(String[] args) throws Exception {
-    FileSystem fs = FileSystem.get(getConf());
-    if (!(fs instanceof DistributedFileSystem)) {
-      System.err.println("GetStoragePolicies can only be used against HDFS. " +
-          "Please check the default FileSystem setting in your configuration.");
-      return 1;
-    }
-    DistributedFileSystem dfs = (DistributedFileSystem) fs;
-
-    try {
-      BlockStoragePolicy[] policies = dfs.getStoragePolicies();
-      System.out.println("Block Storage Policies:");
-      for (BlockStoragePolicy policy : policies) {
-        if (policy != null) {
-          System.out.println("\t" + policy);
-        }
-      }
-    } catch (IOException e) {
-      String[] content = e.getLocalizedMessage().split("\n");
-      System.err.println("GetStoragePolicies: " + content[0]);
-      return 1;
-    }
-    return 0;
-  }
-
-  public static void main(String[] args) throws Exception {
-    int rc = ToolRunner.run(new GetStoragePolicies(), args);
-    System.exit(rc);
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
new file mode 100644
index 0000000000000..d1b6017d7e6ca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This class implements block storage policy operations.
+ */
+public class StoragePolicyAdmin extends Configured implements Tool {
+
+  public static void main(String[] argsArray) throws Exception {
+    final StoragePolicyAdmin admin = new StoragePolicyAdmin(new
+        Configuration());
+    System.exit(admin.run(argsArray));
+  }
+
+  public StoragePolicyAdmin(Configuration conf) {
+    super(conf);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length == 0) {
+      AdminHelper.printUsage(false, "storagepolicies", COMMANDS);
+      return 1;
+    }
+    final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+        COMMANDS);
+    if (command == null) {
+      System.err.println("Can't understand command '" + args[0] + "'");
+      if (!args[0].startsWith("-")) {
+        System.err.println("Command names must start with dashes.");
+      }
+      AdminHelper.printUsage(false, "storagepolicies", COMMANDS);
+      return 1;
+    }
+    final List<String> argsList = new LinkedList<>();
+    argsList.addAll(Arrays.asList(args).subList(1, args.length));
+    try {
+      return command.run(getConf(), argsList);
+    } catch (IllegalArgumentException e) {
+      System.err.println(AdminHelper.prettifyException(e));
+      return -1;
+    }
+  }
+
+  /** Command to list all the existing storage policies */
+  private static class ListStoragePoliciesCommand
+      implements AdminHelper.Command {
+    @Override
+    public String getName() {
+      return "-listPolicies";
+    }
+
+    @Override
+    public String getShortUsage() {
+      return "[" + getName() + "]\n";
+    }
+
+    @Override
+    public String getLongUsage() {
+      return getShortUsage() + "\n" +
+          "List all the existing block storage policies.\n";
+    }
+
+    @Override
+    public int run(Configuration conf, List<String> args) throws IOException {
+      final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+      try {
+        BlockStoragePolicy[] policies = dfs.getStoragePolicies();
+        System.out.println("Block Storage Policies:");
+        for (BlockStoragePolicy policy : policies) {
+          if (policy != null) {
+            System.out.println("\t" + policy);
+          }
+        }
+      } catch (IOException e) {
+        System.err.println(AdminHelper.prettifyException(e));
+        return 2;
+      }
+      return 0;
+    }
+  }
+
+  /** Command to get the storage policy of a file/directory */
+  private static class GetStoragePolicyCommand implements AdminHelper.Command {
+    @Override
+    public String getName() {
+      return "-getStoragePolicy";
+    }
+
+    @Override
+    public String getShortUsage() {
+      return "[" + getName() + " -path <path>]\n";
+    }
+
+    @Override
+    public String getLongUsage() {
+      final TableListing listing = AdminHelper.getOptionDescriptionListing();
+      listing.addRow("<path>",
+          "The path of the file/directory for getting the storage policy");
+      return getShortUsage() + "\n" +
+          "Get the storage policy of a file/directory.\n\n" +
+          listing.toString();
+    }
+
+    @Override
+    public int run(Configuration conf, List<String> args) throws IOException {
+      final String path = StringUtils.popOptionWithArgument("-path", args);
+      if (path == null) {
+        System.err.println("Please specify the path with -path.\nUsage:" +
+            getLongUsage());
+        return 1;
+      }
+
+      final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+      try {
+        HdfsFileStatus status = dfs.getClient().getFileInfo(path);
+        if (status == null) {
+          System.err.println("File/Directory does not exist: " + path);
+          return 2;
+        }
+        byte storagePolicyId = status.getStoragePolicy();
+        if (storagePolicyId == BlockStoragePolicySuite.ID_UNSPECIFIED) {
+          System.out.println("The storage policy of " + path + " is unspecified");
+          return 0;
+        }
+        BlockStoragePolicy[] policies = dfs.getStoragePolicies();
+        for (BlockStoragePolicy p : policies) {
+          if (p.getId() == storagePolicyId) {
+            System.out.println("The storage policy of " + path + ":\n" + p);
+            return 0;
+          }
+        }
+      } catch (Exception e) {
+        System.err.println(AdminHelper.prettifyException(e));
+        return 2;
+      }
+      System.err.println("Cannot identify the storage policy for " + path);
+      return 2;
+    }
+  }
+
+  /** Command to set the storage policy to a file/directory */
+  private static class SetStoragePolicyCommand implements AdminHelper.Command {
+    @Override
+    public String getName() {
+      return "-setStoragePolicy";
+    }
+
+    @Override
+    public String getShortUsage() {
+      return "[" + getName() + " -path <path> -policy <policy>]\n";
+    }
+
+    @Override
+    public String getLongUsage() {
+      TableListing listing = AdminHelper.getOptionDescriptionListing();
+      listing.addRow("<path>", "The path of the file/directory to set storage" +
+          " policy");
+      listing.addRow("<policy>", "The name of the block storage policy");
+      return getShortUsage() + "\n" +
+          "Set the storage policy to a file/directory.\n\n" +
+          listing.toString();
+    }
+
+    @Override
+    public int run(Configuration conf, List<String> args) throws IOException {
+      final String path = StringUtils.popOptionWithArgument("-path", args);
+      if (path == null) {
+        System.err.println("Please specify the path for setting the storage " +
+            "policy.\nUsage: " + getLongUsage());
+        return 1;
+      }
+
+      final String policyName = StringUtils.popOptionWithArgument("-policy",
+          args);
+      if (policyName == null) {
+        System.err.println("Please specify the policy name.\nUsage: " +
+            getLongUsage());
+        return 1;
+      }
+
+      final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+      try {
+        dfs.setStoragePolicy(new Path(path), policyName);
+        System.out.println("Set storage policy " + policyName + " on " + path);
+      } catch (Exception e) {
+        System.err.println(AdminHelper.prettifyException(e));
+        return 2;
+      }
+      return 0;
+    }
+  }
+
+  private static final AdminHelper.Command[] COMMANDS = {
+      new ListStoragePoliciesCommand(),
+      new SetStoragePolicyCommand(),
+      new GetStoragePolicyCommand()
+  };
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
index eb6cae3d58a7a..bc5ff5608c812 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DelimitedImageVisitor.java
@@ -144,7 +144,7 @@ void visit(ImageElement element, String value) throws IOException {
     
     // Special case of file size, which is sum of the num bytes in each block
     if(element == ImageElement.NUM_BYTES)
-      fileSize += Long.valueOf(value);
+      fileSize += Long.parseLong(value);
     
     if(elements.containsKey(element) && element != ImageElement.NUM_BYTES)
       elements.put(element, value);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
index eb93c87fc1eba..43fcd69a839a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
@@ -17,11 +17,7 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.base.Charsets;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelFutureListener;
@@ -30,19 +26,31 @@
 import io.netty.channel.group.ChannelGroup;
 import io.netty.handler.codec.http.DefaultFullHttpResponse;
 import io.netty.handler.codec.http.DefaultHttpResponse;
-import static io.netty.handler.codec.http.HttpResponseStatus.*;
-
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpResponseStatus;
-import static io.netty.handler.codec.http.HttpVersion.*;
 import io.netty.handler.codec.http.QueryStringDecoder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.web.JsonUtil;
-import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
+import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX_LENGTH;
 /**
  * Implement the read-only WebHDFS API for fsimage.
  */
@@ -67,7 +75,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
     if (request.getMethod() != HttpMethod.GET) {
       DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
         METHOD_NOT_ALLOWED);
-      resp.headers().set("Connection", "close");
+      resp.headers().set(CONNECTION, CLOSE);
       ctx.write(resp).addListener(ChannelFutureListener.CLOSE);
       return;
     }
@@ -77,24 +85,29 @@ public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
 
     final String content;
     String path = getPath(decoder);
-    if ("GETFILESTATUS".equals(op)) {
-      content = image.getFileStatus(path);
-    } else if ("LISTSTATUS".equals(op)) {
-      content = image.listStatus(path);
-    } else if ("GETACLSTATUS".equals(op)) {
-      content = image.getAclStatus(path);
-    } else {
-      throw new IllegalArgumentException("Invalid value for webhdfs parameter" + " \"op\"");
+    switch (op) {
+      case "GETFILESTATUS":
+        content = image.getFileStatus(path);
+        break;
+      case "LISTSTATUS":
+        content = image.listStatus(path);
+        break;
+      case "GETACLSTATUS":
+        content = image.getAclStatus(path);
+        break;
+      default:
+        throw new IllegalArgumentException(
+            "Invalid value for webhdfs parameter" + " \"op\"");
     }
 
     LOG.info("op=" + op + " target=" + path);
 
     DefaultFullHttpResponse resp = new DefaultFullHttpResponse(
             HTTP_1_1, HttpResponseStatus.OK,
-            Unpooled.wrappedBuffer(content.getBytes()));
-    resp.headers().set("Content-Type", "application/json");
-    resp.headers().set("Content-Length", resp.content().readableBytes());
-    resp.headers().set("Connection", "close");
+            Unpooled.wrappedBuffer(content.getBytes(Charsets.UTF_8)));
+    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
+    resp.headers().set(CONTENT_LENGTH, resp.content().readableBytes());
+    resp.headers().set(CONNECTION, CLOSE);
     ctx.write(resp).addListener(ChannelFutureListener.CLOSE);
   }
 
@@ -109,19 +122,19 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
     Exception e = cause instanceof Exception ? (Exception) cause : new
       Exception(cause);
     final String output = JsonUtil.toJsonString(e);
-    ByteBuf content = Unpooled.wrappedBuffer(output.getBytes());
+    ByteBuf content = Unpooled.wrappedBuffer(output.getBytes(Charsets.UTF_8));
     final DefaultFullHttpResponse resp = new DefaultFullHttpResponse(
             HTTP_1_1, INTERNAL_SERVER_ERROR, content);
 
-    resp.headers().set("Content-Type", "application/json");
+    resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
     if (e instanceof IllegalArgumentException) {
       resp.setStatus(BAD_REQUEST);
     } else if (e instanceof FileNotFoundException) {
       resp.setStatus(NOT_FOUND);
     }
 
-    resp.headers().set("Content-Length", resp.content().readableBytes());
-    resp.headers().set("Connection", "close");
+    resp.headers().set(CONTENT_LENGTH, resp.content().readableBytes());
+    resp.headers().set(CONNECTION, CLOSE);
     ctx.write(resp).addListener(ChannelFutureListener.CLOSE);
   }
 
@@ -134,11 +147,11 @@ private static String getOp(QueryStringDecoder decoder) {
   private static String getPath(QueryStringDecoder decoder)
           throws FileNotFoundException {
     String path = decoder.path();
-    if (path.startsWith("/webhdfs/v1/")) {
-      return path.substring(11);
+    if (path.startsWith(WEBHDFS_PREFIX)) {
+      return path.substring(WEBHDFS_PREFIX_LENGTH);
     } else {
       throw new FileNotFoundException("Path: " + path + " should " +
-              "start with \"/webhdfs/v1/\"");
+              "start with " + WEBHDFS_PREFIX);
     }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index ff665e7798d15..fd291061923a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -34,10 +34,12 @@
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.CodedInputStream;
 import com.google.protobuf.InvalidProtocolBufferException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
@@ -46,6 +48,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -108,17 +111,15 @@ static FSImageLoader load(String inputFile) throws IOException {
     }
 
     FsImageProto.FileSummary summary = FSImageUtil.loadSummary(file);
-    FileInputStream fin = null;
 
-    try {
+
+    try (FileInputStream fin = new FileInputStream(file.getFD())) {
       // Map to record INodeReference to the referred id
       ImmutableList<Long> refIdList = null;
       String[] stringTable = null;
       byte[][] inodes = null;
       Map<Long, long[]> dirmap = null;
 
-      fin = new FileInputStream(file.getFD());
-
       ArrayList<FsImageProto.FileSummary.Section> sections =
           Lists.newArrayList(summary.getSectionsList());
       Collections.sort(sections,
@@ -166,8 +167,6 @@ public int compare(FsImageProto.FileSummary.Section s1,
         }
       }
       return new FSImageLoader(stringTable, inodes, dirmap);
-    } finally {
-      IOUtils.cleanup(null, fin);
     }
   }
 
@@ -238,7 +237,7 @@ private static byte[][] loadINodeSection(InputStream in)
     return inodes;
   }
 
-  private static String[] loadStringTable(InputStream in) throws
+  static String[] loadStringTable(InputStream in) throws
   IOException {
     FsImageProto.StringTableSection s = FsImageProto.StringTableSection
         .parseDelimitedFrom(in);
@@ -314,27 +313,15 @@ private List<Map<String, Object>> getFileStatusList(String path)
    * @throws IOException if failed to serialize fileStatus to JSON.
    */
   String getAclStatus(String path) throws IOException {
-    StringBuilder sb = new StringBuilder();
-    List<AclEntry> aclEntryList = getAclEntryList(path);
     PermissionStatus p = getPermissionStatus(path);
-    sb.append("{\"AclStatus\":{\"entries\":[");
-    int i = 0;
-    for (AclEntry aclEntry : aclEntryList) {
-      if (i++ != 0) {
-        sb.append(',');
-      }
-      sb.append('"');
-      sb.append(aclEntry.toString());
-      sb.append('"');
-    }
-    sb.append("],\"group\": \"");
-    sb.append(p.getGroupName());
-    sb.append("\",\"owner\": \"");
-    sb.append(p.getUserName());
-    sb.append("\",\"stickyBit\": ");
-    sb.append(p.getPermission().getStickyBit());
-    sb.append("}}\n");
-    return sb.toString();
+    List<AclEntry> aclEntryList = getAclEntryList(path);
+    FsPermission permission = p.getPermission();
+    AclStatus.Builder builder = new AclStatus.Builder();
+    builder.owner(p.getUserName()).group(p.getGroupName())
+        .addEntries(aclEntryList).setPermission(permission)
+        .stickyBit(permission.getStickyBit());
+    AclStatus aclStatus = builder.build();
+    return JsonUtil.toJsonString(aclStatus);
   }
 
   private List<AclEntry> getAclEntryList(String path) throws IOException {
@@ -492,7 +479,7 @@ private long lookup(String path) throws IOException {
     }
   }
 
-  private long getFileSize(FsImageProto.INodeSection.INodeFile f) {
+  static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
     long size = 0;
     for (HdfsProtos.BlockProto p : f.getBlocksList()) {
       size += p.getNumBytes();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
index 61c36506097d7..056ad96a5ba1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
@@ -21,7 +21,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 import java.io.RandomAccessFile;
 
 import org.apache.hadoop.conf.Configuration;
@@ -30,7 +30,6 @@
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
 
 import com.google.common.base.Preconditions;
@@ -67,7 +66,7 @@ final class FileDistributionCalculator {
   private final Configuration conf;
   private final long maxSize;
   private final int steps;
-  private final PrintWriter out;
+  private final PrintStream out;
 
   private final int[] distribution;
   private int totalFiles;
@@ -77,7 +76,7 @@ final class FileDistributionCalculator {
   private long maxFileSize;
 
   FileDistributionCalculator(Configuration conf, long maxSize, int steps,
-      PrintWriter out) {
+      PrintStream out) {
     this.conf = conf;
     this.maxSize = maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize;
     this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
@@ -96,9 +95,7 @@ void visit(RandomAccessFile file) throws IOException {
     }
 
     FileSummary summary = FSImageUtil.loadSummary(file);
-    FileInputStream in = null;
-    try {
-      in = new FileInputStream(file.getFD());
+    try (FileInputStream in = new FileInputStream(file.getFD())) {
       for (FileSummary.Section s : summary.getSectionsList()) {
         if (SectionName.fromString(s.getName()) != SectionName.INODE) {
           continue;
@@ -111,8 +108,6 @@ void visit(RandomAccessFile file) throws IOException {
         run(is);
         output();
       }
-    } finally {
-      IOUtils.cleanup(null, in);
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
index f293db44d3d67..146d00a85ee20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
@@ -159,10 +159,10 @@ void visit(ImageElement element, String value) throws IOException {
         current.path = (value.equals("") ? "/" : value);
         break;
       case REPLICATION:
-        current.replication = Integer.valueOf(value);
+        current.replication = Integer.parseInt(value);
         break;
       case NUM_BYTES:
-        current.fileSize += Long.valueOf(value);
+        current.fileSize += Long.parseLong(value);
         break;
       default:
         break;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
index 6e303a9f16159..7d229dbf98d73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
@@ -135,7 +135,7 @@ void visit(ImageElement element, String value) throws IOException {
         perms = value;
         break;
       case REPLICATION:
-        replication = Integer.valueOf(value);
+        replication = Integer.parseInt(value);
         break;
       case USER_NAME:
         username = value;
@@ -144,7 +144,7 @@ void visit(ImageElement element, String value) throws IOException {
         group = value;
         break;
       case NUM_BYTES:
-        filesize += Long.valueOf(value);
+        filesize += Long.parseLong(value);
         break;
       case MODIFICATION_TIME:
         modTime = value;
@@ -173,6 +173,6 @@ void visitEnclosingElement(ImageElement element,
     if(element == ImageElement.INODE)
       newLine();
     else if (element == ImageElement.BLOCKS)
-      numBlocks = Integer.valueOf(value);
+      numBlocks = Integer.parseInt(value);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index f02acaed64956..659036691e3e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -18,9 +18,8 @@
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
 import java.io.EOFException;
-import java.io.File;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 import java.io.RandomAccessFile;
 
 import org.apache.commons.cli.CommandLine;
@@ -33,7 +32,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
 
 /**
@@ -67,6 +65,10 @@ public class OfflineImageViewerPB {
       + "    -step defines the granularity of the distribution. (2MB by default)\n"
       + "  * Web: Run a viewer to expose read-only WebHDFS API.\n"
       + "    -addr specifies the address to listen. (localhost:5978 by default)\n"
+      + "  * Delimited: Generate a text file with all of the elements common\n"
+      + "    to both inodes and inodes-under-construction, separated by a\n"
+      + "    delimiter. The default delimiter is \\t, though this may be\n"
+      + "    changed via the -delimiter argument.\n"
       + "\n"
       + "Required command line arguments:\n"
       + "-i,--inputFile <arg>   FSImage file to process.\n"
@@ -76,8 +78,12 @@ public class OfflineImageViewerPB {
       + "                       file exists, it will be overwritten.\n"
       + "                       (output to stdout by default)\n"
       + "-p,--processor <arg>   Select which type of processor to apply\n"
-      + "                       against image file. (XML|FileDistribution|Web)\n"
+      + "                       against image file. (XML|FileDistribution|Web|Delimited)\n"
       + "                       (Web by default)\n"
+      + "-delimiter <arg>       Delimiting string to use with Delimited processor\n"
+      + "-t,--temp <arg>        Use temporary dir to cache intermediate result to generate\n"
+      + "                       Delimited outputs. If not set, Delimited processor constructs\n"
+      + "                       the namespace in memory before outputting text."
       + "-h,--help              Display usage information and exit\n";
 
   /**
@@ -99,6 +105,8 @@ private static Options buildOptions() {
     options.addOption("maxSize", true, "");
     options.addOption("step", true, "");
     options.addOption("addr", true, "");
+    options.addOption("delimiter", true, "");
+    options.addOption("t", "temp", true, "");
 
     return options;
   }
@@ -143,37 +151,43 @@ public static int run(String[] args) throws Exception {
     String inputFile = cmd.getOptionValue("i");
     String processor = cmd.getOptionValue("p", "Web");
     String outputFile = cmd.getOptionValue("o", "-");
-
-    PrintWriter out = outputFile.equals("-") ?
-        new PrintWriter(System.out) : new PrintWriter(new File(outputFile));
+    String delimiter = cmd.getOptionValue("delimiter",
+        PBImageDelimitedTextWriter.DEFAULT_DELIMITER);
+    String tempPath = cmd.getOptionValue("t", "");
 
     Configuration conf = new Configuration();
-    try {
-      if (processor.equals("FileDistribution")) {
-        long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
-        int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
-        new FileDistributionCalculator(conf, maxSize, step, out)
-            .visit(new RandomAccessFile(inputFile, "r"));
-      } else if (processor.equals("XML")) {
-        new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
-            "r"));
-      } else if (processor.equals("Web")) {
-        String addr = cmd.getOptionValue("addr", "localhost:5978");
-        WebImageViewer viewer = new WebImageViewer(NetUtils.createSocketAddr
-                (addr));
-        try {
-          viewer.start(inputFile);
-        } finally {
-          viewer.close();
-        }
+    try (PrintStream out = outputFile.equals("-") ?
+        System.out : new PrintStream(outputFile, "UTF-8")) {
+      switch (processor) {
+        case "FileDistribution":
+          long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
+          int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
+          new FileDistributionCalculator(conf, maxSize, step, out).visit(
+              new RandomAccessFile(inputFile, "r"));
+          break;
+        case "XML":
+          new PBImageXmlWriter(conf, out).visit(
+              new RandomAccessFile(inputFile, "r"));
+          break;
+        case "Web":
+          String addr = cmd.getOptionValue("addr", "localhost:5978");
+          try (WebImageViewer viewer = new WebImageViewer(
+              NetUtils.createSocketAddr(addr))) {
+            viewer.start(inputFile);
+          }
+          break;
+        case "Delimited":
+          try (PBImageDelimitedTextWriter writer =
+              new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
+            writer.visit(new RandomAccessFile(inputFile, "r"));
+          }
+          break;
       }
       return 0;
     } catch (EOFException e) {
       System.err.println("Input file ended unexpectedly. Exiting");
     } catch (IOException e) {
       System.err.println("Encountered exception.  Exiting: " + e.getMessage());
-    } finally {
-      IOUtils.cleanup(null, out);
     }
     return -1;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
new file mode 100644
index 0000000000000..350967d1ff8b2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeSymlink;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * A PBImageDelimitedTextWriter generates a text representation of the PB fsimage,
+ * with each element separated by a delimiter string.  All of the elements
+ * common to both inodes and inodes-under-construction are included. When
+ * processing an fsimage with a layout version that did not include an
+ * element, such as AccessTime, the output file will include a column
+ * for the value, but no value will be included.
+ *
+ * Individual block information for each file is not currently included.
+ *
+ * The default delimiter is tab, as this is an unlikely value to be included in
+ * an inode path or other text metadata. The delimiter value can be via the
+ * constructor.
+ */
+public class PBImageDelimitedTextWriter extends PBImageTextWriter {
+  static final String DEFAULT_DELIMITER = "\t";
+  private static final String DATE_FORMAT="yyyy-MM-dd HH:mm";
+  private final SimpleDateFormat dateFormatter =
+      new SimpleDateFormat(DATE_FORMAT);
+
+  private final String delimiter;
+
+  PBImageDelimitedTextWriter(PrintStream out, String delimiter, String tempPath)
+      throws IOException {
+    super(out, tempPath);
+    this.delimiter = delimiter;
+  }
+
+  private String formatDate(long date) {
+    return dateFormatter.format(new Date(date));
+  }
+
+  private void append(StringBuffer buffer, int field) {
+    buffer.append(delimiter);
+    buffer.append(field);
+  }
+
+  private void append(StringBuffer buffer, long field) {
+    buffer.append(delimiter);
+    buffer.append(field);
+  }
+
+  private void append(StringBuffer buffer, String field) {
+    buffer.append(delimiter);
+    buffer.append(field);
+  }
+
+  @Override
+  public String getEntry(String parent, INode inode) {
+    StringBuffer buffer = new StringBuffer();
+    String path = new File(parent, inode.getName().toStringUtf8()).toString();
+    buffer.append(path);
+    PermissionStatus p = null;
+
+    switch (inode.getType()) {
+    case FILE:
+      INodeFile file = inode.getFile();
+      p = getPermission(file.getPermission());
+      append(buffer, file.getReplication());
+      append(buffer, formatDate(file.getModificationTime()));
+      append(buffer, formatDate(file.getAccessTime()));
+      append(buffer, file.getPreferredBlockSize());
+      append(buffer, file.getBlocksCount());
+      append(buffer, FSImageLoader.getFileSize(file));
+      append(buffer, 0);  // NS_QUOTA
+      append(buffer, 0);  // DS_QUOTA
+      break;
+    case DIRECTORY:
+      INodeDirectory dir = inode.getDirectory();
+      p = getPermission(dir.getPermission());
+      append(buffer, 0);  // Replication
+      append(buffer, formatDate(dir.getModificationTime()));
+      append(buffer, formatDate(0));  // Access time.
+      append(buffer, 0);  // Block size.
+      append(buffer, 0);  // Num blocks.
+      append(buffer, 0);  // Num bytes.
+      append(buffer, dir.getNsQuota());
+      append(buffer, dir.getDsQuota());
+      break;
+    case SYMLINK:
+      INodeSymlink s = inode.getSymlink();
+      p = getPermission(s.getPermission());
+      append(buffer, 0);  // Replication
+      append(buffer, formatDate(s.getModificationTime()));
+      append(buffer, formatDate(s.getAccessTime()));
+      append(buffer, 0);  // Block size.
+      append(buffer, 0);  // Num blocks.
+      append(buffer, 0);  // Num bytes.
+      append(buffer, 0);  // NS_QUOTA
+      append(buffer, 0);  // DS_QUOTA
+      break;
+    default:
+      break;
+    }
+    assert p != null;
+    append(buffer, p.getPermission().toString());
+    append(buffer, p.getUserName());
+    append(buffer, p.getGroupName());
+    return buffer.toString();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
new file mode 100644
index 0000000000000..0da263d87ae3f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -0,0 +1,586 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.io.LimitInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
+import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
+import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Time;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.Options;
+import org.iq80.leveldb.WriteBatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.io.RandomAccessFile;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class reads the protobuf-based fsimage and generates text output
+ * for each inode to {@link PBImageTextWriter#out}. The sub-class can override
+ * {@link getEntry()} to generate formatted string for each inode.
+ *
+ * Since protobuf-based fsimage does not guarantee the order of inodes and
+ * directories, PBImageTextWriter runs two-phase scans:
+ *
+ * <ol>
+ *   <li>The first phase, PBImageTextWriter scans the INode sections to reads the
+ *   filename of each directory. It also scans the INode_Dir sections to loads
+ *   the relationships between a directory and its children. It uses these metadata
+ *   to build FS namespace and stored in {@link MetadataMap}</li>
+ *   <li>The second phase, PBImageTextWriter re-scans the INode sections. For each
+ *   inode, it looks up the path of the parent directory in the {@link MetadataMap},
+ *   and generate output.</li>
+ * </ol>
+ *
+ * Two various of {@link MetadataMap} are provided. {@link InMemoryMetadataDB}
+ * stores all metadata in memory (O(n) memory) while
+ * {@link LevelDBMetadataMap} stores metadata in LevelDB on disk (O(1) memory).
+ * User can choose between them based on the time/space tradeoffs.
+ */
+abstract class PBImageTextWriter implements Closeable {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(PBImageTextWriter.class);
+
+  /**
+   * This metadata map is used to construct the namespace before generating
+   * text outputs.
+   *
+   * It contains two mapping relationships:
+   * <p>
+   *   <li>It maps each inode (inode Id) to its parent directory (inode Id).</li>
+   *   <li>It maps each directory from its inode Id.</li>
+   * </p>
+   */
+  private static interface MetadataMap extends Closeable {
+    /**
+     * Associate an inode with its parent directory.
+     */
+    public void putDirChild(long parentId, long childId) throws IOException;
+
+    /**
+     * Associate a directory with its inode Id.
+     */
+    public void putDir(INode dir) throws IOException;
+
+    /** Get the full path of the parent directory for the given inode. */
+    public String getParentPath(long inode) throws IOException;
+
+    /** Synchronize metadata to persistent storage, if possible */
+    public void sync() throws IOException;
+  }
+
+  /**
+   * Maintain all the metadata in memory.
+   */
+  private static class InMemoryMetadataDB implements MetadataMap {
+    /**
+     * Represent a directory in memory.
+     */
+    private static class Dir {
+      private final long inode;
+      private Dir parent = null;
+      private String name;
+      private String path = null;  // cached full path of the directory.
+
+      Dir(long inode, String name) {
+        this.inode = inode;
+        this.name = name;
+      }
+
+      private void setParent(Dir parent) {
+        Preconditions.checkState(this.parent == null);
+        this.parent = parent;
+      }
+
+      /**
+       * Returns the full path of this directory.
+       */
+      private String getPath() {
+        if (this.parent == null) {
+          return "/";
+        }
+        if (this.path == null) {
+          this.path = new File(parent.getPath(), name).toString();
+          this.name = null;
+        }
+        return this.path;
+      }
+
+      @Override
+      public boolean equals(Object o) {
+        return o instanceof Dir && inode == ((Dir) o).inode;
+      }
+
+      @Override
+      public int hashCode() {
+        return Long.valueOf(inode).hashCode();
+      }
+    }
+
+    /** INode Id to Dir object mapping */
+    private Map<Long, Dir> dirMap = new HashMap<>();
+
+    /** Children to parent directory INode ID mapping. */
+    private Map<Long, Dir> dirChildMap = new HashMap<>();
+
+    InMemoryMetadataDB() {
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    @Override
+    public void putDirChild(long parentId, long childId) {
+      Dir parent = dirMap.get(parentId);
+      Dir child = dirMap.get(childId);
+      if (child != null) {
+        child.setParent(parent);
+      }
+      Preconditions.checkState(!dirChildMap.containsKey(childId));
+      dirChildMap.put(childId, parent);
+    }
+
+    @Override
+    public void putDir(INode p) {
+      Preconditions.checkState(!dirMap.containsKey(p.getId()));
+      Dir dir = new Dir(p.getId(), p.getName().toStringUtf8());
+      dirMap.put(p.getId(), dir);
+    }
+
+    public String getParentPath(long inode) throws IOException {
+      if (inode == INodeId.ROOT_INODE_ID) {
+        return "";
+      }
+      Dir parent = dirChildMap.get(inode);
+      Preconditions.checkState(parent != null,
+          "Can not find parent directory for INode: %s", inode);
+      return parent.getPath();
+    }
+
+    @Override
+    public void sync() {
+    }
+  }
+
+  /**
+   * A MetadataMap that stores metadata in LevelDB.
+   */
+  private static class LevelDBMetadataMap implements MetadataMap {
+    /**
+     * Store metadata in LevelDB.
+     */
+    private static class LevelDBStore implements Closeable {
+      private DB db = null;
+      private WriteBatch batch = null;
+      private int writeCount = 0;
+      private static final int BATCH_SIZE = 1024;
+
+      LevelDBStore(final File dbPath) throws IOException {
+        Options options = new Options();
+        options.createIfMissing(true);
+        options.errorIfExists(true);
+        db = JniDBFactory.factory.open(dbPath, options);
+        batch = db.createWriteBatch();
+      }
+
+      @Override
+      public void close() throws IOException {
+        if (batch != null) {
+          IOUtils.cleanup(null, batch);
+          batch = null;
+        }
+        IOUtils.cleanup(null, db);
+        db = null;
+      }
+
+      public void put(byte[] key, byte[] value) throws IOException {
+        batch.put(key, value);
+        writeCount++;
+        if (writeCount >= BATCH_SIZE) {
+          sync();
+        }
+      }
+
+      public byte[] get(byte[] key) throws IOException {
+        return db.get(key);
+      }
+
+      public void sync() throws IOException {
+        try {
+          db.write(batch);
+        } finally {
+          batch.close();
+          batch = null;
+        }
+        batch = db.createWriteBatch();
+        writeCount = 0;
+      }
+    }
+
+    /**
+     * A LRU cache for directory path strings.
+     *
+     * The key of this LRU cache is the inode of a directory.
+     */
+    private static class DirPathCache extends LinkedHashMap<Long, String> {
+      private final static int CAPACITY = 16 * 1024;
+
+      DirPathCache() {
+        super(CAPACITY);
+      }
+
+      @Override
+      protected boolean removeEldestEntry(Map.Entry<Long, String> entry) {
+        return super.size() > CAPACITY;
+      }
+    }
+
+    /** Map the child inode to the parent directory inode. */
+    private LevelDBStore dirChildMap = null;
+    /** Directory entry map */
+    private LevelDBStore dirMap = null;
+    private DirPathCache dirPathCache = new DirPathCache();
+
+    LevelDBMetadataMap(String baseDir) throws IOException {
+      File dbDir = new File(baseDir);
+      if (dbDir.exists()) {
+        FileUtils.deleteDirectory(dbDir);
+      }
+      if (!dbDir.mkdirs()) {
+        throw new IOException("Failed to mkdir on " + dbDir);
+      }
+      try {
+        dirChildMap = new LevelDBStore(new File(dbDir, "dirChildMap"));
+        dirMap = new LevelDBStore(new File(dbDir, "dirMap"));
+      } catch (IOException e) {
+        LOG.error("Failed to open LevelDBs", e);
+        IOUtils.cleanup(null, this);
+      }
+    }
+
+    @Override
+    public void close() throws IOException {
+      IOUtils.cleanup(null, dirChildMap, dirMap);
+      dirChildMap = null;
+      dirMap = null;
+    }
+
+    private static byte[] toBytes(long value) {
+      return ByteBuffer.allocate(8).putLong(value).array();
+    }
+
+    private static byte[] toBytes(String value)
+        throws UnsupportedEncodingException {
+      return value.getBytes("UTF-8");
+    }
+
+    private static long toLong(byte[] bytes) {
+      Preconditions.checkArgument(bytes.length == 8);
+      return ByteBuffer.wrap(bytes).getLong();
+    }
+
+    private static String toString(byte[] bytes) throws IOException {
+      try {
+        return new String(bytes, "UTF-8");
+      } catch (UnsupportedEncodingException e) {
+        throw new IOException(e);
+      }
+    }
+
+    @Override
+    public void putDirChild(long parentId, long childId) throws IOException {
+      dirChildMap.put(toBytes(childId), toBytes(parentId));
+    }
+
+    @Override
+    public void putDir(INode dir) throws IOException {
+      Preconditions.checkArgument(dir.hasDirectory(),
+          "INode %s (%s) is not a directory.", dir.getId(), dir.getName());
+      dirMap.put(toBytes(dir.getId()), toBytes(dir.getName().toStringUtf8()));
+    }
+
+    @Override
+    public String getParentPath(long inode) throws IOException {
+      if (inode == INodeId.ROOT_INODE_ID) {
+        return "/";
+      }
+      byte[] bytes = dirChildMap.get(toBytes(inode));
+      Preconditions.checkState(bytes != null && bytes.length == 8,
+          "Can not find parent directory for inode %s, "
+              + "fsimage might be corrupted", inode);
+      long parent = toLong(bytes);
+      if (!dirPathCache.containsKey(parent)) {
+        bytes = dirMap.get(toBytes(parent));
+        if (parent != INodeId.ROOT_INODE_ID) {
+          Preconditions.checkState(bytes != null,
+              "Can not find parent directory for inode %s, "
+                  + ", the fsimage might be corrupted.", parent);
+        }
+        String parentName = toString(bytes);
+        String parentPath =
+            new File(getParentPath(parent), parentName).toString();
+        dirPathCache.put(parent, parentPath);
+      }
+      return dirPathCache.get(parent);
+    }
+
+    @Override
+    public void sync() throws IOException {
+      dirChildMap.sync();
+      dirMap.sync();
+    }
+  }
+
+  private String[] stringTable;
+  private PrintStream out;
+  private MetadataMap metadataMap = null;
+
+  /**
+   * Construct a PB FsImage writer to generate text file.
+   * @param out the writer to output text information of fsimage.
+   * @param tempPath the path to store metadata. If it is empty, store metadata
+   *                 in memory instead.
+   */
+  PBImageTextWriter(PrintStream out, String tempPath) throws IOException {
+    this.out = out;
+    if (tempPath.isEmpty()) {
+      metadataMap = new InMemoryMetadataDB();
+    } else {
+      metadataMap = new LevelDBMetadataMap(tempPath);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    IOUtils.cleanup(null, metadataMap);
+  }
+
+  /**
+   * Get text output for the given inode.
+   * @param parent the path of parent directory
+   * @param inode the INode object to output.
+   */
+  abstract protected String getEntry(String parent, INode inode);
+
+  public void visit(RandomAccessFile file) throws IOException {
+    Configuration conf = new Configuration();
+    if (!FSImageUtil.checkFileFormat(file)) {
+      throw new IOException("Unrecognized FSImage");
+    }
+
+    FileSummary summary = FSImageUtil.loadSummary(file);
+
+    try (FileInputStream fin = new FileInputStream(file.getFD())) {
+      InputStream is;
+      ArrayList<FileSummary.Section> sections =
+          Lists.newArrayList(summary.getSectionsList());
+      Collections.sort(sections,
+          new Comparator<FileSummary.Section>() {
+            @Override
+            public int compare(FsImageProto.FileSummary.Section s1,
+                FsImageProto.FileSummary.Section s2) {
+              FSImageFormatProtobuf.SectionName n1 =
+                  FSImageFormatProtobuf.SectionName.fromString(s1.getName());
+              FSImageFormatProtobuf.SectionName n2 =
+                  FSImageFormatProtobuf.SectionName.fromString(s2.getName());
+              if (n1 == null) {
+                return n2 == null ? 0 : -1;
+              } else if (n2 == null) {
+                return -1;
+              } else {
+                return n1.ordinal() - n2.ordinal();
+              }
+            }
+          });
+
+      for (FileSummary.Section section : sections) {
+        fin.getChannel().position(section.getOffset());
+        is = FSImageUtil.wrapInputStreamForCompression(conf,
+            summary.getCodec(), new BufferedInputStream(new LimitInputStream(
+                fin, section.getLength())));
+        switch (SectionName.fromString(section.getName())) {
+        case STRING_TABLE:
+          stringTable = FSImageLoader.loadStringTable(is);
+          break;
+        default:
+          break;
+        }
+      }
+
+      loadDirectories(fin, sections, summary, conf);
+      loadINodeDirSection(fin, sections, summary, conf);
+      metadataMap.sync();
+      output(conf, summary, fin, sections);
+    }
+  }
+
+  private void output(Configuration conf, FileSummary summary,
+      FileInputStream fin, ArrayList<FileSummary.Section> sections)
+      throws IOException {
+    InputStream is;
+    long startTime = Time.monotonicNow();
+    for (FileSummary.Section section : sections) {
+      if (SectionName.fromString(section.getName()) == SectionName.INODE) {
+        fin.getChannel().position(section.getOffset());
+        is = FSImageUtil.wrapInputStreamForCompression(conf,
+            summary.getCodec(), new BufferedInputStream(new LimitInputStream(
+                fin, section.getLength())));
+        outputINodes(is);
+      }
+    }
+    long timeTaken = Time.monotonicNow() - startTime;
+    LOG.debug("Time to output inodes: {}ms", timeTaken);
+  }
+
+  protected PermissionStatus getPermission(long perm) {
+    return FSImageFormatPBINode.Loader.loadPermission(perm, stringTable);
+  }
+
+  /** Load the directories in the INode section. */
+  private void loadDirectories(
+      FileInputStream fin, List<FileSummary.Section> sections,
+      FileSummary summary, Configuration conf)
+      throws IOException {
+    LOG.info("Loading directories");
+    long startTime = Time.monotonicNow();
+    for (FileSummary.Section section : sections) {
+      if (SectionName.fromString(section.getName())
+          == SectionName.INODE) {
+        fin.getChannel().position(section.getOffset());
+        InputStream is = FSImageUtil.wrapInputStreamForCompression(conf,
+            summary.getCodec(), new BufferedInputStream(new LimitInputStream(
+                fin, section.getLength())));
+        loadDirectoriesInINodeSection(is);
+      }
+    }
+    long timeTaken = Time.monotonicNow() - startTime;
+    LOG.info("Finished loading directories in {}ms", timeTaken);
+  }
+
+  private void loadINodeDirSection(
+      FileInputStream fin, List<FileSummary.Section> sections,
+      FileSummary summary, Configuration conf)
+      throws IOException {
+    LOG.info("Loading INode directory section.");
+    long startTime = Time.monotonicNow();
+    for (FileSummary.Section section : sections) {
+      if (SectionName.fromString(section.getName())
+          == SectionName.INODE_DIR) {
+        fin.getChannel().position(section.getOffset());
+        InputStream is = FSImageUtil.wrapInputStreamForCompression(conf,
+            summary.getCodec(), new BufferedInputStream(
+                new LimitInputStream(fin, section.getLength())));
+        buildNamespace(is);
+      }
+    }
+    long timeTaken = Time.monotonicNow() - startTime;
+    LOG.info("Finished loading INode directory section in {}ms", timeTaken);
+  }
+
+  /**
+   * Load the filenames of the directories from the INode section.
+   */
+  private void loadDirectoriesInINodeSection(InputStream in) throws IOException {
+    INodeSection s = INodeSection.parseDelimitedFrom(in);
+    LOG.info("Loading directories in INode section.");
+    int numDirs = 0;
+    for (int i = 0; i < s.getNumInodes(); ++i) {
+      INode p = INode.parseDelimitedFrom(in);
+      if (LOG.isDebugEnabled() && i % 10000 == 0) {
+        LOG.debug("Scanned {} inodes.", i);
+      }
+      if (p.hasDirectory()) {
+        metadataMap.putDir(p);
+        numDirs++;
+      }
+    }
+    LOG.info("Found {} directories in INode section.", numDirs);
+  }
+
+  /**
+   * Scan the INodeDirectory section to construct the namespace.
+   */
+  private void buildNamespace(InputStream in) throws IOException {
+    int count = 0;
+    while (true) {
+      FsImageProto.INodeDirectorySection.DirEntry e =
+          FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
+      if (e == null) {
+        break;
+      }
+      count++;
+      if (LOG.isDebugEnabled() && count % 10000 == 0) {
+        LOG.debug("Scanned {} directories.", count);
+      }
+      long parentId = e.getParent();
+      // Referred INode is not support for now.
+      for (int i = 0; i < e.getChildrenCount(); i++) {
+        long childId = e.getChildren(i);
+        metadataMap.putDirChild(parentId, childId);
+      }
+      Preconditions.checkState(e.getRefChildrenCount() == 0);
+    }
+    LOG.info("Scanned {} INode directories to build namespace.", count);
+  }
+
+  private void outputINodes(InputStream in) throws IOException {
+    INodeSection s = INodeSection.parseDelimitedFrom(in);
+    LOG.info("Found {} INodes in the INode section", s.getNumInodes());
+    for (int i = 0; i < s.getNumInodes(); ++i) {
+      INode p = INode.parseDelimitedFrom(in);
+      String parentPath = metadataMap.getParentPath(p.getId());
+      out.println(getEntry(parentPath, p));
+
+      if (LOG.isDebugEnabled() && i % 100000 == 0) {
+        LOG.debug("Outputted {} INodes.", i);
+      }
+    }
+    LOG.info("Outputted {} INodes.", s.getNumInodes());
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 3e3f0217ab34a..f3fe886089705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -21,7 +21,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -50,7 +50,6 @@
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
 import org.apache.hadoop.hdfs.util.XMLUtils;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
 
 import com.google.common.collect.Lists;
@@ -62,10 +61,10 @@
 @InterfaceAudience.Private
 public final class PBImageXmlWriter {
   private final Configuration conf;
-  private final PrintWriter out;
+  private final PrintStream out;
   private String[] stringTable;
 
-  public PBImageXmlWriter(Configuration conf, PrintWriter out) {
+  public PBImageXmlWriter(Configuration conf, PrintStream out) {
     this.conf = conf;
     this.out = out;
   }
@@ -76,9 +75,7 @@ public void visit(RandomAccessFile file) throws IOException {
     }
 
     FileSummary summary = FSImageUtil.loadSummary(file);
-    FileInputStream fin = null;
-    try {
-      fin = new FileInputStream(file.getFD());
+    try (FileInputStream fin = new FileInputStream(file.getFD())) {
       out.print("<?xml version=\"1.0\"?>\n<fsimage>");
 
       ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary
@@ -140,8 +137,6 @@ public int compare(FileSummary.Section s1, FileSummary.Section s2) {
         }
       }
       out.print("</fsimage>\n");
-    } finally {
-      IOUtils.cleanup(null, fin);
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java
new file mode 100644
index 0000000000000..5b29c4308b8bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ReferenceCountMap.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Class for de-duplication of instances. <br>
+ * Hold the references count to a single instance. If there are no references
+ * then the entry will be removed.<br>
+ * Type E should implement {@link ReferenceCounter}<br>
+ * Note: This class is NOT thread-safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ReferenceCountMap<E extends ReferenceCountMap.ReferenceCounter> {
+
+  private Map<E, E> referenceMap = new HashMap<E, E>();
+
+  /**
+   * Add the reference. If the instance already present, just increase the
+   * reference count.
+   * 
+   * @param key Key to put in reference map
+   * @return Referenced instance
+   */
+  public E put(E key) {
+    E value = referenceMap.get(key);
+    if (value == null) {
+      value = key;
+      referenceMap.put(key, value);
+    }
+    value.incrementAndGetRefCount();
+    return value;
+  }
+
+  /**
+   * Delete the reference. Decrease the reference count for the instance, if
+   * any. On all references removal delete the instance from the map.
+   * 
+   * @param key Key to remove the reference.
+   */
+  public void remove(E key) {
+    E value = referenceMap.get(key);
+    if (value != null && value.decrementAndGetRefCount() == 0) {
+      referenceMap.remove(key);
+    }
+  }
+
+  /**
+   * Get entries in the reference Map.
+   * 
+   * @return
+   */
+  @VisibleForTesting
+  public ImmutableList<E> getEntries() {
+    return new ImmutableList.Builder<E>().addAll(referenceMap.keySet()).build();
+  }
+
+  /**
+   * Get the reference count for the key
+   */
+  public long getReferenceCount(E key) {
+    ReferenceCounter counter = referenceMap.get(key);
+    if (counter != null) {
+      return counter.getRefCount();
+    }
+    return 0;
+  }
+
+  /**
+   * Get the number of unique elements
+   */
+  public int getUniqueElementsSize() {
+    return referenceMap.size();
+  }
+
+  /**
+   * Clear the contents
+   */
+  @VisibleForTesting
+  public void clear() {
+    referenceMap.clear();
+  }
+
+  /**
+   * Interface for the reference count holder
+   */
+  public static interface ReferenceCounter {
+    public int getRefCount();
+
+    public int incrementAndGetRefCount();
+
+    public int decrementAndGetRefCount();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
index 2792460d53119..e36f0f7e0897b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
@@ -22,15 +22,6 @@ public interface RwLock {
   /** Acquire read lock. */
   public void readLock();
   
-  /**
-   * Acquire the long read lock, unless interrupted while waiting. The long
-   * read lock should also serve to block all concurrent writers.
-   **/
-  void longReadLockInterruptibly() throws InterruptedException;
-  
-  /** Release the long read lock. */
-  public void longReadUnlock();
-
   /** Release read lock. */
   public void readUnlock();
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 0a2ae2652dbf2..aa6100cc318e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -655,6 +655,16 @@ public static String toJsonString(final AclStatus status) {
     m.put("group", status.getGroup());
     m.put("stickyBit", status.isStickyBit());
     m.put("entries", status.getEntries());
+    FsPermission perm = status.getPermission();
+    if (perm != null) {
+      m.put("permission", toString(perm));
+      if (perm.getAclBit()) {
+        m.put("aclBit", true);
+      }
+      if (perm.getEncryptedBit()) {
+        m.put("encBit", true);
+      }
+    }
     final Map<String, Map<String, Object>> finalMap =
         new TreeMap<String, Map<String, Object>>();
     finalMap.put(AclStatus.class.getSimpleName(), m);
@@ -673,7 +683,12 @@ public static AclStatus toAclStatus(final Map<?, ?> json) {
     aclStatusBuilder.owner((String) m.get("owner"));
     aclStatusBuilder.group((String) m.get("group"));
     aclStatusBuilder.stickyBit((Boolean) m.get("stickyBit"));
-
+    String permString = (String) m.get("permission");
+    if (permString != null) {
+      final FsPermission permission = toFsPermission(permString,
+          (Boolean) m.get("aclBit"), (Boolean) m.get("encBit"));
+      aclStatusBuilder.setPermission(permission);
+    }
     final Object[] entries = (Object[]) m.get("entries");
 
     List<AclEntry> aclEntryList = new ArrayList<AclEntry>();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 559efdb1ceffd..938f7c77f6473 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -312,16 +312,20 @@ private Path makeAbsolute(Path f) {
     if (in == null) {
       throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
     }
-    final String contentType = c.getContentType();
-    if (contentType != null) {
-      final MediaType parsed = MediaType.valueOf(contentType);
-      if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
-        throw new IOException("Content-Type \"" + contentType
-            + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
-            + "\" (parsed=\"" + parsed + "\")");
+    try {
+      final String contentType = c.getContentType();
+      if (contentType != null) {
+        final MediaType parsed = MediaType.valueOf(contentType);
+        if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
+          throw new IOException("Content-Type \"" + contentType
+              + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
+              + "\" (parsed=\"" + parsed + "\")");
+        }
       }
+      return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
+    } finally {
+      in.close();
     }
-    return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
   }
 
   private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
@@ -1156,6 +1160,14 @@ public FSDataOutputStream append(final Path f, final int bufferSize,
     ).run();
   }
 
+  @Override
+  public boolean truncate(Path f, long newLength) throws IOException {
+    statistics.incrementWriteOps(1);
+
+    final HttpOpParam.Op op = PostOpParam.Op.TRUNCATE;
+    return new FsPathBooleanRunner(op, f, new NewLengthParam(newLength)).run();
+  }
+
   @Override
   public boolean delete(Path f, boolean recursive) throws IOException {
     final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/NewLengthParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/NewLengthParam.java
new file mode 100644
index 0000000000000..83aba9ea20e66
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/NewLengthParam.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** NewLength parameter. */
+public class NewLengthParam extends LongParam {
+  /** Parameter name. */
+  public static final String NAME = "newlength";
+  /** Default parameter value. */
+  public static final String DEFAULT = NULL;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public NewLengthParam(final Long value) {
+    super(DOMAIN, value, 0L, null);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public NewLengthParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
index 54034f0e818c5..13f792e7dfe77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
@@ -27,6 +27,8 @@ public static enum Op implements HttpOpParam.Op {
 
     CONCAT(false, HttpURLConnection.HTTP_OK),
 
+    TRUNCATE(false, HttpURLConnection.HTTP_OK),
+
     NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED);
 
     final boolean doOutputAndRedirect;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c
index 2373aa78024c0..eb7115c5684d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/exception.c
@@ -79,6 +79,11 @@ static const struct ExceptionInfo gExceptionInfo[] = {
         0,
         EDQUOT,
     },
+    {
+        "java.lang.UnsupportedOperationException",
+        0,
+        ENOTSUP,
+    },
     {
         "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException",
         0,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
index dc8f39d5ce896..95d8f019defa2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
@@ -181,7 +181,38 @@ int hdfsFileGetReadStatistics(hdfsFile file,
 int64_t hdfsReadStatisticsGetRemoteBytesRead(
                             const struct hdfsReadStatistics *stats)
 {
-  return stats->totalBytesRead - stats->totalLocalBytesRead;
+    return stats->totalBytesRead - stats->totalLocalBytesRead;
+}
+
+int hdfsFileClearReadStatistics(hdfsFile file)
+{
+    jthrowable jthr;
+    int ret;
+    JNIEnv* env = getJNIEnv();
+
+    if (env == NULL) {
+        errno = EINTERNAL;
+        return EINTERNAL;
+    }
+    if (file->type != HDFS_STREAM_INPUT) {
+        ret = EINVAL;
+        goto done;
+    }
+    jthr = invokeMethod(env, NULL, INSTANCE, file->file,
+                  "org/apache/hadoop/hdfs/client/HdfsDataInputStream",
+                  "clearReadStatistics", "()V");
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "hdfsFileClearReadStatistics: clearReadStatistics failed");
+        goto done;
+    }
+    ret = 0;
+done:
+    if (ret) {
+        errno = ret;
+        return ret;
+    }
+    return 0;
 }
 
 void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
index 0625da3759193..072051f2614c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
@@ -118,6 +118,19 @@ extern  "C" {
     int64_t hdfsReadStatisticsGetRemoteBytesRead(
                             const struct hdfsReadStatistics *stats);
 
+    /**
+     * Clear the read statistics for a file.
+     *
+     * @param file      The file to clear the read statistics of.
+     *
+     * @return          0 on success; the error code otherwise.
+     *                  EINVAL: the file is not open for reading.
+     *                  ENOTSUP: the file does not support clearing the read
+     *                  statistics.
+     *                  Errno will also be set to this code on failure.
+     */
+    int hdfsFileClearReadStatistics(hdfsFile file);
+
     /**
      * Free some HDFS read statistics.
      *
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
index 2c42fa5f204c3..b37ebcce0b6c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
@@ -17,8 +17,6 @@
  */
 
 #include "exception.h"
-#include "hdfs.h"
-#include "hdfs_test.h"
 #include "jni_helper.h"
 #include "native_mini_dfs.h"
 #include "platform.h"
@@ -32,6 +30,10 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#ifndef EINTERNAL
+#define EINTERNAL 255
+#endif
+
 #define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
 #define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
 #define HADOOP_CONF     "org/apache/hadoop/conf/Configuration"
@@ -52,6 +54,25 @@ struct NativeMiniDfsCluster {
     char domainSocketPath[PATH_MAX];
 };
 
+static int hdfsDisableDomainSocketSecurity(void)
+{
+    jthrowable jthr;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    jthr = invokeMethod(env, NULL, STATIC, NULL,
+            "org/apache/hadoop/net/unix/DomainSocket",
+            "disableBindPathValidation", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "DomainSocket#disableBindPathValidation");
+        return -1;
+    }
+    return 0;
+}
+
 static jthrowable nmdConfigureShortCircuit(JNIEnv *env,
               struct NativeMiniDfsCluster *cl, jobject cobj)
 {
@@ -345,29 +366,10 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
     return ret;
 }
 
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
-                            struct hdfsBuilder *bld)
-{
-    int ret;
-    tPort port;
-
-    hdfsBuilderSetNameNode(bld, "localhost");
-    port = (tPort)nmdGetNameNodePort(cl);
-    if (port < 0) {
-      fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
-      return EIO;
-    }
-    hdfsBuilderSetNameNodePort(bld, port);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl) {
     if (cl->domainSocketPath[0]) {
-      ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
-      if (ret) {
-          return ret;
-      }
-      ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
-                            cl->domainSocketPath);
-      if (ret) {
-          return ret;
-      }
+        return cl->domainSocketPath;
     }
-    return 0;
+
+    return NULL;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
index 41d69c2966a22..ce8b1cfdab050 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
@@ -21,6 +21,10 @@
 
 #include <jni.h> /* for jboolean */
 
+#ifdef __cplusplus
+extern  "C" {
+#endif
+
 struct hdfsBuilder;
 struct NativeMiniDfsCluster; 
 
@@ -110,13 +114,16 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
                                int *port, const char **hostName);
 
 /**
- * Configure the HDFS builder appropriately to connect to this cluster.
+ * Get domain socket path set for this cluster.
  *
- * @param bld       The hdfs builder
+ * @param cl        The cluster
  *
- * @return          the port, or a negative error code
+ * @return          A const string of domain socket path, or NULL if not set.
  */
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
-                            struct hdfsBuilder *bld);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
index 3774417c83732..92941cfb35587 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
@@ -201,6 +201,37 @@ static int createZeroCopyTestFile(hdfsFS fs, char *testFileName,
     return 0;
 }
 
+static int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
+                            struct hdfsBuilder *bld) {
+    int ret;
+    tPort port;
+    const char *domainSocket;
+
+    hdfsBuilderSetNameNode(bld, "localhost");
+    port = (tPort) nmdGetNameNodePort(cl);
+    if (port < 0) {
+      fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
+      return EIO;
+    }
+    hdfsBuilderSetNameNodePort(bld, port);
+
+    domainSocket = hdfsGetDomainSocketPath(cl);
+
+    if (domainSocket) {
+      ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
+      if (ret) {
+        return ret;
+      }
+      ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
+                                  domainSocket);
+      if (ret) {
+        return ret;
+      }
+    }
+    return 0;
+}
+
+
 /**
  * Test that we can write a file with libhdfs and then read it back
  */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
index 016f0b19dde36..17ff7a8367d26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
@@ -205,6 +205,10 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
     errno = 0;
     EXPECT_UINT64_EQ((uint64_t)expected, readStats->totalBytesRead);
     hdfsFileFreeReadStatistics(readStats);
+    EXPECT_ZERO(hdfsFileClearReadStatistics(file));
+    EXPECT_ZERO(hdfsFileGetReadStatistics(file, &readStats));
+    EXPECT_UINT64_EQ((uint64_t)0, readStats->totalBytesRead);
+    hdfsFileFreeReadStatistics(readStats);
     EXPECT_ZERO(memcmp(paths->prefix, tmp, expected));
     EXPECT_ZERO(hdfsCloseFile(fs, file));
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
index 2c1d3cb9f3e6f..bd6f76cb4955f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
@@ -66,6 +66,7 @@ enum CreateFlagProto {
   OVERWRITE = 0x02; // Truncate/overwrite a file. Same as POSIX O_TRUNC
   APPEND = 0x04;    // Append to a file
   LAZY_PERSIST = 0x10; // File with reduced durability guarantees.
+  NEW_BLOCK = 0x20; // Write data to a new block when appending
 }
 
 message CreateRequestProto {
@@ -86,6 +87,7 @@ message CreateResponseProto {
 message AppendRequestProto {
   required string src = 1;
   required string clientName = 2;
+  optional uint32 flag = 3; // bits set using CreateFlag
 }
 
 message AppendResponseProto {
@@ -198,6 +200,16 @@ message ConcatRequestProto {
 message ConcatResponseProto { // void response
 }
 
+message TruncateRequestProto {
+  required string src = 1;
+  required uint64 newLength = 2;
+  required string clientName = 3;
+}
+
+message TruncateResponseProto {
+  required bool result = 1;
+}
+
 message RenameRequestProto {
   required string src = 1;
   required string dst = 2;
@@ -560,6 +572,7 @@ message SetQuotaRequestProto {
   required string path = 1;
   required uint64 namespaceQuota = 2;
   required uint64 diskspaceQuota = 3;
+  optional StorageTypeProto storageType = 4;
 }
 
 message SetQuotaResponseProto { // void response
@@ -722,6 +735,7 @@ service ClientNamenodeProtocol {
   rpc reportBadBlocks(ReportBadBlocksRequestProto)
       returns(ReportBadBlocksResponseProto);
   rpc concat(ConcatRequestProto) returns(ConcatResponseProto);
+  rpc truncate(TruncateRequestProto) returns(TruncateResponseProto);
   rpc rename(RenameRequestProto) returns(RenameResponseProto);
   rpc rename2(Rename2RequestProto) returns(Rename2ResponseProto);
   rpc delete(DeleteRequestProto) returns(DeleteResponseProto);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
index 47f79bed1697c..1a2177798823a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
@@ -59,6 +59,8 @@ message UpdateReplicaUnderRecoveryRequestProto {
   required ExtendedBlockProto block = 1; // Block identifier
   required uint64 recoveryId = 2;        // New genstamp of the replica
   required uint64 newLength = 3;         // New length of the replica
+  // New blockId for copy (truncate), default is 0.
+  optional uint64 newBlockId = 4 [default = 0];
 }
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/acl.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/acl.proto
index e940142e339a7..57cc855786720 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/acl.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/acl.proto
@@ -58,6 +58,7 @@ message AclStatusProto {
   required string group          = 2;
   required bool sticky           = 3;
   repeated AclEntryProto entries = 4;
+  optional FsPermissionProto permission = 5;
 }
 
 message AclEditLogProto {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
index fd1ba8a0980dc..95126882d66e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
@@ -210,6 +210,7 @@ message PacketHeaderProto {
   optional bool syncBlock = 5 [default = false];
 }
 
+// Status is a 4-bit enum
 enum Status {
   SUCCESS = 0;
   ERROR = 1;
@@ -220,7 +221,7 @@ enum Status {
   CHECKSUM_OK = 6;
   ERROR_UNSUPPORTED = 7;
   OOB_RESTART = 8;            // Quick restart
-  OOB_INTERRUPTED = 9;        // Interrupted
+  OOB_RESERVED1 = 9;          // Reserved
   OOB_RESERVED2 = 10;         // Reserved
   OOB_RESERVED3 = 11;         // Reserved
   IN_PROGRESS = 12;
@@ -228,7 +229,7 @@ enum Status {
 
 message PipelineAckProto {
   required sint64 seqno = 1;
-  repeated Status status = 2;
+  repeated uint32 reply = 2;
   optional uint64 downstreamAckTimeNanos = 3 [default = 0];
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 588f6c8612296..643a0345737d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -270,6 +270,7 @@ message SnapshotDiffSection {
     optional uint64 fileSize = 2;
     optional bytes name = 3;
     optional INodeSection.INodeFile snapshotCopy = 4;
+    repeated BlockProto blocks = 5;
   }
 
   message DiffEntry {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index 04a8f3f0fb0c2..97906b164028e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -554,8 +554,9 @@ enum ReplicaStateProto {
  * Block that needs to be recovered with at a given location
  */
 message RecoveringBlockProto {
-  required uint64 newGenStamp = 1;      // New genstamp post recovery
-  required LocatedBlockProto block = 2; // Block to be recovered
+  required uint64 newGenStamp = 1;        // New genstamp post recovery
+  required LocatedBlockProto block = 2;   // Block to be recovered
+  optional BlockProto truncateBlock = 3;  // New block for recovery (truncate)
 }
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
index e51c02cf771e3..5b78fe62de6ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
@@ -78,6 +78,7 @@ message CreateEventProto {
   optional int32 replication = 7;
   optional string symlinkTarget = 8;
   optional bool overwrite = 9;
+  optional int64 defaultBlockSize = 10 [default=0];
 }
 
 message CloseEventProto {
@@ -88,6 +89,7 @@ message CloseEventProto {
 
 message AppendEventProto {
   required string path = 1;
+  optional bool newBlock = 2 [default = false];
 }
 
 message RenameEventProto {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 06d7ba81de468..966f5f0ff4c43 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -30,16 +30,6 @@
   <description>version of this configuration file</description>
 </property>
 
-<property>
-  <name>dfs.namenode.logging.level</name>
-  <value>info</value>
-  <description>
-    The logging level for dfs namenode. Other values are "dir" (trace
-    namespace mutations), "block" (trace block under/over replications
-    and block creations/deletions), or "all".
-  </description>
-</property>
-
 <property>
   <name>dfs.namenode.rpc-address</name>
   <value></value>
@@ -154,14 +144,6 @@
   </description>
 </property>
 
-<property>
-  <name>dfs.https.enable</name>
-  <value>false</value>
-  <description>
-    Deprecated. Use "dfs.http.policy" instead.
-  </description>
-</property>
-
 <property>
   <name>dfs.http.policy</name>
   <value>HTTP_ONLY</value>
@@ -314,7 +296,8 @@
   <name>dfs.namenode.fs-limits.max-directory-items</name>
   <value>1048576</value>
   <description>Defines the maximum number of items that a directory may
-      contain.  A value of 0 will disable the check.</description>
+      contain. Cannot set the property to a value less than 1 or more than
+      6400000.</description>
 </property>
 
 <property>
@@ -1003,6 +986,26 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.scan.period.hours</name>
+  <value>0</value>
+  <description>
+        If this is 0 or negative, the DataNode's block scanner will be
+        disabled.  If this is positive, the DataNode will not scan any
+        individual block more than once in the specified scan period.
+  </description>
+</property>
+
+<property>
+  <name>dfs.block.scanner.volume.bytes.per.second</name>
+  <value>1048576</value>
+  <description>
+        If this is 0, the DataNode's block scanner will be disabled.  If this
+        is positive, this is the number of bytes per second that the DataNode's
+        block scanner will try to scan from each volume.
+  </description>
+</property>
+
 <property>
   <name>dfs.datanode.readahead.bytes</name>
   <value>4193404</value>
@@ -1223,14 +1226,6 @@
   </description>
 </property>
 
-<property>
-  <name>dfs.support.append</name>
-  <value>true</value>
-  <description>
-    Does HDFS allow appends to files?
-  </description>
-</property>
-
 <property>
   <name>dfs.client.use.datanode.hostname</name>
   <value>false</value>
@@ -2252,4 +2247,12 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.namenode.blocks.per.postponedblocks.rescan</name>
+  <value>10000</value>
+  <description>Number of blocks to rescan for each iteration of
+    postponedMisreplicatedBlocks.
+  </description>
+</property>
+
 </configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs
new file mode 100644
index 0000000000000..5eb9e4826467b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs
@@ -0,0 +1,36 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile hdfs
+
+function _hdfs_hadoop_classpath
+{
+  #
+  # get all of the hdfs jars+config in the path
+  #
+  # developers
+  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+    hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
+  fi
+
+  # put hdfs in classpath if present
+  if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
+    hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
+  fi
+
+  hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
+  hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 4971e90e0936c..9c83f3a0192dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -153,13 +153,10 @@
 {#nn}
 <table class="table table-bordered table-striped">
   <tr><th> Configured Capacity:</th><td>{Total|fmt_bytes}</td></tr>
-  <tr><th> DFS Used:</th><td>{Used|fmt_bytes}</td></tr>
+  <tr><th> DFS Used:</th><td>{Used|fmt_bytes} ({PercentUsed|fmt_percentage})</td></tr>
   <tr><th> Non DFS Used:</th><td>{NonDfsUsedSpace|fmt_bytes}</td></tr>
-  <tr><th> DFS Remaining:</th><td>{Free|fmt_bytes}</td></tr>
-  <tr><th> DFS Used%:</th><td>{PercentUsed|fmt_percentage}</td></tr>
-  <tr><th> DFS Remaining%:</th><td>{PercentRemaining|fmt_percentage}</td></tr>
-  <tr><th> Block Pool Used:</th><td>{BlockPoolUsedSpace|fmt_bytes}</td></tr>
-  <tr><th> Block Pool Used%:</th><td>{PercentBlockPoolUsed|fmt_percentage}</td></tr>
+  <tr><th> DFS Remaining:</th><td>{Free|fmt_bytes} ({PercentRemaining|fmt_percentage})</td></tr>
+  <tr><th> Block Pool Used:</th><td>{BlockPoolUsedSpace|fmt_bytes} ({PercentBlockPoolUsed|fmt_percentage})</td></tr>
   <tr><th> DataNodes usages% (Min/Median/Max/stdDev): </th>
 	<td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td></tr>
 {/nn}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
index 69674c74aeae3..5336ea3fa4cc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
@@ -189,7 +189,7 @@ hdfs mover [-p <files/dirs> | -f <local file name>]
   * Command:
 
 +------------------------------------------+
-hdfs storagepolicies
+hdfs storagepolicies -listPolicies
 +------------------------------------------+
 
   * Arguments: none.
@@ -201,16 +201,16 @@ hdfs storagepolicies
   * Command:
 
 +------------------------------------------+
-hdfs dfsadmin -setStoragePolicy <path> <policyName>
+hdfs storagepolicies -setStoragePolicy -path <path> -policy <policy>
 +------------------------------------------+
 
   * Arguments:
 
-*----------------------+-----------------------------------------------------+
-| <<<\<path\>>>>       | The path referring to either a directory or a file. |
-*----------------------+-----------------------------------------------------+
-| <<<\<policyName\>>>> | The name of the storage policy.                     |
-*----------------------+-----------------------------------------------------+
+*--------------------------+-----------------------------------------------------+
+| <<<-path \<path\>>>>     | The path referring to either a directory or a file. |
+*--------------------------+-----------------------------------------------------+
+| <<<-policy \<policy\>>>> | The name of the storage policy.                     |
+*--------------------------+-----------------------------------------------------+
 
   []
 
@@ -221,13 +221,13 @@ hdfs dfsadmin -setStoragePolicy <path> <policyName>
   * Command:
 
 +------------------------------------------+
-hdfs dfsadmin -getStoragePolicy <path>
+hdfs storagepolicies -getStoragePolicy -path <path>
 +------------------------------------------+
 
   * Arguments:
 
-*----------------------+-----------------------------------------------------+
-| <<<\<path\>>>>       | The path referring to either a directory or a file. |
-*----------------------+-----------------------------------------------------+
+*----------------------------+-----------------------------------------------------+
+| <<<-path \<path\>>>>       | The path referring to either a directory or a file. |
+*----------------------------+-----------------------------------------------------+
 
   []
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
index 3d27b8e218621..17aaf3ce59739 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
@@ -32,113 +32,114 @@ HDFS Federation
 
   * <<Namespace>>
 
-    * Consists of directories, files and blocks
+    * Consists of directories, files and blocks.
 
-    * It supports all the namespace related file system operations such as 
+    * It supports all the namespace related file system operations such as
       create, delete, modify and list files and directories.
 
-  * <<Block Storage Service>> has two parts
+  * <<Block Storage Service>>, which has two parts:
 
-    * Block Management (which is done in Namenode)
+    * Block Management (performed in the Namenode)
 
-      * Provides datanode cluster membership by handling registrations, and 
+      * Provides Datanode cluster membership by handling registrations, and
         periodic heart beats.
 
       * Processes block reports and maintains location of blocks.
 
-      * Supports block related operations such as create, delete, modify and 
+      * Supports block related operations such as create, delete, modify and
         get block location.
 
-      * Manages replica placement and replication of a block for under 
-        replicated blocks and deletes blocks that are over replicated.
+      * Manages replica placement, block replication for under
+        replicated blocks, and deletes blocks that are over replicated.
 
-    * Storage - is provided by datanodes by storing blocks on the local file 
-      system and allows read/write access.
+    * Storage - is provided by Datanodes by storing blocks on the local file
+      system and allowing read/write access.
+
+  The prior HDFS architecture allows only a single namespace for the
+  entire cluster. In that configuration, a single Namenode manages the
+  namespace. HDFS Federation addresses this limitation by adding
+  support for multiple Namenodes/namespaces to HDFS.
 
-  The prior HDFS architecture allows only a single namespace for the 
-  entire cluster. A single Namenode manages this namespace. HDFS 
-  Federation addresses limitation of the prior architecture by adding 
-  support multiple Namenodes/namespaces to HDFS file system.
-    
 * {Multiple Namenodes/Namespaces}
 
-  In order to scale the name service horizontally, federation uses multiple 
-  independent Namenodes/namespaces. The Namenodes are federated, that is, the 
-  Namenodes are independent and don’t require coordination with each other. 
-  The datanodes are used as common storage for blocks by all the Namenodes. 
-  Each datanode registers with all the Namenodes in the cluster. Datanodes 
-  send periodic heartbeats and block reports and handles commands from the 
-  Namenodes.
+  In order to scale the name service horizontally, federation uses multiple
+  independent Namenodes/namespaces. The Namenodes are federated; the
+  Namenodes are independent and do not require coordination with each other.
+  The Datanodes are used as common storage for blocks by all the Namenodes.
+  Each Datanode registers with all the Namenodes in the cluster. Datanodes
+  send periodic heartbeats and block reports. They also handle
+  commands from the Namenodes.
 
-  Users may use {{{./ViewFs.html}ViewFs}} to create personalized namespace views,
-  where ViewFs is analogous to client side mount tables in some Unix/Linux systems.
+  Users may use {{{./ViewFs.html}ViewFs}} to create personalized namespace views.
+  ViewFs is analogous to client side mount tables in some Unix/Linux systems.
 
 [./images/federation.gif] HDFS Federation Architecture
 
 
   <<Block Pool>>
 
-  A Block Pool is a set of blocks that belong to a single namespace. 
-  Datanodes store blocks for all the block pools in the cluster.
-  It is managed independently of other block pools. This allows a namespace 
-  to generate Block IDs for new blocks without the need for coordination 
-  with the other namespaces. The failure of a Namenode does not prevent 
-  the datanode from serving other Namenodes in the cluster.
+  A Block Pool is a set of blocks that belong to a single namespace.
+  Datanodes store blocks for all the block pools in the cluster.  Each
+  Block Pool is managed independently. This allows a namespace to
+  generate Block IDs for new blocks without the need for coordination
+  with the other namespaces. A Namenode failure does not prevent the
+  Datanode from serving other Namenodes in the cluster.
 
-  A Namespace and its block pool together are called Namespace Volume. 
-  It is a self-contained unit of management. When a Namenode/namespace 
-  is deleted, the corresponding block pool at the datanodes is deleted.
+  A Namespace and its block pool together are called Namespace Volume.
+  It is a self-contained unit of management. When a Namenode/namespace
+  is deleted, the corresponding block pool at the Datanodes is deleted.
   Each namespace volume is upgraded as a unit, during cluster upgrade.
 
   <<ClusterID>>
 
-  A new identifier <<ClusterID>> is added to identify all the nodes in 
-  the cluster.  When a Namenode is formatted, this identifier is provided 
-  or auto generated. This ID should be used for formatting the other 
-  Namenodes into the cluster.
+  A <<ClusterID>> identifier is used to identify all the nodes in the
+  cluster.  When a Namenode is formatted, this identifier is either
+  provided or auto generated. This ID should be used for formatting
+  the other Namenodes into the cluster.
 
 ** Key Benefits
 
-  * Namespace Scalability - HDFS cluster storage scales horizontally but 
-    the namespace does not. Large deployments or deployments using lot 
-    of small files benefit from scaling the namespace by adding more 
-    Namenodes to the cluster
+  * Namespace Scalability - Federation adds namespace horizontal
+    scaling. Large deployments or deployments using lot of small files
+    benefit from namespace scaling by allowing more Namenodes to be
+    added to the cluster.
 
-  * Performance - File system operation throughput is limited by a single
-    Namenode in the prior architecture. Adding more Namenodes to the cluster
-    scales the file system read/write operations throughput.
+  * Performance - File system throughput is not limited by a single
+    Namenode. Adding more Namenodes to the cluster scales the file
+    system read/write throughput.
 
-  * Isolation - A single Namenode offers no isolation in multi user 
-    environment. An experimental application can overload the Namenode 
-    and slow down production critical applications. With multiple Namenodes, 
-    different categories of applications and users can be isolated to 
-    different namespaces.
+  * Isolation - A single Namenode offers no isolation in a multi user
+    environment. For example, an experimental application can overload
+    the Namenode and slow down production critical applications. By using
+    multiple Namenodes, different categories of applications and users
+    can be isolated to different namespaces.
 
 * {Federation Configuration}
 
-  Federation configuration is <<backward compatible>> and allows existing 
-  single Namenode configuration to work without any change. The new 
-  configuration is designed such that all the nodes in the cluster have 
-  same configuration without the need for deploying different configuration 
-  based on the type of the node in the cluster.
+  Federation configuration is <<backward compatible>> and allows
+  existing single Namenode configurations to work without any
+  change. The new configuration is designed such that all the nodes in
+  the cluster have the same configuration without the need for
+  deploying different configurations based on the type of the node in
+  the cluster.
 
-  A new abstraction called <<<NameServiceID>>> is added with
-  federation. The Namenode and its corresponding secondary/backup/checkpointer
-  nodes belong to this. To support single configuration file, the Namenode and
-  secondary/backup/checkpointer configuration parameters are suffixed with
-  <<<NameServiceID>>> and are added to the same configuration file.
+  Federation adds a new <<<NameServiceID>>> abstraction. A Namenode
+  and its corresponding secondary/backup/checkpointer nodes all belong
+  to a NameServiceId. In order to support a single configuration file,
+  the Namenode and secondary/backup/checkpointer configuration
+  parameters are suffixed with the <<<NameServiceID>>>.
 
 
 ** Configuration:
 
-  <<Step 1>>: Add the following parameters to your configuration:
-  <<<dfs.nameservices>>>: Configure with list of comma separated 
-  NameServiceIDs. This will be used by Datanodes to determine all the 
+  <<Step 1>>: Add the <<<dfs.nameservices>>> parameter to your
+  configuration and configure it with a list of comma separated
+  NameServiceIDs. This will be used by the Datanodes to determine the
   Namenodes in the cluster.
-  
-  <<Step 2>>: For each Namenode and Secondary Namenode/BackupNode/Checkpointer 
-  add the following configuration suffixed with the corresponding 
-  <<<NameServiceID>>> into the common configuration file.
+
+  <<Step 2>>: For each Namenode and Secondary Namenode/BackupNode/Checkpointer
+  add the following configuration parameters suffixed with the corresponding
+  <<<NameServiceID>>> into the common configuration file:
 
 *---------------------+--------------------------------------------+
 || Daemon             || Configuration Parameter                   |
@@ -159,8 +160,8 @@ HDFS Federation
 | BackupNode          | <<<dfs.namenode.backup.address>>>          |
 |                     | <<<dfs.secondary.namenode.keytab.file>>>   |
 *---------------------+--------------------------------------------+
-    
-  Here is an example configuration with two namenodes:
+
+  Here is an example configuration with two Namenodes:
 
 ----
 <configuration>
@@ -199,60 +200,58 @@ HDFS Federation
 
 ** Formatting Namenodes
 
-  <<Step 1>>: Format a namenode using the following command:
-  
+  <<Step 1>>: Format a Namenode using the following command:
+
 ----
-> $HADOOP_PREFIX_HOME/bin/hdfs namenode -format [-clusterId <cluster_id>]
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format [-clusterId <cluster_id>]
 ----
-  Choose a unique cluster_id, which will not conflict other clusters in 
-  your environment. If it is not provided, then a unique ClusterID is 
+  Choose a unique cluster_id which will not conflict other clusters in
+  your environment. If a cluster_id is not provided, then a unique one is
   auto generated.
 
-  <<Step 2>>: Format additional namenode using the following command:
+  <<Step 2>>: Format additional Namenodes using the following command:
 
 ----
-> $HADOOP_PREFIX_HOME/bin/hdfs namenode -format -clusterId <cluster_id>
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format -clusterId <cluster_id>
 ----
-  Note that the cluster_id in step 2 must be same as that of the 
-  cluster_id in step 1. If they are different, the additional Namenodes 
+  Note that the cluster_id in step 2 must be same as that of the
+  cluster_id in step 1. If they are different, the additional Namenodes
   will not be part of the federated cluster.
 
 ** Upgrading from an older release and configuring federation
 
-  Older releases supported a single Namenode. 
-  Upgrade the cluster to newer release to enable federation
+  Older releases only support a single Namenode.
+  Upgrade the cluster to newer release in order to enable federation
   During upgrade you can provide a ClusterID as follows:
 
 ----
-> $HADOOP_PREFIX_HOME/bin/hdfs start namenode --config $HADOOP_CONF_DIR  -upgrade -clusterId <cluster_ID>
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode -upgrade -clusterId <cluster_ID>
 ----
-  If ClusterID is not provided, it is auto generated.
+  If cluster_id is not provided, it is auto generated.
 
 ** Adding a new Namenode to an existing HDFS cluster
 
-  Follow the following steps:
+  Perform the following steps:
 
-  * Add configuration parameter <<<dfs.nameservices>>> to the configuration.
+  * Add <<<dfs.nameservices>>> to the configuration.
 
-  * Update the configuration with NameServiceID suffix. Configuration 
-    key names have changed post release 0.20. You must use new configuration 
-    parameter names, for federation.
+  * Update the configuration with the NameServiceID suffix. Configuration
+    key names changed post release 0.20. You must use the new configuration
+    parameter names in order to use federation.
 
-  * Add new Namenode related config to the configuration files.
+  * Add the new Namenode related config to the configuration file.
 
   * Propagate the configuration file to the all the nodes in the cluster.
 
-  * Start the new Namenode, Secondary/Backup.
+  * Start the new Namenode and Secondary/Backup.
 
-  * Refresh the datanodes to pickup the newly added Namenode by running 
-    the following command:
+  * Refresh the Datanodes to pickup the newly added Namenode by running
+    the following command against all the Datanodes in the cluster:
 
 ----
-> $HADOOP_PREFIX_HOME/bin/hdfs dfadmin -refreshNameNode <datanode_host_name>:<datanode_rpc_port>
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNameNode <datanode_host_name>:<datanode_rpc_port>
 ----
 
-  * The above command must be run against all the datanodes in the cluster.
-
 * {Managing the cluster}
 
 **  Starting and stopping cluster
@@ -260,83 +259,81 @@ HDFS Federation
   To start the cluster run the following command:
 
 ----
-> $HADOOP_PREFIX_HOME/bin/start-dfs.sh
+[hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
 ----
 
   To stop the cluster run the following command:
 
 ----
-> $HADOOP_PREFIX_HOME/bin/stop-dfs.sh
+[hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
 ----
 
-  These commands can be run from any node where the HDFS configuration is 
-  available.  The command uses configuration to determine the Namenodes 
-  in the cluster and starts the Namenode process on those nodes. The 
-  datanodes are started on nodes specified in the <<<slaves>>> file. The 
-  script can be used as reference for building your own scripts for 
-  starting and stopping the cluster.
+  These commands can be run from any node where the HDFS configuration is
+  available.  The command uses the configuration to determine the Namenodes
+  in the cluster and then starts the Namenode process on those nodes. The
+  Datanodes are started on the nodes specified in the <<<slaves>>> file. The
+  script can be used as a reference for building your own scripts to
+  start and stop the cluster.
 
 **  Balancer
 
-  Balancer has been changed to work with multiple Namenodes in the cluster to 
-  balance the cluster. Balancer can be run using the command:
+  The Balancer has been changed to work with multiple
+  Namenodes. The Balancer can be run using the command:
 
 ----
-"$HADOOP_PREFIX"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer [-policy <policy>]
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start balancer [-policy <policy>]
 ----
 
-  Policy could be:
+  The policy parameter can be any of the following:
 
-  * <<<datanode>>> - this is the <default> policy. This balances the storage at 
-    the datanode level. This is similar to balancing policy from prior releases.
+  * <<<datanode>>> - this is the <default> policy. This balances the storage at
+    the Datanode level. This is similar to balancing policy from prior releases.
 
-  * <<<blockpool>>> - this balances the storage at the block pool level. 
-    Balancing at block pool level balances storage at the datanode level also.
+  * <<<blockpool>>> - this balances the storage at the block pool
+    level which also balances at the Datanode level.
 
   Note that Balancer only balances the data and does not balance the namespace.
   For the complete command usage, see {{{../hadoop-common/CommandsManual.html#balancer}balancer}}.
 
 ** Decommissioning
 
-  Decommissioning is similar to prior releases. The nodes that need to be 
-  decomissioned are added to the exclude file at all the Namenode. Each 
-  Namenode decommissions its Block Pool. When all the Namenodes finish 
-  decommissioning a datanode, the datanode is considered to be decommissioned.
+  Decommissioning is similar to prior releases. The nodes that need to be
+  decomissioned are added to the exclude file at all of the Namenodes. Each
+  Namenode decommissions its Block Pool. When all the Namenodes finish
+  decommissioning a Datanode, the Datanode is considered decommissioned.
 
-  <<Step 1>>: To distributed an exclude file to all the Namenodes, use the 
+  <<Step 1>>: To distribute an exclude file to all the Namenodes, use the
   following command:
 
 ----
-"$HADOOP_PREFIX"/bin/distributed-exclude.sh <exclude_file>
+[hdfs]$ $HADOOP_PREFIX/sbin/distribute-exclude.sh <exclude_file>
 ----
 
-  <<Step 2>>: Refresh all the Namenodes to pick up the new exclude file.
+  <<Step 2>>: Refresh all the Namenodes to pick up the new exclude file:
 
 ----
-"$HADOOP_PREFIX"/bin/refresh-namenodes.sh
+[hdfs]$ $HADOOP_PREFIX/sbin/refresh-namenodes.sh
 ----
- 
-  The above command uses HDFS configuration to determine the Namenodes 
-  configured in the cluster and refreshes all the Namenodes to pick up 
+
+  The above command uses HDFS configuration to determine the
+  configured Namenodes in the cluster and refreshes them to pick up
   the new exclude file.
 
 ** Cluster Web Console
 
-  Similar to Namenode status web page, a Cluster Web Console is added in 
-  federation to monitor the federated cluster at 
+  Similar to the Namenode status web page, when using federation a
+  Cluster Web Console is available to monitor the federated cluster at
   <<<http://<any_nn_host:port>/dfsclusterhealth.jsp>>>.
   Any Namenode in the cluster can be used to access this web page.
 
-  The web page provides the following information:
+  The Cluster Web Console provides the following information:
 
-  * Cluster summary that shows number of files, number of blocks and 
-    total configured storage capacity, available and used storage information 
+  * A cluster summary that shows the number of files, number of blocks,
+    total configured storage capacity, and the available and used storage
     for the entire cluster.
 
-  * Provides list of Namenodes and summary that includes number of files,
-    blocks, missing blocks, number of live and dead data nodes for each 
-    Namenode. It also provides a link to conveniently access Namenode web UI.
-
-  * It also provides decommissioning status of datanodes.
-
+  * A list of Namenodes and a summary that includes the number of files,
+    blocks, missing blocks, and live and dead data nodes for each
+    Namenode. It also provides a link to access each Namenode's web UI.
 
+  * The decommissioning status of Datanodes.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 281221dab03f7..941a8eef7a873 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -18,7 +18,7 @@
 
 HDFS Commands Guide
 
-%{toc|section=1|fromDepth=2|toDepth=4}
+%{toc|section=1|fromDepth=2|toDepth=3}
 
 * Overview
 
@@ -26,39 +26,37 @@ HDFS Commands Guide
    hdfs script without any arguments prints the description for all
    commands.
 
-   Usage: <<<hdfs [--config confdir] [--loglevel loglevel] [COMMAND]
-          [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
-
-   Hadoop has an option parsing framework that employs parsing generic options
-   as well as running classes.
-
-*-----------------------+---------------+
-|| COMMAND_OPTION       || Description
-*-----------------------+---------------+
-| <<<--config confdir>>>| Overwrites the default Configuration directory.
-|                       | Default is <<<${HADOOP_HOME}/conf>>>.
-*-----------------------+---------------+
-| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
-|                       | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
-|                       | Default is INFO.
-*-----------------------+---------------+
-| GENERIC_OPTIONS       | The common set of options supported by multiple
-|                       | commands. Full list is
-|                       | {{{../hadoop-common/CommandsManual.html#Generic_Options}here}}.
-*-----------------------+---------------+
-| COMMAND_OPTIONS       | Various commands with their options are described in
-|                       | the following sections. The commands have been
-|                       | grouped into {{{User Commands}}} and
-|                       | {{{Administration Commands}}}.
-*-----------------------+---------------+
-
-* User Commands
+ Usage: <<<hdfs [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
+
+  Hadoop has an option parsing framework that employs parsing generic options as
+  well as running classes.
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description                   |
+*-------------------------+-------------+
+| SHELL_OPTIONS | The common set of shell options. These are documented on the {{{../../hadoop-project-dist/hadoop-common/CommandsManual.html#Shell Options}Commands Manual}} page.
+*-------------------------+----+
+| GENERIC_OPTIONS | The common set of options supported by multiple commands. See the Hadoop {{{../../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic Options}Commands Manual}} for more information.
+*------------------+---------------+
+| COMMAND COMMAND_OPTIONS | Various commands with their options are described
+|                         | in the following sections. The commands have been
+|                         | grouped into {{User Commands}} and
+|                         | {{Administration Commands}}.
+*-------------------------+--------------+
+
+* {User Commands}
 
    Commands useful for users of a hadoop cluster.
 
+** <<<classpath>>>
+
+  Usage: <<<hdfs classpath>>>
+
+  Prints the class path needed to get the Hadoop jar and the required libraries
+
 ** <<<dfs>>>
 
-   Usage: <<<hdfs dfs [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
+   Usage: <<<hdfs dfs [COMMAND [COMMAND_OPTIONS]]>>>
 
    Run a filesystem command on the file system supported in Hadoop.
    The various COMMAND_OPTIONS can be found at
@@ -66,97 +64,307 @@ HDFS Commands Guide
 
 ** <<<fetchdt>>>
 
-   Gets Delegation Token from a NameNode.
-   See {{{./HdfsUserGuide.html#fetchdt}fetchdt}} for more info.
-
-   Usage: <<<hdfs fetchdt [GENERIC_OPTIONS]
-          [--webservice <namenode_http_addr>] <path> >>>
+   Usage: <<<hdfs fetchdt [--webservice <namenode_http_addr>] <path> >>>
 
 *------------------------------+---------------------------------------------+
 || COMMAND_OPTION              || Description
 *------------------------------+---------------------------------------------+
-| <fileName>                   | File name to store the token into.
-*------------------------------+---------------------------------------------+
 | --webservice <https_address> | use http protocol instead of RPC
 *------------------------------+---------------------------------------------+
+| <fileName>                   | File name to store the token into.
+*------------------------------+---------------------------------------------+
+
+
+   Gets Delegation Token from a NameNode.
+   See {{{./HdfsUserGuide.html#fetchdt}fetchdt}} for more info.
 
 ** <<<fsck>>>
 
-   Runs a HDFS filesystem checking utility.
-   See {{{./HdfsUserGuide.html#fsck}fsck}} for more info.
+   Usage:
 
-   Usage: <<<hdfs fsck [GENERIC_OPTIONS] <path>
-          [-list-corruptfileblocks | 
+---
+   hdfs fsck <path>
+          [-list-corruptfileblocks |
           [-move | -delete | -openforwrite]
           [-files [-blocks [-locations | -racks]]]
-          [-includeSnapshots] [-showprogress]>>>
+          [-includeSnapshots] [-showprogress]
+---
 
 *------------------------+---------------------------------------------+
 ||  COMMAND_OPTION       || Description
 *------------------------+---------------------------------------------+
 |   <path>               | Start checking from this path.
 *------------------------+---------------------------------------------+
-| -move                  | Move corrupted files to /lost+found.
-*------------------------+---------------------------------------------+
 | -delete                | Delete corrupted files.
 *------------------------+---------------------------------------------+
 | -files                 | Print out files being checked.
 *------------------------+---------------------------------------------+
-| -openforwrite          | Print out files opened for write.
+| -files -blocks         | Print out the block report
+*------------------------+---------------------------------------------+
+| -files -blocks -locations | Print out locations for every block.
 *------------------------+---------------------------------------------+
-|                        | Include snapshot data if the given path 
-| -includeSnapshots      | indicates a snapshottable directory or 
+| -files -blocks -racks  | Print out network topology for data-node locations.
+*------------------------+---------------------------------------------+
+|                        | Include snapshot data if the given path
+| -includeSnapshots      | indicates a snapshottable directory or
 |                        | there are snapshottable directories under it.
 *------------------------+---------------------------------------------+
-| -list-corruptfileblocks| Print out list of missing blocks and 
+| -list-corruptfileblocks| Print out list of missing blocks and
 |                        | files they belong to.
 *------------------------+---------------------------------------------+
-| -blocks                | Print out block report.
-*------------------------+---------------------------------------------+
-| -locations             | Print out locations for every block.
+| -move                  | Move corrupted files to /lost+found.
 *------------------------+---------------------------------------------+
-| -racks                 | Print out network topology for data-node locations.
+| -openforwrite          | Print out files opened for write.
 *------------------------+---------------------------------------------+
 | -showprogress          | Print out dots for progress in output. Default is OFF
 |                        | (no progress).
 *------------------------+---------------------------------------------+
 
-** <<<version>>>
 
-   Prints the version.
+   Runs the HDFS filesystem checking utility.
+   See {{{./HdfsUserGuide.html#fsck}fsck}} for more info.
+
+** <<<getconf>>>
+
+   Usage:
+
+---
+   hdfs getconf -namenodes
+   hdfs getconf -secondaryNameNodes
+   hdfs getconf -backupNodes
+   hdfs getconf -includeFile
+   hdfs getconf -excludeFile
+   hdfs getconf -nnRpcAddresses
+   hdfs getconf -confKey [key]
+---
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+|	-namenodes			| gets list of namenodes in the cluster.
+*------------------------+---------------------------------------------+
+|	-secondaryNameNodes	| gets list of secondary namenodes in the cluster.
+*------------------------+---------------------------------------------+
+|	-backupNodes		| gets list of backup nodes in the cluster.
+*------------------------+---------------------------------------------+
+|	-includeFile		| gets the include file path that defines the datanodes that can join the cluster.
+*------------------------+---------------------------------------------+
+|	-excludeFile		| gets the exclude file path that defines the datanodes that need to decommissioned.
+*------------------------+---------------------------------------------+
+|	-nnRpcAddresses		| gets the namenode rpc addresses
+*------------------------+---------------------------------------------+
+|	-confKey [key]		| gets a specific key from the configuration
+*------------------------+---------------------------------------------+
+
+ Gets configuration information from the configuration directory, post-processing.
+
+** <<<groups>>>
+
+   Usage: <<<hdfs groups [username ...]>>>
+
+   Returns the group information given one or more usernames.
+
+** <<<lsSnapshottableDir>>>
+
+   Usage: <<<hdfs lsSnapshottableDir [-help]>>>
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+| -help | print help
+*------------------------+---------------------------------------------+
+
+  Get the list of snapshottable directories.  When this is run as a super user,
+  it returns all snapshottable directories.  Otherwise it returns those directories
+  that are owned by the current user.
+
+** <<<jmxget>>>
+
+   Usage: <<<hdfs jmxget [-localVM ConnectorURL | -port port | -server mbeanserver | -service service]>>>
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+| -help | print help
+*------------------------+---------------------------------------------+
+| -localVM ConnectorURL | connect to the VM on the same machine
+*------------------------+---------------------------------------------+
+| -port <mbean server port>    |  specify mbean server port, if missing
+|                             | it will try to connect to MBean Server in
+|                              |  the same VM
+*------------------------+---------------------------------------------+
+| -service  | specify jmx service, either DataNode or NameNode, the default
+*------------------------+---------------------------------------------+
+
+    Dump JMX information from a service.
+
+** <<<oev>>>
+
+  Usage: <<<hdfs oev [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE>>>
+
+*** Required command line arguments:
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+|-i,--inputFile <arg> | edits file to process, xml (case
+                       | insensitive) extension means XML format,
+                       | any other filename means binary format
+*------------------------+---------------------------------------------+
+| -o,--outputFile <arg> | Name of output file. If the specified
+                      | file exists, it will be overwritten,
+                      | format of the file is determined
+                      | by -p option
+*------------------------+---------------------------------------------+
+
+*** Optional command line arguments:
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+| -f,--fix-txids        | Renumber the transaction IDs in the input,
+                      | so that there are no gaps or invalid transaction IDs.
+*------------------------+---------------------------------------------+
+| -h,--help             | Display usage information and exit
+*------------------------+---------------------------------------------+
+| -r,--recover          | When reading binary edit logs, use recovery
+                      | mode.  This will give you the chance to skip
+                      | corrupt parts of the edit log.
+*------------------------+---------------------------------------------+
+| -p,--processor <arg>  | Select which type of processor to apply
+                      | against image file, currently supported
+                      | processors are: binary (native binary format
+                      | that Hadoop uses), xml (default, XML
+                      | format), stats (prints statistics about
+                      | edits file)
+*------------------------+---------------------------------------------+
+| -v,--verbose          | More verbose output, prints the input and
+                      | output filenames, for processors that write
+                      | to a file, also output to screen. On large
+                      | image files this will dramatically increase
+                      | processing time (default is false).
+*------------------------+---------------------------------------------+
+
+  Hadoop offline edits viewer.
+
+** <<<oiv>>>
+
+  Usage: <<<hdfs oiv [OPTIONS] -i INPUT_FILE>>>
+
+*** Required command line arguments:
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+|-i,--inputFile <arg> | edits file to process, xml (case
+                       | insensitive) extension means XML format,
+                       | any other filename means binary format
+*------------------------+---------------------------------------------+
+
+*** Optional command line arguments:
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+| -h,--help             | Display usage information and exit
+*------------------------+---------------------------------------------+
+| -o,--outputFile <arg> | Name of output file. If the specified
+                      | file exists, it will be overwritten,
+                      | format of the file is determined
+                      | by -p option
+*------------------------+---------------------------------------------+
+| -p,--processor <arg>  | Select which type of processor to apply
+                      | against image file, currently supported
+                      | processors are: binary (native binary format
+                      | that Hadoop uses), xml (default, XML
+                      | format), stats (prints statistics about
+                      | edits file)
+*------------------------+---------------------------------------------+
+
+  Hadoop Offline Image Viewer for newer image files.
+
+** <<<oiv_legacy>>>
+
+  Usage: <<<hdfs oiv_legacy [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE>>>
+
+*------------------------+---------------------------------------------+
+||  COMMAND_OPTION       || Description
+*------------------------+---------------------------------------------+
+| -h,--help             | Display usage information and exit
+*------------------------+---------------------------------------------+
+|-i,--inputFile <arg> | edits file to process, xml (case
+                       | insensitive) extension means XML format,
+                       | any other filename means binary format
+*------------------------+---------------------------------------------+
+| -o,--outputFile <arg> | Name of output file. If the specified
+                      | file exists, it will be overwritten,
+                      | format of the file is determined
+                      | by -p option
+*------------------------+---------------------------------------------+
+
+  Hadoop offline image viewer for older versions of Hadoop.
+
+
+** <<<snapshotDiff>>>
+
+    Usage:  <<<hdfs snapshotDiff <path> <fromSnapshot> <toSnapshot> >>>
+
+    Determine the difference between HDFS snapshots. See the
+    {{{./HdfsSnapshots.html#Get_Snapshots_Difference_Report}HDFS Snapshot Documentation}} for more information.
+
+** <<<version>>>
 
    Usage: <<<hdfs version>>>
 
+   Prints the version.
+
 * Administration Commands
 
    Commands useful for administrators of a hadoop cluster.
 
 ** <<<balancer>>>
 
-   Runs a cluster balancing utility. An administrator can simply press Ctrl-C
-   to stop the rebalancing process. See
-   {{{./HdfsUserGuide.html#Balancer}Balancer}} for more details.
-
    Usage: <<<hdfs balancer [-threshold <threshold>] [-policy <policy>]>>>
 
 *------------------------+----------------------------------------------------+
 || COMMAND_OPTION        | Description
 *------------------------+----------------------------------------------------+
-| -threshold <threshold> | Percentage of disk capacity. This overwrites the
-|                        | default threshold.
-*------------------------+----------------------------------------------------+
 | -policy <policy>       | <<<datanode>>> (default): Cluster is balanced if
 |                        | each datanode is balanced. \
 |                        | <<<blockpool>>>: Cluster is balanced if each block
 |                        | pool in each datanode is balanced.
 *------------------------+----------------------------------------------------+
+| -threshold <threshold> | Percentage of disk capacity. This overwrites the
+|                        | default threshold.
+*------------------------+----------------------------------------------------+
+
+   Runs a cluster balancing utility. An administrator can simply press Ctrl-C
+   to stop the rebalancing process. See
+   {{{./HdfsUserGuide.html#Balancer}Balancer}} for more details.
 
    Note that the <<<blockpool>>> policy is more strict than the <<<datanode>>>
    policy.
 
-** <<<datanode>>>
+** <<<cacheadmin>>>
 
-   Runs a HDFS datanode.
+    Usage: <<<hdfs cacheadmin -addDirective -path <path> -pool <pool-name> [-force] [-replication <replication>] [-ttl <time-to-live>]>>>
+
+    See the {{{./CentralizedCacheManagement.html#cacheadmin_command-line_interface}HDFS Cache Administration Documentation}} for more information.
+
+** <<<crypto>>>
+
+    Usage:
+
+---
+  hdfs crypto -createZone -keyName <keyName> -path <path>
+  hdfs crypto -help <command-name>
+  hdfs crypto -listZones
+---
+
+  See the {{{./TransparentEncryption.html#crypto_command-line_interface}HDFS Transparent Encryption Documentation}} for more information.
+
+
+** <<<datanode>>>
 
    Usage: <<<hdfs datanode [-regular | -rollback | -rollingupgrace rollback]>>>
 
@@ -172,12 +380,14 @@ HDFS Commands Guide
 | -rollingupgrade rollback | Rollback a rolling upgrade operation.
 *-----------------+-----------------------------------------------------------+
 
+   Runs a HDFS datanode.
+
 ** <<<dfsadmin>>>
 
-   Runs a HDFS dfsadmin client.
+  Usage:
 
-+------------------------------------------+
-   Usage: hdfs dfsadmin [GENERIC_OPTIONS]
+------------------------------------------
+    hdfs dfsadmin [GENERIC_OPTIONS]
           [-report [-live] [-dead] [-decommissioning]]
           [-safemode enter | leave | get | wait]
           [-saveNamespace]
@@ -210,7 +420,7 @@ HDFS Commands Guide
           [-getDatanodeInfo <datanode_host:ipc_port>]
           [-triggerBlockReport [-incremental] <datanode_host:ipc_port>]
           [-help [cmd]]
-+------------------------------------------+
+------------------------------------------
 
 *-----------------+-----------------------------------------------------------+
 || COMMAND_OPTION || Description
@@ -323,11 +533,11 @@ HDFS Commands Guide
 *-----------------+-----------------------------------------------------------+
 | -allowSnapshot \<snapshotDir\> | Allowing snapshots of a directory to be
                   | created. If the operation completes successfully, the
-                  | directory becomes snapshottable.
+                  | directory becomes snapshottable. See the {{{./HdfsSnapshots.html}HDFS Snapshot Documentation}} for more information.
 *-----------------+-----------------------------------------------------------+
 | -disallowSnapshot \<snapshotDir\> | Disallowing snapshots of a directory to
                   | be created. All snapshots of the directory must be deleted
-                  | before disallowing snapshots.
+                  | before disallowing snapshots. See the {{{./HdfsSnapshots.html}HDFS Snapshot Documentation}} for more information.
 *-----------------+-----------------------------------------------------------+
 | -fetchImage \<local directory\> | Downloads the most recent fsimage from the
                   | NameNode and saves it in the specified local directory.
@@ -351,30 +561,68 @@ HDFS Commands Guide
                   | is specified.
 *-----------------+-----------------------------------------------------------+
 
-** <<<mover>>>
+   Runs a HDFS dfsadmin client.
 
-   Runs the data migration utility.
-   See {{{./ArchivalStorage.html#Mover_-_A_New_Data_Migration_Tool}Mover}} for more details.
+** <<<haadmin>>>
 
-   Usage: <<<hdfs mover [-p <files/dirs> | -f <local file name>]>>>
+    Usage:
+    
+---
+    hdfs haadmin -checkHealth <serviceId>
+    hdfs haadmin -failover [--forcefence] [--forceactive] <serviceId> <serviceId>
+    hdfs haadmin -getServiceState <serviceId>
+    hdfs haadmin -help <command>
+    hdfs haadmin -transitionToActive <serviceId> [--forceactive]
+    hdfs haadmin -transitionToStandby <serviceId>
+---
 
 *--------------------+--------------------------------------------------------+
 || COMMAND_OPTION    || Description
 *--------------------+--------------------------------------------------------+
-| -p \<files/dirs\>  | Specify a space separated list of HDFS files/dirs to migrate.
+| -checkHealth | check the health of the given NameNode
+*--------------------+--------------------------------------------------------+
+| -failover | initiate a failover between two NameNodes
+*--------------------+--------------------------------------------------------+
+| -getServiceState | determine whether the given NameNode is Active or Standby
+*--------------------+--------------------------------------------------------+
+| -transitionToActive  | transition the state of the given NameNode to Active (Warning: No fencing is done)
+*--------------------+--------------------------------------------------------+
+| -transitionToStandby | transition the state of the given NameNode to Standby (Warning: No fencing is done)
+*--------------------+--------------------------------------------------------+
+
+  See {{{./HDFSHighAvailabilityWithNFS.html#Administrative_commands}HDFS HA with NFS}} or
+  {{{./HDFSHighAvailabilityWithQJM.html#Administrative_commands}HDFS HA with QJM}} for more
+  information on this command.
+
+** <<<journalnode>>>
+
+   Usage: <<<hdfs journalnode>>>
+
+   This comamnd starts a journalnode for use with {{{./HDFSHighAvailabilityWithQJM.html#Administrative_commands}HDFS HA with QJM}}.
+
+** <<<mover>>>
+
+   Usage: <<<hdfs mover [-p <files/dirs> | -f <local file name>]>>>
+
+*--------------------+--------------------------------------------------------+
+|| COMMAND_OPTION    || Description
 *--------------------+--------------------------------------------------------+
 | -f \<local file\>  | Specify a local file containing a list of HDFS files/dirs to migrate.
+*--------------------+--------------------------------------------------------+
+| -p \<files/dirs\>  | Specify a space separated list of HDFS files/dirs to migrate.
 *--------------------+--------------------------------------------------------+
 
+   Runs the data migration utility.
+   See {{{./ArchivalStorage.html#Mover_-_A_New_Data_Migration_Tool}Mover}} for more details.
+
   Note that, when both -p and -f options are omitted, the default path is the root directory.
 
 ** <<<namenode>>>
 
-   Runs the namenode. More info about the upgrade, rollback and finalize is at
-   {{{./HdfsUserGuide.html#Upgrade_and_Rollback}Upgrade Rollback}}.
+   Usage:
 
-+------------------------------------------+
-   Usage: hdfs namenode [-backup] |
+------------------------------------------
+  hdfs namenode [-backup] |
           [-checkpoint] |
           [-format [-clusterid cid ] [-force] [-nonInteractive] ] |
           [-upgrade [-clusterid cid] [-renameReserved<k-v pairs>] ] |
@@ -387,7 +635,7 @@ HDFS Commands Guide
           [-bootstrapStandby] |
           [-recover [-force] ] |
           [-metadataVersion ]
-+------------------------------------------+
+------------------------------------------
 
 *--------------------+--------------------------------------------------------+
 || COMMAND_OPTION    || Description
@@ -443,11 +691,23 @@ HDFS Commands Guide
                      | metadata versions of the software and the image.
 *--------------------+--------------------------------------------------------+
 
-** <<<secondarynamenode>>>
+   Runs the namenode. More info about the upgrade, rollback and finalize is at
+   {{{./HdfsUserGuide.html#Upgrade_and_Rollback}Upgrade Rollback}}.
 
-   Runs the HDFS secondary namenode.
-   See {{{./HdfsUserGuide.html#Secondary_NameNode}Secondary Namenode}}
-   for more info.
+
+** <<<nfs3>>>
+
+   Usage: <<<hdfs nfs3>>>
+
+   This comamnd starts the NFS3 gateway for use with the {{{./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service}HDFS NFS3 Service}}.
+
+** <<<portmap>>>
+
+   Usage: <<<hdfs portmap>>>
+
+   This comamnd starts the RPC portmap for use with the {{{./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service}HDFS NFS3 Service}}.
+
+** <<<secondarynamenode>>>
 
    Usage: <<<hdfs secondarynamenode [-checkpoint [force]] | [-format] |
           [-geteditsize]>>>
@@ -464,3 +724,71 @@ HDFS Commands Guide
 | -geteditsize         | Prints the number of uncheckpointed transactions on
                        | the NameNode.
 *----------------------+------------------------------------------------------+
+
+   Runs the HDFS secondary namenode.
+   See {{{./HdfsUserGuide.html#Secondary_NameNode}Secondary Namenode}}
+   for more info.
+
+
+** <<<storagepolicies>>>
+
+    Usage: <<<hdfs storagepolicies>>>
+
+    Lists out all storage policies.  See the {{{./ArchivalStorage.html}HDFS Storage Policy Documentation}} for more information.
+
+** <<<zkfc>>>
+
+
+   Usage: <<<hdfs zkfc [-formatZK [-force] [-nonInteractive]]>>>
+
+*----------------------+------------------------------------------------------+
+|| COMMAND_OPTION      || Description
+*----------------------+------------------------------------------------------+
+| -formatZK | Format the Zookeeper instance
+*----------------------+------------------------------------------------------+
+| -h | Display help
+*----------------------+------------------------------------------------------+
+
+   This comamnd starts a Zookeeper Failover Controller process for use with {{{./HDFSHighAvailabilityWithQJM.html#Administrative_commands}HDFS HA with QJM}}.
+
+
+* Debug Commands
+
+   Useful commands to help administrators debug HDFS issues, like validating
+   block files and calling recoverLease.
+
+** <<<verify>>>
+
+   Usage: <<<hdfs dfs verify [-meta <metadata-file>] [-block <block-file>]>>>
+
+*------------------------+----------------------------------------------------+
+|| COMMAND_OPTION        | Description
+*------------------------+----------------------------------------------------+
+| -block <block-file>    | Optional parameter to specify the absolute path for
+|                        | the block file on the local file system of the data
+|                        | node.
+*------------------------+----------------------------------------------------+
+| -meta <metadata-file>  | Absolute path for the metadata file on the local file
+|                        | system of the data node.
+*------------------------+----------------------------------------------------+
+
+   Verify HDFS metadata and block files.  If a block file is specified, we
+   will verify that the checksums in the metadata file match the block
+   file.
+
+** <<<recoverLease>>>
+
+   Usage: <<<hdfs dfs recoverLease [-path <path>] [-retries <num-retries>]>>>
+
+*-------------------------------+--------------------------------------------+
+|| COMMAND_OPTION               || Description
+*-------------------------------+---------------------------------------------+
+| [-path <path>]                | HDFS path for which to recover the lease.
+*-------------------------------+---------------------------------------------+
+| [-retries <num-retries>]      | Number of times the client will retry calling
+|                               | recoverLease. The default number of retries
+|                               | is 1.
+*-------------------------------+---------------------------------------------+
+
+   Recover the lease on the specified path.  The path must reside on an
+   HDFS filesystem.  The default number of retries is 1.
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm
index b404e23ea06be..14776b5b6f3d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm
@@ -25,7 +25,7 @@ HDFS High Availability
   This guide provides an overview of the HDFS High Availability (HA) feature and
   how to configure and manage an HA HDFS cluster, using NFS for the shared
   storage required by the NameNodes.
- 
+
   This document assumes that the reader has a general understanding of
   general components and node types in an HDFS cluster. Please refer to the
   HDFS Architecture guide for details.
@@ -44,7 +44,7 @@ HDFS High Availability
   an HDFS cluster. Each cluster had a single NameNode, and if that machine or
   process became unavailable, the cluster as a whole would be unavailable
   until the NameNode was either restarted or brought up on a separate machine.
-  
+
   This impacted the total availability of the HDFS cluster in two major ways:
 
     * In the case of an unplanned event such as a machine crash, the cluster would
@@ -52,7 +52,7 @@ HDFS High Availability
 
     * Planned maintenance events such as software or hardware upgrades on the
       NameNode machine would result in windows of cluster downtime.
-  
+
   The HDFS High Availability feature addresses the above problems by providing
   the option of running two redundant NameNodes in the same cluster in an
   Active/Passive configuration with a hot standby. This allows a fast failover to
@@ -67,7 +67,7 @@ HDFS High Availability
   for all client operations in the cluster, while the Standby is simply acting
   as a slave, maintaining enough state to provide a fast failover if
   necessary.
-  
+
   In order for the Standby node to keep its state synchronized with the Active
   node, the current implementation requires that the two nodes both have access
   to a directory on a shared storage device (eg an NFS mount from a NAS). This
@@ -80,12 +80,12 @@ HDFS High Availability
   a failover, the Standby will ensure that it has read all of the edits from the
   shared storage before promoting itself to the Active state. This ensures that
   the namespace state is fully synchronized before a failover occurs.
-  
+
   In order to provide a fast failover, it is also necessary that the Standby node
   have up-to-date information regarding the location of blocks in the cluster.
   In order to achieve this, the DataNodes are configured with the location of
   both NameNodes, and send block location information and heartbeats to both.
-  
+
   It is vital for the correct operation of an HA cluster that only one of the
   NameNodes be Active at a time. Otherwise, the namespace state would quickly
   diverge between the two, risking data loss or other incorrect results.  In
@@ -116,7 +116,7 @@ HDFS High Availability
     network, and power). Beacuse of this, it is recommended that the shared storage
     server be a high-quality dedicated NAS appliance rather than a simple Linux
     server.
-  
+
   Note that, in an HA cluster, the Standby NameNode also performs checkpoints of
   the namespace state, and thus it is not necessary to run a Secondary NameNode,
   CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an
@@ -133,7 +133,7 @@ HDFS High Availability
   The new configuration is designed such that all the nodes in the cluster may
   have the same configuration without the need for deploying different
   configuration files to different machines based on the type of the node.
- 
+
   Like HDFS Federation, HA clusters reuse the <<<nameservice ID>>> to identify a
   single HDFS instance that may in fact consist of multiple HA NameNodes. In
   addition, a new abstraction called <<<NameNode ID>>> is added with HA. Each
@@ -330,7 +330,7 @@ HDFS High Availability
       <<dfs_namenode_rpc-address>> will contain the RPC address of the target node, even
       though the configuration may specify that variable as
       <<dfs.namenode.rpc-address.ns1.nn1>>.
-      
+
       Additionally, the following variables referring to the target node to be fenced
       are also available:
 
@@ -345,7 +345,7 @@ HDFS High Availability
 *-----------------------:-----------------------------------+
 | $target_namenodeid    | the namenode ID of the NN to be fenced |
 *-----------------------:-----------------------------------+
-      
+
       These environment variables may also be used as substitutions in the shell
       command itself. For example:
 
@@ -355,7 +355,7 @@ HDFS High Availability
   <value>shell(/path/to/my/script.sh --nameservice=$target_nameserviceid $target_host:$target_port)</value>
 </property>
 ---
-      
+
       If the shell command returns an exit
       code of 0, the fencing is determined to be successful. If it returns any other
       exit code, the fencing was not successful and the next fencing method in the
@@ -386,7 +386,7 @@ HDFS High Availability
 
     * If you are setting up a fresh HDFS cluster, you should first run the format
     command (<hdfs namenode -format>) on one of NameNodes.
-  
+
     * If you have already formatted the NameNode, or are converting a
     non-HA-enabled cluster to be HA-enabled, you should now copy over the
     contents of your NameNode metadata directories to the other, unformatted
@@ -394,7 +394,7 @@ HDFS High Availability
     unformatted NameNode. Running this command will also ensure that the shared
     edits directory (as configured by <<dfs.namenode.shared.edits.dir>>) contains
     sufficient edits transactions to be able to start both NameNodes.
-  
+
     * If you are converting a non-HA NameNode to be HA, you should run the
     command "<hdfs -initializeSharedEdits>", which will initialize the shared
     edits directory with the edits data from the local NameNode edits directories.
@@ -484,7 +484,7 @@ Usage: DFSHAAdmin [-ns <nameserviceId>]
   of coordination data, notifying clients of changes in that data, and
   monitoring clients for failures. The implementation of automatic HDFS failover
   relies on ZooKeeper for the following things:
-  
+
     * <<Failure detection>> - each of the NameNode machines in the cluster
     maintains a persistent session in ZooKeeper. If the machine crashes, the
     ZooKeeper session will expire, notifying the other NameNode that a failover
@@ -585,7 +585,7 @@ Usage: DFSHAAdmin [-ns <nameserviceId>]
   from one of the NameNode hosts.
 
 ----
-$ hdfs zkfc -formatZK
+[hdfs]$ $HADOOP_PREFIX/bin/zkfc -formatZK
 ----
 
   This will create a znode in ZooKeeper inside of which the automatic failover
@@ -605,7 +605,7 @@ $ hdfs zkfc -formatZK
   can start the daemon by running:
 
 ----
-$ hadoop-daemon.sh start zkfc
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc
 ----
 
 ** Securing access to ZooKeeper
@@ -646,7 +646,7 @@ digest:hdfs-zkfcs:mypassword
   a command like the following:
 
 ----
-$ java -cp $ZK_HOME/lib/*:$ZK_HOME/zookeeper-3.4.2.jar org.apache.zookeeper.server.auth.DigestAuthenticationProvider hdfs-zkfcs:mypassword
+[hdfs]$ java -cp $ZK_HOME/lib/*:$ZK_HOME/zookeeper-3.4.2.jar org.apache.zookeeper.server.auth.DigestAuthenticationProvider hdfs-zkfcs:mypassword
 output: hdfs-zkfcs:mypassword->hdfs-zkfcs:P/OQvnYyU/nF/mGYvB/xurX8dYs=
 ----
 
@@ -726,24 +726,24 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
   using the same <<<hdfs haadmin>>> command. It will perform a coordinated
   failover.
 
- 
+
 * BookKeeper as a Shared storage (EXPERIMENTAL)
 
-   One option for shared storage for the NameNode is BookKeeper. 
+   One option for shared storage for the NameNode is BookKeeper.
   BookKeeper achieves high availability and strong durability guarantees by replicating
-  edit log entries across multiple storage nodes. The edit log can be striped across 
-  the storage nodes for high performance. Fencing is supported in the protocol, i.e, 
+  edit log entries across multiple storage nodes. The edit log can be striped across
+  the storage nodes for high performance. Fencing is supported in the protocol, i.e,
   BookKeeper will not allow two writers to write the single edit log.
 
   The meta data for BookKeeper is stored in ZooKeeper.
   In current HA architecture, a Zookeeper cluster is required for ZKFC. The same cluster can be
   for BookKeeper metadata.
 
-  For more details on building a BookKeeper cluster, please refer to the 
+  For more details on building a BookKeeper cluster, please refer to the
    {{{http://zookeeper.apache.org/bookkeeper/docs/trunk/bookkeeperConfig.html }BookKeeper documentation}}
 
  The BookKeeperJournalManager is an implementation of the HDFS JournalManager interface, which allows custom write ahead logging implementations to be plugged into the HDFS NameNode.
- 
+
  **<<BookKeeper Journal Manager>>
 
    To use BookKeeperJournalManager, add the following to hdfs-site.xml.
@@ -772,12 +772,12 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
    classpath. We explain how to generate a jar file with the journal manager and
    its dependencies, and how to put it into the classpath below.
 
- *** <<More configuration options>> 
+ *** <<More configuration options>>
 
-     * <<dfs.namenode.bookkeeperjournal.output-buffer-size>> - 
+     * <<dfs.namenode.bookkeeperjournal.output-buffer-size>> -
        Number of bytes a bookkeeper journal stream will buffer before
        forcing a flush. Default is 1024.
-     
+
 ----
        <property>
          <name>dfs.namenode.bookkeeperjournal.output-buffer-size</name>
@@ -785,7 +785,7 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
        </property>
 ----
 
-     * <<dfs.namenode.bookkeeperjournal.ensemble-size>> - 
+     * <<dfs.namenode.bookkeeperjournal.ensemble-size>> -
        Number of bookkeeper servers in edit log ensembles. This
        is the number of bookkeeper servers which need to be available
        for the edit log to be writable. Default is 3.
@@ -797,7 +797,7 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
        </property>
 ----
 
-     * <<dfs.namenode.bookkeeperjournal.quorum-size>> - 
+     * <<dfs.namenode.bookkeeperjournal.quorum-size>> -
        Number of bookkeeper servers in the write quorum. This is the
        number of bookkeeper servers which must have acknowledged the
        write of an entry before it is considered written. Default is 2.
@@ -809,7 +809,7 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
        </property>
 ----
 
-     * <<dfs.namenode.bookkeeperjournal.digestPw>> - 
+     * <<dfs.namenode.bookkeeperjournal.digestPw>> -
        Password to use when creating edit log segments.
 
 ----
@@ -819,9 +819,9 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
        </property>
 ----
 
-     * <<dfs.namenode.bookkeeperjournal.zk.session.timeout>> - 
+     * <<dfs.namenode.bookkeeperjournal.zk.session.timeout>> -
        Session timeout for Zookeeper client from BookKeeper Journal Manager.
-       Hadoop recommends that this value should be less than the ZKFC 
+       Hadoop recommends that this value should be less than the ZKFC
        session timeout value. Default value is 3000.
 
 ----
@@ -838,22 +838,22 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
 
      $ mvn clean package -Pdist
 
-     This will generate a jar with the BookKeeperJournalManager, 
+     This will generate a jar with the BookKeeperJournalManager,
      hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar
 
      Note that the -Pdist part of the build command is important, this would
-     copy the dependent bookkeeper-server jar under 
+     copy the dependent bookkeeper-server jar under
      hadoop-hdfs/src/contrib/bkjournal/target/lib.
 
  *** <<Putting the BookKeeperJournalManager in the NameNode classpath>>
 
     To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal and
-    bookkeeper-server jar, mentioned above, into the lib directory of hdfs. In the 
+    bookkeeper-server jar, mentioned above, into the lib directory of hdfs. In the
     standard distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
 
     cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
 
- *** <<Current limitations>> 
+ *** <<Current limitations>>
 
       1) Security in BookKeeper. BookKeeper does not support SASL nor SSL for
          connections between the NameNode and BookKeeper storage nodes.
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
index ff6a42c259234..a3a9f12b1cfb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
@@ -25,7 +25,7 @@ HDFS High Availability Using the Quorum Journal Manager
   This guide provides an overview of the HDFS High Availability (HA) feature
   and how to configure and manage an HA HDFS cluster, using the Quorum Journal
   Manager (QJM) feature.
- 
+
   This document assumes that the reader has a general understanding of
   general components and node types in an HDFS cluster. Please refer to the
   HDFS Architecture guide for details.
@@ -44,7 +44,7 @@ HDFS High Availability Using the Quorum Journal Manager
   an HDFS cluster. Each cluster had a single NameNode, and if that machine or
   process became unavailable, the cluster as a whole would be unavailable
   until the NameNode was either restarted or brought up on a separate machine.
-  
+
   This impacted the total availability of the HDFS cluster in two major ways:
 
     * In the case of an unplanned event such as a machine crash, the cluster would
@@ -52,7 +52,7 @@ HDFS High Availability Using the Quorum Journal Manager
 
     * Planned maintenance events such as software or hardware upgrades on the
       NameNode machine would result in windows of cluster downtime.
-  
+
   The HDFS High Availability feature addresses the above problems by providing
   the option of running two redundant NameNodes in the same cluster in an
   Active/Passive configuration with a hot standby. This allows a fast failover to
@@ -67,7 +67,7 @@ HDFS High Availability Using the Quorum Journal Manager
   for all client operations in the cluster, while the Standby is simply acting
   as a slave, maintaining enough state to provide a fast failover if
   necessary.
-  
+
   In order for the Standby node to keep its state synchronized with the Active
   node, both nodes communicate with a group of separate daemons called
   "JournalNodes" (JNs). When any namespace modification is performed by the
@@ -78,12 +78,12 @@ HDFS High Availability Using the Quorum Journal Manager
   failover, the Standby will ensure that it has read all of the edits from the
   JounalNodes before promoting itself to the Active state. This ensures that the
   namespace state is fully synchronized before a failover occurs.
-  
+
   In order to provide a fast failover, it is also necessary that the Standby node
   have up-to-date information regarding the location of blocks in the cluster.
   In order to achieve this, the DataNodes are configured with the location of
   both NameNodes, and send block location information and heartbeats to both.
-  
+
   It is vital for the correct operation of an HA cluster that only one of the
   NameNodes be Active at a time. Otherwise, the namespace state would quickly
   diverge between the two, risking data loss or other incorrect results.  In
@@ -113,7 +113,7 @@ HDFS High Availability Using the Quorum Journal Manager
     you should run an odd number of JNs, (i.e. 3, 5, 7, etc.). Note that when
     running with N JournalNodes, the system can tolerate at most (N - 1) / 2
     failures and continue to function normally.
-  
+
   Note that, in an HA cluster, the Standby NameNode also performs checkpoints of
   the namespace state, and thus it is not necessary to run a Secondary NameNode,
   CheckpointNode, or BackupNode in an HA cluster. In fact, to do so would be an
@@ -130,7 +130,7 @@ HDFS High Availability Using the Quorum Journal Manager
   The new configuration is designed such that all the nodes in the cluster may
   have the same configuration without the need for deploying different
   configuration files to different machines based on the type of the node.
- 
+
   Like HDFS Federation, HA clusters reuse the <<<nameservice ID>>> to identify a
   single HDFS instance that may in fact consist of multiple HA NameNodes. In
   addition, a new abstraction called <<<NameNode ID>>> is added with HA. Each
@@ -347,7 +347,7 @@ HDFS High Availability Using the Quorum Journal Manager
       <<dfs_namenode_rpc-address>> will contain the RPC address of the target node, even
       though the configuration may specify that variable as
       <<dfs.namenode.rpc-address.ns1.nn1>>.
-      
+
       Additionally, the following variables referring to the target node to be fenced
       are also available:
 
@@ -362,7 +362,7 @@ HDFS High Availability Using the Quorum Journal Manager
 *-----------------------:-----------------------------------+
 | $target_namenodeid    | the namenode ID of the NN to be fenced |
 *-----------------------:-----------------------------------+
-      
+
       These environment variables may also be used as substitutions in the shell
       command itself. For example:
 
@@ -372,7 +372,7 @@ HDFS High Availability Using the Quorum Journal Manager
   <value>shell(/path/to/my/script.sh --nameservice=$target_nameserviceid $target_host:$target_port)</value>
 </property>
 ---
-      
+
       If the shell command returns an exit
       code of 0, the fencing is determined to be successful. If it returns any other
       exit code, the fencing was not successful and the next fencing method in the
@@ -424,7 +424,7 @@ HDFS High Availability Using the Quorum Journal Manager
 
     * If you are setting up a fresh HDFS cluster, you should first run the format
     command (<hdfs namenode -format>) on one of NameNodes.
-  
+
     * If you have already formatted the NameNode, or are converting a
     non-HA-enabled cluster to be HA-enabled, you should now copy over the
     contents of your NameNode metadata directories to the other, unformatted
@@ -432,7 +432,7 @@ HDFS High Availability Using the Quorum Journal Manager
     unformatted NameNode. Running this command will also ensure that the
     JournalNodes (as configured by <<dfs.namenode.shared.edits.dir>>) contain
     sufficient edits transactions to be able to start both NameNodes.
-  
+
     * If you are converting a non-HA NameNode to be HA, you should run the
     command "<hdfs -initializeSharedEdits>", which will initialize the
     JournalNodes with the edits data from the local NameNode edits directories.
@@ -522,7 +522,7 @@ Usage: DFSHAAdmin [-ns <nameserviceId>]
   of coordination data, notifying clients of changes in that data, and
   monitoring clients for failures. The implementation of automatic HDFS failover
   relies on ZooKeeper for the following things:
-  
+
     * <<Failure detection>> - each of the NameNode machines in the cluster
     maintains a persistent session in ZooKeeper. If the machine crashes, the
     ZooKeeper session will expire, notifying the other NameNode that a failover
@@ -623,7 +623,7 @@ Usage: DFSHAAdmin [-ns <nameserviceId>]
   from one of the NameNode hosts.
 
 ----
-$ hdfs zkfc -formatZK
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs zkfc -formatZK
 ----
 
   This will create a znode in ZooKeeper inside of which the automatic failover
@@ -643,7 +643,7 @@ $ hdfs zkfc -formatZK
   can start the daemon by running:
 
 ----
-$ hadoop-daemon.sh start zkfc
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start zkfc
 ----
 
 ** Securing access to ZooKeeper
@@ -684,7 +684,7 @@ digest:hdfs-zkfcs:mypassword
   a command like the following:
 
 ----
-$ java -cp $ZK_HOME/lib/*:$ZK_HOME/zookeeper-3.4.2.jar org.apache.zookeeper.server.auth.DigestAuthenticationProvider hdfs-zkfcs:mypassword
+[hdfs]$ java -cp $ZK_HOME/lib/*:$ZK_HOME/zookeeper-3.4.2.jar org.apache.zookeeper.server.auth.DigestAuthenticationProvider hdfs-zkfcs:mypassword
 output: hdfs-zkfcs:mypassword->hdfs-zkfcs:P/OQvnYyU/nF/mGYvB/xurX8dYs=
 ----
 
@@ -786,18 +786,18 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6ONESbM=:rwcda
     operations, the operation will fail.
 
     [[3]] Start one of the NNs with the <<<'-upgrade'>>> flag.
-  
+
     [[4]] On start, this NN will not enter the standby state as usual in an HA
     setup. Rather, this NN will immediately enter the active state, perform an
     upgrade of its local storage dirs, and also perform an upgrade of the shared
     edit log.
-  
+
     [[5]] At this point the other NN in the HA pair will be out of sync with
     the upgraded NN. In order to bring it back in sync and once again have a highly
     available setup, you should re-bootstrap this NameNode by running the NN with
     the <<<'-bootstrapStandby'>>> flag. It is an error to start this second NN with
     the <<<'-upgrade'>>> flag.
-  
+
   Note that if at any time you want to restart the NameNodes before finalizing
   or rolling back the upgrade, you should start the NNs as normal, i.e. without
   any special startup flag.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
index 152a9850ff74e..8555b2316b961 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
@@ -36,15 +36,15 @@ HDFS NFS Gateway
      HDFS file system.
 
    * Users can stream data directly to HDFS through the mount point. File
-     append is supported but random write is not supported. 
+     append is supported but random write is not supported.
 
   The NFS gateway machine needs the same thing to run an HDFS client like Hadoop JAR files, HADOOP_CONF directory.
-  The NFS gateway can be on the same host as DataNode, NameNode, or any HDFS client. 
+  The NFS gateway can be on the same host as DataNode, NameNode, or any HDFS client.
 
 
 * {Configuration}
 
-   The NFS-gateway uses proxy user to proxy all the users accessing the NFS mounts. 
+   The NFS-gateway uses proxy user to proxy all the users accessing the NFS mounts.
    In non-secure mode, the user running the gateway is the proxy user, while in secure mode the
    user in Kerberos keytab is the proxy user. Suppose the proxy user is 'nfsserver'
    and users belonging to the groups 'users-group1'
@@ -57,10 +57,10 @@ HDFS NFS Gateway
   <name>hadoop.proxyuser.nfsserver.groups</name>
   <value>root,users-group1,users-group2</value>
   <description>
-         The 'nfsserver' user is allowed to proxy all members of the 'users-group1' and 
+         The 'nfsserver' user is allowed to proxy all members of the 'users-group1' and
          'users-group2' groups. Note that in most cases you will need to include the
          group "root" because the user "root" (which usually belonges to "root" group) will
-         generally be the user that initially executes the mount on the NFS client system. 
+         generally be the user that initially executes the mount on the NFS client system.
          Set this to '*' to allow nfsserver user to proxy any group.
   </description>
 </property>
@@ -78,7 +78,7 @@ HDFS NFS Gateway
 ----
 
    The above are the only required configuration for the NFS gateway in non-secure mode. For Kerberized
-   hadoop clusters, the following configurations need to be added to hdfs-site.xml for the gateway (NOTE: replace 
+   hadoop clusters, the following configurations need to be added to hdfs-site.xml for the gateway (NOTE: replace
    string "nfsserver" with the proxy user name and ensure the user contained in the keytab is
    also the same proxy user):
 
@@ -95,7 +95,7 @@ HDFS NFS Gateway
     <value>nfsserver/_HOST@YOUR-REALM.COM</value>
   </property>
 ----
-  
+
    The rest of the NFS gateway configurations are optional for both secure and non-secure mode.
 
    The AIX NFS client has a {{{https://issues.apache.org/jira/browse/HDFS-6549}few known issues}}
@@ -119,11 +119,11 @@ HDFS NFS Gateway
 
    It's strongly recommended for the users to update a few configuration properties based on their use
    cases. All the following configuration properties can be added or updated in hdfs-site.xml.
-  
-   * If the client mounts the export with access time update allowed, make sure the following 
-    property is not disabled in the configuration file. Only NameNode needs to restart after 
+
+   * If the client mounts the export with access time update allowed, make sure the following
+    property is not disabled in the configuration file. Only NameNode needs to restart after
     this property is changed. On some Unix systems, the user can disable access time update
-    by mounting the export with "noatime". If the export is mounted with "noatime", the user 
+    by mounting the export with "noatime". If the export is mounted with "noatime", the user
     doesn't need to change the following property and thus no need to restart namenode.
 
 ----
@@ -149,11 +149,11 @@ HDFS NFS Gateway
       this property is updated.
 
 ----
-  <property>    
+  <property>
     <name>nfs.dump.dir</name>
     <value>/tmp/.hdfs-nfs</value>
   </property>
----- 
+----
 
   * By default, the export can be mounted by any client. To better control the access,
     users can update the following property. The value string contains machine name and
@@ -161,7 +161,7 @@ HDFS NFS Gateway
     characters. The machine name format can be a single host, a Java regular expression, or an IPv4 address. The
     access privilege uses rw or ro to specify read/write or read-only access of the machines to exports. If the access
     privilege is not provided, the default is read-only. Entries are separated by ";".
-    For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;". Only the NFS gateway needs to restart after 
+    For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;". Only the NFS gateway needs to restart after
     this property is updated.
 
 ----
@@ -171,22 +171,22 @@ HDFS NFS Gateway
 </property>
 ----
 
-  * JVM and log settings. You can export JVM settings (e.g., heap size and GC log) in 
-   HADOOP_NFS3_OPTS. More NFS related settings can be found in hadoop-env.sh. 
-   To get NFS debug trace, you can edit the log4j.property file 
+  * JVM and log settings. You can export JVM settings (e.g., heap size and GC log) in
+   HADOOP_NFS3_OPTS. More NFS related settings can be found in hadoop-env.sh.
+   To get NFS debug trace, you can edit the log4j.property file
    to add the following. Note, debug trace, especially for ONCRPC, can be very verbose.
 
     To change logging level:
 
------------------------------------------------ 
+-----------------------------------------------
     log4j.logger.org.apache.hadoop.hdfs.nfs=DEBUG
------------------------------------------------ 
+-----------------------------------------------
 
     To get more details of ONCRPC requests:
 
------------------------------------------------ 
+-----------------------------------------------
     log4j.logger.org.apache.hadoop.oncrpc=DEBUG
------------------------------------------------ 
+-----------------------------------------------
 
 
 * {Start and stop NFS gateway service}
@@ -195,53 +195,39 @@ HDFS NFS Gateway
   The NFS gateway process has both nfsd and mountd. It shares the HDFS root "/" as the
   only export. It is recommended to use the portmap included in NFS gateway package. Even
   though NFS gateway works with portmap/rpcbind provide by most Linux distributions, the
-  package included portmap is needed on some Linux systems such as REHL6.2 due to an 
+  package included portmap is needed on some Linux systems such as REHL6.2 due to an
   {{{https://bugzilla.redhat.com/show_bug.cgi?id=731542}rpcbind bug}}. More detailed discussions can
   be found in {{{https://issues.apache.org/jira/browse/HDFS-4763}HDFS-4763}}.
 
-   [[1]] Stop nfs/rpcbind/portmap services provided by the platform (commands can be different on various Unix platforms):
-      
+   [[1]] Stop nfsv3 and rpcbind/portmap services provided by the platform (commands can be different on various Unix platforms):
+
 -------------------------
-     service nfs stop
-      
-     service rpcbind stop
+[root]> service nfs stop
+[root]> service rpcbind stop
 -------------------------
 
-
-   [[2]] Start package included portmap (needs root privileges):
+   [[2]] Start Hadoop's portmap (needs root privileges):
 
 -------------------------
-     hdfs portmap
-  
-     OR
-
-     hadoop-daemon.sh start portmap
+[root]> $HADOOP_PREFIX/bin/hdfs --daemon start portmap
 -------------------------
 
    [[3]] Start mountd and nfsd.
-   
+
      No root privileges are required for this command. In non-secure mode, the NFS gateway
-     should be started by the proxy user mentioned at the beginning of this user guide. 
-     While in secure mode, any user can start NFS gateway 
+     should be started by the proxy user mentioned at the beginning of this user guide.
+     While in secure mode, any user can start NFS gateway
      as long as the user has read access to the Kerberos keytab defined in "nfs.keytab.file".
 
 -------------------------
-     hdfs nfs3
-
-     OR
-
-     hadoop-daemon.sh start nfs3
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start nfs3
 -------------------------
 
-     Note, if the hadoop-daemon.sh script starts the NFS gateway, its log can be found in the hadoop log folder.
-
-
    [[4]] Stop NFS gateway services.
 
 -------------------------
-      hadoop-daemon.sh stop nfs3
-
-      hadoop-daemon.sh stop portmap
+[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop nfs3
+[root]> $HADOOP_PREFIX/bin/hdfs --daemon stop portmap
 -------------------------
 
   Optionally, you can forgo running the Hadoop-provided portmap daemon and
@@ -263,7 +249,7 @@ HDFS NFS Gateway
     [[1]] Execute the following command to verify if all the services are up and running:
 
 -------------------------
-       rpcinfo -p $nfs_server_ip
+[root]> rpcinfo -p $nfs_server_ip
 -------------------------
 
      You should see output similar to the following:
@@ -293,11 +279,11 @@ HDFS NFS Gateway
     [[2]]  Verify if the HDFS namespace is exported and can be mounted.
 
 -------------------------
-        showmount -e $nfs_server_ip                         
+[root]> showmount -e $nfs_server_ip
 -------------------------
 
       You should see output similar to the following:
-     
+
 -------------------------
         Exports list on $nfs_server_ip :
 
@@ -307,22 +293,22 @@ HDFS NFS Gateway
 
 * {Mount the export “/”}
 
-  Currently NFS v3 only uses TCP as the transportation protocol. 
+  Currently NFS v3 only uses TCP as the transportation protocol.
   NLM is not supported so mount option "nolock" is needed. It's recommended to use
-  hard mount. This is because, even after the client sends all data to 
-  NFS gateway, it may take NFS gateway some extra time to transfer data to HDFS 
+  hard mount. This is because, even after the client sends all data to
+  NFS gateway, it may take NFS gateway some extra time to transfer data to HDFS
   when writes were reorderd by NFS client Kernel.
- 
-  If soft mount has to be used, the user should give it a relatively 
+
+  If soft mount has to be used, the user should give it a relatively
   long timeout (at least no less than the default timeout on the host) .
 
   The users can mount the HDFS namespace as shown below:
 
--------------------------------------------------------------------  
-       mount -t nfs -o vers=3,proto=tcp,nolock,noacl $server:/  $mount_point
+-------------------------------------------------------------------
+ [root]>mount -t nfs -o vers=3,proto=tcp,nolock,noacl $server:/  $mount_point
 -------------------------------------------------------------------
 
-  Then the users can access HDFS as part of the local file system except that, 
+  Then the users can access HDFS as part of the local file system except that,
   hard link and random write are not supported yet. To optimize the performance
   of large file I/O, one can increase the NFS transfer size(rsize and wsize) during mount.
   By default, NFS gateway supports 1MB as the maximum transfer size. For larger data
@@ -347,7 +333,7 @@ HDFS NFS Gateway
 * {User authentication and mapping}
 
   NFS gateway in this release uses AUTH_UNIX style authentication. When the user on NFS client
-  accesses the mount point, NFS client passes the UID to NFS gateway. 
+  accesses the mount point, NFS client passes the UID to NFS gateway.
   NFS gateway does a lookup to find user name from the UID, and then passes the
   username to the HDFS along with the HDFS requests.
   For example, if the NFS client has current user as "admin", when the user accesses
@@ -358,7 +344,7 @@ HDFS NFS Gateway
   The system administrator must ensure that the user on NFS client host has the same
   name and UID as that on the NFS gateway host. This is usually not a problem if
   the same user management system (e.g., LDAP/NIS) is used to create and deploy users on
-  HDFS nodes and NFS client node. In case the user account is created manually on different hosts, one might need to 
+  HDFS nodes and NFS client node. In case the user account is created manually on different hosts, one might need to
   modify UID (e.g., do "usermod -u 123 myusername") on either NFS client or NFS gateway host
   in order to make it the same on both sides. More technical details of RPC AUTH_UNIX can be found
   in {{{http://tools.ietf.org/html/rfc1057}RPC specification}}.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
index 54cd2ed0a4bac..662f8b81236a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
@@ -919,6 +919,7 @@ Transfer-Encoding: chunked
         ], 
         "group": "supergroup", 
         "owner": "hadoop", 
+        "permission":"775",
         "stickyBit": false
     }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
index 5040a3b6983ad..296c8d2dfbd64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
@@ -113,7 +113,8 @@ private static byte[] byteBufferToArray(ByteBuffer buf) {
     return resultArray;
   }
   
-  private static final int BLOCK_SIZE = 4096;
+  private static final int BLOCK_SIZE = 
+      (int) NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize();
   
   public static HdfsConfiguration initZeroCopyTest() {
     Assume.assumeTrue(NativeIO.isAvailable());
@@ -140,7 +141,7 @@ public void testZeroCopyReads() throws Exception {
     MiniDFSCluster cluster = null;
     final Path TEST_PATH = new Path("/a");
     FSDataInputStream fsIn = null;
-    final int TEST_FILE_LENGTH = 12345;
+    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
     
     FileSystem fs = null;
     try {
@@ -163,15 +164,15 @@ public void testZeroCopyReads() throws Exception {
       IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
       fsIn.close();
       fsIn = fs.open(TEST_PATH);
-      ByteBuffer result = fsIn.read(null, 4096,
+      ByteBuffer result = fsIn.read(null, BLOCK_SIZE,
           EnumSet.of(ReadOption.SKIP_CHECKSUMS));
-      Assert.assertEquals(4096, result.remaining());
+      Assert.assertEquals(BLOCK_SIZE, result.remaining());
       HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn;
-      Assert.assertEquals(4096,
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalBytesRead());
-      Assert.assertEquals(4096,
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
-      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, 4096),
+      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE),
           byteBufferToArray(result));
       fsIn.releaseBuffer(result);
     } finally {
@@ -187,7 +188,7 @@ public void testShortZeroCopyReads() throws Exception {
     MiniDFSCluster cluster = null;
     final Path TEST_PATH = new Path("/a");
     FSDataInputStream fsIn = null;
-    final int TEST_FILE_LENGTH = 12345;
+    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
     
     FileSystem fs = null;
     try {
@@ -210,24 +211,24 @@ public void testShortZeroCopyReads() throws Exception {
       fsIn.close();
       fsIn = fs.open(TEST_PATH);
 
-      // Try to read 8192, but only get 4096 because of the block size.
+      // Try to read (2 * ${BLOCK_SIZE}), but only get ${BLOCK_SIZE} because of the block size.
       HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn;
       ByteBuffer result =
-        dfsIn.read(null, 8192, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
-      Assert.assertEquals(4096, result.remaining());
-      Assert.assertEquals(4096,
+        dfsIn.read(null, 2 * BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
+      Assert.assertEquals(BLOCK_SIZE, result.remaining());
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalBytesRead());
-      Assert.assertEquals(4096,
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
-      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, 4096),
+      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE),
           byteBufferToArray(result));
       dfsIn.releaseBuffer(result);
       
-      // Try to read 4097, but only get 4096 because of the block size.
+      // Try to read (1 + ${BLOCK_SIZE}), but only get ${BLOCK_SIZE} because of the block size.
       result = 
-          dfsIn.read(null, 4097, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
-      Assert.assertEquals(4096, result.remaining());
-      Assert.assertArrayEquals(Arrays.copyOfRange(original, 4096, 8192),
+          dfsIn.read(null, 1 + BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
+      Assert.assertEquals(BLOCK_SIZE, result.remaining());
+      Assert.assertArrayEquals(Arrays.copyOfRange(original, BLOCK_SIZE, 2 * BLOCK_SIZE),
           byteBufferToArray(result));
       dfsIn.releaseBuffer(result);
     } finally {
@@ -243,7 +244,7 @@ public void testZeroCopyReadsNoFallback() throws Exception {
     MiniDFSCluster cluster = null;
     final Path TEST_PATH = new Path("/a");
     FSDataInputStream fsIn = null;
-    final int TEST_FILE_LENGTH = 12345;
+    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
     
     FileSystem fs = null;
     try {
@@ -269,18 +270,18 @@ public void testZeroCopyReadsNoFallback() throws Exception {
       HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn;
       ByteBuffer result;
       try {
-        result = dfsIn.read(null, 4097, EnumSet.noneOf(ReadOption.class));
+        result = dfsIn.read(null, BLOCK_SIZE + 1, EnumSet.noneOf(ReadOption.class));
         Assert.fail("expected UnsupportedOperationException");
       } catch (UnsupportedOperationException e) {
         // expected
       }
-      result = dfsIn.read(null, 4096, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
-      Assert.assertEquals(4096, result.remaining());
-      Assert.assertEquals(4096,
+      result = dfsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
+      Assert.assertEquals(BLOCK_SIZE, result.remaining());
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalBytesRead());
-      Assert.assertEquals(4096,
+      Assert.assertEquals(BLOCK_SIZE,
           dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
-      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, 4096),
+      Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE),
           byteBufferToArray(result));
     } finally {
       if (fsIn != null) fsIn.close();
@@ -330,7 +331,7 @@ public void testZeroCopyMmapCache() throws Exception {
     HdfsConfiguration conf = initZeroCopyTest();
     MiniDFSCluster cluster = null;
     final Path TEST_PATH = new Path("/a");
-    final int TEST_FILE_LENGTH = 16385;
+    final int TEST_FILE_LENGTH = 5 * BLOCK_SIZE;
     final int RANDOM_SEED = 23453;
     final String CONTEXT = "testZeroCopyMmapCacheContext";
     FSDataInputStream fsIn = null;
@@ -360,10 +361,10 @@ public void testZeroCopyMmapCache() throws Exception {
     final ShortCircuitCache cache = ClientContext.get(
         CONTEXT, new DFSClient.Conf(conf)). getShortCircuitCache();
     cache.accept(new CountingVisitor(0, 5, 5, 0));
-    results[0] = fsIn.read(null, 4096,
+    results[0] = fsIn.read(null, BLOCK_SIZE,
         EnumSet.of(ReadOption.SKIP_CHECKSUMS));
     fsIn.seek(0);
-    results[1] = fsIn.read(null, 4096,
+    results[1] = fsIn.read(null, BLOCK_SIZE,
         EnumSet.of(ReadOption.SKIP_CHECKSUMS));
 
     // The mmap should be of the first block of the file.
@@ -386,9 +387,9 @@ public void visit(int numOutstandingMmaps,
     });
 
     // Read more blocks.
-    results[2] = fsIn.read(null, 4096,
+    results[2] = fsIn.read(null, BLOCK_SIZE,
         EnumSet.of(ReadOption.SKIP_CHECKSUMS));
-    results[3] = fsIn.read(null, 4096,
+    results[3] = fsIn.read(null, BLOCK_SIZE,
         EnumSet.of(ReadOption.SKIP_CHECKSUMS));
 
     // we should have 3 mmaps, 1 evictable
@@ -592,7 +593,7 @@ public void testZeroCopyReadOfCachedData() throws Exception {
     BlockReaderTestUtil.enableBlockReaderFactoryTracing();
     BlockReaderTestUtil.enableHdfsCachingTracing();
 
-    final int TEST_FILE_LENGTH = 16385;
+    final int TEST_FILE_LENGTH = BLOCK_SIZE;
     final Path TEST_PATH = new Path("/a");
     final int RANDOM_SEED = 23453;
     HdfsConfiguration conf = initZeroCopyTest();
@@ -601,7 +602,8 @@ public void testZeroCopyReadOfCachedData() throws Exception {
     final String CONTEXT = "testZeroCopyReadOfCachedData";
     conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT);
     conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
-        DFSTestUtil.roundUpToMultiple(TEST_FILE_LENGTH, 4096));
+        DFSTestUtil.roundUpToMultiple(TEST_FILE_LENGTH,
+          (int) NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize()));
     MiniDFSCluster cluster = null;
     ByteBuffer result = null, result2 = null;
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
index 3ce2893b8b343..0c3abec90c13b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
@@ -23,7 +23,6 @@
 import java.io.IOException;
 import java.net.URI;
 
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -48,7 +47,7 @@
 abstract public class TestSymlinkHdfs extends SymlinkBaseTest {
 
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
   }
 
   protected static MiniDFSCluster cluster;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java
index f589d7e27a4b1..76c32bfbaa9c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java
@@ -43,8 +43,7 @@
  * by the Text command.
  */
 public class TestHdfsTextCommand {
-  private static final String TEST_ROOT_DIR =
-    System.getProperty("test.build.data", "build/test/data/") + "/testText";
+  private static final String TEST_ROOT_DIR = "/test/data/testText";
   private static final Path AVRO_FILENAME = new Path(TEST_ROOT_DIR, "weather.avro");
   private static MiniDFSCluster cluster;
   private static FileSystem fs;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AppendTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AppendTestUtil.java
index eab44be13d2d4..de4da5f9cfe09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AppendTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AppendTestUtil.java
@@ -18,9 +18,11 @@
 package org.apache.hadoop.hdfs;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Arrays;
 import java.util.Random;
 
 import org.apache.commons.logging.Log;
@@ -80,6 +82,27 @@ public static byte[] randomBytes(long seed, int size) {
     return b;
   }
 
+  /** @return a random file partition of length n. */
+  public static int[] randomFilePartition(int n, int parts) {
+    int[] p = new int[parts];
+    for(int i = 0; i < p.length; i++) {
+      p[i] = nextInt(n - i - 1) + 1;
+    }
+    Arrays.sort(p);
+    for(int i = 1; i < p.length; i++) {
+      if (p[i] <= p[i - 1]) {
+        p[i] = p[i - 1] + 1;
+      }
+    }
+    
+    LOG.info("partition=" + Arrays.toString(p));
+    assertTrue("i=0", p[0] > 0 && p[0] < n);
+    for(int i = 1; i < p.length; i++) {
+      assertTrue("i=" + i, p[i] > p[i - 1] && p[i] < n);
+    }
+    return p;
+  }
+
   static void sleep(long ms) {
     try {
       Thread.sleep(ms);
@@ -136,6 +159,22 @@ public static void check(FileSystem fs, Path p, long length) throws IOException
     }
   }
 
+  public static void check(DistributedFileSystem fs, Path p, int position,
+      int length) throws IOException {
+    byte[] buf = new byte[length];
+    int i = 0;
+    try {
+      FSDataInputStream in = fs.open(p);
+      in.read(position, buf, 0, buf.length);
+      for(i = position; i < length + position; i++) {
+        assertEquals((byte) i, buf[i - position]);
+      }
+      in.close();
+    } catch(IOException ioe) {
+      throw new IOException("p=" + p + ", length=" + length + ", i=" + i, ioe);
+    }
+  }
+
   /**
    *  create a buffer that contains the entire test file data.
    */
@@ -157,6 +196,11 @@ public static FSDataOutputStream createFile(FileSystem fileSys, Path name, int r
         (short) repl, BLOCK_SIZE);
   }
 
+  public static void checkFullFile(FileSystem fs, Path file, int len,
+      final byte[] compareContent) throws IOException {
+    checkFullFile(fs, file, len, compareContent, file.toString());
+  }
+
   /**
    *  Compare the content of a file created from FileSystem and Path with
    *  the specified byte[] buffer's content
@@ -164,6 +208,18 @@ public static FSDataOutputStream createFile(FileSystem fileSys, Path name, int r
    */
   public static void checkFullFile(FileSystem fs, Path name, int len,
                             final byte[] compareContent, String message) throws IOException {
+    checkFullFile(fs, name, len, compareContent, message, true);
+  }
+
+  public static void checkFullFile(FileSystem fs, Path name, int len,
+      final byte[] compareContent, String message,
+      boolean checkFileStatus) throws IOException {
+    if (checkFileStatus) {
+      final FileStatus status = fs.getFileStatus(name);
+      assertEquals("len=" + len + " but status.getLen()=" + status.getLen(),
+          len, status.getLen());
+    }
+
     FSDataInputStream stm = fs.open(name);
     byte[] actual = new byte[len];
     stm.readFully(0, actual);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index a51c42cd4f643..e8fc662107f40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -23,8 +23,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.base.Supplier;
 import com.google.common.collect.Lists;
-
 import com.google.common.collect.Maps;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -47,8 +47,8 @@
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -63,6 +63,7 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.ha
         .ConfiguredFailoverProxyProvider;
@@ -81,6 +82,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.log4j.Level;
 import org.junit.Assume;
 import org.mockito.internal.util.reflection.Whitebox;
 
@@ -687,11 +689,6 @@ public static Token<BlockTokenIdentifier> getBlockToken(
     return ((DFSOutputStream) out.getWrappedStream()).getBlockToken();
   }
 
-  static void setLogLevel2All(org.apache.commons.logging.Log log) {
-    ((org.apache.commons.logging.impl.Log4JLogger)log
-        ).getLogger().setLevel(org.apache.log4j.Level.ALL);
-  }
-
   public static String readFile(File f) throws IOException {
     StringBuilder b = new StringBuilder();
     BufferedReader in = new BufferedReader(new FileReader(f));
@@ -822,14 +819,17 @@ public List<String> getGroups(String user) throws IOException {
    * Get a FileSystem instance as specified user in a doAs block.
    */
   static public FileSystem getFileSystemAs(UserGroupInformation ugi, 
-                                   final Configuration conf) throws IOException, 
-                                                        InterruptedException {
-    return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
-      @Override
-      public FileSystem run() throws Exception {
-        return FileSystem.get(conf);
-      }
-    });
+      final Configuration conf) throws IOException {
+    try {
+      return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
+        @Override
+        public FileSystem run() throws Exception {
+          return FileSystem.get(conf);
+        }
+      });
+    } catch (InterruptedException e) {
+      throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+    }
   }
 
   public static byte[] generateSequentialBytes(int start, int length) {
@@ -1137,6 +1137,9 @@ public static void runOperations(MiniDFSCluster cluster,
     FSDataOutputStream s = filesystem.create(pathFileCreate);
     // OP_CLOSE 9
     s.close();
+    // OP_APPEND 47
+    FSDataOutputStream s2 = filesystem.append(pathFileCreate, 4096, null);
+    s2.close();
     // OP_SET_STORAGE_POLICY 45
     filesystem.setStoragePolicy(pathFileCreate,
         HdfsConstants.HOT_STORAGE_POLICY_NAME);
@@ -1199,7 +1202,13 @@ public static void runOperations(MiniDFSCluster cluster,
     DFSTestUtil.createFile(filesystem, pathConcatFiles[1], length, replication,
         seed);
     filesystem.concat(pathConcatTarget, pathConcatFiles);
-    
+
+    // OP_TRUNCATE 46
+    length = blockSize * 2;
+    DFSTestUtil.createFile(filesystem, pathFileCreate, length, replication,
+        seed);
+    filesystem.truncate(pathFileCreate, blockSize);
+
     // OP_SYMLINK 17
     Path pathSymlink = new Path("/file_symlink");
     fc.createSymlink(pathConcatTarget, pathSymlink, false);
@@ -1512,12 +1521,12 @@ public static void createKey(String keyName, MiniDFSCluster cluster,
   public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn,
       ExtendedBlock blk) {
     BlockManager bm0 = nn.getNamesystem().getBlockManager();
-    BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
+    BlockInfoContiguous storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
     assertTrue("Block " + blk + " should be under construction, " +
         "got: " + storedBlock,
-        storedBlock instanceof BlockInfoUnderConstruction);
-    BlockInfoUnderConstruction ucBlock =
-      (BlockInfoUnderConstruction)storedBlock;
+        storedBlock instanceof BlockInfoContiguousUnderConstruction);
+    BlockInfoContiguousUnderConstruction ucBlock =
+      (BlockInfoContiguousUnderConstruction)storedBlock;
     // We expect that the replica with the most recent heart beat will be
     // the one to be in charge of the synchronization / recovery protocol.
     final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations();
@@ -1636,4 +1645,29 @@ public Boolean get() {
       }
     }, 100, waitTime);
   }
+
+ /**
+   * Change the length of a block at datanode dnIndex
+   */
+  public static boolean changeReplicaLength(MiniDFSCluster cluster,
+      ExtendedBlock blk, int dnIndex, int lenDelta) throws IOException {
+    File blockFile = cluster.getBlockFile(dnIndex, blk);
+    if (blockFile != null && blockFile.exists()) {
+      RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
+      raFile.setLength(raFile.length()+lenDelta);
+      raFile.close();
+      return true;
+    }
+    LOG.info("failed to change length of block " + blk);
+    return false;
+  }
+
+  public static void setNameNodeLogLevel(Level level) {
+    GenericTestUtils.setLogLevel(FSNamesystem.LOG, level);
+    GenericTestUtils.setLogLevel(BlockManager.LOG, level);
+    GenericTestUtils.setLogLevel(LeaseManager.LOG, level);
+    GenericTestUtils.setLogLevel(NameNode.LOG, level);
+    GenericTestUtils.setLogLevel(NameNode.stateChangeLog, level);
+    GenericTestUtils.setLogLevel(NameNode.blockStateChangeLog, level);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index e3c6fc5f32a0d..33bd4e5b50031 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1191,7 +1191,9 @@ public void waitClusterUp() throws IOException {
         } catch (InterruptedException e) {
         }
         if (++i > 10) {
-          throw new IOException("Timed out waiting for Mini HDFS Cluster to start");
+          final String msg = "Timed out waiting for Mini HDFS Cluster to start";
+          LOG.error(msg);
+          throw new IOException(msg);
         }
       }
     }
@@ -1876,7 +1878,7 @@ public String readBlockOnDataNode(int i, ExtendedBlock block)
    * @return true if a replica was corrupted, false otherwise
    * Types: delete, write bad data, truncate
    */
-  public static boolean corruptReplica(int i, ExtendedBlock blk)
+  public boolean corruptReplica(int i, ExtendedBlock blk)
       throws IOException {
     File blockFile = getBlockFile(i, blk);
     return corruptBlock(blockFile);
@@ -1913,7 +1915,7 @@ public static boolean corruptBlockByDeletingBlockFile(File blockFile)
     return blockFile.delete();
   }
 
-  public static boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
+  public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
       long newGenStamp) throws IOException {
     File blockFile = getBlockFile(dnIndex, blk);
     File metaFile = FsDatasetUtil.findMetaFile(blockFile);
@@ -1923,6 +1925,9 @@ public static boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
 
   /*
    * Shutdown a particular datanode
+   * @param i node index
+   * @return null if the node index is out of range, else the properties of the
+   * removed node
    */
   public synchronized DataNodeProperties stopDataNode(int i) {
     if (i < 0 || i >= dataNodes.size()) {
@@ -1941,18 +1946,20 @@ public synchronized DataNodeProperties stopDataNode(int i) {
 
   /*
    * Shutdown a datanode by name.
+   * @return the removed datanode or null if there was no match
    */
   public synchronized DataNodeProperties stopDataNode(String dnName) {
-    int i;
-    for (i = 0; i < dataNodes.size(); i++) {
+    int node = -1;
+    for (int i = 0; i < dataNodes.size(); i++) {
       DataNode dn = dataNodes.get(i).datanode;
       LOG.info("DN name=" + dnName + " found DN=" + dn +
           " with name=" + dn.getDisplayName());
       if (dnName.equals(dn.getDatanodeId().getXferAddr())) {
+        node = i;
         break;
       }
     }
-    return stopDataNode(i);
+    return stopDataNode(node);
   }
 
   /**
@@ -2429,7 +2436,7 @@ public File getInstanceStorageDir(int dnIndex, int dirIndex) {
    * @param dirIndex directory index.
    * @return Storage directory
    */
-  public static File getStorageDir(int dnIndex, int dirIndex) {
+  public File getStorageDir(int dnIndex, int dirIndex) {
     return new File(getBaseDirectory(), getStorageDirPath(dnIndex, dirIndex));
   }
 
@@ -2440,8 +2447,8 @@ public static File getStorageDir(int dnIndex, int dirIndex) {
    * @param dirIndex directory index.
    * @return storage directory path
    */
-  private static String getStorageDirPath(int dnIndex, int dirIndex) {
-    return "data/data" + (2 * dnIndex + 1 + dirIndex);
+  private String getStorageDirPath(int dnIndex, int dirIndex) {
+    return "data/data" + (storagesPerDatanode * dnIndex + 1 + dirIndex);
   }
 
   /**
@@ -2570,10 +2577,10 @@ public File[] getAllBlockFiles(ExtendedBlock block) {
    * @param dnIndex Index of the datanode to get block files for
    * @param block block for which corresponding files are needed
    */
-  public static File getBlockFile(int dnIndex, ExtendedBlock block) {
+  public File getBlockFile(int dnIndex, ExtendedBlock block) {
     // Check for block file in the two storage directories of the datanode
     for (int i = 0; i <=1 ; i++) {
-      File storageDir = MiniDFSCluster.getStorageDir(dnIndex, i);
+      File storageDir = getStorageDir(dnIndex, i);
       File blockFile = getBlockFile(storageDir, block);
       if (blockFile.exists()) {
         return blockFile;
@@ -2588,10 +2595,10 @@ public static File getBlockFile(int dnIndex, ExtendedBlock block) {
    * @param dnIndex Index of the datanode to get block files for
    * @param block block for which corresponding files are needed
    */
-  public static File getBlockMetadataFile(int dnIndex, ExtendedBlock block) {
+  public File getBlockMetadataFile(int dnIndex, ExtendedBlock block) {
     // Check for block file in the two storage directories of the datanode
     for (int i = 0; i <=1 ; i++) {
-      File storageDir = MiniDFSCluster.getStorageDir(dnIndex, i);
+      File storageDir = getStorageDir(dnIndex, i);
       File blockMetaFile = getBlockMetadataFile(storageDir, block);
       if (blockMetaFile.exists()) {
         return blockMetaFile;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java
index 9b4fef6ae9407..29c32f53896f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java
@@ -160,8 +160,8 @@ public void runBlockReaderLocalTest(BlockReaderLocalTest test,
       fsIn.close();
       fsIn = null;
       ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
-      File dataFile = MiniDFSCluster.getBlockFile(0, block);
-      File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block);
+      File dataFile = cluster.getBlockFile(0, block);
+      File metaFile = cluster.getBlockMetadataFile(0, block);
 
       ShortCircuitCache shortCircuitCache =
           ClientContext.getFromConf(conf).getShortCircuitCache();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 3d417e65ee023..b8810470e4195 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -67,6 +67,7 @@ public class TestBlockStoragePolicy {
   static final short REPLICATION = 3;
 
   static final byte COLD = HdfsConstants.COLD_STORAGE_POLICY_ID;
+  static final byte EC = HdfsConstants.EC_STORAGE_POLICY_ID;
   static final byte WARM = HdfsConstants.WARM_STORAGE_POLICY_ID;
   static final byte HOT  = HdfsConstants.HOT_STORAGE_POLICY_ID;
   static final byte ONESSD  = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
@@ -114,6 +115,9 @@ public void testDefaultPolicies() {
     expectedPolicyStrings.put(COLD,
         "BlockStoragePolicy{COLD:" + COLD + ", storageTypes=[ARCHIVE], " +
             "creationFallbacks=[], replicationFallbacks=[]}");
+    expectedPolicyStrings.put(EC,
+        "BlockStoragePolicy{EC:" + EC + ", storageTypes=[DISK], " +
+            "creationFallbacks=[], replicationFallbacks=[ARCHIVE]}");
     expectedPolicyStrings.put(WARM,
         "BlockStoragePolicy{WARM:" + WARM + ", storageTypes=[DISK, ARCHIVE], " +
             "creationFallbacks=[DISK, ARCHIVE], " +
@@ -1156,13 +1160,15 @@ public void testGetAllStoragePolicies() throws Exception {
     final DistributedFileSystem fs = cluster.getFileSystem();
     try {
       BlockStoragePolicy[] policies = fs.getStoragePolicies();
-      Assert.assertEquals(6, policies.length);
+      Assert.assertEquals(7, policies.length);
       Assert.assertEquals(POLICY_SUITE.getPolicy(COLD).toString(),
           policies[0].toString());
-      Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
+      Assert.assertEquals(POLICY_SUITE.getPolicy(EC).toString(),
           policies[1].toString());
-      Assert.assertEquals(POLICY_SUITE.getPolicy(HOT).toString(),
+      Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
           policies[2].toString());
+      Assert.assertEquals(POLICY_SUITE.getPolicy(HOT).toString(),
+          policies[3].toString());
     } finally {
       IOUtils.cleanup(null, fs);
       cluster.shutdown();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 82db11074f468..4f449d1bdda0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -21,6 +21,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.XAttrSetFlag;
@@ -71,7 +72,7 @@ private static long checkTxid(EventBatch batch, long prevTxid){
    */
   @Test
   public void testOpcodeCount() {
-    Assert.assertEquals(47, FSEditLogOpCodes.values().length);
+    Assert.assertEquals(49, FSEditLogOpCodes.values().length);
   }
 
 
@@ -109,7 +110,8 @@ public void testBasic() throws IOException, URISyntaxException,
       os.write(new byte[BLOCK_SIZE]);
       os.close(); // CloseOp -> CloseEvent
       // AddOp -> AppendEvent
-      os = client.append("/file2", BLOCK_SIZE, null, null);
+      os = client.append("/file2", BLOCK_SIZE, EnumSet.of(CreateFlag.APPEND),
+          null, null);
       os.write(new byte[BLOCK_SIZE]);
       os.close(); // CloseOp -> CloseEvent
       Thread.sleep(10); // so that the atime will get updated on the next line
@@ -170,6 +172,7 @@ public void testBasic() throws IOException, URISyntaxException,
       Assert.assertTrue(ce.getReplication() > 0);
       Assert.assertTrue(ce.getSymlinkTarget() == null);
       Assert.assertTrue(ce.getOverwrite());
+      Assert.assertEquals(BLOCK_SIZE, ce.getDefaultBlockSize());
 
       // CloseOp
       batch = waitForNextEvents(eis);
@@ -181,12 +184,14 @@ public void testBasic() throws IOException, URISyntaxException,
       Assert.assertTrue(ce2.getFileSize() > 0);
       Assert.assertTrue(ce2.getTimestamp() > 0);
 
-      // AddOp
+      // AppendOp
       batch = waitForNextEvents(eis);
       Assert.assertEquals(1, batch.getEvents().length);
       txid = checkTxid(batch, txid);
       Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND);
-      Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2"));
+      Event.AppendEvent append2 = (Event.AppendEvent)batch.getEvents()[0];
+      Assert.assertEquals("/file2", append2.getPath());
+      Assert.assertFalse(append2.toNewBlock());
 
       // CloseOp
       batch = waitForNextEvents(eis);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 59c0b2c3a0895..ee0407602847b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -52,6 +52,8 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
 import org.junit.Test;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
@@ -1515,7 +1517,7 @@ public Object run() throws Exception {
   
   @Test (timeout = 30000)
   public void testGet() throws IOException {
-    DFSTestUtil.setLogLevel2All(FSInputChecker.LOG);
+    GenericTestUtils.setLogLevel(FSInputChecker.LOG, Level.ALL);
 
     final String fname = "testGet.txt";
     Path root = new Path("/test/get");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
index 88ad0cc2dac7c..ad907f6374870 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
@@ -24,7 +24,6 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
@@ -91,6 +90,10 @@ private static class ReferenceFileInfo {
     }
   }
   
+  public interface ClusterVerifier {
+    public void verifyClusterPostUpgrade(final MiniDFSCluster cluster) throws IOException;
+  }
+
   final LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
   Iterator<ReferenceFileInfo> refIter;
   
@@ -119,7 +122,7 @@ void unpackStorage(String tarFileName, String referenceName)
       if (line.length() <= 0 || line.startsWith("#")) {
         continue;
       }
-      String[] arr = line.split("\\s+\t\\s+");
+      String[] arr = line.split("\\s+");
       if (arr.length < 1) {
         continue;
       }
@@ -288,7 +291,7 @@ public void testFailOnPreUpgradeImage() throws IOException {
   public void testUpgradeFromRel22Image() throws IOException {
     unpackStorage(HADOOP22_IMAGE, HADOOP_DFS_DIR_TXT);
     upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
-        numDataNodes(4));
+        numDataNodes(4), null);
   }
   
   /**
@@ -316,7 +319,7 @@ public void testUpgradeFromCorruptRel22Image() throws IOException {
     // Upgrade should now fail
     try {
       upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
-          numDataNodes(4));
+          numDataNodes(4), null);
       fail("Upgrade did not fail with bad MD5");
     } catch (IOException ioe) {
       String msg = StringUtils.stringifyException(ioe);
@@ -573,7 +576,7 @@ static void recoverAllLeases(DFSClient dfs,
     } while (dirList.hasMore());
   }
   
-  void upgradeAndVerify(MiniDFSCluster.Builder bld)
+  void upgradeAndVerify(MiniDFSCluster.Builder bld, ClusterVerifier verifier)
       throws IOException {
     MiniDFSCluster cluster = null;
     try {
@@ -592,6 +595,10 @@ void upgradeAndVerify(MiniDFSCluster.Builder bld)
       }
       recoverAllLeases(dfsClient, new Path("/"));
       verifyFileSystem(dfs);
+
+      if (verifier != null) {
+        verifier.verifyClusterPostUpgrade(cluster);
+      }
     } finally {
       if (cluster != null) { cluster.shutdown(); }
     } 
@@ -611,6 +618,6 @@ public void testUpgradeFromRel1BBWImage() throws IOException {
         "data1");
     upgradeAndVerify(new MiniDFSCluster.Builder(conf).
           numDataNodes(1).enableManagedDfsDirsRedundancy(false).
-          manageDataDfsDirs(false));
+          manageDataDfsDirs(false), null);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java
index eae8ea7681bd7..1563b72260818 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java
@@ -29,7 +29,6 @@
 import static org.junit.Assert.fail;
 
 import java.io.InputStream;
-import java.io.PrintWriter;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -46,7 +45,6 @@
 import org.junit.Test;
 
 import com.google.common.base.Supplier;
-import com.google.common.io.NullOutputStream;
 
 public class TestDataTransferKeepalive {
   final Configuration conf = new HdfsConfiguration();
@@ -223,7 +221,7 @@ public void testManyClosedSocketsInCache() throws Exception {
         stms[i] = fs.open(TEST_FILE);
       }
       for (InputStream stm : stms) {
-        IOUtils.copyBytes(stm, new NullOutputStream(), 1024);
+        IOUtils.copyBytes(stm, new IOUtils.NullOutputStream(), 1024);
       }
     } finally {
       IOUtils.cleanup(null, stms);
@@ -245,9 +243,7 @@ public void testManyClosedSocketsInCache() throws Exception {
   private void assertXceiverCount(int expected) {
     int count = getXceiverCountWithoutServer();
     if (count != expected) {
-      ReflectionUtils.printThreadInfo(
-          new PrintWriter(System.err),
-          "Thread dumps");
+      ReflectionUtils.printThreadInfo(System.err, "Thread dumps");
       fail("Expected " + expected + " xceivers, found " +
           count);
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index 3586551011d66..55cd8e26dc134 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -160,7 +160,9 @@ private void writeZeroLengthPacket(ExtendedBlock block, String description)
 
     //ok finally write a block with 0 len
     sendResponse(Status.SUCCESS, "", null, recvOut);
-    new PipelineAck(100, new Status[]{Status.SUCCESS}).write(recvOut);
+    new PipelineAck(100, new int[] {PipelineAck.combineHeader
+      (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write
+      (recvOut);
     sendRecvData(description, false);
   }
   
@@ -393,7 +395,8 @@ public void testDataTransferProtocol() throws IOException {
     hdr.write(sendOut);
 
     sendResponse(Status.SUCCESS, "", null, recvOut);
-    new PipelineAck(100, new Status[]{Status.ERROR}).write(recvOut);
+    new PipelineAck(100, new int[] {PipelineAck.combineHeader
+      (PipelineAck.ECN.DISABLED, Status.ERROR)}).write(recvOut);
     sendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, 
                  true);
 
@@ -414,7 +417,8 @@ public void testDataTransferProtocol() throws IOException {
     sendOut.flush();
     //ok finally write a block with 0 len
     sendResponse(Status.SUCCESS, "", null, recvOut);
-    new PipelineAck(100, new Status[]{Status.SUCCESS}).write(recvOut);
+    new PipelineAck(100, new int[] {PipelineAck.combineHeader
+      (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write(recvOut);
     sendRecvData("Writing a zero len block blockid " + newBlockId, false);
     
     /* Test OP_READ_BLOCK */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java
deleted file mode 100644
index 1b4b3172394ce..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java
+++ /dev/null
@@ -1,496 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.util.Random;
-import java.util.concurrent.TimeoutException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/**
- * This test verifies that block verification occurs on the datanode
- */
-public class TestDatanodeBlockScanner {
-  
-  private static final Log LOG = 
-                 LogFactory.getLog(TestDatanodeBlockScanner.class);
-  
-  private static final long TIMEOUT = 20000; // 20 sec.
-  
-  private static final Pattern pattern =
-             Pattern.compile(".*?(blk_[-]*\\d+).*?scan time\\s*:\\s*(\\d+)");
-  
-  private static final Pattern pattern_blockVerify =
-             Pattern.compile(".*?(SCAN_PERIOD)\\s*:\\s*(\\d+.*?)");
-  
-  static {
-    ((Log4JLogger)FSNamesystem.auditLog).getLogger().setLevel(Level.WARN);
-  }
-  /**
-   * This connects to datanode and fetches block verification data.
-   * It repeats this until the given block has a verification time > newTime.
-   * @param newTime - validation timestamps before newTime are "old", the
-   *            result of previous validations.  This method waits until a "new"
-   *            validation timestamp is obtained.  If no validator runs soon
-   *            enough, the method will time out.
-   * @return - the new validation timestamp
-   * @throws IOException
-   * @throws TimeoutException
-   */
-  private static long waitForVerification(int infoPort, FileSystem fs, 
-                          Path file, int blocksValidated, 
-                          long newTime, long timeout) 
-  throws IOException, TimeoutException {
-    URL url = new URL("http://localhost:" + infoPort +
-                      "/blockScannerReport?listblocks");
-    long lastWarnTime = Time.monotonicNow();
-    if (newTime <= 0) newTime = 1L;
-    long verificationTime = 0;
-    
-    String block = DFSTestUtil.getFirstBlock(fs, file).getBlockName();
-    long failtime = (timeout <= 0) ? Long.MAX_VALUE 
-        : Time.monotonicNow() + timeout;
-    while (verificationTime < newTime) {
-      if (failtime < Time.monotonicNow()) {
-        throw new TimeoutException("failed to achieve block verification after "
-            + timeout + " msec.  Current verification timestamp = "
-            + verificationTime + ", requested verification time > " 
-            + newTime);
-      }
-      String response = DFSTestUtil.urlGet(url);
-      if(blocksValidated >= 0) {
-        for(Matcher matcher = pattern_blockVerify.matcher(response); matcher.find();) {
-          if (block.equals(matcher.group(1))) {
-            assertEquals(1, blocksValidated);
-            break;
-          }
-        }
-      }
-      for(Matcher matcher = pattern.matcher(response); matcher.find();) {
-        if (block.equals(matcher.group(1))) {
-          verificationTime = Long.parseLong(matcher.group(2));
-          break;
-        }
-      }
-      
-      if (verificationTime < newTime) {
-        long now = Time.monotonicNow();
-        if ((now - lastWarnTime) >= 5*1000) {
-          LOG.info("Waiting for verification of " + block);
-          lastWarnTime = now; 
-        }
-        try {
-          Thread.sleep(500);
-        } catch (InterruptedException ignored) {}
-      }
-    }
-    
-    return verificationTime;
-  }
-
-  @Test
-  public void testDatanodeBlockScanner() throws IOException, TimeoutException {
-    long startTime = Time.monotonicNow();
-    
-    Configuration conf = new HdfsConfiguration();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
-    cluster.waitActive();
-    
-    FileSystem fs = cluster.getFileSystem();
-    Path file1 = new Path("/tmp/testBlockVerification/file1");
-    Path file2 = new Path("/tmp/testBlockVerification/file2");
-    
-    /*
-     * Write the first file and restart the cluster.
-     */
-    DFSTestUtil.createFile(fs, file1, 10, (short)1, 0);
-    cluster.shutdown();
-
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(1)
-                                .format(false).build();
-    cluster.waitActive();
-    
-    DFSClient dfsClient =  new DFSClient(new InetSocketAddress("localhost", 
-                                         cluster.getNameNodePort()), conf);
-    fs = cluster.getFileSystem();
-    DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
-    
-    /*
-     * The cluster restarted. The block should be verified by now.
-     */
-    assertTrue(waitForVerification(dn.getInfoPort(), fs, file1, 1, startTime,
-        TIMEOUT) >= startTime);
-    
-    /*
-     * Create a new file and read the block. The block should be marked 
-     * verified since the client reads the block and verifies checksum. 
-     */
-    DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
-    IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
-                      conf, true); 
-    assertTrue(waitForVerification(dn.getInfoPort(), fs, file2, 2, startTime,
-        TIMEOUT) >= startTime);
-    
-    cluster.shutdown();
-  }
-
-  public static boolean corruptReplica(ExtendedBlock blk, int replica) throws IOException {
-    return MiniDFSCluster.corruptReplica(replica, blk);
-  }
-
-  @Test
-  public void testBlockCorruptionPolicy() throws Exception {
-    Configuration conf = new HdfsConfiguration();
-    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
-    Random random = new Random();
-    FileSystem fs = null;
-    int rand = random.nextInt(3);
-
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
-    cluster.waitActive();
-    fs = cluster.getFileSystem();
-    Path file1 = new Path("/tmp/testBlockVerification/file1");
-    DFSTestUtil.createFile(fs, file1, 1024, (short)3, 0);
-    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
-    
-    DFSTestUtil.waitReplication(fs, file1, (short)3);
-    assertFalse(DFSTestUtil.allBlockReplicasCorrupt(cluster, file1, 0));
-
-    // Corrupt random replica of block 
-    assertTrue(MiniDFSCluster.corruptReplica(rand, block));
-
-    // Restart the datanode hoping the corrupt block to be reported
-    cluster.restartDataNode(rand);
-
-    // We have 2 good replicas and block is not corrupt
-    DFSTestUtil.waitReplication(fs, file1, (short)2);
-    assertFalse(DFSTestUtil.allBlockReplicasCorrupt(cluster, file1, 0));
-  
-    // Corrupt all replicas. Now, block should be marked as corrupt
-    // and we should get all the replicas 
-    assertTrue(MiniDFSCluster.corruptReplica(0, block));
-    assertTrue(MiniDFSCluster.corruptReplica(1, block));
-    assertTrue(MiniDFSCluster.corruptReplica(2, block));
-
-    // Trigger each of the DNs to scan this block immediately.
-    // The block pool scanner doesn't run frequently enough on its own
-    // to notice these, and due to HDFS-1371, the client won't report
-    // bad blocks to the NN when all replicas are bad.
-    for (DataNode dn : cluster.getDataNodes()) {
-      DataNodeTestUtils.runBlockScannerForBlock(dn, block);
-    }
-
-    // We now have the blocks to be marked as corrupt and we get back all
-    // its replicas
-    DFSTestUtil.waitReplication(fs, file1, (short)3);
-    assertTrue(DFSTestUtil.allBlockReplicasCorrupt(cluster, file1, 0));
-    cluster.shutdown();
-  }
-  
-  /**
-   * testBlockCorruptionRecoveryPolicy.
-   * This tests recovery of corrupt replicas, first for one corrupt replica
-   * then for two. The test invokes blockCorruptionRecoveryPolicy which
-   * 1. Creates a block with desired number of replicas
-   * 2. Corrupts the desired number of replicas and restarts the datanodes
-   *    containing the corrupt replica. Additionaly we also read the block
-   *    in case restarting does not report corrupt replicas.
-   *    Restarting or reading from the datanode would trigger reportBadBlocks 
-   *    to namenode.
-   *    NameNode adds it to corruptReplicasMap and neededReplication
-   * 3. Test waits until all corrupt replicas are reported, meanwhile
-   *    Re-replciation brings the block back to healthy state
-   * 4. Test again waits until the block is reported with expected number
-   *    of good replicas.
-   */
-  @Test
-  public void testBlockCorruptionRecoveryPolicy1() throws Exception {
-    // Test recovery of 1 corrupt replica
-    LOG.info("Testing corrupt replica recovery for one corrupt replica");
-    blockCorruptionRecoveryPolicy(4, (short)3, 1);
-  }
-
-  @Test
-  public void testBlockCorruptionRecoveryPolicy2() throws Exception {
-    // Test recovery of 2 corrupt replicas
-    LOG.info("Testing corrupt replica recovery for two corrupt replicas");
-    blockCorruptionRecoveryPolicy(5, (short)3, 2);
-  }
-  
-  private void blockCorruptionRecoveryPolicy(int numDataNodes, 
-                                             short numReplicas,
-                                             int numCorruptReplicas) 
-                                             throws Exception {
-    Configuration conf = new HdfsConfiguration();
-    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 30L);
-    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 3);
-    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 3L);
-    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
-    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 5L);
-
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
-    cluster.waitActive();
-    FileSystem fs = cluster.getFileSystem();
-    Path file1 = new Path("/tmp/testBlockCorruptRecovery/file");
-    DFSTestUtil.createFile(fs, file1, 1024, numReplicas, 0);
-    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
-    final int ITERATIONS = 10;
-
-    // Wait until block is replicated to numReplicas
-    DFSTestUtil.waitReplication(fs, file1, numReplicas);
-
-    for (int k = 0; ; k++) {
-      // Corrupt numCorruptReplicas replicas of block 
-      int[] corruptReplicasDNIDs = new int[numCorruptReplicas];
-      for (int i=0, j=0; (j != numCorruptReplicas) && (i < numDataNodes); i++) {
-        if (corruptReplica(block, i)) {
-          corruptReplicasDNIDs[j++] = i;
-          LOG.info("successfully corrupted block " + block + " on node " 
-                   + i + " " + cluster.getDataNodes().get(i).getDisplayName());
-        }
-      }
-      
-      // Restart the datanodes containing corrupt replicas 
-      // so they would be reported to namenode and re-replicated
-      // They MUST be restarted in reverse order from highest to lowest index,
-      // because the act of restarting them removes them from the ArrayList
-      // and causes the indexes of all nodes above them in the list to change.
-      for (int i = numCorruptReplicas - 1; i >= 0 ; i--) {
-        LOG.info("restarting node with corrupt replica: position " 
-            + i + " node " + corruptReplicasDNIDs[i] + " " 
-            + cluster.getDataNodes().get(corruptReplicasDNIDs[i]).getDisplayName());
-        cluster.restartDataNode(corruptReplicasDNIDs[i]);
-      }
-
-      // Loop until all corrupt replicas are reported
-      try {
-        DFSTestUtil.waitCorruptReplicas(fs, cluster.getNamesystem(), file1, 
-                                        block, numCorruptReplicas);
-      } catch(TimeoutException e) {
-        if (k > ITERATIONS) {
-          throw e;
-        }
-        LOG.info("Timed out waiting for corrupt replicas, trying again, iteration " + k);
-        continue;
-      }
-      break;
-    }
-    
-    // Loop until the block recovers after replication
-    DFSTestUtil.waitReplication(fs, file1, numReplicas);
-    assertFalse(DFSTestUtil.allBlockReplicasCorrupt(cluster, file1, 0));
-
-    // Make sure the corrupt replica is invalidated and removed from
-    // corruptReplicasMap
-    DFSTestUtil.waitCorruptReplicas(fs, cluster.getNamesystem(), file1, 
-        block, 0);
-    cluster.shutdown();
-  }
-  
-  /** Test if NameNode handles truncated blocks in block report */
-  @Test
-  public void testTruncatedBlockReport() throws Exception {
-    final Configuration conf = new HdfsConfiguration();
-    final short REPLICATION_FACTOR = (short)2;
-    final Path fileName = new Path("/file1");
-
-    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 3L);
-    conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 3);
-    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 3L);
-    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
-
-    long startTime = Time.monotonicNow();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-                                               .numDataNodes(REPLICATION_FACTOR)
-                                               .build();
-    cluster.waitActive();
-    
-    ExtendedBlock block;
-    try {
-      FileSystem fs = cluster.getFileSystem();
-      DFSTestUtil.createFile(fs, fileName, 1, REPLICATION_FACTOR, 0);
-      DFSTestUtil.waitReplication(fs, fileName, REPLICATION_FACTOR);
-      block = DFSTestUtil.getFirstBlock(fs, fileName);
-    } finally {
-      cluster.shutdown();
-    }
-
-    // Restart cluster and confirm block is verified on datanode 0,
-    // then truncate it on datanode 0.
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(REPLICATION_FACTOR)
-                                .format(false)
-                                .build();
-    cluster.waitActive();
-    try {
-      FileSystem fs = cluster.getFileSystem();
-      int infoPort = cluster.getDataNodes().get(0).getInfoPort();
-      assertTrue(waitForVerification(infoPort, fs, fileName, 1, startTime, TIMEOUT) >= startTime);
-      
-      // Truncate replica of block
-      if (!changeReplicaLength(block, 0, -1)) {
-        throw new IOException(
-            "failed to find or change length of replica on node 0 "
-            + cluster.getDataNodes().get(0).getDisplayName());
-      }      
-    } finally {
-      cluster.shutdown();
-    }
-
-    // Restart the cluster, add a node, and check that the truncated block is 
-    // handled correctly
-    cluster = new MiniDFSCluster.Builder(conf)
-                                .numDataNodes(REPLICATION_FACTOR)
-                                .format(false)
-                                .build();
-    cluster.startDataNodes(conf, 1, true, null, null);
-    cluster.waitActive();  // now we have 3 datanodes
-
-    // Assure the cluster has left safe mode.
-    cluster.waitClusterUp();
-    assertFalse("failed to leave safe mode", 
-        cluster.getNameNode().isInSafeMode());
-
-    try {
-      // wait for truncated block be detected by block scanner,
-      // and the block to be replicated
-      DFSTestUtil.waitReplication(
-          cluster.getFileSystem(), fileName, REPLICATION_FACTOR);
-      
-      // Make sure that truncated block will be deleted
-      waitForBlockDeleted(block, 0, TIMEOUT);
-    } finally {
-      cluster.shutdown();
-    }
-  }
-  
-  /**
-   * Change the length of a block at datanode dnIndex
-   */
-  static boolean changeReplicaLength(ExtendedBlock blk, int dnIndex,
-      int lenDelta) throws IOException {
-    File blockFile = MiniDFSCluster.getBlockFile(dnIndex, blk);
-    if (blockFile != null && blockFile.exists()) {
-      RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
-      raFile.setLength(raFile.length()+lenDelta);
-      raFile.close();
-      return true;
-    }
-    LOG.info("failed to change length of block " + blk);
-    return false;
-  }
-  
-  private static void waitForBlockDeleted(ExtendedBlock blk, int dnIndex,
-      long timeout) throws TimeoutException, InterruptedException {
-    File blockFile = MiniDFSCluster.getBlockFile(dnIndex, blk);
-    long failtime = Time.monotonicNow()
-                    + ((timeout > 0) ? timeout : Long.MAX_VALUE);
-    while (blockFile != null && blockFile.exists()) {
-      if (failtime < Time.monotonicNow()) {
-        throw new TimeoutException("waited too long for blocks to be deleted: "
-            + blockFile.getPath() + (blockFile.exists() ? " still exists; " : " is absent; "));
-      }
-      Thread.sleep(100);
-      blockFile = MiniDFSCluster.getBlockFile(dnIndex, blk);
-    }
-  }
-  
-  private static final String BASE_PATH = (new File("/data/current/finalized"))
-      .getAbsolutePath();
-  
-  @Test
-  public void testReplicaInfoParsing() throws Exception {
-    testReplicaInfoParsingSingle(BASE_PATH);
-    testReplicaInfoParsingSingle(BASE_PATH + "/subdir1");
-    testReplicaInfoParsingSingle(BASE_PATH + "/subdir1/subdir2/subdir3");
-  }
-  
-  private static void testReplicaInfoParsingSingle(String subDirPath) {
-    File testFile = new File(subDirPath);
-    assertEquals(BASE_PATH, ReplicaInfo.parseBaseDir(testFile).baseDirPath);
-  }
-
-  @Test
-  public void testDuplicateScans() throws Exception {
-    long startTime = Time.monotonicNow();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration())
-        .numDataNodes(1).build();
-    FileSystem fs = null;
-    try {
-      fs = cluster.getFileSystem();
-      DataNode dataNode = cluster.getDataNodes().get(0);
-      int infoPort = dataNode.getInfoPort();
-      long scanTimeBefore = 0, scanTimeAfter = 0;
-      for (int i = 1; i < 10; i++) {
-        Path fileName = new Path("/test" + i);
-        DFSTestUtil.createFile(fs, fileName, 1024, (short) 1, 1000L);
-        waitForVerification(infoPort, fs, fileName, i, startTime, TIMEOUT);
-        if (i > 1) {
-          scanTimeAfter = DataNodeTestUtils.getLatestScanTime(dataNode,
-              DFSTestUtil.getFirstBlock(fs, new Path("/test" + (i - 1))));
-          assertFalse("scan time shoud not be 0", scanTimeAfter == 0);
-          assertEquals("There should not be duplicate scan", scanTimeBefore,
-              scanTimeAfter);
-        }
-
-        scanTimeBefore = DataNodeTestUtils.getLatestScanTime(dataNode,
-            DFSTestUtil.getFirstBlock(fs, new Path("/test" + i)));
-      }
-      cluster.restartDataNode(0);
-      Thread.sleep(10000);
-      dataNode = cluster.getDataNodes().get(0);
-      scanTimeAfter = DataNodeTestUtils.getLatestScanTime(dataNode,
-          DFSTestUtil.getFirstBlock(fs, new Path("/test" + (9))));
-      assertEquals("There should not be duplicate scan", scanTimeBefore,
-          scanTimeAfter);
-    } finally {
-      IOUtils.closeStream(fs);
-      cluster.shutdown();
-    }
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
index 28687723aa9a1..bf26295d15c96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
@@ -22,8 +22,6 @@
 
 import java.io.IOException;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -33,10 +31,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.Test;
 
@@ -45,13 +41,10 @@
  */
 public class TestDatanodeDeath {
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)InterDatanodeProtocol.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
   }
 
   static final int blockSize = 8192;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
index 0966301cb4e76..343320c0326fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeLayoutUpgrade.java
@@ -43,6 +43,6 @@ public void testUpgradeToIdBasedLayout() throws IOException {
         System.getProperty("test.build.data") + File.separator +
             "dfs" + File.separator + "name");
     upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf).numDataNodes(1)
-    .manageDataDfsDirs(false).manageNameDfsDirs(false));
+    .manageDataDfsDirs(false).manageNameDfsDirs(false), null);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeStartupFixesLegacyStorageIDs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeStartupFixesLegacyStorageIDs.java
new file mode 100644
index 0000000000000..e262abcb6a8a9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeStartupFixesLegacyStorageIDs.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.TestDFSUpgradeFromImage.ClusterVerifier;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * The test verifies that legacy storage IDs in older DataNode
+ * images are replaced with UUID-based storage IDs. The startup may
+ * or may not involve a Datanode Layout upgrade. Each test case uses
+ * the following resource files.
+ *
+ *    1. testCaseName.tgz - NN and DN directories corresponding
+ *                          to a specific layout version.
+ *    2. testCaseName.txt - Text file listing the checksum of each file
+ *                          in the cluster and overall checksum. See
+ *                          TestUpgradeFromImage for the file format.
+ *
+ * If any test case is renamed then the corresponding resource files must
+ * also be renamed.
+ */
+public class TestDatanodeStartupFixesLegacyStorageIDs {
+
+  /**
+   * Perform a upgrade using the test image corresponding to
+   * testCaseName.
+   *
+   * @param testCaseName
+   * @param expectedStorageId if null, then the upgrade generates a new
+   *                          unique storage ID.
+   * @throws IOException
+   */
+  private static void runLayoutUpgradeTest(final String testCaseName,
+                                           final String expectedStorageId)
+      throws IOException {
+    TestDFSUpgradeFromImage upgrade = new TestDFSUpgradeFromImage();
+    upgrade.unpackStorage(testCaseName + ".tgz", testCaseName + ".txt");
+    Configuration conf = new Configuration(TestDFSUpgradeFromImage.upgradeConf);
+    initStorageDirs(conf, testCaseName);
+    upgradeAndVerify(upgrade, conf, new ClusterVerifier() {
+      @Override
+      public void verifyClusterPostUpgrade(MiniDFSCluster cluster) throws IOException {
+        // Verify that a GUID-based storage ID was generated.
+        final String bpid = cluster.getNamesystem().getBlockPoolId();
+        StorageReport[] reports =
+            cluster.getDataNodes().get(0).getFSDataset().getStorageReports(bpid);
+        assertThat(reports.length, is(1));
+        final String storageID = reports[0].getStorage().getStorageID();
+        assertTrue(DatanodeStorage.isValidStorageId(storageID));
+
+        if (expectedStorageId != null) {
+          assertThat(storageID, is(expectedStorageId));
+        }
+      }
+    });
+  }
+
+  private static void initStorageDirs(final Configuration conf,
+                                      final String testName) {
+    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+             System.getProperty("test.build.data") + File.separator +
+                 testName + File.separator + "dfs" + File.separator + "data");
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+             System.getProperty("test.build.data") + File.separator +
+                 testName + File.separator + "dfs" + File.separator + "name");
+
+  }
+
+  private static void upgradeAndVerify(final TestDFSUpgradeFromImage upgrade,
+                                       final Configuration conf,
+                                       final ClusterVerifier verifier)
+      throws IOException{
+    upgrade.upgradeAndVerify(new MiniDFSCluster.Builder(conf)
+                                 .numDataNodes(1)
+                                 .manageDataDfsDirs(false)
+                                 .manageNameDfsDirs(false), verifier);
+  }
+
+  /**
+   * Upgrade from 2.2 (no storage IDs per volume) correctly generates
+   * GUID-based storage IDs. Test case for HDFS-7575.
+   */
+  @Test (timeout=300000)
+  public void testUpgradeFrom22FixesStorageIDs() throws IOException {
+    runLayoutUpgradeTest(GenericTestUtils.getMethodName(), null);
+  }
+
+  /**
+   * Startup from a 2.6-layout that has legacy storage IDs correctly
+   * generates new storage IDs.
+   * Test case for HDFS-7575.
+   */
+  @Test (timeout=300000)
+  public void testUpgradeFrom22via26FixesStorageIDs() throws IOException {
+    runLayoutUpgradeTest(GenericTestUtils.getMethodName(), null);
+  }
+
+  /**
+   * Startup from a 2.6-layout that already has unique storage IDs does
+   * not regenerate the storage IDs.
+   * Test case for HDFS-7575.
+   */
+  @Test (timeout=300000)
+  public void testUpgradeFrom26PreservesStorageIDs() throws IOException {
+    // StorageId present in the image testUpgradeFrom26PreservesStorageId.tgz
+    runLayoutUpgradeTest(GenericTestUtils.getMethodName(),
+                         "DS-a0e39cfa-930f-4abd-813c-e22b59223774");
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 603bf6e0cb224..13eb4cf97e027 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -538,6 +538,19 @@ private void doRenameEncryptionZone(FSTestWrapper wrapper) throws Exception {
         !wrapper.exists(pathFooBaz) && wrapper.exists(pathFooBar));
     assertEquals("Renamed file contents not the same",
         contents, DFSTestUtil.readFile(fs, pathFooBarFile));
+
+    // Verify that we can rename an EZ root
+    final Path newFoo = new Path(testRoot, "newfoo");
+    assertTrue("Rename of EZ root", fs.rename(pathFoo, newFoo));
+    assertTrue("Rename of EZ root failed",
+        !wrapper.exists(pathFoo) && wrapper.exists(newFoo));
+
+    // Verify that we can't rename an EZ root onto itself
+    try {
+      wrapper.rename(newFoo, newFoo);
+    } catch (IOException e) {
+      assertExceptionContains("are the same", e);
+    }
   }
 
   @Test(timeout = 60000)
@@ -1265,11 +1278,11 @@ public void testOfflineImageViewerOnEncryptionZones() throws Exception {
     }
 
     // Run the XML OIV processor
-    StringWriter output = new StringWriter();
-    PrintWriter pw = new PrintWriter(output);
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    PrintStream pw = new PrintStream(output);
     PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), pw);
     v.visit(new RandomAccessFile(originalFsimage, "r"));
-    final String xml = output.getBuffer().toString();
+    final String xml = output.toString();
     SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
     parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
index a8c6779b733a9..0d4435779aaf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFetchImage.java
@@ -26,20 +26,27 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.io.MD5Hash;
+import org.junit.AfterClass;
 import org.junit.Test;
 
 public class TestFetchImage {
   
   private static final File FETCHED_IMAGE_FILE = new File(
-      System.getProperty("build.test.dir"), "fetched-image-dir");
+      System.getProperty("test.build.dir"), "target/fetched-image-dir");
   // Shamelessly stolen from NNStorage.
   private static final Pattern IMAGE_REGEX = Pattern.compile("fsimage_(\\d+)");
 
+  @AfterClass
+  public static void cleanup() {
+    FileUtil.fullyDelete(FETCHED_IMAGE_FILE);
+  }
+
   /**
    * Download a few fsimages using `hdfs dfsadmin -fetchImage ...' and verify
    * the results.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index 34c701d0bbdb6..af404cd82c8a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -25,10 +25,12 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.EnumSet;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.HardLink;
@@ -101,9 +103,10 @@ private void checkFile(FileSystem fileSys, Path name, int repl)
       System.arraycopy(fileContents, 0, expected, 0, expected.length);
     }
     // do a sanity check. Read the file
+    // do not check file status since the file is not yet closed.
     AppendTestUtil.checkFullFile(fileSys, name,
         AppendTestUtil.NUM_BLOCKS * AppendTestUtil.BLOCK_SIZE,
-        expected, "Read 1");
+        expected, "Read 1", false);
   }
 
   /**
@@ -344,7 +347,46 @@ public void testAppendTwice() throws Exception {
       cluster.shutdown();
     }
   }
+
+  /** Test two consecutive appends on a file with a full block. */
+  @Test
+  public void testAppend2Twice() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+    final DistributedFileSystem fs1 = cluster.getFileSystem();
+    final FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(conf);
+    try {
+      final Path p = new Path("/testAppendTwice/foo");
+      final int len = 1 << 16;
+      final byte[] fileContents = AppendTestUtil.initBuffer(len);
+
+      {
+        // create a new file with a full block.
+        FSDataOutputStream out = fs2.create(p, true, 4096, (short)1, len);
+        out.write(fileContents, 0, len);
+        out.close();
+      }
   
+      //1st append does not add any data so that the last block remains full
+      //and the last block in INodeFileUnderConstruction is a BlockInfo
+      //but not BlockInfoUnderConstruction.
+      ((DistributedFileSystem) fs2).append(p,
+          EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+
+      // 2nd append should get AlreadyBeingCreatedException
+      fs1.append(p);
+      Assert.fail();
+    } catch(RemoteException re) {
+      AppendTestUtil.LOG.info("Got an exception:", re);
+      Assert.assertEquals(AlreadyBeingCreatedException.class.getName(),
+          re.getClassName());
+    } finally {
+      fs2.close();
+      fs1.close();
+      cluster.shutdown();
+    }
+  }
+
   /** Tests appending after soft-limit expires. */
   @Test
   public void testAppendAfterSoftLimit() 
@@ -386,6 +428,54 @@ public void testAppendAfterSoftLimit()
     }
   }
 
+  /** Tests appending after soft-limit expires. */
+  @Test
+  public void testAppend2AfterSoftLimit() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+    //Set small soft-limit for lease
+    final long softLimit = 1L;
+    final long hardLimit = 9999999L;
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+        .build();
+    cluster.setLeasePeriod(softLimit, hardLimit);
+    cluster.waitActive();
+
+    DistributedFileSystem fs = cluster.getFileSystem();
+    DistributedFileSystem fs2 = new DistributedFileSystem();
+    fs2.initialize(fs.getUri(), conf);
+
+    final Path testPath = new Path("/testAppendAfterSoftLimit");
+    final byte[] fileContents = AppendTestUtil.initBuffer(32);
+
+    // create a new file without closing
+    FSDataOutputStream out = fs.create(testPath);
+    out.write(fileContents);
+
+    //Wait for > soft-limit
+    Thread.sleep(250);
+
+    try {
+      FSDataOutputStream appendStream2 = fs2.append(testPath,
+          EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+      appendStream2.write(fileContents);
+      appendStream2.close();
+      assertEquals(fileContents.length, fs.getFileStatus(testPath).getLen());
+      // make sure we now have 1 block since the first writer was revoked
+      LocatedBlocks blks = fs.getClient().getLocatedBlocks(testPath.toString(),
+          0L);
+      assertEquals(1, blks.getLocatedBlocks().size());
+      for (LocatedBlock blk : blks.getLocatedBlocks()) {
+        assertEquals(fileContents.length, blk.getBlockSize());
+      }
+    } finally {
+      fs.close();
+      fs2.close();
+      cluster.shutdown();
+    }
+  }
+
   /**
    * Old replica of the block should not be accepted as valid for append/read
    */
@@ -439,4 +529,77 @@ public void testFailedAppendBlockRejection() throws Exception {
     }
   }
 
+  /**
+   * Old replica of the block should not be accepted as valid for append/read
+   */
+  @Test
+  public void testMultiAppend2() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    conf.set("dfs.client.block.write.replace-datanode-on-failure.enable",
+        "false");
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
+        .build();
+    DistributedFileSystem fs = null;
+    final String hello = "hello\n";
+    try {
+      fs = cluster.getFileSystem();
+      Path path = new Path("/test");
+      FSDataOutputStream out = fs.create(path);
+      out.writeBytes(hello);
+      out.close();
+
+      // stop one datanode
+      DataNodeProperties dnProp = cluster.stopDataNode(0);
+      String dnAddress = dnProp.datanode.getXferAddress().toString();
+      if (dnAddress.startsWith("/")) {
+        dnAddress = dnAddress.substring(1);
+      }
+
+      // append again to bump genstamps
+      for (int i = 0; i < 2; i++) {
+        out = fs.append(path,
+            EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+        out.writeBytes(hello);
+        out.close();
+      }
+
+      // re-open and make the block state as underconstruction
+      out = fs.append(path, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK),
+          4096, null);
+      cluster.restartDataNode(dnProp, true);
+      // wait till the block report comes
+      Thread.sleep(2000);
+      out.writeBytes(hello);
+      out.close();
+      // check the block locations
+      LocatedBlocks blocks = fs.getClient().getLocatedBlocks(path.toString(), 0L);
+      // since we append the file 3 time, we should be 4 blocks
+      assertEquals(4, blocks.getLocatedBlocks().size());
+      for (LocatedBlock block : blocks.getLocatedBlocks()) {
+        assertEquals(hello.length(), block.getBlockSize());
+      }
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < 4; i++) {
+        sb.append(hello);
+      }
+      final byte[] content = sb.toString().getBytes();
+      AppendTestUtil.checkFullFile(fs, path, content.length, content,
+          "Read /test");
+
+      // restart namenode to make sure the editlog can be properly applied
+      cluster.restartNameNode(true);
+      cluster.waitActive();
+      AppendTestUtil.checkFullFile(fs, path, content.length, content,
+          "Read /test");
+      blocks = fs.getClient().getLocatedBlocks(path.toString(), 0L);
+      // since we append the file 3 time, we should be 4 blocks
+      assertEquals(4, blocks.getLocatedBlocks().size());
+      for (LocatedBlock block : blocks.getLocatedBlocks()) {
+        assertEquals(hello.length(), block.getBlockSize());
+      }
+    } finally {
+      IOUtils.closeStream(fs);
+      cluster.shutdown();
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
index eecd23b7b79ab..dd4fe14440640 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -24,22 +25,23 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
 
-import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.Test;
 
@@ -50,12 +52,9 @@
 public class TestFileAppend2 {
 
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
   }
 
   static final int numBlocks = 5;
@@ -67,11 +66,7 @@ public class TestFileAppend2 {
   final int numberOfFiles = 50;
   final int numThreads = 10;
   final int numAppendsPerThread = 20;
-/***
-  int numberOfFiles = 1;
-  int numThreads = 1;
-  int numAppendsPerThread = 2000;
-****/
+
   Workload[] workload = null;
   final ArrayList<Path> testFiles = new ArrayList<Path>();
   volatile static boolean globalStatus = true;
@@ -229,16 +224,170 @@ public void testSimpleAppend() throws IOException {
     }
   }
 
+  /**
+   * Creates one file, writes a few bytes to it and then closed it.
+   * Reopens the same file for appending using append2 API, write all blocks and
+   * then close. Verify that all data exists in file.
+   */
+  @Test
+  public void testSimpleAppend2() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    if (simulatedStorage) {
+      SimulatedFSDataset.setFactory(conf);
+    }
+    conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 50);
+    fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    try {
+      { // test appending to a file.
+        // create a new file.
+        Path file1 = new Path("/simpleAppend.dat");
+        FSDataOutputStream stm = AppendTestUtil.createFile(fs, file1, 1);
+        System.out.println("Created file simpleAppend.dat");
+
+        // write to file
+        int mid = 186;   // io.bytes.per.checksum bytes
+        System.out.println("Writing " + mid + " bytes to file " + file1);
+        stm.write(fileContents, 0, mid);
+        stm.close();
+        System.out.println("Wrote and Closed first part of file.");
+
+        // write to file
+        int mid2 = 607;   // io.bytes.per.checksum bytes
+        System.out.println("Writing " + mid + " bytes to file " + file1);
+        stm = fs.append(file1,
+            EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+        stm.write(fileContents, mid, mid2-mid);
+        stm.close();
+        System.out.println("Wrote and Closed second part of file.");
+
+        // write the remainder of the file
+        stm = fs.append(file1,
+            EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+        // ensure getPos is set to reflect existing size of the file
+        assertTrue(stm.getPos() > 0);
+        System.out.println("Writing " + (AppendTestUtil.FILE_SIZE - mid2) +
+            " bytes to file " + file1);
+        stm.write(fileContents, mid2, AppendTestUtil.FILE_SIZE - mid2);
+        System.out.println("Written second part of file");
+        stm.close();
+        System.out.println("Wrote and Closed second part of file.");
+
+        // verify that entire file is good
+        AppendTestUtil.checkFullFile(fs, file1, AppendTestUtil.FILE_SIZE,
+            fileContents, "Read 2");
+        // also make sure there three different blocks for the file
+        List<LocatedBlock> blocks = fs.getClient().getLocatedBlocks(
+            file1.toString(), 0L).getLocatedBlocks();
+        assertEquals(12, blocks.size()); // the block size is 1024
+        assertEquals(mid, blocks.get(0).getBlockSize());
+        assertEquals(mid2 - mid, blocks.get(1).getBlockSize());
+        for (int i = 2; i < 11; i++) {
+          assertEquals(AppendTestUtil.BLOCK_SIZE, blocks.get(i).getBlockSize());
+        }
+        assertEquals((AppendTestUtil.FILE_SIZE - mid2)
+            % AppendTestUtil.BLOCK_SIZE, blocks.get(11).getBlockSize());
+      }
+
+      { // test appending to an non-existing file.
+        FSDataOutputStream out = null;
+        try {
+          out = fs.append(new Path("/non-existing.dat"),
+              EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+          fail("Expected to have FileNotFoundException");
+        } catch(java.io.FileNotFoundException fnfe) {
+          System.out.println("Good: got " + fnfe);
+          fnfe.printStackTrace(System.out);
+        } finally {
+          IOUtils.closeStream(out);
+        }
+      }
+
+      { // test append permission.
+        // set root to all writable
+        Path root = new Path("/");
+        fs.setPermission(root, new FsPermission((short)0777));
+        fs.close();
+
+        // login as a different user
+        final UserGroupInformation superuser =
+          UserGroupInformation.getCurrentUser();
+        String username = "testappenduser";
+        String group = "testappendgroup";
+        assertFalse(superuser.getShortUserName().equals(username));
+        assertFalse(Arrays.asList(superuser.getGroupNames()).contains(group));
+        UserGroupInformation appenduser = UserGroupInformation
+            .createUserForTesting(username, new String[] { group });
+
+        fs = (DistributedFileSystem) DFSTestUtil.getFileSystemAs(appenduser,
+            conf);
+
+        // create a file
+        Path dir = new Path(root, getClass().getSimpleName());
+        Path foo = new Path(dir, "foo.dat");
+        FSDataOutputStream out = null;
+        int offset = 0;
+        try {
+          out = fs.create(foo);
+          int len = 10 + AppendTestUtil.nextInt(100);
+          out.write(fileContents, offset, len);
+          offset += len;
+        } finally {
+          IOUtils.closeStream(out);
+        }
+
+        // change dir and foo to minimal permissions.
+        fs.setPermission(dir, new FsPermission((short)0100));
+        fs.setPermission(foo, new FsPermission((short)0200));
+
+        // try append, should success
+        out = null;
+        try {
+          out = fs.append(foo,
+              EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+          int len = 10 + AppendTestUtil.nextInt(100);
+          out.write(fileContents, offset, len);
+          offset += len;
+        } finally {
+          IOUtils.closeStream(out);
+        }
+
+        // change dir and foo to all but no write on foo.
+        fs.setPermission(foo, new FsPermission((short)0577));
+        fs.setPermission(dir, new FsPermission((short)0777));
+
+        // try append, should fail
+        out = null;
+        try {
+          out = fs.append(foo,
+              EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+          fail("Expected to have AccessControlException");
+        } catch(AccessControlException ace) {
+          System.out.println("Good: got " + ace);
+          ace.printStackTrace(System.out);
+        } finally {
+          IOUtils.closeStream(out);
+        }
+      }
+    } finally {
+      fs.close();
+      cluster.shutdown();
+    }
+  }
+
   //
   // an object that does a bunch of appends to files
   //
   class Workload extends Thread {
     private final int id;
     private final MiniDFSCluster cluster;
+    private final boolean appendToNewBlock;
 
-    Workload(MiniDFSCluster cluster, int threadIndex) {
+    Workload(MiniDFSCluster cluster, int threadIndex, boolean append2) {
       id = threadIndex;
       this.cluster = cluster;
+      this.appendToNewBlock = append2;
     }
 
     // create a bunch of files. Write to them and then verify.
@@ -261,7 +410,7 @@ public void run() {
         long len = 0;
         int sizeToAppend = 0;
         try {
-          FileSystem fs = cluster.getFileSystem();
+          DistributedFileSystem fs = cluster.getFileSystem();
 
           // add a random number of bytes to file
           len = fs.getFileStatus(testfile).getLen();
@@ -285,7 +434,9 @@ public void run() {
                              " appending " + sizeToAppend + " bytes " +
                              " to file " + testfile +
                              " of size " + len);
-          FSDataOutputStream stm = fs.append(testfile);
+          FSDataOutputStream stm = appendToNewBlock ? fs.append(testfile,
+                  EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)
+              : fs.append(testfile);
           stm.write(fileContents, (int)len, sizeToAppend);
           stm.close();
 
@@ -298,7 +449,7 @@ public void run() {
                                  " expected size " + (len + sizeToAppend) +
                                  " waiting for namenode metadata update.");
               Thread.sleep(5000);
-            } catch (InterruptedException e) {;}
+            } catch (InterruptedException e) {}
           }
 
           assertTrue("File " + testfile + " size is " + 
@@ -306,7 +457,7 @@ public void run() {
                      " but expected " + (len + sizeToAppend),
                     fs.getFileStatus(testfile).getLen() == (len + sizeToAppend));
 
-          AppendTestUtil.checkFullFile(fs, testfile, (int)(len + sizeToAppend),
+          AppendTestUtil.checkFullFile(fs, testfile, (int) (len + sizeToAppend),
               fileContents, "Read 2");
         } catch (Throwable e) {
           globalStatus = false;
@@ -331,10 +482,8 @@ public void run() {
 
   /**
    * Test that appends to files at random offsets.
-   * @throws IOException an exception might be thrown
    */
-  @Test
-  public void testComplexAppend() throws IOException {
+  private void testComplexAppend(boolean appendToNewBlock) throws IOException {
     fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
     Configuration conf = new HdfsConfiguration();
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 2000);
@@ -366,7 +515,7 @@ public void testComplexAppend() throws IOException {
       // Create threads and make them run workload concurrently.
       workload = new Workload[numThreads];
       for (int i = 0; i < numThreads; i++) {
-        workload[i] = new Workload(cluster, i);
+        workload[i] = new Workload(cluster, i, appendToNewBlock);
         workload[i].start();
       }
 
@@ -390,4 +539,14 @@ public void testComplexAppend() throws IOException {
     //
     assertTrue("testComplexAppend Worker encountered exceptions.", globalStatus);
   }
+
+  @Test
+  public void testComplexAppend() throws IOException {
+    testComplexAppend(false);
+  }
+
+  @Test
+  public void testComplexAppend2() throws IOException {
+    testComplexAppend(true);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
index d5de0ff4d0945..1b4d92d4349ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
@@ -24,20 +24,21 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.util.EnumSet;
+import java.util.List;
 
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.mockito.invocation.InvocationOnMock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 import org.mockito.stubbing.Answer;
 
-import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSClientAdapter;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -46,24 +47,20 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
 import org.apache.log4j.Level;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 /** This class implements some of tests posted in HADOOP-2658. */
 public class TestFileAppend3  {
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)InterDatanodeProtocol.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
   }
 
   static final long BLOCK_SIZE = 64 * 1024;
@@ -121,6 +118,32 @@ public void testTC1() throws Exception {
     AppendTestUtil.check(fs, p, len1 + len2);
   }
 
+  @Test
+  public void testTC1ForAppend2() throws Exception {
+    final Path p = new Path("/TC1/foo2");
+
+    //a. Create file and write one block of data. Close file.
+    final int len1 = (int) BLOCK_SIZE;
+    {
+      FSDataOutputStream out = fs.create(p, false, buffersize, REPLICATION,
+          BLOCK_SIZE);
+      AppendTestUtil.write(out, 0, len1);
+      out.close();
+    }
+
+    // Reopen file to append. Append half block of data. Close file.
+    final int len2 = (int) BLOCK_SIZE / 2;
+    {
+      FSDataOutputStream out = fs.append(p,
+          EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+      AppendTestUtil.write(out, len1, len2);
+      out.close();
+    }
+
+    // b. Reopen file and read 1.5 blocks worth of data. Close file.
+    AppendTestUtil.check(fs, p, len1 + len2);
+  }
+
   /**
    * TC2: Append on non-block boundary.
    * @throws IOException an exception might be thrown
@@ -152,6 +175,40 @@ public void testTC2() throws Exception {
     AppendTestUtil.check(fs, p, len1 + len2);
   }
 
+  @Test
+  public void testTC2ForAppend2() throws Exception {
+    final Path p = new Path("/TC2/foo2");
+
+    //a. Create file with one and a half block of data. Close file.
+    final int len1 = (int) (BLOCK_SIZE + BLOCK_SIZE / 2);
+    {
+      FSDataOutputStream out = fs.create(p, false, buffersize, REPLICATION,
+          BLOCK_SIZE);
+      AppendTestUtil.write(out, 0, len1);
+      out.close();
+    }
+
+    AppendTestUtil.check(fs, p, len1);
+
+    //   Reopen file to append quarter block of data. Close file.
+    final int len2 = (int) BLOCK_SIZE / 4;
+    {
+      FSDataOutputStream out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK),
+          4096, null);
+      AppendTestUtil.write(out, len1, len2);
+      out.close();
+    }
+
+    // b. Reopen file and read 1.75 blocks of data. Close file.
+    AppendTestUtil.check(fs, p, len1 + len2);
+    List<LocatedBlock> blocks = fs.getClient().getLocatedBlocks(
+        p.toString(), 0L).getLocatedBlocks();
+    Assert.assertEquals(3, blocks.size());
+    Assert.assertEquals(BLOCK_SIZE, blocks.get(0).getBlockSize());
+    Assert.assertEquals(BLOCK_SIZE / 2, blocks.get(1).getBlockSize());
+    Assert.assertEquals(BLOCK_SIZE / 4, blocks.get(2).getBlockSize());
+  }
+
   /**
    * TC5: Only one simultaneous append.
    * @throws IOException an exception might be thrown
@@ -179,18 +236,63 @@ public void testTC5() throws Exception {
       AppendTestUtil.LOG.info("GOOD: got an exception", ioe);
     }
 
+    try {
+      ((DistributedFileSystem) AppendTestUtil
+          .createHdfsWithDifferentUsername(conf)).append(p,
+          EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+      fail("This should fail.");
+    } catch(IOException ioe) {
+      AppendTestUtil.LOG.info("GOOD: got an exception", ioe);
+    }
+
     //d. On Machine M1, close file.
     out.close();        
   }
 
+  @Test
+  public void testTC5ForAppend2() throws Exception {
+    final Path p = new Path("/TC5/foo2");
+
+    // a. Create file on Machine M1. Write half block to it. Close file.
+    {
+      FSDataOutputStream out = fs.create(p, false, buffersize, REPLICATION,
+          BLOCK_SIZE);
+      AppendTestUtil.write(out, 0, (int)(BLOCK_SIZE/2));
+      out.close();
+    }
+
+    // b. Reopen file in "append" mode on Machine M1.
+    FSDataOutputStream out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK),
+        4096, null);
+
+    // c. On Machine M2, reopen file in "append" mode. This should fail.
+    try {
+      ((DistributedFileSystem) AppendTestUtil
+          .createHdfsWithDifferentUsername(conf)).append(p,
+          EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null);
+      fail("This should fail.");
+    } catch(IOException ioe) {
+      AppendTestUtil.LOG.info("GOOD: got an exception", ioe);
+    }
+
+    try {
+      AppendTestUtil.createHdfsWithDifferentUsername(conf).append(p);
+      fail("This should fail.");
+    } catch(IOException ioe) {
+      AppendTestUtil.LOG.info("GOOD: got an exception", ioe);
+    }
+
+    // d. On Machine M1, close file.
+    out.close();
+  }
+
   /**
    * TC7: Corrupted replicas are present.
    * @throws IOException an exception might be thrown
    */
-  @Test
-  public void testTC7() throws Exception {
+  private void testTC7(boolean appendToNewBlock) throws Exception {
     final short repl = 2;
-    final Path p = new Path("/TC7/foo");
+    final Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1"));
     System.out.println("p=" + p);
     
     //a. Create file with replication factor of 2. Write half block of data. Close file.
@@ -224,7 +326,8 @@ public void testTC7() throws Exception {
     //c. Open file in "append mode".  Append a new block worth of data. Close file.
     final int len2 = (int)BLOCK_SIZE; 
     {
-      FSDataOutputStream out = fs.append(p);
+      FSDataOutputStream out = appendToNewBlock ?
+          fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null) : fs.append(p);
       AppendTestUtil.write(out, len1, len2);
       out.close();
     }
@@ -233,13 +336,21 @@ public void testTC7() throws Exception {
     AppendTestUtil.check(fs, p, len1 + len2);
   }
 
+  @Test
+  public void testTC7() throws Exception {
+    testTC7(false);
+  }
+
+  @Test
+  public void testTC7ForAppend2() throws Exception {
+    testTC7(true);
+  }
+
   /**
    * TC11: Racing rename
-   * @throws IOException an exception might be thrown
    */
-  @Test
-  public void testTC11() throws Exception {
-    final Path p = new Path("/TC11/foo");
+  private void testTC11(boolean appendToNewBlock) throws Exception {
+    final Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1"));
     System.out.println("p=" + p);
 
     //a. Create file and write one block of data. Close file.
@@ -251,7 +362,9 @@ public void testTC11() throws Exception {
     }
 
     //b. Reopen file in "append" mode. Append half block of data.
-    FSDataOutputStream out = fs.append(p);
+    FSDataOutputStream out = appendToNewBlock ?
+        fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null) :
+        fs.append(p);
     final int len2 = (int)BLOCK_SIZE/2; 
     AppendTestUtil.write(out, len1, len2);
     out.hflush();
@@ -283,13 +396,21 @@ public void testTC11() throws Exception {
     }
   }
 
+  @Test
+  public void testTC11() throws Exception {
+    testTC11(false);
+  }
+
+  @Test
+  public void testTC11ForAppend2() throws Exception {
+    testTC11(true);
+  }
+
   /** 
    * TC12: Append to partial CRC chunk
-   * @throws IOException an exception might be thrown
    */
-  @Test
-  public void testTC12() throws Exception {
-    final Path p = new Path("/TC12/foo");
+  private void testTC12(boolean appendToNewBlock) throws Exception {
+    final Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1"));
     System.out.println("p=" + p);
     
     //a. Create file with a block size of 64KB
@@ -305,23 +426,43 @@ public void testTC12() throws Exception {
     //b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file.
     final int len2 = 5877; 
     {
-      FSDataOutputStream out = fs.append(p);
+      FSDataOutputStream out = appendToNewBlock ?
+          fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null) :
+          fs.append(p);
       AppendTestUtil.write(out, len1, len2);
       out.close();
     }
 
     //c. Reopen file and read 25687+5877 bytes of data from file. Close file.
     AppendTestUtil.check(fs, p, len1 + len2);
+    if (appendToNewBlock) {
+      LocatedBlocks blks = fs.dfs.getLocatedBlocks(p.toString(), 0);
+      Assert.assertEquals(2, blks.getLocatedBlocks().size());
+      Assert.assertEquals(len1, blks.getLocatedBlocks().get(0).getBlockSize());
+      Assert.assertEquals(len2, blks.getLocatedBlocks().get(1).getBlockSize());
+      AppendTestUtil.check(fs, p, 0, len1);
+      AppendTestUtil.check(fs, p, len1, len2);
+    }
   }
-  
-  /** Append to a partial CRC chunk and 
-   * the first write does not fill up the partial CRC trunk
-   * *
-   * @throws IOException
-   */
+
   @Test
-  public void testAppendToPartialChunk() throws IOException {
-    final Path p = new Path("/partialChunk/foo");
+  public void testTC12() throws Exception {
+    testTC12(false);
+  }
+
+  @Test
+  public void testTC12ForAppend2() throws Exception {
+    testTC12(true);
+  }
+
+  /**
+   * Append to a partial CRC chunk and the first write does not fill up the
+   * partial CRC trunk
+   */
+  private void testAppendToPartialChunk(boolean appendToNewBlock)
+      throws IOException {
+    final Path p = new Path("/partialChunk/foo"
+        + (appendToNewBlock ? "0" : "1"));
     final int fileLen = 513;
     System.out.println("p=" + p);
     
@@ -336,7 +477,9 @@ public void testAppendToPartialChunk() throws IOException {
     System.out.println("Wrote 1 byte and closed the file " + p);
 
     // append to file
-    stm = fs.append(p);
+    stm = appendToNewBlock ?
+        fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null) :
+        fs.append(p);
     // Append to a partial CRC trunk
     stm.write(fileContents, 1, 1);
     stm.hflush();
@@ -345,7 +488,9 @@ public void testAppendToPartialChunk() throws IOException {
     System.out.println("Append 1 byte and closed the file " + p);
 
     // write the remainder of the file
-    stm = fs.append(p);
+    stm = appendToNewBlock ?
+        fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null) :
+        fs.append(p);
 
     // ensure getPos is set to reflect existing size of the file
     assertEquals(2, stm.getPos());
@@ -444,4 +589,14 @@ public void run() {
     // if append was called with a stale file stat.
     doSmallAppends(file, fs, 20);
   }
+
+  @Test
+  public void testAppendToPartialChunk() throws IOException {
+    testAppendToPartialChunk(false);
+  }
+
+  @Test
+  public void testAppendToPartialChunkforAppend2() throws IOException {
+    testAppendToPartialChunk(true);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
index 32a41966dc3a9..ca25018fc9d51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java
@@ -45,10 +45,7 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -72,11 +69,9 @@ public class TestFileAppend4 {
   final boolean simulatedStorage = false;
 
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
   }
 
   @Before
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
index 0bca23d1edc8e..a2b344cb65b94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
@@ -99,10 +99,11 @@ public void testAppendRestart() throws Exception {
       // OP_ADD to create file
       // OP_ADD_BLOCK for first block
       // OP_CLOSE to close file
-      // OP_ADD to reopen file
+      // OP_APPEND to reopen file
       // OP_ADD_BLOCK for second block
       // OP_CLOSE to close file
-      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+      assertEquals(1, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+      assertEquals(1, (int)counts.get(FSEditLogOpCodes.OP_APPEND).held);
       assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_ADD_BLOCK).held);
       assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
 
@@ -112,13 +113,14 @@ public void testAppendRestart() throws Exception {
       // OP_ADD to create file
       // OP_ADD_BLOCK for first block
       // OP_CLOSE to close file
-      // OP_ADD to re-establish the lease
+      // OP_APPEND to re-establish the lease
       // OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block)
       // OP_ADD_BLOCK at the start of the second block
       // OP_CLOSE to close file
       // Total: 2 OP_ADDs, 1 OP_UPDATE_BLOCKS, 2 OP_ADD_BLOCKs, and 2 OP_CLOSEs
        //       in addition to the ones above
-      assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_ADD).held);
+      assertEquals(2, (int)counts.get(FSEditLogOpCodes.OP_APPEND).held);
       assertEquals(1, (int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
       assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_ADD_BLOCK).held);
       assertEquals(2+2, (int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
index a7c6a69ac9be4..7d3946ac78abc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
@@ -29,9 +29,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.fs.FileSystem;
@@ -39,27 +36,27 @@
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
 import org.apache.log4j.Level;
 import org.junit.Test;
+import org.slf4j.Logger;
 
 /**
  * A JUnit test for corrupted file handling.
  */
 public class TestFileCorruption {
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
   }
-  static Log LOG = ((Log4JLogger)NameNode.stateChangeLog);
+  static Logger LOG = NameNode.stateChangeLog;
 
   /** check if DFS can handle corrupted blocks properly */
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index 3a399f3f5eb61..a0129da1f4a4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -64,7 +64,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -85,7 +84,6 @@
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
@@ -405,8 +403,7 @@ public FileSystem run() throws Exception {
         fs2.create(p, false);
         fail("Did not throw!");
       } catch (IOException abce) {
-        GenericTestUtils.assertExceptionContains("already being created by",
-            abce);
+        GenericTestUtils.assertExceptionContains("Failed to CREATE_FILE", abce);
       }
       // NameNodeProxies' createNNProxyWithClientProtocol has 5 retries.
       assertCounter("AlreadyBeingCreatedExceptionNumOps",
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationDelete.java
index 46569c76fce09..47ce947cf13bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationDelete.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationDelete.java
@@ -20,23 +20,16 @@
 
 import java.io.IOException;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.log4j.Level;
 import org.junit.Test;
 
 public class TestFileCreationDelete {
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 2af86bd6bb048..cc898527882c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -167,9 +167,7 @@ public void testReadSelectNonStaleDatanode() throws Exception {
       if (stm != null) {
         stm.close();
       }
-      if (client != null) {
-        client.close();
-      }
+      client.close();
       cluster.shutdown();
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
index 9ada95f91af64..a33ad18c6b38b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
@@ -31,7 +31,9 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.log4j.Level;
 import org.junit.Test;
 
@@ -121,7 +123,66 @@ public void hSyncUpdateLength_00() throws IOException {
       cluster.shutdown();
     }
   }
-  
+
+  /**
+   * Test hsync with END_BLOCK flag.
+   */
+  @Test
+  public void hSyncEndBlock_00() throws IOException {
+    final int preferredBlockSize = 1024;
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, preferredBlockSize);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
+        .build();
+    DistributedFileSystem fileSystem = cluster.getFileSystem();
+    FSDataOutputStream stm = null;
+    try {
+      Path path = new Path("/" + fName);
+      stm = fileSystem.create(path, true, 4096, (short) 2,
+          AppendTestUtil.BLOCK_SIZE);
+      System.out.println("Created file " + path.toString());
+      ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
+          .of(SyncFlag.END_BLOCK));
+      long currentFileLength = fileSystem.getFileStatus(path).getLen();
+      assertEquals(0L, currentFileLength);
+      LocatedBlocks blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
+      assertEquals(0, blocks.getLocatedBlocks().size());
+
+      // write a block and call hsync(end_block) at the block boundary
+      stm.write(new byte[preferredBlockSize]);
+      ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
+          .of(SyncFlag.END_BLOCK));
+      currentFileLength = fileSystem.getFileStatus(path).getLen();
+      assertEquals(preferredBlockSize, currentFileLength);
+      blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
+      assertEquals(1, blocks.getLocatedBlocks().size());
+
+      // call hsync then call hsync(end_block) immediately
+      stm.write(new byte[preferredBlockSize / 2]);
+      stm.hsync();
+      ((DFSOutputStream) stm.getWrappedStream()).hsync(EnumSet
+          .of(SyncFlag.END_BLOCK));
+      currentFileLength = fileSystem.getFileStatus(path).getLen();
+      assertEquals(preferredBlockSize + preferredBlockSize / 2,
+          currentFileLength);
+      blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
+      assertEquals(2, blocks.getLocatedBlocks().size());
+
+      stm.write(new byte[preferredBlockSize / 4]);
+      stm.hsync();
+      currentFileLength = fileSystem.getFileStatus(path).getLen();
+      assertEquals(preferredBlockSize + preferredBlockSize / 2
+          + preferredBlockSize / 4, currentFileLength);
+      blocks = fileSystem.dfs.getLocatedBlocks(path.toString(), 0);
+      assertEquals(3, blocks.getLocatedBlocks().size());
+    } finally {
+      IOUtils.cleanup(null, stm, fileSystem);
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
   /**
    * The test calls
    * {@link #doTheJob(Configuration, String, long, short, boolean, EnumSet)}
@@ -133,6 +194,29 @@ public void hSyncUpdateLength_01() throws IOException {
         (short) 2, true, EnumSet.of(SyncFlag.UPDATE_LENGTH));
   }
 
+  /**
+   * The test calls
+   * {@link #doTheJob(Configuration, String, long, short, boolean, EnumSet)}
+   * while requiring the semantic of {@link SyncFlag#END_BLOCK}.
+   */
+  @Test
+  public void hSyncEndBlock_01() throws IOException {
+    doTheJob(new HdfsConfiguration(), fName, AppendTestUtil.BLOCK_SIZE,
+        (short) 2, true, EnumSet.of(SyncFlag.END_BLOCK));
+  }
+
+  /**
+   * The test calls
+   * {@link #doTheJob(Configuration, String, long, short, boolean, EnumSet)}
+   * while requiring the semantic of {@link SyncFlag#END_BLOCK} and
+   * {@link SyncFlag#UPDATE_LENGTH}.
+   */
+  @Test
+  public void hSyncEndBlockAndUpdateLength() throws IOException {
+    doTheJob(new HdfsConfiguration(), fName, AppendTestUtil.BLOCK_SIZE,
+        (short) 2, true, EnumSet.of(SyncFlag.END_BLOCK, SyncFlag.UPDATE_LENGTH));
+  }
+
   /**
    * The test calls
    * {@link #doTheJob(Configuration, String, long, short, boolean, EnumSet)}
@@ -152,7 +236,20 @@ public void hSyncUpdateLength_02() throws IOException {
     doTheJob(conf, fName, customBlockSize, (short) 2, true,
         EnumSet.of(SyncFlag.UPDATE_LENGTH));
   }
-  
+
+  @Test
+  public void hSyncEndBlock_02() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    int customPerChecksumSize = 512;
+    int customBlockSize = customPerChecksumSize * 3;
+    // Modify defaul filesystem settings
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, customPerChecksumSize);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, customBlockSize);
+
+    doTheJob(conf, fName, customBlockSize, (short) 2, true,
+        EnumSet.of(SyncFlag.END_BLOCK));
+  }
+
   /**
    * The test calls
    * {@link #doTheJob(Configuration, String, long, short, boolean, EnumSet)}
@@ -173,7 +270,20 @@ public void hSyncUpdateLength_03() throws IOException {
     doTheJob(conf, fName, customBlockSize, (short) 2, true,
         EnumSet.of(SyncFlag.UPDATE_LENGTH));
   }
-  
+
+  @Test
+  public void hSyncEndBlock_03() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    int customPerChecksumSize = 400;
+    int customBlockSize = customPerChecksumSize * 3;
+    // Modify defaul filesystem settings
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, customPerChecksumSize);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, customBlockSize);
+
+    doTheJob(conf, fName, customBlockSize, (short) 2, true,
+        EnumSet.of(SyncFlag.END_BLOCK));
+  }
+
   /**
    * The method starts new cluster with defined Configuration; creates a file
    * with specified block_size and writes 10 equal sections in it; it also calls
@@ -197,12 +307,13 @@ public static void doTheJob(Configuration conf, final String fileName,
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
                                                .numDataNodes(replicas).build();
     // Make sure we work with DFS in order to utilize all its functionality
-    DistributedFileSystem fileSystem =
-        cluster.getFileSystem();
+    DistributedFileSystem fileSystem = cluster.getFileSystem();
 
     FSDataInputStream is;
     try {
       Path path = new Path(fileName);
+      final String pathName = new Path(fileSystem.getWorkingDirectory(), path)
+          .toUri().getPath();
       FSDataOutputStream stm = fileSystem.create(path, false, 4096, replicas,
           block_size);
       System.out.println("Created file " + fileName);
@@ -210,7 +321,8 @@ public static void doTheJob(Configuration conf, final String fileName,
       int tenth = AppendTestUtil.FILE_SIZE/SECTIONS;
       int rounding = AppendTestUtil.FILE_SIZE - tenth * SECTIONS;
       for (int i=0; i<SECTIONS; i++) {
-        System.out.println("Writing " + (tenth * i) + " to " + (tenth * (i+1)) + " section to file " + fileName);
+        System.out.println("Writing " + (tenth * i) + " to "
+            + (tenth * (i + 1)) + " section to file " + fileName);
         // write to the file
         stm.write(fileContent, tenth * i, tenth);
         
@@ -227,7 +339,11 @@ public static void doTheJob(Configuration conf, final String fileName,
           assertEquals(
             "File size doesn't match for hsync/hflush with updating the length",
             tenth * (i + 1), currentFileLength);
+        } else if (isSync && syncFlags.contains(SyncFlag.END_BLOCK)) {
+          LocatedBlocks blocks = fileSystem.dfs.getLocatedBlocks(pathName, 0);
+          assertEquals(i + 1, blocks.getLocatedBlocks().size());
         }
+
         byte [] toRead = new byte[tenth];
         byte [] expected = new byte[tenth];
         System.arraycopy(fileContent, tenth * i, expected, 0, tenth);
@@ -374,8 +490,8 @@ public void testHFlushInterrupted() throws Exception {
 
 
       // verify that entire file is good
-      AppendTestUtil.checkFullFile(fs, p, 4,
-        fileContents, "Failed to deal with thread interruptions");
+      AppendTestUtil.checkFullFile(fs, p, 4, fileContents,
+          "Failed to deal with thread interruptions", false);
     } finally {
       cluster.shutdown();
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
index b84989f8765bb..15580a5ae2ce2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
@@ -22,8 +22,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.util.EnumSet;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,6 +42,7 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestInterDatanodeProtocol;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.After;
 import org.junit.Test;
@@ -124,7 +127,8 @@ public void testBlockSynchronization() throws Exception {
     }
 
     DataNode.LOG.info("dfs.dfs.clientName=" + dfs.dfs.clientName);
-    cluster.getNameNodeRpc().append(filestr, dfs.dfs.clientName);
+    cluster.getNameNodeRpc().append(filestr, dfs.dfs.clientName,
+        new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
 
     // expire lease to trigger block recovery.
     waitLeaseRecovery(cluster);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
index 34b2188b4795d..78ac19cd861d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java
@@ -79,7 +79,7 @@ public void testMissingBlocksAlert()
 
       // Corrupt the block
       ExtendedBlock block = DFSTestUtil.getFirstBlock(dfs, corruptFile);
-      assertTrue(TestDatanodeBlockScanner.corruptReplica(block, 0));
+      assertTrue(cluster.corruptReplica(0, block));
 
       // read the file so that the corrupt block is reported to NN
       FSDataInputStream in = dfs.open(corruptFile); 
@@ -124,8 +124,7 @@ public void testMissingBlocksAlert()
       DFSTestUtil.createFile(dfs, replOneFile, fileLen, (short)1, 0);
       ExtendedBlock replOneBlock = DFSTestUtil.getFirstBlock(
           dfs, replOneFile);
-      assertTrue(TestDatanodeBlockScanner.corruptReplica(
-          replOneBlock, 0));
+      assertTrue(cluster.corruptReplica(0, replOneBlock));
 
       // read the file so that the corrupt block is reported to NN
       in = dfs.open(replOneFile);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMultiThreadedHflush.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMultiThreadedHflush.java
index 92c7672f04bbe..a839d85823b62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMultiThreadedHflush.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMultiThreadedHflush.java
@@ -32,12 +32,11 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.metrics2.util.Quantile;
 import org.apache.hadoop.metrics2.util.SampleQuantiles;
+import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
-import com.google.common.base.Stopwatch;
-
 /**
  * This class tests hflushing concurrently from many threads.
  */
@@ -100,10 +99,10 @@ public void run() {
     }
 
     private void doAWrite() throws IOException {
-      Stopwatch sw = new Stopwatch().start();
+      StopWatch sw = new StopWatch().start();
       stm.write(toWrite);
       stm.hflush();
-      long micros = sw.elapsedTime(TimeUnit.MICROSECONDS);
+      long micros = sw.now(TimeUnit.MICROSECONDS);
       quantiles.insert(micros);
     }
   }
@@ -276,12 +275,12 @@ public int run(String args[]) throws Exception {
       int replication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
           DFSConfigKeys.DFS_REPLICATION_DEFAULT);
       
-      Stopwatch sw = new Stopwatch().start();
+      StopWatch sw = new StopWatch().start();
       test.doMultithreadedWrites(conf, p, numThreads, writeSize, numWrites,
           replication);
       sw.stop();
   
-      System.out.println("Finished in " + sw.elapsedMillis() + "ms");
+      System.out.println("Finished in " + sw.now(TimeUnit.MILLISECONDS) + "ms");
       System.out.println("Latency quantiles (in microseconds):\n" +
           test.quantiles);
       return 0;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPipelines.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPipelines.java
index e0bcb0ff6711b..a82f04ff529af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPipelines.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPipelines.java
@@ -26,7 +26,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
@@ -35,8 +34,6 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.datanode.Replica;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.After;
@@ -159,9 +156,8 @@ private static void setConfiguration() {
   }
 
   private static void initLoggers() {
-    ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) DFSClient.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRenameWhileOpen.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
index 6ed675d816509..949fc74de2ace 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRenameWhileOpen.java
@@ -22,25 +22,18 @@
 
 import java.io.IOException;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.log4j.Level;
 import org.junit.Test;
 import org.mockito.Mockito;
 
 public class TestRenameWhileOpen {
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   //TODO: un-comment checkFullFile once the lease recovery is done
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 23464b24997d3..d116f82d5090c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -349,7 +349,6 @@ public void testPendingReplicationRetry() throws IOException {
           0, Long.MAX_VALUE).get(0).getBlock();
       
       cluster.shutdown();
-      cluster = null;
       
       for (int i=0; i<25; i++) {
         buffer[i] = '0';
@@ -358,7 +357,7 @@ public void testPendingReplicationRetry() throws IOException {
       int fileCount = 0;
       // Choose 3 copies of block file - delete 1 and corrupt the remaining 2
       for (int dnIndex=0; dnIndex<3; dnIndex++) {
-        File blockFile = MiniDFSCluster.getBlockFile(dnIndex, block);
+        File blockFile = cluster.getBlockFile(dnIndex, block);
         LOG.info("Checking for file " + blockFile);
         
         if (blockFile != null && blockFile.exists()) {
@@ -445,7 +444,7 @@ private void changeBlockLen(MiniDFSCluster cluster, int lenDelta)
 
     // Change the length of a replica
     for (int i=0; i<cluster.getDataNodes().size(); i++) {
-      if (TestDatanodeBlockScanner.changeReplicaLength(block, i, lenDelta)) {
+      if (DFSTestUtil.changeReplicaLength(cluster, block, i, lenDelta)) {
         break;
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 5c78dbe304ab3..2d5bef2b6abf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -292,9 +293,13 @@ public void runFsFun(String msg, FSRun f) {
     try {
       f.run(fs);
       fail(msg);
-     } catch (IOException ioe) {
-       assertTrue(ioe.getMessage().contains("safe mode"));
-     }
+    } catch (RemoteException re) {
+      assertEquals(SafeModeException.class.getName(), re.getClassName());
+      GenericTestUtils.assertExceptionContains(
+          "Name node is in safe mode", re);
+    } catch (IOException ioe) {
+      fail(msg + " " + StringUtils.stringifyException(ioe));
+    }
   }
 
   /**
@@ -341,6 +346,12 @@ public void run(FileSystem fs) throws IOException {
         DFSTestUtil.appendFile(fs, file1, "new bytes");
       }});
 
+    runFsFun("Truncate file while in SM", new FSRun() {
+      @Override
+      public void run(FileSystem fs) throws IOException {
+        fs.truncate(file1, 0);
+      }});
+
     runFsFun("Delete file while in SM", new FSRun() {
       @Override
       public void run(FileSystem fs) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
index b7fdccfb33ef6..7f318df30341f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java
@@ -97,7 +97,7 @@ public FsDatasetChecker(DataStorage storage, Configuration conf) {
      * correctly propagate the hint to FsDatasetSpi.
      */
     @Override
-    public synchronized ReplicaInPipelineInterface createRbw(
+    public synchronized ReplicaHandler createRbw(
         StorageType storageType, ExtendedBlock b, boolean allowLazyPersist)
         throws IOException {
       assertThat(b.getLocalBlock().getNumBytes(), is(EXPECTED_BLOCK_LENGTH));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
index bbaf3ed0e3e4f..dac26a08b9890 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
@@ -476,7 +476,7 @@ public static void createDataNodeVersionFile(File[] parent,
     for (int i = 0; i < parent.length; i++) {
       File versionFile = new File(parent[i], "VERSION");
       StorageDirectory sd = new StorageDirectory(parent[i].getParentFile());
-      storage.createStorageID(sd);
+      storage.createStorageID(sd, false);
       storage.writeProperties(versionFile, sd);
       versionFiles[i] = versionFile;
       File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
index 6a87cdfa8d610..64ca3faf8ddb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/TestPacketReceiver.java
@@ -27,6 +27,8 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import com.google.common.primitives.Ints;
+
 import static org.junit.Assert.*;
 
 public class TestPacketReceiver {
@@ -38,7 +40,7 @@ private byte[] prepareFakePacket(byte[] data, byte[] sums) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream dos = new DataOutputStream(baos);
     
-    int packetLen = data.length + sums.length + 4;
+    int packetLen = data.length + sums.length + Ints.BYTES;
     PacketHeader header = new PacketHeader(
         packetLen, OFFSET_IN_BLOCK, SEQNO, false, data.length, false);
     header.write(dos);
@@ -87,6 +89,7 @@ private void doTestReceiveAndMirror(PacketReceiver pr,
     PacketHeader header = pr.getHeader();
     assertEquals(SEQNO, header.getSeqno());
     assertEquals(OFFSET_IN_BLOCK, header.getOffsetInBlock());
+    assertEquals(dataLen + checksumsLen + Ints.BYTES, header.getPacketLen());
     
     // Mirror the packet to an output stream and make sure it matches
     // the packet we sent.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
index 0d860b4f064eb..52e5384dbcdfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransfer.java
@@ -25,16 +25,22 @@
 
 import java.io.IOException;
 
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -110,12 +116,28 @@ public void testClientAndServerDoNotHaveCommonQop() throws Exception {
   public void testServerSaslNoClientSasl() throws Exception {
     HdfsConfiguration clusterConf = createSecureConfig(
       "authentication,integrity,privacy");
+    // Set short retry timeouts so this test runs faster
+    clusterConf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10);
     startCluster(clusterConf);
     HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
     clientConf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, "");
-    exception.expect(IOException.class);
-    exception.expectMessage("could only be replicated to 0 nodes");
-    doTest(clientConf);
+
+    LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(
+        LogFactory.getLog(DataNode.class));
+    try {
+      doTest(clientConf);
+      Assert.fail("Should fail if SASL data transfer protection is not " +
+          "configured or not supported in client");
+    } catch (IOException e) {
+      GenericTestUtils.assertMatches(e.getMessage(), 
+          "could only be replicated to 0 nodes");
+    } finally {
+      logs.stopCapturing();
+    }
+
+    GenericTestUtils.assertMatches(logs.getOutput(),
+        "Failed to read expected SASL data transfer protection " +
+        "handshake from client at");
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
index 1bf3add16de91..5b5c450be37d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
@@ -27,6 +27,7 @@
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
@@ -38,8 +39,6 @@
 import org.apache.hadoop.hdfs.qjournal.client.IPCLoggerChannel;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
-import org.apache.hadoop.hdfs.qjournal.server.Journal;
-import org.apache.hadoop.hdfs.qjournal.server.JournalNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -48,12 +47,12 @@
 import org.apache.hadoop.test.MetricsAsserts;
 import org.apache.hadoop.test.PathUtils;
 import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StopWatch;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
-import com.google.common.base.Stopwatch;
 import com.google.common.primitives.Bytes;
 import com.google.common.primitives.Ints;
 
@@ -320,11 +319,11 @@ private void doPerfTest(int editsSize, int numEdits) throws Exception {
     ch.setEpoch(1);
     ch.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION).get();
     
-    Stopwatch sw = new Stopwatch().start();
+    StopWatch sw = new StopWatch().start();
     for (int i = 1; i < numEdits; i++) {
       ch.sendEdits(1L, i, 1, data).get();
     }
-    long time = sw.elapsedMillis();
+    long time = sw.now(TimeUnit.MILLISECONDS);
     
     System.err.println("Wrote " + numEdits + " batches of " + editsSize +
         " bytes in " + time + "ms");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
index 6ee6e545416d0..f51757c9e8328 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
@@ -26,7 +26,6 @@
 import java.util.concurrent.TimeoutException;
 
 import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -43,9 +42,6 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.log4j.Level;
 import org.junit.Assert;
 import org.junit.Test;
@@ -57,9 +53,7 @@ public class TestBalancerWithMultipleNameNodes {
   static final Log LOG = Balancer.LOG;
   {
     ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 2755b2989dd02..fccd308e70a1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -238,6 +238,14 @@ public static DatanodeStorageInfo updateStorage(DatanodeDescriptor dn,
     return dn.updateStorage(s);
   }
 
+  /**
+   * Call heartbeat check function of HeartbeatManager
+   * @param bm the BlockManager to manipulate
+   */
+  public static void rescanPostponedMisreplicatedBlocks(BlockManager bm) {
+    bm.rescanPostponedMisreplicatedBlocks();
+  }
+
   public static DatanodeDescriptor getLocalDatanodeDescriptor(
       boolean initializeStorage) {
     DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.getLocalDatanodeID());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 41c8f8a517f77..7425c6a7d9bb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -49,7 +49,7 @@ public class TestBlockInfo {
 
   @Test
   public void testAddStorage() throws Exception {
-    BlockInfo blockInfo = new BlockInfo((short) 3);
+    BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
 
     final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
 
@@ -67,11 +67,11 @@ public void testReplaceStorage() throws Exception {
     final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
     final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
     final int NUM_BLOCKS = 10;
-    BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
+    BlockInfoContiguous[] blockInfos = new BlockInfoContiguous[NUM_BLOCKS];
 
     // Create a few dummy blocks and add them to the first storage.
     for (int i = 0; i < NUM_BLOCKS; ++i) {
-      blockInfos[i] = new BlockInfo((short) 3);
+      blockInfos[i] = new BlockInfoContiguous((short) 3);
       storage1.addBlock(blockInfos[i]);
     }
 
@@ -90,14 +90,14 @@ public void testBlockListMoveToHead() throws Exception {
 
     DatanodeStorageInfo dd = DFSTestUtil.createDatanodeStorageInfo("s1", "1.1.1.1");
     ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
-    ArrayList<BlockInfo> blockInfoList = new ArrayList<BlockInfo>();
+    ArrayList<BlockInfoContiguous> blockInfoList = new ArrayList<BlockInfoContiguous>();
     int headIndex;
     int curIndex;
 
     LOG.info("Building block list...");
     for (int i = 0; i < MAX_BLOCKS; i++) {
       blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
-      blockInfoList.add(new BlockInfo(blockList.get(i), (short) 3));
+      blockInfoList.add(new BlockInfoContiguous(blockList.get(i), (short) 3));
       dd.addBlock(blockInfoList.get(i));
 
       // index of the datanode should be 0
@@ -108,7 +108,7 @@ public void testBlockListMoveToHead() throws Exception {
     // list length should be equal to the number of blocks we inserted
     LOG.info("Checking list length...");
     assertEquals("Length should be MAX_BLOCK", MAX_BLOCKS, dd.numBlocks());
-    Iterator<BlockInfo> it = dd.getBlockIterator();
+    Iterator<BlockInfoContiguous> it = dd.getBlockIterator();
     int len = 0;
     while (it.hasNext()) {
       it.next();
@@ -130,7 +130,7 @@ public void testBlockListMoveToHead() throws Exception {
     // move head of the list to the head - this should not change the list
     LOG.info("Moving head to the head...");
 
-    BlockInfo temp = dd.getBlockListHeadForTesting();
+    BlockInfoContiguous temp = dd.getBlockListHeadForTesting();
     curIndex = 0;
     headIndex = 0;
     dd.moveBlockToHead(temp, curIndex, headIndex);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
index 4c3644809a061..453f411ac21e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
@@ -39,7 +39,7 @@ public void testInitializeBlockRecovery() throws Exception {
     DatanodeDescriptor dd3 = s3.getDatanodeDescriptor();
 
     dd1.isAlive = dd2.isAlive = dd3.isAlive = true;
-    BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
+    BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction(
         new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP),
         (short) 3,
         BlockUCState.UNDER_CONSTRUCTION,
@@ -51,7 +51,7 @@ public void testInitializeBlockRecovery() throws Exception {
     dd2.setLastUpdate(currentTime - 1 * 1000);
     dd3.setLastUpdate(currentTime - 2 * 1000);
     blockInfo.initializeBlockRecovery(1);
-    BlockInfoUnderConstruction[] blockInfoRecovery = dd2.getLeaseRecoveryCommand(1);
+    BlockInfoContiguousUnderConstruction[] blockInfoRecovery = dd2.getLeaseRecoveryCommand(1);
     assertEquals(blockInfoRecovery[0], blockInfo);
 
     // Recovery attempt #2.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInitialEncoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInitialEncoding.java
new file mode 100644
index 0000000000000..a84f67b232b0c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInitialEncoding.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.EC_STORAGE_POLICY_NAME;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.EC_STORAGE_POLICY_ID;
+import static org.junit.Assert.assertEquals;
+
+public class TestBlockInitialEncoding {
+  private final int NUM_OF_DATANODES = 3;
+  private Configuration conf;
+  private MiniDFSCluster cluster;
+  private DistributedFileSystem fs;
+  private static final int BLOCK_SIZE = 1024;
+  private HdfsAdmin dfsAdmin;
+  private FSNamesystem namesystem;
+
+  @Before
+  public void setupCluster() throws IOException {
+    conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    cluster = new MiniDFSCluster.Builder(conf).
+        numDataNodes(NUM_OF_DATANODES).build();
+    cluster.waitActive();
+    fs = cluster.getFileSystem();
+    dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
+    namesystem = cluster.getNamesystem();
+  }
+
+  @After
+  public void shutdownCluster() throws IOException {
+    cluster.shutdown();
+  }
+
+  @Test
+  public void testBlockInitialEncoding()
+      throws IOException, InterruptedException {
+    final Path testDir = new Path("/test");
+    fs.mkdir(testDir, FsPermission.getDirDefault());
+    dfsAdmin.setStoragePolicy(testDir, EC_STORAGE_POLICY_NAME);
+    final Path ECFilePath = new Path("/test/foo.ec");
+    DFSTestUtil.createFile(fs, ECFilePath, 4 * BLOCK_SIZE, (short) 3, 0);
+    INode inode = namesystem.getFSDirectory().getINode(ECFilePath.toString());
+    assertEquals(EC_STORAGE_POLICY_ID, inode.getStoragePolicyID());
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 3df890ff07d25..c7dfcf96a8e7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -141,7 +141,7 @@ public void testBasicReplication() throws Exception {
   private void doBasicTest(int testIndex) {
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
 
     DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo);
     assertEquals(2, pipeline.length);
@@ -173,7 +173,7 @@ private void doTestTwoOfThreeNodesDecommissioned(int testIndex) throws Exception
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission two of the nodes (A1, A2)
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1);
@@ -217,7 +217,7 @@ private void doTestAllNodesHoldingReplicasDecommissioned(int testIndex) throws E
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission all of the nodes
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 3);
@@ -270,7 +270,7 @@ private void doTestOneOfTwoRacksDecommissioned(int testIndex) throws Exception {
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission all of the nodes in rack A
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 2);
@@ -329,7 +329,7 @@ public void testSufficientlyReplBlocksUsesNewRack() throws Exception {
   private void doTestSufficientlyReplBlocksUsesNewRack(int testIndex) {
     // Originally on only nodes in rack A.
     List<DatanodeDescriptor> origNodes = rackA;
-    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
     DatanodeStorageInfo pipeline[] = scheduleSingleReplication(blockInfo);
     
     assertEquals(2, pipeline.length); // single new copy
@@ -372,7 +372,7 @@ private void doTestSingleRackClusterIsSufficientlyReplicated(int testIndex,
    * Tell the block manager that replication is completed for the given
    * pipeline.
    */
-  private void fulfillPipeline(BlockInfo blockInfo,
+  private void fulfillPipeline(BlockInfoContiguous blockInfo,
       DatanodeStorageInfo[] pipeline) throws IOException {
     for (int i = 1; i < pipeline.length; i++) {
       DatanodeStorageInfo storage = pipeline[i];
@@ -381,9 +381,9 @@ private void fulfillPipeline(BlockInfo blockInfo,
     }
   }
 
-  private BlockInfo blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
+  private BlockInfoContiguous blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
     Block block = new Block(blkId);
-    BlockInfo blockInfo = new BlockInfo(block, (short) 3);
+    BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3);
 
     for (DatanodeDescriptor dn : nodes) {
       for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
@@ -425,10 +425,10 @@ private List<DatanodeDescriptor> startDecommission(int ... indexes) {
     return nodes;
   }
   
-  private BlockInfo addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
+  private BlockInfoContiguous addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     Mockito.doReturn((short)3).when(bc).getBlockReplication();
-    BlockInfo blockInfo = blockOnNodes(blockId, nodes);
+    BlockInfoContiguous blockInfo = blockOnNodes(blockId, nodes);
 
     bm.blocksMap.addBlockCollection(blockInfo, bc);
     return blockInfo;
@@ -571,11 +571,13 @@ public void testSafeModeIBR() throws Exception {
     reset(node);
     bm.getDatanodeManager().registerDatanode(nodeReg);
     verify(node).updateRegInfo(nodeReg);
-    assertEquals(0, ds.getBlockReportCount()); // ready for report again
     // send block report, should be processed after restart
     reset(node);
     bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
-        new BlockListAsLongs(null, null));
+                     new BlockListAsLongs(null, null));
+    // Reinitialize as registration with empty storage list pruned
+    // node.storageMap.
+    ds = node.getStorageInfos()[0];
     assertEquals(1, ds.getBlockReportCount());
   }
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index 1ee4b250b200b..1bc7cdce4ff20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -209,7 +209,7 @@ public void testCorruptBlockRereplicatedAcrossRacks() throws Exception {
 
       // Corrupt a replica of the block
       int dnToCorrupt = DFSTestUtil.firstDnWithBlock(cluster, b);
-      assertTrue(MiniDFSCluster.corruptReplica(dnToCorrupt, b));
+      assertTrue(cluster.corruptReplica(dnToCorrupt, b));
 
       // Restart the datanode so blocks are re-scanned, and the corrupt
       // block is detected.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
index fe639e4b2d8ce..7cdb42376a54c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
@@ -58,8 +58,8 @@ public void testGetInvalidateBlocks() throws Exception {
   public void testBlocksCounter() throws Exception {
     DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true);
     assertEquals(0, dd.numBlocks());
-    BlockInfo blk = new BlockInfo(new Block(1L), (short) 1);
-    BlockInfo blk1 = new BlockInfo(new Block(2L), (short) 2);
+    BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short) 1);
+    BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short) 2);
     DatanodeStorageInfo[] storages = dd.getStorageInfos();
     assertTrue(storages.length > 0);
     // add first block
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
index 988a0ed7713d9..efd1febbda35d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
@@ -171,7 +171,7 @@ public void testHeartbeatBlockRecovery() throws Exception {
               dd1.getStorageInfos()[0],
               dd2.getStorageInfos()[0],
               dd3.getStorageInfos()[0]};
-          BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
+          BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction(
               new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), (short) 3,
               BlockUCState.UNDER_RECOVERY, storages);
           dd1.addBlockToBeRecovered(blockInfo);
@@ -193,7 +193,7 @@ public void testHeartbeatBlockRecovery() throws Exception {
           // More than the default stale interval of 30 seconds.
           dd2.setLastUpdate(System.currentTimeMillis() - 40 * 1000);
           dd3.setLastUpdate(System.currentTimeMillis());
-          blockInfo = new BlockInfoUnderConstruction(
+          blockInfo = new BlockInfoContiguousUnderConstruction(
               new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), (short) 3,
               BlockUCState.UNDER_RECOVERY, storages);
           dd1.addBlockToBeRecovered(blockInfo);
@@ -214,7 +214,7 @@ public void testHeartbeatBlockRecovery() throws Exception {
           // More than the default stale interval of 30 seconds.
           dd2.setLastUpdate(System.currentTimeMillis() - 40 * 1000);
           dd3.setLastUpdate(System.currentTimeMillis() - 80 * 1000);
-          blockInfo = new BlockInfoUnderConstruction(
+          blockInfo = new BlockInfoContiguousUnderConstruction(
               new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), (short) 3,
               BlockUCState.UNDER_RECOVERY, storages);
           dd1.addBlockToBeRecovered(blockInfo);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
new file mode 100644
index 0000000000000..88300de3c2a04
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.stat.inference.TestUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+
+public class TestNameNodePrunesMissingStorages {
+  static final Log LOG = LogFactory.getLog(TestNameNodePrunesMissingStorages.class);
+
+
+  private static void runTest(final String testCaseName,
+                              final boolean createFiles,
+                              final int numInitialStorages,
+                              final int expectedStoragesAfterTest) throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+
+    try {
+      cluster = new MiniDFSCluster
+          .Builder(conf)
+          .numDataNodes(1)
+          .storagesPerDatanode(numInitialStorages)
+          .build();
+      cluster.waitActive();
+
+      final DataNode dn0 = cluster.getDataNodes().get(0);
+
+      // Ensure NN knows about the storage.
+      final DatanodeID dnId = dn0.getDatanodeId();
+      final DatanodeDescriptor dnDescriptor =
+          cluster.getNamesystem().getBlockManager().getDatanodeManager().getDatanode(dnId);
+      assertThat(dnDescriptor.getStorageInfos().length, is(numInitialStorages));
+
+      final String bpid = cluster.getNamesystem().getBlockPoolId();
+      final DatanodeRegistration dnReg = dn0.getDNRegistrationForBP(bpid);
+      DataNodeTestUtils.triggerBlockReport(dn0);
+
+      if (createFiles) {
+        final Path path = new Path("/", testCaseName);
+        DFSTestUtil.createFile(
+            cluster.getFileSystem(), path, 1024, (short) 1, 0x1BAD5EED);
+        DataNodeTestUtils.triggerBlockReport(dn0);
+      }
+
+      // Generate a fake StorageReport that is missing one storage.
+      final StorageReport reports[] =
+          dn0.getFSDataset().getStorageReports(bpid);
+      final StorageReport prunedReports[] = new StorageReport[numInitialStorages - 1];
+      System.arraycopy(reports, 0, prunedReports, 0, prunedReports.length);
+
+      // Stop the DataNode and send fake heartbeat with missing storage.
+      cluster.stopDataNode(0);
+      cluster.getNameNodeRpc().sendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, 0);
+
+      // Check that the missing storage was pruned.
+      assertThat(dnDescriptor.getStorageInfos().length, is(expectedStoragesAfterTest));
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
+  /**
+   * Test that the NameNode prunes empty storage volumes that are no longer
+   * reported by the DataNode.
+   * @throws IOException
+   */
+  @Test (timeout=300000)
+  public void testUnusedStorageIsPruned() throws IOException {
+    // Run the test with 1 storage, after the text expect 0 storages.
+    runTest(GenericTestUtils.getMethodName(), false, 1, 0);
+  }
+
+  /**
+   * Verify that the NameNode does not prune storages with blocks.
+   * @throws IOException
+   */
+  @Test (timeout=300000)
+  public void testStorageWithBlocksIsNotPruned() throws IOException {
+    // Run the test with 1 storage, after the text still expect 1 storage.
+    runTest(GenericTestUtils.getMethodName(), true, 1, 1);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
index 1c3f75a68d4b0..2942d0fb1843f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
@@ -34,7 +34,6 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
-import org.apache.hadoop.hdfs.TestDatanodeBlockScanner;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -54,6 +53,7 @@ public class TestOverReplicatedBlocks {
   @Test
   public void testProcesOverReplicateBlock() throws Exception {
     Configuration conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
     conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
     conf.set(
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,
@@ -68,16 +68,17 @@ public void testProcesOverReplicateBlock() throws Exception {
       
       // corrupt the block on datanode 0
       ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
-      assertTrue(TestDatanodeBlockScanner.corruptReplica(block, 0));
+      assertTrue(cluster.corruptReplica(0, block));
       DataNodeProperties dnProps = cluster.stopDataNode(0);
       // remove block scanner log to trigger block scanning
-      File scanLog = new File(MiniDFSCluster.getFinalizedDir(
+      File scanCursor = new File(new File(MiniDFSCluster.getFinalizedDir(
           cluster.getInstanceStorageDir(0, 0),
-          cluster.getNamesystem().getBlockPoolId()).getParent().toString()
-          + "/../dncp_block_verification.log.prev");
+          cluster.getNamesystem().getBlockPoolId()).getParent()).getParent(),
+          "scanner.cursor");
       //wait for one minute for deletion to succeed;
-      for(int i=0; !scanLog.delete(); i++) {
-        assertTrue("Could not delete log file in one minute", i < 60);
+      for(int i = 0; !scanCursor.delete(); i++) {
+        assertTrue("Could not delete " + scanCursor.getAbsolutePath() +
+            " in one minute", i < 60);
         try {
           Thread.sleep(1000);
         } catch (InterruptedException ignored) {}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
index e10d8243ab95b..84ae771fbdac5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
@@ -19,7 +19,6 @@
 
 import java.text.SimpleDateFormat;
 
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.Path;
@@ -29,6 +28,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.Assert;
@@ -42,7 +42,7 @@
  */
 public class TestPendingInvalidateBlock {
   {
-    ((Log4JLogger)BlockManager.LOG).getLogger().setLevel(Level.DEBUG);    
+    GenericTestUtils.setLogLevel(BlockManager.LOG, Level.DEBUG);
   }
 
   private static final int BLOCKSIZE = 1024;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index e4834d658cb1d..1c008acc25494 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1167,7 +1167,7 @@ public void testAddStoredBlockDoesNotCauseSkippedReplication()
     // block under construction, the BlockManager will realize the expected
     // replication has been achieved and remove it from the under-replicated
     // queue.
-    BlockInfoUnderConstruction info = new BlockInfoUnderConstruction(block1, (short) 1);
+    BlockInfoContiguousUnderConstruction info = new BlockInfoContiguousUnderConstruction(block1, (short) 1);
     BlockCollection bc = mock(BlockCollection.class);
     when(bc.getBlockReplication()).thenReturn((short)1);
     bm.addBlockCollection(info, bc);
@@ -1209,7 +1209,7 @@ public void testAddStoredBlockDoesNotCauseSkippedReplication()
     chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
     assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
 
-    final BlockInfo info = new BlockInfo(block1, (short) 1);
+    final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
     final BlockCollection mbc = mock(BlockCollection.class);
     when(mbc.getLastBlock()).thenReturn(info);
     when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
@@ -1223,7 +1223,7 @@ public void testAddStoredBlockDoesNotCauseSkippedReplication()
 
     DatanodeStorageInfo[] storageAry = {new DatanodeStorageInfo(
         dataNodes[0], new DatanodeStorage("s1"))};
-    final BlockInfoUnderConstruction ucBlock =
+    final BlockInfoContiguousUnderConstruction ucBlock =
         info.convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION,
             storageAry);
     DatanodeStorageInfo storage = mock(DatanodeStorageInfo.class);
@@ -1231,15 +1231,15 @@ public void testAddStoredBlockDoesNotCauseSkippedReplication()
     when(dn.isDecommissioned()).thenReturn(true);
     when(storage.getState()).thenReturn(DatanodeStorage.State.NORMAL);
     when(storage.getDatanodeDescriptor()).thenReturn(dn);
-    when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
-    when(storage.addBlock(any(BlockInfo.class))).thenReturn
+    when(storage.removeBlock(any(BlockInfoContiguous.class))).thenReturn(true);
+    when(storage.addBlock(any(BlockInfoContiguous.class))).thenReturn
         (DatanodeStorageInfo.AddBlockResult.ADDED);
     ucBlock.addStorage(storage);
 
-    when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
+    when(mbc.setLastBlock((BlockInfoContiguous) any(), (DatanodeStorageInfo[]) any()))
     .thenReturn(ucBlock);
 
-    bm.convertLastBlockToUnderConstruction(mbc);
+    bm.convertLastBlockToUnderConstruction(mbc, 0L);
 
     // Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
     // from QUEUE_VERY_UNDER_REPLICATED.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
index e9557da229ebc..a22a71e1ce24a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
@@ -30,6 +30,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import org.apache.commons.logging.Log;
@@ -51,7 +52,6 @@
 import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -82,8 +82,8 @@ public abstract class BlockReportTestBase {
 
   private static short REPL_FACTOR = 1;
   private static final int RAND_LIMIT = 2000;
-  private static final long DN_RESCAN_INTERVAL = 5000;
-  private static final long DN_RESCAN_EXTRA_WAIT = 2 * DN_RESCAN_INTERVAL;
+  private static final long DN_RESCAN_INTERVAL = 1;
+  private static final long DN_RESCAN_EXTRA_WAIT = 3 * DN_RESCAN_INTERVAL;
   private static final int DN_N0 = 0;
   private static final int FILE_START = 0;
 
@@ -294,7 +294,7 @@ public void blockReport_02() throws IOException {
       }
     }
 
-    waitTil(DN_RESCAN_EXTRA_WAIT);
+    waitTil(TimeUnit.SECONDS.toMillis(DN_RESCAN_EXTRA_WAIT));
 
     // all blocks belong to the same file, hence same BP
     String poolId = cluster.getNamesystem().getBlockPoolId();
@@ -809,10 +809,9 @@ public boolean accept(File file, String s) {
   }
 
   private static void initLoggers() {
-    ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) BlockReportTestBase.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(BlockReportTestBase.LOG, Level.ALL);
   }
 
   private Block findBlock(Path path, long size) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
index f50afd463255f..fd51e523d9cde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
@@ -113,30 +113,6 @@ public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
     return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf,
         dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname);
   }
-  
-  public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
-    BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
-    bpScanner.verifyBlock(new ExtendedBlock(b.getBlockPoolId(),
-        new BlockPoolSliceScanner.BlockScanInfo(b.getLocalBlock())));
-  }
-
-  private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,
-      ExtendedBlock b) {
-    DataBlockScanner scanner = dn.getBlockScanner();
-    BlockPoolSliceScanner bpScanner = scanner.getBPScanner(b.getBlockPoolId());
-    return bpScanner;
-  }
-
-  public static long getLatestScanTime(DataNode dn, ExtendedBlock b) {
-    BlockPoolSliceScanner scanner = getBlockPoolScanner(dn, b);
-    return scanner.getLastScanTime(b.getLocalBlock());
-  }
-
-  public static void shutdownBlockScanner(DataNode dn) {
-    if (dn.blockScanner != null) {
-      dn.blockScanner.shutdown();
-    }
-  }
 
   /**
    * This method is used for testing. 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index e03b756c21085..6ff4603a04d50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -43,13 +44,12 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
@@ -147,7 +147,7 @@ private class BInfo implements ReplicaInPipelineInterface {
         oStream = null;
       }
     }
-
+    
     @Override
     public String getStorageUuid() {
       return storage.getStorageUuid();
@@ -431,6 +431,11 @@ static class SimulatedVolume implements FsVolumeSpi {
       this.storage = storage;
     }
 
+    @Override
+    public FsVolumeReference obtainReference() throws ClosedChannelException {
+      return null;
+    }
+
     @Override
     public String getStorageID() {
       return storage.getStorageUuid();
@@ -478,6 +483,22 @@ public void reserveSpaceForRbw(long bytesToReserve) {
     @Override
     public void releaseReservedSpace(long bytesToRelease) {
     }
+
+    @Override
+    public BlockIterator newBlockIterator(String bpid, String name) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public BlockIterator loadBlockIterator(String bpid, String name)
+        throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public FsDatasetSpi getDataset() {
+      throw new UnsupportedOperationException();
+    }
   }
 
   private final Map<String, Map<Block, BInfo>> blockMap
@@ -489,7 +510,7 @@ public void releaseReservedSpace(long bytesToRelease) {
   public SimulatedFSDataset(DataStorage storage, Configuration conf) {
     if (storage != null) {
       for (int i = 0; i < storage.getNumStorageDirs(); ++i) {
-        storage.createStorageID(storage.getStorageDir(i));
+        storage.createStorageID(storage.getStorageDir(i), false);
       }
       this.datanodeUuid = storage.getDatanodeUuid();
     } else {
@@ -557,12 +578,12 @@ public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException{
   }
 
   synchronized BlockListAsLongs getBlockReport(String bpid) {
-    final List<Block> blocks = new ArrayList<Block>();
+    final List<Replica> blocks = new ArrayList<Replica>();
     final Map<Block, BInfo> map = blockMap.get(bpid);
     if (map != null) {
       for (BInfo b : map.values()) {
         if (b.isFinalized()) {
-          blocks.add(b.theBlock);
+          blocks.add(b);
         }
       }
     }
@@ -780,8 +801,8 @@ public String toString() {
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipelineInterface append(ExtendedBlock b,
-      long newGS, long expectedBlockLen) throws IOException {
+  public synchronized ReplicaHandler append(
+      ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException {
     final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
     BInfo binfo = map.get(b.getLocalBlock());
     if (binfo == null || !binfo.isFinalized()) {
@@ -789,12 +810,12 @@ public synchronized ReplicaInPipelineInterface append(ExtendedBlock b,
           + " is not valid, and cannot be appended to.");
     }
     binfo.unfinalizeBlock();
-    return binfo;
+    return new ReplicaHandler(binfo, null);
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipelineInterface recoverAppend(ExtendedBlock b,
-      long newGS, long expectedBlockLen) throws IOException {
+  public synchronized ReplicaHandler recoverAppend(
+      ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException {
     final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
     BInfo binfo = map.get(b.getLocalBlock());
     if (binfo == null) {
@@ -807,7 +828,7 @@ public synchronized ReplicaInPipelineInterface recoverAppend(ExtendedBlock b,
     map.remove(b);
     binfo.theBlock.setGenerationStamp(newGS);
     map.put(binfo.theBlock, binfo);
-    return binfo;
+    return new ReplicaHandler(binfo, null);
   }
 
   @Override // FsDatasetSpi
@@ -829,8 +850,9 @@ public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
   }
   
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipelineInterface recoverRbw(ExtendedBlock b,
-      long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException {
+  public synchronized ReplicaHandler recoverRbw(
+      ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
+      throws IOException {
     final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
     BInfo binfo = map.get(b.getLocalBlock());
     if ( binfo == null) {
@@ -844,18 +866,18 @@ public synchronized ReplicaInPipelineInterface recoverRbw(ExtendedBlock b,
     map.remove(b);
     binfo.theBlock.setGenerationStamp(newGS);
     map.put(binfo.theBlock, binfo);
-    return binfo;
+    return new ReplicaHandler(binfo, null);
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipelineInterface createRbw(
+  public synchronized ReplicaHandler createRbw(
       StorageType storageType, ExtendedBlock b,
       boolean allowLazyPersist) throws IOException {
     return createTemporary(storageType, b);
   }
 
   @Override // FsDatasetSpi
-  public synchronized ReplicaInPipelineInterface createTemporary(
+  public synchronized ReplicaHandler createTemporary(
       StorageType storageType, ExtendedBlock b) throws IOException {
     if (isValidBlock(b)) {
           throw new ReplicaAlreadyExistsException("Block " + b + 
@@ -868,7 +890,7 @@ public synchronized ReplicaInPipelineInterface createTemporary(
     final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
     BInfo binfo = new BInfo(b.getBlockPoolId(), b.getLocalBlock(), true);
     map.put(binfo.theBlock, binfo);
-    return binfo;
+    return new ReplicaHandler(binfo, null);
   }
 
   synchronized InputStream getBlockInputStream(ExtendedBlock b
@@ -1106,6 +1128,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
   @Override // FsDatasetSpi
   public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
                                         long recoveryId,
+                                        long newBlockId,
                                         long newlength) {
     // Caller does not care about the exact Storage UUID returned.
     return datanodeUuid;
@@ -1229,11 +1252,6 @@ public Map<String, Object> getVolumeInfoMap() {
     throw new UnsupportedOperationException();
   }
 
-  @Override
-  public RollingLogs createRollingLogs(String bpid, String prefix) {
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public FsVolumeSpi getVolume(ExtendedBlock b) {
     return volume;
@@ -1252,7 +1270,7 @@ public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block,
 
   @Override
   public void onCompleteLazyPersist(String bpId, long blockId,
-      long creationTime, File[] savedFiles, FsVolumeImpl targetVolume) {
+      long creationTime, File[] savedFiles, FsVolumeSpi targetVolume) {
     throw new UnsupportedOperationException();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
index e71c0ea982a8e..1152c74477c47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
@@ -114,7 +114,7 @@ public void testBlockHasMultipleReplicasOnSameDN() throws IOException {
     }
 
     for (int i = 0; i < cluster.getStoragesPerDatanode(); ++i) {
-      BlockListAsLongs bll = new BlockListAsLongs(blocks, null);
+      BlockListAsLongs bll = new BlockListAsLongs(blocks);
       FsVolumeSpi v = dn.getFSDataset().getVolumes().get(i);
       DatanodeStorage dns = new DatanodeStorage(v.getStorageID());
       reports[i] = new StorageBlockReport(dns, bll.getBlockListAsLongs());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index 987b4803cfac2..c7a4084989e8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -56,7 +56,6 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.StorageType;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -219,10 +218,10 @@ private void testSyncReplicas(ReplicaRecoveryInfo replica1,
     syncList.add(record1);
     syncList.add(record2);
     
-    when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), 
-        anyLong())).thenReturn("storage1");
-    when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(), 
-        anyLong())).thenReturn("storage2");
+    when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
+        anyLong(), anyLong())).thenReturn("storage1");
+    when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
+        anyLong(), anyLong())).thenReturn("storage2");
     dn.syncBlock(rBlock, syncList);
   }
   
@@ -245,8 +244,10 @@ public void testFinalizedReplicas () throws IOException {
     InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
-    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);    
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
+    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
 
     // two finalized replicas have different length
     replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
@@ -284,8 +285,10 @@ public void testFinalizedRbwReplicas() throws IOException {
     InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
-    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
+    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
     
     // rbw replica has a different length from the finalized one
     replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
@@ -297,9 +300,10 @@ public void testFinalizedRbwReplicas() throws IOException {
     dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
     verify(dn2, never()).updateReplicaUnderRecovery(
-        block, RECOVERY_ID, REPLICA_LEN1);
+        block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
   }
   
   /**
@@ -323,9 +327,10 @@ public void testFinalizedRwrReplicas() throws IOException {
     InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
     verify(dn2, never()).updateReplicaUnderRecovery(
-        block, RECOVERY_ID, REPLICA_LEN1);
+        block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
     
     // rbw replica has a different length from the finalized one
     replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
@@ -337,9 +342,10 @@ public void testFinalizedRwrReplicas() throws IOException {
     dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
+        REPLICA_LEN1);
     verify(dn2, never()).updateReplicaUnderRecovery(
-        block, RECOVERY_ID, REPLICA_LEN1);
+        block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
   }
   
   /**
@@ -362,8 +368,8 @@ public void testRBWReplicas() throws IOException {
 
     long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
     testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
-    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);    
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
+    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
   }
   
   /**
@@ -385,9 +391,9 @@ public void testRBW_RWRReplicas() throws IOException {
     InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);
 
     testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, REPLICA_LEN1);
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
     verify(dn2, never()).updateReplicaUnderRecovery(
-        block, RECOVERY_ID, REPLICA_LEN1);    
+        block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
   }
   
   /**
@@ -411,8 +417,8 @@ public void testRWRReplicas() throws IOException {
     long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
     testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
     
-    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);
-    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, minLen);    
+    verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
+    verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
   }  
 
   private Collection<RecoveringBlock> initRecoveringBlocks() throws IOException {
@@ -513,7 +519,7 @@ public void testFailedReplicaUpdate() throws IOException {
     }
     DataNode spyDN = spy(dn);
     doThrow(new IOException()).when(spyDN).updateReplicaUnderRecovery(
-        block, RECOVERY_ID, block.getNumBytes());
+        block, RECOVERY_ID, BLOCK_ID, block.getNumBytes());
     try {
       spyDN.syncBlock(rBlock, initBlockRecords(spyDN));
       fail("Sync should fail");
@@ -556,7 +562,7 @@ public void testNotMatchedReplicaID() throws IOException {
       LOG.debug("Running " + GenericTestUtils.getMethodName());
     }
     ReplicaInPipelineInterface replicaInfo = dn.data.createRbw(
-        StorageType.DEFAULT, block, false);
+        StorageType.DEFAULT, block, false).getReplica();
     ReplicaOutputStreams streams = null;
     try {
       streams = replicaInfo.createStreams(true,
@@ -634,7 +640,8 @@ public void run() {
           recoveryInitResult.get());
       
       dataNode.updateReplicaUnderRecovery(block.getBlock(), block.getBlock()
-          .getGenerationStamp() + 1, block.getBlockSize());
+          .getGenerationStamp() + 1, block.getBlock().getBlockId(),
+          block.getBlockSize());
     } finally {
       if (null != cluster) {
         cluster.shutdown();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
new file mode 100644
index 0000000000000..b727263e2a9ce
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
@@ -0,0 +1,684 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND;
+import static org.apache.hadoop.hdfs.server.datanode.BlockScanner.Conf.INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS;
+import static org.apache.hadoop.hdfs.server.datanode.BlockScanner.Conf.INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER;
+import static org.apache.hadoop.hdfs.server.datanode.BlockScanner.Conf.INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.ScanResultHandler;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.BlockIterator;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
+import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.Statistics;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestBlockScanner {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestBlockScanner.class);
+
+  @Before
+  public void before() {
+    BlockScanner.Conf.allowUnitTestSettings = true;
+    GenericTestUtils.setLogLevel(BlockScanner.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(VolumeScanner.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(FsVolumeImpl.LOG, Level.ALL);
+  }
+
+  private static void disableBlockScanner(Configuration conf) {
+    conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND, 0L);
+  }
+
+  private static class TestContext implements Closeable {
+    final int numNameServices;
+    final MiniDFSCluster cluster;
+    final DistributedFileSystem[] dfs;
+    final String[] bpids;
+    final DataNode datanode;
+    final BlockScanner blockScanner;
+    final FsDatasetSpi<? extends FsVolumeSpi> data;
+    final List<? extends FsVolumeSpi> volumes;
+
+    TestContext(Configuration conf, int numNameServices) throws Exception {
+      this.numNameServices = numNameServices;
+      MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).
+          numDataNodes(1).
+          storagesPerDatanode(1);
+      if (numNameServices > 1) {
+        bld.nnTopology(MiniDFSNNTopology.
+              simpleFederatedTopology(numNameServices));
+      }
+      cluster = bld.build();
+      cluster.waitActive();
+      dfs = new DistributedFileSystem[numNameServices];
+      for (int i = 0; i < numNameServices; i++) {
+        dfs[i] = cluster.getFileSystem(i);
+      }
+      bpids = new String[numNameServices];
+      for (int i = 0; i < numNameServices; i++) {
+        bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
+      }
+      datanode = cluster.getDataNodes().get(0);
+      blockScanner = datanode.getBlockScanner();
+      for (int i = 0; i < numNameServices; i++) {
+        dfs[i].mkdirs(new Path("/test"));
+      }
+      data = datanode.getFSDataset();
+      volumes = data.getVolumes();
+    }
+
+    @Override
+    public void close() throws IOException {
+      if (cluster != null) {
+        for (int i = 0; i < numNameServices; i++) {
+          dfs[i].delete(new Path("/test"), true);
+        }
+        cluster.shutdown();
+      }
+    }
+
+    public void createFiles(int nsIdx, int numFiles, int length)
+          throws Exception {
+      for (int blockIdx = 0; blockIdx < numFiles; blockIdx++) {
+        DFSTestUtil.createFile(dfs[nsIdx], getPath(blockIdx), length,
+            (short)1, 123L);
+      }
+    }
+
+    public Path getPath(int fileIdx) {
+      return new Path("/test/" + fileIdx);
+    }
+
+    public ExtendedBlock getFileBlock(int nsIdx, int fileIdx)
+          throws Exception {
+      return DFSTestUtil.getFirstBlock(dfs[nsIdx], getPath(fileIdx));
+    }
+  }
+
+  /**
+   * Test iterating through a bunch of blocks in a volume using a volume
+   * iterator.<p/>
+   *
+   * We will rewind the iterator when about halfway through the blocks.
+   *
+   * @param numFiles        The number of files to create.
+   * @param maxStaleness    The maximum staleness to allow with the iterator.
+   * @throws Exception
+   */
+  private void testVolumeIteratorImpl(int numFiles,
+              long maxStaleness) throws Exception {
+    Configuration conf = new Configuration();
+    disableBlockScanner(conf);
+    TestContext ctx = new TestContext(conf, 1);
+    ctx.createFiles(0, numFiles, 1);
+    assertEquals(1, ctx.volumes.size());
+    FsVolumeSpi volume = ctx.volumes.get(0);
+    ExtendedBlock savedBlock = null, loadedBlock = null;
+    boolean testedRewind = false, testedSave = false, testedLoad = false;
+    int blocksProcessed = 0, savedBlocksProcessed = 0;
+    try {
+      BPOfferService bpos[] = ctx.datanode.getAllBpOs();
+      assertEquals(1, bpos.length);
+      BlockIterator iter = volume.newBlockIterator(ctx.bpids[0], "test");
+      assertEquals(ctx.bpids[0], iter.getBlockPoolId());
+      iter.setMaxStalenessMs(maxStaleness);
+      while (true) {
+        HashSet<ExtendedBlock> blocks = new HashSet<ExtendedBlock>();
+        for (int blockIdx = 0; blockIdx < numFiles; blockIdx++) {
+          blocks.add(ctx.getFileBlock(0, blockIdx));
+        }
+        while (true) {
+          ExtendedBlock block = iter.nextBlock();
+          if (block == null) {
+            break;
+          }
+          blocksProcessed++;
+          LOG.info("BlockIterator for {} found block {}, blocksProcessed = {}",
+              volume, block, blocksProcessed);
+          if (testedSave && (savedBlock == null)) {
+            savedBlock = block;
+          }
+          if (testedLoad && (loadedBlock == null)) {
+            loadedBlock = block;
+            // The block that we get back right after loading the iterator
+            // should be the same block we got back right after saving
+            // the iterator.
+            assertEquals(savedBlock, loadedBlock);
+          }
+          boolean blockRemoved = blocks.remove(block);
+          assertTrue("Found unknown block " + block, blockRemoved);
+          if (blocksProcessed > (numFiles / 3)) {
+            if (!testedSave) {
+              LOG.info("Processed {} blocks out of {}.  Saving iterator.",
+                  blocksProcessed, numFiles);
+              iter.save();
+              testedSave = true;
+              savedBlocksProcessed = blocksProcessed;
+            }
+          }
+          if (blocksProcessed > (numFiles / 2)) {
+            if (!testedRewind) {
+              LOG.info("Processed {} blocks out of {}.  Rewinding iterator.",
+                  blocksProcessed, numFiles);
+              iter.rewind();
+              break;
+            }
+          }
+          if (blocksProcessed > ((2 * numFiles) / 3)) {
+            if (!testedLoad) {
+              LOG.info("Processed {} blocks out of {}.  Loading iterator.",
+                  blocksProcessed, numFiles);
+              iter = volume.loadBlockIterator(ctx.bpids[0], "test");
+              iter.setMaxStalenessMs(maxStaleness);
+              break;
+            }
+          }
+        }
+        if (!testedRewind) {
+          testedRewind = true;
+          blocksProcessed = 0;
+          LOG.info("Starting again at the beginning...");
+          continue;
+        }
+        if (!testedLoad) {
+          testedLoad = true;
+          blocksProcessed = savedBlocksProcessed;
+          LOG.info("Starting again at the load point...");
+          continue;
+        }
+        assertEquals(numFiles, blocksProcessed);
+        break;
+      }
+    } finally {
+      ctx.close();
+    }
+  }
+
+  @Test(timeout=60000)
+  public void testVolumeIteratorWithoutCaching() throws Exception {
+    testVolumeIteratorImpl(5, 0);
+  }
+
+  @Test(timeout=60000)
+  public void testVolumeIteratorWithCaching() throws Exception {
+    testVolumeIteratorImpl(600, 100);
+  }
+
+  @Test(timeout=60000)
+  public void testDisableVolumeScanner() throws Exception {
+    Configuration conf = new Configuration();
+    disableBlockScanner(conf);
+    TestContext ctx = new TestContext(conf, 1);
+    try {
+      Assert.assertFalse(ctx.datanode.getBlockScanner().isEnabled());
+    } finally {
+      ctx.close();
+    }
+  }
+
+  public static class TestScanResultHandler extends ScanResultHandler {
+    static class Info {
+      boolean shouldRun = false;
+      final Set<ExtendedBlock> badBlocks = new HashSet<ExtendedBlock>();
+      final Set<ExtendedBlock> goodBlocks = new HashSet<ExtendedBlock>();
+      long blocksScanned = 0;
+      Semaphore sem = null;
+    }
+
+    private VolumeScanner scanner;
+
+    final static ConcurrentHashMap<String, Info> infos =
+        new ConcurrentHashMap<String, Info>();
+
+    static Info getInfo(FsVolumeSpi volume) {
+      Info newInfo = new Info();
+      Info prevInfo = infos.
+          putIfAbsent(volume.getStorageID(), newInfo);
+      return prevInfo == null ? newInfo : prevInfo;
+    }
+
+    @Override
+    public void setup(VolumeScanner scanner) {
+      this.scanner = scanner;
+      Info info = getInfo(scanner.volume);
+      LOG.info("about to start scanning.");
+      synchronized (info) {
+        while (!info.shouldRun) {
+          try {
+            info.wait();
+          } catch (InterruptedException e) {
+          }
+        }
+      }
+      LOG.info("starting scanning.");
+    }
+
+    @Override
+    public void handle(ExtendedBlock block, IOException e) {
+      LOG.info("handling block {} (exception {})", block, e);
+      Info info = getInfo(scanner.volume);
+      Semaphore sem;
+      synchronized (info) {
+        sem = info.sem;
+      }
+      if (sem != null) {
+        try {
+          sem.acquire();
+        } catch (InterruptedException ie) {
+          throw new RuntimeException("interrupted");
+        }
+      }
+      synchronized (info) {
+        if (!info.shouldRun) {
+          throw new RuntimeException("stopping volumescanner thread.");
+        }
+        if (e == null) {
+          info.goodBlocks.add(block);
+        } else {
+          info.badBlocks.add(block);
+        }
+        info.blocksScanned++;
+      }
+    }
+  }
+
+  private void testScanAllBlocksImpl(final boolean rescan) throws Exception {
+    Configuration conf = new Configuration();
+    conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND, 1048576L);
+    if (rescan) {
+      conf.setLong(INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS, 100L);
+    } else {
+      conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
+    }
+    conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+        TestScanResultHandler.class.getName());
+    final TestContext ctx = new TestContext(conf, 1);
+    final int NUM_EXPECTED_BLOCKS = 10;
+    ctx.createFiles(0, NUM_EXPECTED_BLOCKS, 1);
+    final Set<ExtendedBlock> expectedBlocks = new HashSet<ExtendedBlock>();
+    for (int i = 0; i < NUM_EXPECTED_BLOCKS; i++) {
+      expectedBlocks.add(ctx.getFileBlock(0, i));
+    }
+    TestScanResultHandler.Info info =
+        TestScanResultHandler.getInfo(ctx.volumes.get(0));
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+    GenericTestUtils.waitFor(new Supplier<Boolean>(){
+      @Override
+      public Boolean get() {
+        TestScanResultHandler.Info info =
+            TestScanResultHandler.getInfo(ctx.volumes.get(0));
+        int numFoundBlocks = 0;
+        StringBuilder foundBlocksBld = new StringBuilder();
+        String prefix = "";
+        synchronized (info) {
+          for (ExtendedBlock block : info.goodBlocks) {
+            assertTrue(expectedBlocks.contains(block));
+            numFoundBlocks++;
+            foundBlocksBld.append(prefix).append(block);
+            prefix = ", ";
+          }
+          LOG.info("numFoundBlocks = {}.  blocksScanned = {}. Found blocks {}",
+              numFoundBlocks, info.blocksScanned, foundBlocksBld.toString());
+          if (rescan) {
+            return (numFoundBlocks == NUM_EXPECTED_BLOCKS) &&
+                     (info.blocksScanned >= 2 * NUM_EXPECTED_BLOCKS);
+          } else {
+            return numFoundBlocks == NUM_EXPECTED_BLOCKS;
+          }
+        }
+      }
+    }, 10, 60000);
+    if (!rescan) {
+      synchronized (info) {
+        assertEquals(NUM_EXPECTED_BLOCKS, info.blocksScanned);
+      }
+      Statistics stats = ctx.blockScanner.getVolumeStats(
+          ctx.volumes.get(0).getStorageID());
+      assertEquals(5 * NUM_EXPECTED_BLOCKS, stats.bytesScannedInPastHour);
+      assertEquals(NUM_EXPECTED_BLOCKS, stats.blocksScannedSinceRestart);
+      assertEquals(NUM_EXPECTED_BLOCKS, stats.blocksScannedInCurrentPeriod);
+      assertEquals(0, stats.scanErrorsSinceRestart);
+      assertEquals(1, stats.scansSinceRestart);
+    }
+    ctx.close();
+  }
+
+  /**
+   * Test scanning all blocks.  Set the scan period high enough that
+   * we shouldn't rescan any block during this test.
+   */
+  @Test(timeout=60000)
+  public void testScanAllBlocksNoRescan() throws Exception {
+    testScanAllBlocksImpl(false);
+  }
+
+  /**
+   * Test scanning all blocks.  Set the scan period high enough that
+   * we should rescan all blocks at least twice during this test.
+   */
+  @Test(timeout=60000)
+  public void testScanAllBlocksWithRescan() throws Exception {
+    testScanAllBlocksImpl(true);
+  }
+
+  /**
+   * Test that we don't scan too many blocks per second.
+   */
+  @Test(timeout=120000)
+  public void testScanRateLimit() throws Exception {
+    Configuration conf = new Configuration();
+    // Limit scan bytes per second dramatically
+    conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND, 4096L);
+    // Scan continuously
+    conf.setLong(INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS, 1L);
+    conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+        TestScanResultHandler.class.getName());
+    final TestContext ctx = new TestContext(conf, 1);
+    final int NUM_EXPECTED_BLOCKS = 5;
+    ctx.createFiles(0, NUM_EXPECTED_BLOCKS, 4096);
+    final TestScanResultHandler.Info info =
+        TestScanResultHandler.getInfo(ctx.volumes.get(0));
+    long startMs = Time.monotonicNow();
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        synchronized (info) {
+          return info.blocksScanned > 0;
+        }
+      }
+    }, 1, 30000);
+    Thread.sleep(2000);
+    synchronized (info) {
+      long endMs = Time.monotonicNow();
+      // Should scan no more than one block a second.
+      long seconds = ((endMs + 999 - startMs) / 1000);
+      long maxBlocksScanned = seconds * 1;
+      assertTrue("The number of blocks scanned is too large.  Scanned " +
+          info.blocksScanned + " blocks; only expected to scan at most " +
+          maxBlocksScanned + " in " + seconds + " seconds.",
+          info.blocksScanned <= maxBlocksScanned);
+    }
+    ctx.close();
+  }
+
+  @Test(timeout=120000)
+  public void testCorruptBlockHandling() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
+    conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+        TestScanResultHandler.class.getName());
+    final TestContext ctx = new TestContext(conf, 1);
+    final int NUM_EXPECTED_BLOCKS = 5;
+    final int CORRUPT_INDEX = 3;
+    ctx.createFiles(0, NUM_EXPECTED_BLOCKS, 4);
+    ExtendedBlock badBlock = ctx.getFileBlock(0, CORRUPT_INDEX);
+    ctx.cluster.corruptBlockOnDataNodes(badBlock);
+    final TestScanResultHandler.Info info =
+        TestScanResultHandler.getInfo(ctx.volumes.get(0));
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        synchronized (info) {
+          return info.blocksScanned == NUM_EXPECTED_BLOCKS;
+        }
+      }
+    }, 3, 30000);
+    synchronized (info) {
+      assertTrue(info.badBlocks.contains(badBlock));
+      for (int i = 0; i < NUM_EXPECTED_BLOCKS; i++) {
+        if (i != CORRUPT_INDEX) {
+          ExtendedBlock block = ctx.getFileBlock(0, i);
+          assertTrue(info.goodBlocks.contains(block));
+        }
+      }
+    }
+    ctx.close();
+  }
+
+  /**
+   * Test that we save the scan cursor when shutting down the datanode, and
+   * restart scanning from there when the datanode is restarted.
+   */
+  @Test(timeout=120000)
+  public void testDatanodeCursor() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
+    conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+        TestScanResultHandler.class.getName());
+    conf.setLong(INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS, 0L);
+    final TestContext ctx = new TestContext(conf, 1);
+    final int NUM_EXPECTED_BLOCKS = 10;
+    ctx.createFiles(0, NUM_EXPECTED_BLOCKS, 1);
+    final TestScanResultHandler.Info info =
+        TestScanResultHandler.getInfo(ctx.volumes.get(0));
+    synchronized (info) {
+      info.sem = new Semaphore(5);
+      info.shouldRun = true;
+      info.notify();
+    }
+    // Scan the first 5 blocks
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        synchronized (info) {
+          return info.blocksScanned == 5;
+        }
+      }
+    }, 3, 30000);
+    synchronized (info) {
+      assertEquals(5, info.goodBlocks.size());
+      assertEquals(5, info.blocksScanned);
+      info.shouldRun = false;
+    }
+    ctx.datanode.shutdown();
+    String vPath = ctx.volumes.get(0).getBasePath();
+    File cursorPath = new File(new File(new File(vPath, "current"),
+          ctx.bpids[0]), "scanner.cursor");
+    assertTrue("Failed to find cursor save file in " +
+        cursorPath.getAbsolutePath(), cursorPath.exists());
+    Set<ExtendedBlock> prevGoodBlocks = new HashSet<ExtendedBlock>();
+    synchronized (info) {
+      info.sem = new Semaphore(4);
+      prevGoodBlocks.addAll(info.goodBlocks);
+      info.goodBlocks.clear();
+    }
+
+    // The block that we were scanning when we shut down the DN won't get
+    // recorded.
+    // After restarting the datanode, we should scan the next 4 blocks.
+    ctx.cluster.restartDataNode(0);
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        synchronized (info) {
+          if (info.blocksScanned != 9) {
+            LOG.info("Waiting for blocksScanned to reach 9.  It is at {}",
+                info.blocksScanned);
+          }
+          return info.blocksScanned == 9;
+        }
+      }
+    }, 3, 30000);
+    synchronized (info) {
+      assertEquals(4, info.goodBlocks.size());
+      info.goodBlocks.addAll(prevGoodBlocks);
+      assertEquals(9, info.goodBlocks.size());
+      assertEquals(9, info.blocksScanned);
+    }
+    ctx.datanode.shutdown();
+
+    // After restarting the datanode, we should not scan any more blocks.
+    // This is because we reached the end of the block pool earlier, and
+    // the scan period is much, much longer than the test time.
+    synchronized (info) {
+      info.sem = null;
+      info.shouldRun = false;
+      info.goodBlocks.clear();
+    }
+    ctx.cluster.restartDataNode(0);
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+    Thread.sleep(3000);
+    synchronized (info) {
+      assertTrue(info.goodBlocks.isEmpty());
+    }
+    ctx.close();
+  }
+
+  @Test(timeout=120000)
+  public void testMultipleBlockPoolScanning() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 100L);
+    conf.set(INTERNAL_VOLUME_SCANNER_SCAN_RESULT_HANDLER,
+        TestScanResultHandler.class.getName());
+    final TestContext ctx = new TestContext(conf, 3);
+
+    // We scan 5 bytes per file (1 byte in file, 4 bytes of checksum)
+    final int BYTES_SCANNED_PER_FILE = 5;
+    final int NUM_FILES[] = new int[] { 1, 5, 10 };
+    int TOTAL_FILES = 0;
+    for (int i = 0; i < NUM_FILES.length; i++) {
+      TOTAL_FILES += NUM_FILES[i];
+    }
+    ctx.createFiles(0, NUM_FILES[0], 1);
+    ctx.createFiles(0, NUM_FILES[1], 1);
+    ctx.createFiles(0, NUM_FILES[2], 1);
+
+    // start scanning
+    final TestScanResultHandler.Info info =
+        TestScanResultHandler.getInfo(ctx.volumes.get(0));
+    synchronized (info) {
+      info.shouldRun = true;
+      info.notify();
+    }
+
+    // Wait for all the block pools to be scanned.
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        synchronized (info) {
+          Statistics stats = ctx.blockScanner.getVolumeStats(
+              ctx.volumes.get(0).getStorageID());
+          if (stats.scansSinceRestart < 3) {
+            LOG.info("Waiting for scansSinceRestart to reach 3 (it is {})",
+                stats.scansSinceRestart);
+            return false;
+          }
+          if (!stats.eof) {
+            LOG.info("Waiting for eof.");
+            return false;
+          }
+          return true;
+        }
+      }
+    }, 3, 30000);
+
+    Statistics stats = ctx.blockScanner.getVolumeStats(
+        ctx.volumes.get(0).getStorageID());
+    assertEquals(TOTAL_FILES, stats.blocksScannedSinceRestart);
+    assertEquals(BYTES_SCANNED_PER_FILE * TOTAL_FILES,
+        stats.bytesScannedInPastHour);
+    ctx.close();
+  }
+
+  @Test(timeout=120000)
+  public void testNextSorted() throws Exception {
+    List<String> arr = new LinkedList<String>();
+    arr.add("1");
+    arr.add("3");
+    arr.add("5");
+    arr.add("7");
+    Assert.assertEquals("3", FsVolumeImpl.nextSorted(arr, "2"));
+    Assert.assertEquals("3", FsVolumeImpl.nextSorted(arr, "1"));
+    Assert.assertEquals("1", FsVolumeImpl.nextSorted(arr, ""));
+    Assert.assertEquals("1", FsVolumeImpl.nextSorted(arr, null));
+    Assert.assertEquals(null, FsVolumeImpl.nextSorted(arr, "9"));
+  }
+
+  @Test(timeout=120000)
+  public void testCalculateNeededBytesPerSec() throws Exception {
+    // If we didn't check anything the last hour, we should scan now.
+    Assert.assertTrue(
+        VolumeScanner.calculateShouldScan("test", 100, 0, 0, 60));
+
+    // If, on average, we checked 101 bytes/s checked during the last hour,
+    // stop checking now.
+    Assert.assertFalse(VolumeScanner.
+        calculateShouldScan("test", 100, 101 * 3600, 1000, 5000));
+
+    // Target is 1 byte / s, but we didn't scan anything in the last minute.
+    // Should scan now.
+    Assert.assertTrue(VolumeScanner.
+        calculateShouldScan("test", 1, 3540, 0, 60));
+
+    // Target is 1000000 byte / s, but we didn't scan anything in the last
+    // minute.  Should scan now.
+    Assert.assertTrue(VolumeScanner.
+        calculateShouldScan("test", 100000L, 354000000L, 0, 60));
+
+    Assert.assertFalse(VolumeScanner.
+        calculateShouldScan("test", 100000L, 365000000L, 0, 60));
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
index 542daa9d899e9..b1df8adb7e29e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
@@ -227,7 +227,7 @@ public void testFadviseAfterWriteThenRead() throws Exception {
       // verify that we dropped everything from the cache during file creation.
       ExtendedBlock block = cluster.getNameNode().getRpcServer().getBlockLocations(
           TEST_PATH, 0, Long.MAX_VALUE).get(0).getBlock();
-      String fadvisedFileName = MiniDFSCluster.getBlockFile(0, block).getName();
+      String fadvisedFileName = cluster.getBlockFile(0, block).getName();
       Stats stats = tracker.getStats(fadvisedFileName);
       stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
       stats.clear();
@@ -272,7 +272,7 @@ public void testClientDefaults() throws Exception {
       // verify that we dropped everything from the cache during file creation.
       ExtendedBlock block = cluster.getNameNode().getRpcServer().getBlockLocations(
           TEST_PATH, 0, Long.MAX_VALUE).get(0).getBlock();
-      String fadvisedFileName = MiniDFSCluster.getBlockFile(0, block).getName();
+      String fadvisedFileName = cluster.getBlockFile(0, block).getName();
       Stats stats = tracker.getStats(fadvisedFileName);
       stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
       stats.clear();
@@ -313,7 +313,7 @@ public void testFadviseSkippedForSmallReads() throws Exception {
       // specify any policy, we should have done drop-behind.
       ExtendedBlock block = cluster.getNameNode().getRpcServer().getBlockLocations(
           TEST_PATH, 0, Long.MAX_VALUE).get(0).getBlock();
-      String fadvisedFileName = MiniDFSCluster.getBlockFile(0, block).getName();
+      String fadvisedFileName = cluster.getBlockFile(0, block).getName();
       Stats stats = tracker.getStats(fadvisedFileName);
       stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
       stats.clear();
@@ -355,7 +355,7 @@ public void testNoFadviseAfterWriteThenRead() throws Exception {
       // verify that we did not drop everything from the cache during file creation.
       ExtendedBlock block = cluster.getNameNode().getRpcServer().getBlockLocations(
           TEST_PATH, 0, Long.MAX_VALUE).get(0).getBlock();
-      String fadvisedFileName = MiniDFSCluster.getBlockFile(0, block).getName();
+      String fadvisedFileName = cluster.getBlockFile(0, block).getName();
       Stats stats = tracker.getStats(fadvisedFileName);
       Assert.assertNull(stats);
       
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
index 9d59496699031..c067b07ae67db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeExit.java
@@ -21,6 +21,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.IOException;
 
@@ -32,6 +33,7 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 /** 
  * Tests if DataNode process exits if all Block Pool services exit. 
@@ -88,4 +90,18 @@ public void testBPServiceExit() throws Exception {
     stopBPServiceThreads(2, dn);
     assertFalse("DataNode should exit", dn.isDatanodeUp());
   }
+
+  @Test
+  public void testSendOOBToPeers() throws Exception {
+    DataNode dn = cluster.getDataNodes().get(0);
+    DataXceiverServer spyXserver = Mockito.spy(dn.getXferServer());
+    NullPointerException e = new NullPointerException();
+    Mockito.doThrow(e).when(spyXserver).sendOOBToPeers();
+    dn.xserver = spyXserver;
+    try {
+      dn.shutdown();
+    } catch (Throwable t) {
+      fail("DataNode shutdown should not have thrown exception " + t);
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index d468493b8784c..3d0bccc47add1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -56,6 +56,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeoutException;
 
 import org.apache.commons.logging.Log;
@@ -568,7 +570,7 @@ private static void assertFileLocksReleased(Collection<String> dirs)
   @Test(timeout=180000)
   public void testRemoveVolumeBeingWritten()
       throws InterruptedException, TimeoutException, ReconfigurationException,
-      IOException {
+      IOException, BrokenBarrierException {
     // test against removing volumes on the different DataNode on the pipeline.
     for (int i = 0; i < 3; i++) {
       testRemoveVolumeBeingWrittenForDatanode(i);
@@ -582,7 +584,7 @@ public void testRemoveVolumeBeingWritten()
    */
   private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
       throws IOException, ReconfigurationException, TimeoutException,
-      InterruptedException {
+      InterruptedException, BrokenBarrierException {
     // Starts DFS cluster with 3 DataNodes to form a pipeline.
     startDFSCluster(1, 3);
 
@@ -599,11 +601,27 @@ private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
     out.write(writeBuf);
     out.hflush();
 
+    final CyclicBarrier barrier = new CyclicBarrier(2);
+
     List<String> oldDirs = getDataDirs(dn);
-    String newDirs = oldDirs.get(1);  // Remove the first volume.
-    dn.reconfigurePropertyImpl(
-        DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+    final String newDirs = oldDirs.get(1);  // Remove the first volume.
+    final List<Exception> exceptions = new ArrayList<>();
+    Thread reconfigThread = new Thread() {
+      public void run() {
+        try {
+          barrier.await();
+          dn.reconfigurePropertyImpl(
+              DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
+        } catch (ReconfigurationException |
+            InterruptedException |
+            BrokenBarrierException e) {
+          exceptions.add(e);
+        }
+      }
+    };
+    reconfigThread.start();
 
+    barrier.await();
     rb.nextBytes(writeBuf);
     out.write(writeBuf);
     out.hflush();
@@ -614,5 +632,44 @@ private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
     // Read the content back
     byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
     assertEquals(BLOCK_SIZE, content.length);
+
+    reconfigThread.join();
+    if (!exceptions.isEmpty()) {
+      throw new IOException(exceptions.get(0).getCause());
+    }
+  }
+
+  @Test(timeout=60000)
+  public void testAddBackRemovedVolume()
+      throws IOException, TimeoutException, InterruptedException,
+      ReconfigurationException {
+    startDFSCluster(1, 2);
+    // Create some data on every volume.
+    createFile(new Path("/test"), 32);
+
+    DataNode dn = cluster.getDataNodes().get(0);
+    Configuration conf = dn.getConf();
+    String oldDataDir = conf.get(DFS_DATANODE_DATA_DIR_KEY);
+    String keepDataDir = oldDataDir.split(",")[0];
+    String removeDataDir = oldDataDir.split(",")[1];
+
+    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, keepDataDir);
+    for (int i = 0; i < cluster.getNumNameNodes(); i++) {
+      String bpid = cluster.getNamesystem(i).getBlockPoolId();
+      BlockPoolSliceStorage bpsStorage =
+          dn.getStorage().getBPStorage(bpid);
+      // Make sure that there is no block pool level storage under removeDataDir.
+      for (int j = 0; j < bpsStorage.getNumStorageDirs(); j++) {
+        Storage.StorageDirectory sd = bpsStorage.getStorageDir(j);
+        assertFalse(sd.getRoot().getAbsolutePath().startsWith(
+            new File(removeDataDir).getAbsolutePath()
+        ));
+      }
+      assertEquals(dn.getStorage().getBPStorage(bpid).getNumStorageDirs(), 1);
+    }
+
+    // Bring the removed directory back. It only successes if all metadata about
+    // this directory were removed from the previous step.
+    dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, oldDataDir);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index b7795b5f4c780..82a168440c42a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -28,6 +28,7 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
 import java.util.LinkedList;
 import java.util.List;
@@ -44,6 +45,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
 import org.apache.hadoop.io.IOUtils;
@@ -539,7 +541,12 @@ private static class TestFsVolumeSpi implements FsVolumeSpi {
     public String[] getBlockPoolList() {
       return new String[0];
     }
-    
+
+    @Override
+    public FsVolumeReference obtainReference() throws ClosedChannelException {
+      return null;
+    }
+
     @Override
     public long getAvailable() throws IOException {
       return 0;
@@ -582,6 +589,22 @@ public void reserveSpaceForRbw(long bytesToReserve) {
     @Override
     public void releaseReservedSpace(long bytesToRelease) {
     }
+
+    @Override
+    public BlockIterator newBlockIterator(String bpid, String name) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public BlockIterator loadBlockIterator(String bpid, String name)
+          throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public FsDatasetSpi getDataset() {
+      throw new UnsupportedOperationException();
+    }
   }
 
   private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
index d3d0244854906..4e73e6e48e994 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
@@ -79,12 +79,13 @@ public class TestIncrementalBrVariations {
   private DatanodeRegistration dn0Reg;  // DataNodeRegistration for dn0
 
   static {
-    ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) BlockManager.blockLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) DataNode.LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) TestIncrementalBrVariations.LOG).getLogger().setLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
+    GenericTestUtils.setLogLevel(BlockManager.blockLog, Level.ALL);
+    GenericTestUtils.setLogLevel(NameNode.blockStateChangeLog, Level.ALL);
+    GenericTestUtils
+        .setLogLevel(LogFactory.getLog(FSNamesystem.class), Level.ALL);
+    GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
+    GenericTestUtils.setLogLevel(TestIncrementalBrVariations.LOG, Level.ALL);
   }
 
   @Before
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java
deleted file mode 100644
index 55b1739de7dc6..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestMultipleNNDataBlockScanner.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import static org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.SLEEP_PERIOD_MS;
-import org.apache.log4j.Level;
-import org.junit.Assert;
-import org.junit.Test;
-import static org.junit.Assert.fail;
-
-
-public class TestMultipleNNDataBlockScanner {
-  private static final Log LOG = 
-    LogFactory.getLog(TestMultipleNNDataBlockScanner.class);
-  Configuration conf;
-  MiniDFSCluster cluster = null;
-  final String[] bpids = new String[3];
-  final FileSystem[] fs = new FileSystem[3];
-  
-  public void setUp() throws IOException {
-    conf = new HdfsConfiguration();
-    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
-    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 100);
-    cluster = new MiniDFSCluster.Builder(conf)
-        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(3))
-        .build();
-    for (int i = 0; i < 3; i++) {
-      cluster.waitActive(i);
-    }
-    for (int i = 0; i < 3; i++) {
-      bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
-    }
-    for (int i = 0; i < 3; i++) {
-      fs[i] = cluster.getFileSystem(i);
-    }
-    // Create 2 files on each namenode with 10 blocks each
-    for (int i = 0; i < 3; i++) {
-      DFSTestUtil.createFile(fs[i], new Path("file1"), 1000, (short) 1, 0);
-      DFSTestUtil.createFile(fs[i], new Path("file2"), 1000, (short) 1, 1);
-    }
-  }
-  
-  @Test(timeout=120000)
-  public void testDataBlockScanner() throws IOException, InterruptedException {
-    setUp();
-    try {
-      DataNode dn = cluster.getDataNodes().get(0);
-      for (int i = 0; i < 3; i++) {
-        long blocksScanned = 0;
-        while (blocksScanned != 20) {
-          blocksScanned = dn.blockScanner.getBlocksScannedInLastRun(bpids[i]);
-          LOG.info("Waiting for all blocks to be scanned for bpid=" + bpids[i]
-              + "; Scanned so far=" + blocksScanned);
-          Thread.sleep(5000);
-        }
-      }
-
-      StringBuilder buffer = new StringBuilder();
-      dn.blockScanner.printBlockReport(buffer, false);
-      LOG.info("Block Report\n" + buffer.toString());
-    } finally {
-      cluster.shutdown();
-    }
-  }
-  
-  @Test(timeout=120000)
-  public void testBlockScannerAfterRefresh() throws IOException,
-      InterruptedException {
-    setUp();
-    try {
-      Configuration dnConf = cluster.getDataNodes().get(0).getConf();
-      Configuration conf = new HdfsConfiguration(dnConf);
-      StringBuilder namenodesBuilder = new StringBuilder();
-
-      String bpidToShutdown = cluster.getNamesystem(2).getBlockPoolId();
-      for (int i = 0; i < 2; i++) {
-        String nsId = DFSUtil.getNamenodeNameServiceId(cluster
-            .getConfiguration(i));
-        namenodesBuilder.append(nsId);
-        namenodesBuilder.append(",");
-      }
-
-      conf.set(DFSConfigKeys.DFS_NAMESERVICES, namenodesBuilder
-          .toString());
-      DataNode dn = cluster.getDataNodes().get(0);
-      dn.refreshNamenodes(conf);
-
-      try {
-        while (true) {
-          dn.blockScanner.getBlocksScannedInLastRun(bpidToShutdown);
-          Thread.sleep(1000);
-        }
-      } catch (IOException ex) {
-        // Expected
-        LOG.info(ex.getMessage());
-      }
-
-      namenodesBuilder.append(DFSUtil.getNamenodeNameServiceId(cluster
-          .getConfiguration(2)));
-      conf.set(DFSConfigKeys.DFS_NAMESERVICES, namenodesBuilder
-          .toString());
-      dn.refreshNamenodes(conf);
-
-      for (int i = 0; i < 3; i++) {
-        long blocksScanned = 0;
-        while (blocksScanned != 20) {
-          blocksScanned = dn.blockScanner.getBlocksScannedInLastRun(bpids[i]);
-          LOG.info("Waiting for all blocks to be scanned for bpid=" + bpids[i]
-              + "; Scanned so far=" + blocksScanned);
-          Thread.sleep(5000);
-        }
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-  
-  @Test(timeout=120000)
-  public void testBlockScannerAfterRestart() throws IOException,
-      InterruptedException {
-    setUp();
-    try {
-      cluster.restartDataNode(0);
-      cluster.waitActive();
-      DataNode dn = cluster.getDataNodes().get(0);
-      for (int i = 0; i < 3; i++) {
-        while (!dn.blockScanner.isInitialized(bpids[i])) {
-          Thread.sleep(1000);
-        }
-        long blocksScanned = 0;
-        while (blocksScanned != 20) {
-          if (dn.blockScanner != null) {
-            blocksScanned = dn.blockScanner.getBlocksScannedInLastRun(bpids[i]);
-            LOG.info("Waiting for all blocks to be scanned for bpid="
-                + bpids[i] + "; Scanned so far=" + blocksScanned);
-          }
-          Thread.sleep(5000);
-        }
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-  
-  @Test(timeout=120000)
-  public void test2NNBlockRescanInterval() throws IOException {
-    ((Log4JLogger)BlockPoolSliceScanner.LOG).getLogger().setLevel(Level.ALL);
-    Configuration conf = new HdfsConfiguration();
-    cluster = new MiniDFSCluster.Builder(conf)
-        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(3))
-        .build();
-
-    try {
-      FileSystem fs = cluster.getFileSystem(1);
-      Path file2 = new Path("/test/testBlockScanInterval");
-      DFSTestUtil.createFile(fs, file2, 30, (short) 1, 0);
-
-      fs = cluster.getFileSystem(0);
-      Path file1 = new Path("/test/testBlockScanInterval");
-      DFSTestUtil.createFile(fs, file1, 30, (short) 1, 0);
-      for (int i = 0; i < 8; i++) {
-        LOG.info("Verifying that the blockscanner scans exactly once");
-        waitAndScanBlocks(1, 1);
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-
-  /**
-   * HDFS-3828: DN rescans blocks too frequently
-   * 
-   * @throws Exception
-   */
-  @Test(timeout=120000)
-  public void testBlockRescanInterval() throws IOException {
-    ((Log4JLogger)BlockPoolSliceScanner.LOG).getLogger().setLevel(Level.ALL);
-    Configuration conf = new HdfsConfiguration();
-    cluster = new MiniDFSCluster.Builder(conf).build();
-
-    try {
-      FileSystem fs = cluster.getFileSystem();
-      Path file1 = new Path("/test/testBlockScanInterval");
-      DFSTestUtil.createFile(fs, file1, 30, (short) 1, 0);
-      for (int i = 0; i < 4; i++) {
-        LOG.info("Verifying that the blockscanner scans exactly once");
-        waitAndScanBlocks(1, 1);
-      }
-    } finally {
-      cluster.shutdown();
-    }
-  }
-
-  void waitAndScanBlocks(long scansLastRun, long scansTotal)
-      throws IOException {
-    // DataBlockScanner will run for every 5 seconds so we are checking for
-    // every 5 seconds
-    int n = 5;
-    String bpid = cluster.getNamesystem(0).getBlockPoolId();
-    DataNode dn = cluster.getDataNodes().get(0);
-    long blocksScanned, total;
-    do {
-      try {
-        Thread.sleep(SLEEP_PERIOD_MS);
-      } catch (InterruptedException e) {
-        fail("Interrupted: " + e);
-      }
-      blocksScanned = dn.blockScanner.getBlocksScannedInLastRun(bpid);
-      total = dn.blockScanner.getTotalScans(bpid);
-      LOG.info("bpid = " + bpid + " blocksScanned = " + blocksScanned + " total=" + total);
-    } while (n-- > 0 && (blocksScanned != scansLastRun || scansTotal != total));
-    Assert.assertEquals(scansTotal, total);
-    Assert.assertEquals(scansLastRun, blocksScanned);
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index 099a0cdae1c1f..b9adce4e76e6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -67,7 +67,7 @@ int addSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId)
       // we pass expected len as zero, - fsdataset should use the sizeof actual
       // data written
       ReplicaInPipelineInterface bInfo = fsdataset.createRbw(
-          StorageType.DEFAULT, b, false);
+          StorageType.DEFAULT, b, false).getReplica();
       ReplicaOutputStreams out = bInfo.createStreams(true,
           DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512));
       try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
new file mode 100644
index 0000000000000..aa868d5749d99
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -0,0 +1,402 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.extdataset;
+
+import java.io.*;
+import java.util.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.datanode.*;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
+import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
+
+  private final DatanodeStorage storage = new DatanodeStorage(
+      DatanodeStorage.generateUuid(), DatanodeStorage.State.NORMAL,
+      StorageType.DEFAULT);
+
+  @Override
+  public List<ExternalVolumeImpl> getVolumes() {
+    return null;
+  }
+
+  @Override
+  public void addVolume(StorageLocation location, List<NamespaceInfo> nsInfos) throws IOException {
+
+  }
+
+  @Override
+  public void removeVolumes(Collection<StorageLocation> volumes) {
+
+  }
+
+  @Override
+  public DatanodeStorage getStorage(String storageUuid) {
+    return null;
+  }
+
+  @Override
+  public StorageReport[] getStorageReports(String bpid) throws IOException {
+    StorageReport[] result = new StorageReport[1];
+    result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
+    return result;
+  }
+
+  @Override
+  public ExternalVolumeImpl getVolume(ExtendedBlock b) {
+    return null;
+  }
+
+  @Override
+  public Map<String, Object> getVolumeInfoMap() {
+    return null;
+  }
+
+  @Override
+  public List<FinalizedReplica> getFinalizedBlocks(String bpid) {
+    return null;
+  }
+
+  @Override
+  public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) {
+    return null;
+  }
+
+  @Override
+  public void checkAndUpdate(String bpid, long blockId, File diskFile,
+      File diskMetaFile, FsVolumeSpi vol) {
+  }
+
+  @Override
+  public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
+      throws IOException {
+    return new LengthInputStream(null, 0);
+  }
+
+  @Override
+  public long getLength(ExtendedBlock b) throws IOException {
+    return 0;
+  }
+
+  @Override
+  @Deprecated
+  public Replica getReplica(String bpid, long blockId) {
+    return new ExternalReplica();
+  }
+
+  @Override
+  public String getReplicaString(String bpid, long blockId) {
+    return null;
+  }
+
+  @Override
+  public Block getStoredBlock(String bpid, long blkid) throws IOException {
+    return new Block();
+  }
+
+  @Override
+  public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset)
+      throws IOException {
+    return null;
+  }
+
+  @Override
+  public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff,
+      long ckoff) throws IOException {
+    return new ReplicaInputStreams(null, null, null);
+  }
+
+  @Override
+  public ReplicaHandler createTemporary(StorageType t, ExtendedBlock b)
+      throws IOException {
+    return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
+  }
+
+  @Override
+  public ReplicaHandler createRbw(StorageType t, ExtendedBlock b, boolean tf)
+      throws IOException {
+    return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
+  }
+
+  @Override
+  public ReplicaHandler recoverRbw(ExtendedBlock b, long newGS,
+      long minBytesRcvd, long maxBytesRcvd) throws IOException {
+    return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
+  }
+
+  @Override
+  public ReplicaInPipelineInterface convertTemporaryToRbw(
+      ExtendedBlock temporary) throws IOException {
+    return new ExternalReplicaInPipeline();
+  }
+
+  @Override
+  public ReplicaHandler append(ExtendedBlock b, long newGS,
+      long expectedBlockLen) throws IOException {
+    return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
+  }
+
+  @Override
+  public ReplicaHandler recoverAppend(ExtendedBlock b, long newGS,
+      long expectedBlockLen) throws IOException {
+    return new ReplicaHandler(new ExternalReplicaInPipeline(), null);
+  }
+
+  @Override
+  public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen)
+      throws IOException {
+    return null;
+  }
+
+  @Override
+  public void finalizeBlock(ExtendedBlock b) throws IOException {
+  }
+
+  @Override
+  public void unfinalizeBlock(ExtendedBlock b) throws IOException {
+  }
+
+  @Override
+  public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
+    final Map<DatanodeStorage, BlockListAsLongs> result =
+	new HashMap<DatanodeStorage, BlockListAsLongs>();
+
+    result.put(storage, new BlockListAsLongs(null, null));
+    return result;
+  }
+
+  @Override
+  public List<Long> getCacheReport(String bpid) {
+    return null;
+  }
+
+  @Override
+  public boolean contains(ExtendedBlock block) {
+    return false;
+  }
+
+  @Override
+  public void checkBlock(ExtendedBlock b, long minLength, ReplicaState state) throws ReplicaNotFoundException, UnexpectedReplicaStateException, FileNotFoundException, EOFException, IOException {
+
+  }
+
+  @Override
+  public boolean isValidBlock(ExtendedBlock b) {
+    return false;
+  }
+
+  @Override
+  public boolean isValidRbw(ExtendedBlock b) {
+    return false;
+  }
+
+  @Override
+  public void invalidate(String bpid, Block[] invalidBlks) throws IOException {
+  }
+
+  @Override
+  public void cache(String bpid, long[] blockIds) {
+  }
+
+  @Override
+  public void uncache(String bpid, long[] blockIds) {
+  }
+
+  @Override
+  public boolean isCached(String bpid, long blockId) {
+    return false;
+  }
+
+  @Override
+  public void checkDataDir() throws DiskErrorException {
+    throw new DiskChecker.DiskErrorException(null);
+  }
+
+  @Override
+  public void shutdown() {
+  }
+
+  @Override
+  public void adjustCrcChannelPosition(ExtendedBlock b,
+      ReplicaOutputStreams outs, int checksumSize) throws IOException {
+  }
+
+  @Override
+  public boolean hasEnoughResource() {
+    return false;
+  }
+
+  @Override
+  public long getReplicaVisibleLength(ExtendedBlock block) throws IOException {
+    return 0;
+  }
+
+  @Override
+  public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
+      throws IOException {
+    return new ReplicaRecoveryInfo(0, 0, 0, ReplicaState.FINALIZED);
+  }
+
+  @Override
+  public String updateReplicaUnderRecovery(ExtendedBlock oldBlock,
+      long recoveryId, long newBlockId, long newLength) throws IOException {
+    return null;
+  }
+
+  @Override
+  public void addBlockPool(String bpid, Configuration conf) throws IOException {
+  }
+
+  @Override
+  public void shutdownBlockPool(String bpid) {
+  }
+
+  @Override
+  public void deleteBlockPool(String bpid, boolean force) throws IOException {
+  }
+
+  @Override
+  public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b)
+      throws IOException {
+    return new BlockLocalPathInfo(null, "file", "metafile");
+  }
+
+  @Override
+  public HdfsBlocksMetadata getHdfsBlocksMetadata(String bpid, long[] blockIds) throws IOException {
+    return new HdfsBlocksMetadata(null, null, null, null);
+  }
+
+  @Override
+  public void enableTrash(String bpid) {
+
+  }
+
+  @Override
+  public void restoreTrash(String bpid) {
+
+  }
+
+  @Override
+  public boolean trashEnabled(String bpid) {
+    return false;
+  }
+
+  @Override
+  public void setRollingUpgradeMarker(String bpid) throws IOException {
+
+  }
+
+  @Override
+  public void clearRollingUpgradeMarker(String bpid) throws IOException {
+
+  }
+
+  @Override
+  public void submitBackgroundSyncFileRangeRequest(ExtendedBlock block, FileDescriptor fd, long offset, long nbytes, int flags) {
+
+  }
+
+  @Override
+  public void onCompleteLazyPersist(String bpId, long blockId, long creationTime, File[] savedFiles, ExternalVolumeImpl targetVolume) {
+
+  }
+
+  @Override
+  public void onFailLazyPersist(String bpId, long blockId) {
+
+  }
+
+  @Override
+  public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block, StorageType targetStorageType) throws IOException {
+    return null;
+  }
+
+  @Override
+  public long getBlockPoolUsed(String bpid) throws IOException {
+    return 0;
+  }
+
+  @Override
+  public long getDfsUsed() throws IOException {
+    return 0;
+  }
+
+  @Override
+  public long getCapacity() throws IOException {
+    return 0;
+  }
+
+  @Override
+  public long getRemaining() throws IOException {
+    return 0;
+  }
+
+  @Override
+  public String getStorageInfo() {
+    return null;
+  }
+
+  @Override
+  public int getNumFailedVolumes() {
+    return 0;
+  }
+
+  @Override
+  public long getCacheUsed() {
+    return 0;
+  }
+
+  @Override
+  public long getCacheCapacity() {
+    return 0;
+  }
+
+  @Override
+  public long getNumBlocksCached() {
+    return 0;
+  }
+
+  @Override
+  public long getNumBlocksFailedToCache() {
+    return 0;
+  }
+
+  @Override
+  public long getNumBlocksFailedToUncache() {
+    return 0;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java
new file mode 100644
index 0000000000000..8ea74324ce94c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.extdataset;
+
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.datanode.Replica;
+
+public class ExternalReplica implements Replica {
+
+  @Override
+  public long getBlockId() {
+    return 0;
+  }
+
+  @Override
+  public long getGenerationStamp() {
+    return 0;
+  }
+
+  @Override
+  public ReplicaState getState() {
+    return ReplicaState.FINALIZED;
+  }
+
+  @Override
+  public long getNumBytes() {
+    return 0;
+  }
+
+  @Override
+  public long getBytesOnDisk() {
+    return 0;
+  }
+
+  @Override
+  public long getVisibleLength() {
+    return 0;
+  }
+
+  @Override
+  public String getStorageUuid() {
+    return null;
+  }
+
+  @Override
+  public boolean isOnTransientStorage() {
+    return false;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
new file mode 100644
index 0000000000000..c3c0197ce7244
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.extdataset;
+
+import java.io.IOException;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
+import org.apache.hadoop.util.DataChecksum;
+
+public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface {
+
+  @Override
+  public void setNumBytes(long bytesReceived) {
+  }
+
+  @Override
+  public long getBytesAcked() {
+    return 0;
+  }
+
+  @Override
+  public void setBytesAcked(long bytesAcked) {
+  }
+
+  @Override
+  public void setLastChecksumAndDataLen(long dataLength, byte[] lastChecksum) {
+  }
+
+  @Override
+  public ChunkChecksum getLastChecksumAndDataLen() {
+    return new ChunkChecksum(0, null);
+  }
+
+  @Override
+  public ReplicaOutputStreams createStreams(boolean isCreate,
+      DataChecksum requestedChecksum) throws IOException {
+    return new ReplicaOutputStreams(null, null, requestedChecksum, false);
+  }
+
+  @Override
+  public long getBlockId() {
+    return 0;
+  }
+
+  @Override
+  public long getGenerationStamp() {
+    return 0;
+  }
+
+  @Override
+  public ReplicaState getState() {
+    return ReplicaState.FINALIZED;
+  }
+
+  @Override
+  public long getNumBytes() {
+    return 0;
+  }
+
+  @Override
+  public long getBytesOnDisk() {
+    return 0;
+  }
+
+  @Override
+  public long getVisibleLength() {
+    return 0;
+  }
+
+  @Override
+  public String getStorageUuid() {
+    return null;
+  }
+
+  @Override
+  public boolean isOnTransientStorage() {
+    return false;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
new file mode 100644
index 0000000000000..0ea33bbc95ee8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.extdataset;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+
+public class ExternalVolumeImpl implements FsVolumeSpi {
+  @Override
+  public FsVolumeReference obtainReference() throws ClosedChannelException {
+    return null;
+  }
+
+  @Override
+  public String[] getBlockPoolList() {
+    return null;
+  }
+
+  @Override
+  public long getAvailable() throws IOException {
+    return 0;
+  }
+
+  @Override
+  public String getBasePath() {
+    return null;
+  }
+
+  @Override
+  public String getPath(String bpid) throws IOException {
+    return null;
+  }
+
+  @Override
+  public File getFinalizedDir(String bpid) throws IOException {
+    return null;
+  }
+
+  @Override
+  public String getStorageID() {
+    return null;
+  }
+
+  @Override
+  public StorageType getStorageType() {
+    return StorageType.DEFAULT;
+  }
+
+  @Override
+  public boolean isTransientStorage() {
+    return false;
+  }
+
+  @Override
+  public void reserveSpaceForRbw(long bytesToReserve) {
+  }
+
+  @Override
+  public void releaseReservedSpace(long bytesToRelease) {
+  }
+
+  @Override
+  public BlockIterator newBlockIterator(String bpid, String name) {
+    return null;
+  }
+
+  @Override
+  public BlockIterator loadBlockIterator(String bpid, String name)
+      throws IOException {
+    return null;
+  }
+
+  @Override
+  public FsDatasetSpi getDataset() {
+    return null;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java
new file mode 100644
index 0000000000000..82a6951413f95
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/TestExternalDataset.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.extdataset;
+
+import org.apache.hadoop.hdfs.server.datanode.Replica;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.junit.Test;
+
+/**
+ * Tests the ability to create external FsDatasetSpi implementations.
+ *
+ * The purpose of this suite of tests is to ensure that it is possible to
+ * construct subclasses of FsDatasetSpi outside the Hadoop tree
+ * (specifically, outside of the org.apache.hadoop.hdfs.server.datanode
+ * package).  This consists of creating subclasses of the two key classes
+ * (FsDatasetSpi and FsVolumeSpi) *and* instances or subclasses of any
+ * classes/interfaces their methods need to produce.  If methods are added
+ * to or changed in any superclasses, or if constructors of other classes
+ * are changed, this package will fail to compile.  In fixing this
+ * compilation error, any new class dependencies should receive the same
+ * treatment.
+ *
+ * It is worth noting what these tests do *not* accomplish.  Just as
+ * important as being able to produce instances of the appropriate classes
+ * is being able to access all necessary methods on those classes as well
+ * as on any additional classes accepted as inputs to FsDatasetSpi's
+ * methods.  It wouldn't be correct to mandate all methods be public, as
+ * that would defeat encapsulation.  Moreover, there is no natural
+ * mechanism that would prevent a manually-constructed list of methods
+ * from becoming stale.  Rather than creating tests with no clear means of
+ * maintaining them, this problem is left unsolved for now.
+ *
+ * Lastly, though merely compiling this package should signal success,
+ * explicit testInstantiate* unit tests are included below so as to have a
+ * tangible means of referring to each case.
+ */
+public class TestExternalDataset {
+
+  /**
+   * Tests instantiating an FsDatasetSpi subclass.
+   */
+  @Test
+  public void testInstantiateDatasetImpl() throws Throwable {
+    FsDatasetSpi<?> inst = new ExternalDatasetImpl();
+  }
+
+  /**
+   * Tests instantiating a Replica subclass.
+   */
+  @Test
+  public void testIntantiateExternalReplica() throws Throwable {
+    Replica inst = new ExternalReplica();
+  }
+
+  /**
+   * Tests instantiating a ReplicaInPipelineInterface subclass.
+   */
+  @Test
+  public void testInstantiateReplicaInPipeline() throws Throwable {
+    ReplicaInPipelineInterface inst = new ExternalReplicaInPipeline();
+  }
+
+  /**
+   * Tests instantiating an FsVolumeSpi subclass.
+   */
+  @Test
+  public void testInstantiateVolumeImpl() throws Throwable {
+    FsVolumeSpi inst = new ExternalVolumeImpl();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeListTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeListTest.java
new file mode 100644
index 0000000000000..691d390d042b8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeListTest.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertNotEquals;
+import static org.mockito.Mockito.mock;
+
+public class FsVolumeListTest {
+
+  private final Configuration conf = new Configuration();
+  private VolumeChoosingPolicy<FsVolumeImpl> blockChooser =
+      new RoundRobinVolumeChoosingPolicy<>();
+  private FsDatasetImpl dataset = null;
+  private String baseDir;
+  private BlockScanner blockScanner;
+
+  @Before
+  public void setUp() {
+    dataset = mock(FsDatasetImpl.class);
+    baseDir = new FileSystemTestHelper().getTestRootDir();
+    Configuration blockScannerConf = new Configuration();
+    blockScannerConf.setInt(DFSConfigKeys.
+        DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
+    blockScanner = new BlockScanner(null, blockScannerConf);
+  }
+
+  @Test
+  public void testGetNextVolumeWithClosedVolume() throws IOException {
+    FsVolumeList volumeList = new FsVolumeList(0, blockScanner, blockChooser);
+    List<FsVolumeImpl> volumes = new ArrayList<>();
+    for (int i = 0; i < 3; i++) {
+      File curDir = new File(baseDir, "nextvolume-" + i);
+      curDir.mkdirs();
+      FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir,
+          conf, StorageType.DEFAULT);
+      volume.setCapacityForTesting(1024 * 1024 * 1024);
+      volumes.add(volume);
+      volumeList.addVolume(volume.obtainReference());
+    }
+
+    // Close the second volume.
+    volumes.get(1).closeAndWait();
+    for (int i = 0; i < 10; i++) {
+      try (FsVolumeReference ref =
+          volumeList.getNextVolume(StorageType.DEFAULT, 128)) {
+        // volume No.2 will not be chosen.
+        assertNotEquals(ref.getVolume(), volumes.get(1));
+      }
+    }
+  }
+
+  @Test
+  public void testCheckDirsWithClosedVolume() throws IOException {
+    FsVolumeList volumeList = new FsVolumeList(0, blockScanner, blockChooser);
+    List<FsVolumeImpl> volumes = new ArrayList<>();
+    for (int i = 0; i < 3; i++) {
+      File curDir = new File(baseDir, "volume-" + i);
+      curDir.mkdirs();
+      FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir,
+          conf, StorageType.DEFAULT);
+      volumes.add(volume);
+      volumeList.addVolume(volume.obtainReference());
+    }
+
+    // Close the 2nd volume.
+    volumes.get(1).closeAndWait();
+    // checkDirs() should ignore the 2nd volume since it is closed.
+    volumeList.checkDirs();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index c7628495a4a1a..6272991e1a202 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -20,7 +20,6 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -36,10 +35,10 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.tools.JMXGet;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.After;
 import org.junit.Rule;
@@ -50,6 +49,8 @@
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.UUID;
 
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
@@ -65,9 +66,8 @@
 public abstract class LazyPersistTestCase {
 
   static {
-    ((Log4JLogger) NameNode.blockStateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger) FsDatasetImpl.LOG).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(FsDatasetImpl.LOG, Level.ALL);
   }
 
   protected static final int BLOCK_SIZE = 5 * 1024 * 1024;
@@ -131,6 +131,48 @@ protected final LocatedBlocks ensureFileReplicasOnStorageType(
     return locatedBlocks;
   }
 
+  /**
+   * Make sure at least one non-transient volume has a saved copy of the replica.
+   * An infinite loop is used to ensure the async lazy persist tasks are completely
+   * done before verification. Caller of ensureLazyPersistBlocksAreSaved expects
+   * either a successful pass or timeout failure.
+   */
+  protected final void ensureLazyPersistBlocksAreSaved(
+      LocatedBlocks locatedBlocks) throws IOException, InterruptedException {
+    final String bpid = cluster.getNamesystem().getBlockPoolId();
+    List<? extends FsVolumeSpi> volumes =
+      cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+    final Set<Long> persistedBlockIds = new HashSet<Long>();
+
+    while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
+      // Take 1 second sleep before each verification iteration
+      Thread.sleep(1000);
+
+      for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+        for (FsVolumeSpi v : volumes) {
+          if (v.isTransientStorage()) {
+            continue;
+          }
+
+          FsVolumeImpl volume = (FsVolumeImpl) v;
+          File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
+
+          long blockId = lb.getBlock().getBlockId();
+          File targetDir =
+            DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
+          File blockFile = new File(targetDir, lb.getBlock().getBlockName());
+          if (blockFile.exists()) {
+            // Found a persisted copy for this block and added to the Set
+            persistedBlockIds.add(blockId);
+          }
+        }
+      }
+    }
+
+    // We should have found a persisted copy for each located block.
+    assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+  }
+
   protected final void makeRandomTestFile(Path path, long length,
       boolean isLazyPersist, long seed) throws IOException {
     DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 956ab78205890..8f87f57306c6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -22,37 +22,46 @@
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.StorageType;
-import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DNConf;
-import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaHandler;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyListOf;
-import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -60,6 +69,7 @@ public class TestFsDatasetImpl {
   private static final String BASE_DIR =
       new FileSystemTestHelper().getTestRootDir();
   private static final int NUM_INIT_VOLUMES = 2;
+  private static final String CLUSTER_ID = "cluser-id";
   private static final String[] BLOCK_POOL_IDS = {"bpid-0", "bpid-1"};
 
   // Use to generate storageUuid
@@ -69,12 +79,11 @@ public class TestFsDatasetImpl {
   private Configuration conf;
   private DataNode datanode;
   private DataStorage storage;
-  private DataBlockScanner scanner;
   private FsDatasetImpl dataset;
 
   private static Storage.StorageDirectory createStorageDirectory(File root) {
     Storage.StorageDirectory sd = new Storage.StorageDirectory(root);
-    dsForStorageUuid.createStorageID(sd);
+    dsForStorageUuid.createStorageID(sd, false);
     return sd;
   }
 
@@ -98,15 +107,16 @@ private static void createStorageDirs(DataStorage storage, Configuration conf,
 
   @Before
   public void setUp() throws IOException {
-    datanode = Mockito.mock(DataNode.class);
-    storage = Mockito.mock(DataStorage.class);
-    scanner = Mockito.mock(DataBlockScanner.class);
+    datanode = mock(DataNode.class);
+    storage = mock(DataStorage.class);
     this.conf = new Configuration();
+    this.conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 0);
     final DNConf dnConf = new DNConf(conf);
 
     when(datanode.getConf()).thenReturn(conf);
     when(datanode.getDnConf()).thenReturn(dnConf);
-    when(datanode.getBlockScanner()).thenReturn(scanner);
+    final BlockScanner disabledBlockScanner = new BlockScanner(datanode, conf);
+    when(datanode.getBlockScanner()).thenReturn(disabledBlockScanner);
 
     createStorageDirs(storage, conf, NUM_INIT_VOLUMES);
     dataset = new FsDatasetImpl(datanode, storage, conf);
@@ -126,10 +136,11 @@ public void testAddVolumes() throws IOException {
     Set<String> expectedVolumes = new HashSet<String>();
     List<NamespaceInfo> nsInfos = Lists.newArrayList();
     for (String bpid : BLOCK_POOL_IDS) {
-      nsInfos.add(new NamespaceInfo(0, "cluster-id", bpid, 1));
+      nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1));
     }
     for (int i = 0; i < numNewVolumes; i++) {
       String path = BASE_DIR + "/newData" + i;
+      expectedVolumes.add(path);
       StorageLocation loc = StorageLocation.parse(path);
       Storage.StorageDirectory sd = createStorageDirectory(new File(path));
       DataStorage.VolumeBuilder builder =
@@ -146,19 +157,22 @@ public void testAddVolumes() throws IOException {
 
     Set<String> actualVolumes = new HashSet<String>();
     for (int i = 0; i < numNewVolumes; i++) {
-      dataset.getVolumes().get(numExistingVolumes + i).getBasePath();
+      actualVolumes.add(
+          dataset.getVolumes().get(numExistingVolumes + i).getBasePath());
     }
     assertEquals(actualVolumes, expectedVolumes);
   }
 
-  @Test
+  @Test(timeout = 30000)
   public void testRemoveVolumes() throws IOException {
     // Feed FsDataset with block metadata.
     final int NUM_BLOCKS = 100;
     for (int i = 0; i < NUM_BLOCKS; i++) {
       String bpid = BLOCK_POOL_IDS[NUM_BLOCKS % BLOCK_POOL_IDS.length];
       ExtendedBlock eb = new ExtendedBlock(bpid, i);
-      dataset.createRbw(StorageType.DEFAULT, eb, false);
+      try (ReplicaHandler replica =
+          dataset.createRbw(StorageType.DEFAULT, eb, false)) {
+      }
     }
     final String[] dataDirs =
         conf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(",");
@@ -192,9 +206,87 @@ public void run() {}
     assertEquals("The replica infos on this volume has been removed from the "
                  + "volumeMap.", NUM_BLOCKS / NUM_INIT_VOLUMES,
                  totalNumReplicas);
+  }
+
+  @Test(timeout = 5000)
+  public void testRemoveNewlyAddedVolume() throws IOException {
+    final int numExistingVolumes = dataset.getVolumes().size();
+    List<NamespaceInfo> nsInfos = new ArrayList<>();
+    for (String bpid : BLOCK_POOL_IDS) {
+      nsInfos.add(new NamespaceInfo(0, CLUSTER_ID, bpid, 1));
+    }
+    String newVolumePath = BASE_DIR + "/newVolumeToRemoveLater";
+    StorageLocation loc = StorageLocation.parse(newVolumePath);
+
+    Storage.StorageDirectory sd = createStorageDirectory(new File(newVolumePath));
+    DataStorage.VolumeBuilder builder =
+        new DataStorage.VolumeBuilder(storage, sd);
+    when(storage.prepareVolume(eq(datanode), eq(loc.getFile()),
+        anyListOf(NamespaceInfo.class)))
+        .thenReturn(builder);
+
+    dataset.addVolume(loc, nsInfos);
+    assertEquals(numExistingVolumes + 1, dataset.getVolumes().size());
+
+    when(storage.getNumStorageDirs()).thenReturn(numExistingVolumes + 1);
+    when(storage.getStorageDir(numExistingVolumes)).thenReturn(sd);
+    List<StorageLocation> volumesToRemove = Arrays.asList(loc);
+    dataset.removeVolumes(volumesToRemove);
+    assertEquals(numExistingVolumes, dataset.getVolumes().size());
+  }
+
+  @Test(timeout = 5000)
+  public void testChangeVolumeWithRunningCheckDirs() throws IOException {
+    RoundRobinVolumeChoosingPolicy<FsVolumeImpl> blockChooser =
+        new RoundRobinVolumeChoosingPolicy<>();
+    final BlockScanner blockScanner = new BlockScanner(datanode, conf);
+    final FsVolumeList volumeList =
+        new FsVolumeList(0, blockScanner, blockChooser);
+    final List<FsVolumeImpl> oldVolumes = new ArrayList<>();
+
+    // Initialize FsVolumeList with 5 mock volumes.
+    final int NUM_VOLUMES = 5;
+    for (int i = 0; i < NUM_VOLUMES; i++) {
+      FsVolumeImpl volume = mock(FsVolumeImpl.class);
+      oldVolumes.add(volume);
+      when(volume.getBasePath()).thenReturn("data" + i);
+      FsVolumeReference ref = mock(FsVolumeReference.class);
+      when(ref.getVolume()).thenReturn(volume);
+      volumeList.addVolume(ref);
+    }
 
-    // Verify that every BlockPool deletes the removed blocks from the volume.
-    verify(scanner, times(BLOCK_POOL_IDS.length))
-        .deleteBlocks(anyString(), any(Block[].class));
+    // When call checkDirs() on the 2nd volume, anther "thread" removes the 5th
+    // volume and add another volume. It does not affect checkDirs() running.
+    final FsVolumeImpl newVolume = mock(FsVolumeImpl.class);
+    final FsVolumeReference newRef = mock(FsVolumeReference.class);
+    when(newRef.getVolume()).thenReturn(newVolume);
+    FsVolumeImpl blockedVolume = volumeList.getVolumes().get(1);
+    doAnswer(new Answer() {
+      @Override
+      public Object answer(InvocationOnMock invocationOnMock)
+          throws Throwable {
+        volumeList.removeVolume(new File("data4"));
+        volumeList.addVolume(newRef);
+        return null;
+      }
+    }).when(blockedVolume).checkDirs();
+
+    FsVolumeImpl brokenVolume = volumeList.getVolumes().get(2);
+    doThrow(new DiskChecker.DiskErrorException("broken"))
+        .when(brokenVolume).checkDirs();
+
+    volumeList.checkDirs();
+
+    // Since FsVolumeImpl#checkDirs() get a snapshot of the list of volumes
+    // before running removeVolume(), it is supposed to run checkDirs() on all
+    // the old volumes.
+    for (FsVolumeImpl volume : oldVolumes) {
+      verify(volume).checkDirs();
+    }
+    // New volume is not visible to checkDirs() process.
+    verify(newVolume, never()).checkDirs();
+    assertTrue(volumeList.getVolumes().contains(newVolume));
+    assertFalse(volumeList.getVolumes().contains(brokenVolume));
+    assertEquals(NUM_VOLUMES - 1, volumeList.getVolumes().size());
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
index 65a51761c9a11..6cc3d7e3f2ea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
@@ -184,8 +184,8 @@ private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception {
       InterDatanodeProtocol idp = DataNodeTestUtils.createInterDatanodeProtocolProxy(
           datanode, datanodeinfo[0], conf, useDnHostname);
       
-      //stop block scanner, so we could compare lastScanTime
-      DataNodeTestUtils.shutdownBlockScanner(datanode);
+      // Stop the block scanners.
+      datanode.getBlockScanner().removeAllVolumeScanners();
 
       //verify BlockMetaDataInfo
       ExtendedBlock b = locatedblock.getBlock();
@@ -198,7 +198,8 @@ private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception {
       //verify updateBlock
       ExtendedBlock newblock = new ExtendedBlock(b.getBlockPoolId(),
           b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
-      idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
+      idp.updateReplicaUnderRecovery(b, recoveryId, b.getBlockId(),
+          newblock.getNumBytes());
       checkMetaInfo(newblock, datanode);
       
       // Verify correct null response trying to init recovery for a missing block
@@ -368,7 +369,8 @@ public void testUpdateReplicaUnderRecovery() throws IOException {
             .getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp());
         try {
           //update should fail
-          fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, newlength);
+          fsdataset.updateReplicaUnderRecovery(tmp, recoveryid,
+              tmp.getBlockId(), newlength);
           Assert.fail();
         } catch(IOException ioe) {
           System.out.println("GOOD: getting " + ioe);
@@ -377,7 +379,8 @@ public void testUpdateReplicaUnderRecovery() throws IOException {
 
       //update
       final String storageID = fsdataset.updateReplicaUnderRecovery(
-          new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, newlength);
+          new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid,
+          rri.getBlockId(), newlength);
       assertTrue(storageID != null);
 
     } finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 771609cf24daa..a4df4ab09b4a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -17,29 +17,25 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
-import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.io.File;
 import java.io.IOException;
+import java.util.EnumSet;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
 import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
@@ -240,13 +236,34 @@ public void testAppendIsDenied() throws IOException {
     makeTestFile(path, BLOCK_SIZE, true);
 
     try {
-      client.append(path.toString(), BUFFER_LENGTH, null, null).close();
+      client.append(path.toString(), BUFFER_LENGTH,
+          EnumSet.of(CreateFlag.APPEND), null, null).close();
       fail("Append to LazyPersist file did not fail as expected");
     } catch (Throwable t) {
       LOG.info("Got expected exception ", t);
     }
   }
 
+  /**
+   * Truncate to lazy persist file is denied.
+   * @throws IOException
+   */
+  @Test
+  public void testTruncateIsDenied() throws IOException {
+    startUpCluster(true, -1);
+    final String METHOD_NAME = GenericTestUtils.getMethodName();
+    Path path = new Path("/" + METHOD_NAME + ".dat");
+
+    makeTestFile(path, BLOCK_SIZE, true);
+
+    try {
+      client.truncate(path.toString(), BLOCK_SIZE/2);
+      fail("Truncate to LazyPersist file did not fail as expected");
+    } catch (Throwable t) {
+      LOG.info("Got expected exception ", t);
+    }
+  }
+
   /**
    * If one or more replicas of a lazyPersist file are lost, then the file
    * must be discarded by the NN, instead of being kept around as a
@@ -304,37 +321,7 @@ public void testLazyPersistBlocksAreSaved()
 
     // Make sure that there is a saved copy of the replica on persistent
     // storage.
-    final String bpid = cluster.getNamesystem().getBlockPoolId();
-    List<? extends FsVolumeSpi> volumes =
-        cluster.getDataNodes().get(0).getFSDataset().getVolumes();
-
-    final Set<Long> persistedBlockIds = new HashSet<Long>();
-
-    // Make sure at least one non-transient volume has a saved copy of
-    // the replica.
-    for (FsVolumeSpi v : volumes) {
-      if (v.isTransientStorage()) {
-        continue;
-      }
-
-      FsVolumeImpl volume = (FsVolumeImpl) v;
-      File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
-
-      for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
-        File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, lb.getBlock().getBlockId());
-        File blockFile = new File(targetDir, lb.getBlock().getBlockName());
-        if (blockFile.exists()) {
-          // Found a persisted copy for this block!
-          boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
-          assertThat(added, is(true));
-        } else {
-          LOG.error(blockFile + " not found");
-        }
-      }
-    }
-
-    // We should have found a persisted copy for each located block.
-    assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+    ensureLazyPersistBlocksAreSaved(locatedBlocks);
   }
 
   /**
@@ -663,6 +650,7 @@ public void testDnRestartWithSavedReplicas()
     LOG.info("Restarting the DataNode");
     cluster.restartDataNode(0, true);
     cluster.waitActive();
+    triggerBlockReport();
 
     // Ensure that the replica is now on persistent storage.
     ensureFileReplicasOnStorageType(path1, DEFAULT);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
index efc6dcb374c27..be6ca2cf08eae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
@@ -178,7 +178,7 @@ private void doShortCircuitReadAfterEvictionTest() throws IOException,
 
     // Verify short-circuit read from RAM_DISK.
     ensureFileReplicasOnStorageType(path1, RAM_DISK);
-    File metaFile = MiniDFSCluster.getBlockMetadataFile(0,
+    File metaFile = cluster.getBlockMetadataFile(0,
         DFSTestUtil.getFirstBlock(fs, path1));
     assertTrue(metaFile.length() <= BlockMetadataHeader.getHeaderSize());
     assertTrue(verifyReadRandomFile(path1, BLOCK_SIZE, SEED));
@@ -188,7 +188,7 @@ private void doShortCircuitReadAfterEvictionTest() throws IOException,
 
     // Verify short-circuit read from RAM_DISK once again.
     ensureFileReplicasOnStorageType(path1, RAM_DISK);
-    metaFile = MiniDFSCluster.getBlockMetadataFile(0,
+    metaFile = cluster.getBlockMetadataFile(0,
         DFSTestUtil.getFirstBlock(fs, path1));
     assertTrue(metaFile.length() <= BlockMetadataHeader.getHeaderSize());
     assertTrue(verifyReadRandomFile(path1, BLOCK_SIZE, SEED));
@@ -201,7 +201,7 @@ private void doShortCircuitReadAfterEvictionTest() throws IOException,
     // Verify short-circuit read still works from DEFAULT storage.  This time,
     // we'll have a checksum written during lazy persistence.
     ensureFileReplicasOnStorageType(path1, DEFAULT);
-    metaFile = MiniDFSCluster.getBlockMetadataFile(0,
+    metaFile = cluster.getBlockMetadataFile(0,
         DFSTestUtil.getFirstBlock(fs, path1));
     assertTrue(metaFile.length() > BlockMetadataHeader.getHeaderSize());
     assertTrue(verifyReadRandomFile(path1, BLOCK_SIZE, SEED));
@@ -251,7 +251,7 @@ public void doShortCircuitReadBlockFileCorruptionTest() throws IOException,
     // Corrupt the lazy-persisted block file, and verify that checksum
     // verification catches it.
     ensureFileReplicasOnStorageType(path1, DEFAULT);
-    MiniDFSCluster.corruptReplica(0, DFSTestUtil.getFirstBlock(fs, path1));
+    cluster.corruptReplica(0, DFSTestUtil.getFirstBlock(fs, path1));
     exception.expect(ChecksumException.class);
     DFSTestUtil.readFileBuffer(fs, path1);
   }
@@ -291,7 +291,7 @@ public void doShortCircuitReadMetaFileCorruptionTest() throws IOException,
     // Corrupt the lazy-persisted checksum file, and verify that checksum
     // verification catches it.
     ensureFileReplicasOnStorageType(path1, DEFAULT);
-    File metaFile = MiniDFSCluster.getBlockMetadataFile(0,
+    File metaFile = cluster.getBlockMetadataFile(0,
         DFSTestUtil.getFirstBlock(fs, path1));
     MiniDFSCluster.corruptBlock(metaFile);
     exception.expect(ChecksumException.class);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index 60c6d0304f5c3..5aafc9cb29bda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hdfs.server.datanode.ReplicaAlreadyExistsException;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaUnderRecovery;
@@ -148,7 +149,8 @@ private ExtendedBlock[] setup(String bpid, FsDatasetImpl dataSet) throws IOExcep
     };
     
     ReplicaMap replicasMap = dataSet.volumeMap;
-    FsVolumeImpl vol = dataSet.volumes.getNextVolume(StorageType.DEFAULT, 0);
+    FsVolumeImpl vol = (FsVolumeImpl) dataSet.volumes
+        .getNextVolume(StorageType.DEFAULT, 0).getVolume();
     ReplicaInfo replicaInfo = new FinalizedReplica(
         blocks[FINALIZED].getLocalBlock(), vol, vol.getCurrentDir().getParentFile());
     replicasMap.add(bpid, replicaInfo);
@@ -157,10 +159,10 @@ private ExtendedBlock[] setup(String bpid, FsDatasetImpl dataSet) throws IOExcep
     
     replicasMap.add(bpid, new ReplicaInPipeline(
         blocks[TEMPORARY].getBlockId(),
-        blocks[TEMPORARY].getGenerationStamp(), vol, 
+        blocks[TEMPORARY].getGenerationStamp(), vol,
         vol.createTmpFile(bpid, blocks[TEMPORARY].getLocalBlock()).getParentFile(), 0));
     
-    replicaInfo = new ReplicaBeingWritten(blocks[RBW].getLocalBlock(), vol, 
+    replicaInfo = new ReplicaBeingWritten(blocks[RBW].getLocalBlock(), vol,
         vol.createRbwFile(bpid, blocks[RBW].getLocalBlock()).getParentFile(), null);
     replicasMap.add(bpid, replicaInfo);
     replicaInfo.getBlockFile().createNewFile();
@@ -489,8 +491,8 @@ private void testWriteToTemporary(FsDatasetImpl dataSet, ExtendedBlock[] blocks)
     long newGenStamp = blocks[NON_EXISTENT].getGenerationStamp() * 10;
     blocks[NON_EXISTENT].setGenerationStamp(newGenStamp);
     try {
-      ReplicaInPipeline replicaInfo =
-                dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+      ReplicaInPipelineInterface replicaInfo =
+          dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]).getReplica();
       Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp);
       Assert.assertTrue(
           replicaInfo.getBlockId() == blocks[NON_EXISTENT].getBlockId());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 3f96c0c5ce001..0349251ad38c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.Storage;
 
@@ -66,10 +66,10 @@ static void addFiles(FSEditLog editLog, int numFiles, short replication,
     INodeDirectory dirInode = new INodeDirectory(inodeId.nextValue(), null, p,
       0L);
     editLog.logMkDir(BASE_PATH, dirInode);
-    BlockInfo[] blocks = new BlockInfo[blocksPerFile];
+    BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile];
     for (int iB = 0; iB < blocksPerFile; ++iB) {
       blocks[iB] = 
-       new BlockInfo(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
+       new BlockInfoContiguous(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
                                replication);
     }
     
@@ -97,7 +97,7 @@ static void addFiles(FSEditLog editLog, int numFiles, short replication,
         editLog.logMkDir(currentDir, dirInode);
       }
       INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
-          p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+          p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
       fileUc.toUnderConstruction("", "");
       editLog.logOpenFile(filePath, fileUc, false, false);
       editLog.logCloseFile(filePath, inode);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index aff133f270d89..f481bc1c64743 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -25,7 +25,9 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -40,6 +42,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.AclException;
 import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -79,6 +82,12 @@ public abstract class FSAclBaseTest {
 
   private FileSystem fs, fsAsBruce, fsAsDiana, fsAsSupergroupMember, fsAsBob;
 
+  protected static void startCluster() throws IOException {
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
+  }
+
   @AfterClass
   public static void shutdown() {
     if (cluster != null) {
@@ -426,7 +435,7 @@ public void testRemoveAclEntriesPathNotFound() throws IOException {
   }
 
   @Test
-  public void testRemoveDefaultAcl() throws IOException {
+  public void testRemoveDefaultAcl() throws Exception {
     FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
     List<AclEntry> aclSpec = Lists.newArrayList(
       aclEntry(ACCESS, USER, ALL),
@@ -443,10 +452,15 @@ public void testRemoveDefaultAcl() throws IOException {
       aclEntry(ACCESS, GROUP, READ_EXECUTE) }, returned);
     assertPermission((short)010770);
     assertAclFeature(true);
+    // restart of the cluster
+    restartCluster();
+    s = fs.getAclStatus(path);
+    AclEntry[] afterRestart = s.getEntries().toArray(new AclEntry[0]);
+    assertArrayEquals(returned, afterRestart);
   }
 
   @Test
-  public void testRemoveDefaultAclOnlyAccess() throws IOException {
+  public void testRemoveDefaultAclOnlyAccess() throws Exception {
     fs.create(path).close();
     fs.setPermission(path, FsPermission.createImmutable((short)0640));
     List<AclEntry> aclSpec = Lists.newArrayList(
@@ -463,10 +477,15 @@ public void testRemoveDefaultAclOnlyAccess() throws IOException {
       aclEntry(ACCESS, GROUP, READ_EXECUTE) }, returned);
     assertPermission((short)010770);
     assertAclFeature(true);
+    // restart of the cluster
+    restartCluster();
+    s = fs.getAclStatus(path);
+    AclEntry[] afterRestart = s.getEntries().toArray(new AclEntry[0]);
+    assertArrayEquals(returned, afterRestart);
   }
 
   @Test
-  public void testRemoveDefaultAclOnlyDefault() throws IOException {
+  public void testRemoveDefaultAclOnlyDefault() throws Exception {
     FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
     List<AclEntry> aclSpec = Lists.newArrayList(
       aclEntry(DEFAULT, USER, "foo", ALL));
@@ -477,10 +496,15 @@ public void testRemoveDefaultAclOnlyDefault() throws IOException {
     assertArrayEquals(new AclEntry[] { }, returned);
     assertPermission((short)0750);
     assertAclFeature(false);
+    // restart of the cluster
+    restartCluster();
+    s = fs.getAclStatus(path);
+    AclEntry[] afterRestart = s.getEntries().toArray(new AclEntry[0]);
+    assertArrayEquals(returned, afterRestart);
   }
 
   @Test
-  public void testRemoveDefaultAclMinimal() throws IOException {
+  public void testRemoveDefaultAclMinimal() throws Exception {
     FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
     fs.removeDefaultAcl(path);
     AclStatus s = fs.getAclStatus(path);
@@ -488,10 +512,15 @@ public void testRemoveDefaultAclMinimal() throws IOException {
     assertArrayEquals(new AclEntry[] { }, returned);
     assertPermission((short)0750);
     assertAclFeature(false);
+    // restart of the cluster
+    restartCluster();
+    s = fs.getAclStatus(path);
+    AclEntry[] afterRestart = s.getEntries().toArray(new AclEntry[0]);
+    assertArrayEquals(returned, afterRestart);
   }
 
   @Test
-  public void testRemoveDefaultAclStickyBit() throws IOException {
+  public void testRemoveDefaultAclStickyBit() throws Exception {
     FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)01750));
     List<AclEntry> aclSpec = Lists.newArrayList(
       aclEntry(ACCESS, USER, ALL),
@@ -508,6 +537,11 @@ public void testRemoveDefaultAclStickyBit() throws IOException {
       aclEntry(ACCESS, GROUP, READ_EXECUTE) }, returned);
     assertPermission((short)011770);
     assertAclFeature(true);
+    // restart of the cluster
+    restartCluster();
+    s = fs.getAclStatus(path);
+    AclEntry[] afterRestart = s.getEntries().toArray(new AclEntry[0]);
+    assertArrayEquals(returned, afterRestart);
   }
 
   @Test(expected=FileNotFoundException.class)
@@ -827,8 +861,8 @@ public void testSetPermissionCannotSetAclBit() throws IOException {
     fs.setPermission(path,
       new FsPermissionExtension(FsPermission.
           createImmutable((short)0755), true, true));
-    INode inode = cluster.getNamesystem().getFSDirectory().getNode(
-      path.toUri().getPath(), false);
+    INode inode = cluster.getNamesystem().getFSDirectory().getINode(
+        path.toUri().getPath(), false);
     assertNotNull(inode);
     FsPermission perm = inode.getFsPermission();
     assertNotNull(perm);
@@ -1137,9 +1171,7 @@ public void testSkipAclEnforcementPermsDisabled() throws Exception {
     assertFilePermissionDenied(fsAsDiana, DIANA, bruceFile);
     try {
       conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
-      destroyFileSystems();
       restartCluster();
-      initFileSystems();
       assertFilePermissionGranted(fsAsDiana, DIANA, bruceFile);
     } finally {
       conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
@@ -1317,6 +1349,233 @@ public void testAccess() throws IOException, InterruptedException {
     }
   }
 
+  @Test
+  public void testEffectiveAccess() throws Exception {
+    Path p1 = new Path("/testEffectiveAccess");
+    fs.mkdirs(p1);
+    // give all access at first
+    fs.setPermission(p1, FsPermission.valueOf("-rwxrwxrwx"));
+    AclStatus aclStatus = fs.getAclStatus(p1);
+    assertEquals("Entries should be empty", 0, aclStatus.getEntries().size());
+    assertEquals("Permission should be carried by AclStatus",
+        fs.getFileStatus(p1).getPermission(), aclStatus.getPermission());
+
+    // Add a named entries with all access
+    fs.modifyAclEntries(p1, Lists.newArrayList(
+        aclEntry(ACCESS, USER, "bruce", ALL),
+        aclEntry(ACCESS, GROUP, "groupY", ALL)));
+    aclStatus = fs.getAclStatus(p1);
+    assertEquals("Entries should contain owner group entry also", 3, aclStatus
+        .getEntries().size());
+
+    // restrict the access
+    fs.setPermission(p1, FsPermission.valueOf("-rwxr-----"));
+    // latest permissions should be reflected as effective permission
+    aclStatus = fs.getAclStatus(p1);
+    List<AclEntry> entries = aclStatus.getEntries();
+    for (AclEntry aclEntry : entries) {
+      if (aclEntry.getName() != null || aclEntry.getType() == GROUP) {
+        assertEquals(FsAction.ALL, aclEntry.getPermission());
+        assertEquals(FsAction.READ, aclStatus.getEffectivePermission(aclEntry));
+      }
+    }
+    fsAsBruce.access(p1, READ);
+    try {
+      fsAsBruce.access(p1, WRITE);
+      fail("Access should not be given");
+    } catch (AccessControlException e) {
+      // expected
+    }
+    fsAsBob.access(p1, READ);
+    try {
+      fsAsBob.access(p1, WRITE);
+      fail("Access should not be given");
+    } catch (AccessControlException e) {
+      // expected
+    }
+  }
+
+  /**
+   * Verify the de-duplication of AclFeatures with same entries.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testDeDuplication() throws Exception {
+    // This test needs to verify the count of the references which is held by
+    // static data structure. So shutting down entire cluster to get the fresh
+    // data.
+    shutdown();
+    AclStorage.getUniqueAclFeatures().clear();
+    startCluster();
+    setUp();
+    int currentSize = 0;
+    Path p1 = new Path("/testDeduplication");
+    {
+      // unique default AclEntries for this test
+      List<AclEntry> aclSpec = Lists.newArrayList(
+          aclEntry(DEFAULT, USER, "testdeduplicateuser", ALL),
+          aclEntry(DEFAULT, GROUP, "testdeduplicategroup", ALL));
+      fs.mkdirs(p1);
+      fs.modifyAclEntries(p1, aclSpec);
+      assertEquals("One more ACL feature should be unique", currentSize + 1,
+          AclStorage.getUniqueAclFeatures().getUniqueElementsSize());
+      currentSize++;
+    }
+    Path child1 = new Path(p1, "child1");
+    AclFeature child1AclFeature;
+    {
+      // new child dir should copy entries from its parent.
+      fs.mkdirs(child1);
+      assertEquals("One more ACL feature should be unique", currentSize + 1,
+          AclStorage.getUniqueAclFeatures().getUniqueElementsSize());
+      child1AclFeature = getAclFeature(child1, cluster);
+      assertEquals("Reference count should be 1", 1,
+          child1AclFeature.getRefCount());
+      currentSize++;
+    }
+    Path child2 = new Path(p1, "child2");
+    {
+      // new child dir should copy entries from its parent. But all entries are
+      // same as its sibling without any more acl changes.
+      fs.mkdirs(child2);
+      assertEquals("existing AclFeature should be re-used", currentSize,
+          AclStorage.getUniqueAclFeatures().getUniqueElementsSize());
+      AclFeature child2AclFeature = getAclFeature(child1, cluster);
+      assertSame("Same Aclfeature should be re-used", child1AclFeature,
+          child2AclFeature);
+      assertEquals("Reference count should be 2", 2,
+          child2AclFeature.getRefCount());
+    }
+    {
+      // modification of ACL on should decrement the original reference count
+      // and increase new one.
+      List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER,
+          "user1", ALL));
+      fs.modifyAclEntries(child1, aclSpec);
+      AclFeature modifiedAclFeature = getAclFeature(child1, cluster);
+      assertEquals("Old Reference count should be 1", 1,
+          child1AclFeature.getRefCount());
+      assertEquals("New Reference count should be 1", 1,
+          modifiedAclFeature.getRefCount());
+
+      // removing the new added ACL entry should refer to old ACLfeature
+      AclEntry aclEntry = new AclEntry.Builder().setScope(ACCESS).setType(USER)
+          .setName("user1").build();
+      fs.removeAclEntries(child1, Lists.newArrayList(aclEntry));
+      assertEquals("Old Reference count should be 2 again", 2,
+          child1AclFeature.getRefCount());
+      assertEquals("New Reference count should be 0", 0,
+          modifiedAclFeature.getRefCount());
+    }
+    {
+      // verify the reference count on deletion of Acls
+      fs.removeAcl(child2);
+      assertEquals("Reference count should be 1", 1,
+          child1AclFeature.getRefCount());
+    }
+    {
+      // verify the reference count on deletion of dir with ACL
+      fs.delete(child1, true);
+      assertEquals("Reference count should be 0", 0,
+          child1AclFeature.getRefCount());
+    }
+
+    Path file1 = new Path(p1, "file1");
+    Path file2 = new Path(p1, "file2");
+    AclFeature fileAclFeature;
+    {
+      // Using same reference on creation of file
+      fs.create(file1).close();
+      fileAclFeature = getAclFeature(file1, cluster);
+      assertEquals("Reference count should be 1", 1,
+          fileAclFeature.getRefCount());
+      fs.create(file2).close();
+      assertEquals("Reference count should be 2", 2,
+          fileAclFeature.getRefCount());
+    }
+    {
+      // modifying ACLs on file should decrease the reference count on old
+      // instance and increase on the new instance
+      List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER,
+          "user1", ALL));
+      // adding new ACL entry
+      fs.modifyAclEntries(file1, aclSpec);
+      AclFeature modifiedFileAcl = getAclFeature(file1, cluster);
+      assertEquals("Old Reference count should be 1", 1,
+          fileAclFeature.getRefCount());
+      assertEquals("New Reference count should be 1", 1,
+          modifiedFileAcl.getRefCount());
+
+      // removing the new added ACL entry should refer to old ACLfeature
+      AclEntry aclEntry = new AclEntry.Builder().setScope(ACCESS).setType(USER)
+          .setName("user1").build();
+      fs.removeAclEntries(file1, Lists.newArrayList(aclEntry));
+      assertEquals("Old Reference count should be 2", 2,
+          fileAclFeature.getRefCount());
+      assertEquals("New Reference count should be 0", 0,
+          modifiedFileAcl.getRefCount());
+    }
+    {
+      // reference count should be decreased on deletion of files with ACLs
+      fs.delete(file2, true);
+      assertEquals("Reference count should be decreased on delete of the file",
+          1, fileAclFeature.getRefCount());
+      fs.delete(file1, true);
+      assertEquals("Reference count should be decreased on delete of the file",
+          0, fileAclFeature.getRefCount());
+
+      // On reference count reaches 0 instance should be removed from map
+      fs.create(file1).close();
+      AclFeature newFileAclFeature = getAclFeature(file1, cluster);
+      assertNotSame("Instance should be different on reference count 0",
+          fileAclFeature, newFileAclFeature);
+      fileAclFeature = newFileAclFeature;
+    }
+    Map<AclFeature, Integer> restartRefCounter = new HashMap<>();
+    // Restart the Namenode to check the references.
+    // Here reference counts will not be same after restart because, while
+    // shutting down namenode will not call any removal of AclFeature.
+    // However this is applicable only in case of tests as in real-cluster JVM
+    // itself will be new.
+    List<AclFeature> entriesBeforeRestart = AclStorage.getUniqueAclFeatures()
+        .getEntries();
+    {
+      //restart by loading edits
+      for (AclFeature aclFeature : entriesBeforeRestart) {
+        restartRefCounter.put(aclFeature, aclFeature.getRefCount());
+      }
+      cluster.restartNameNode(true);
+      List<AclFeature> entriesAfterRestart = AclStorage.getUniqueAclFeatures()
+          .getEntries();
+      assertEquals("Entries before and after should be same",
+          entriesBeforeRestart, entriesAfterRestart);
+      for (AclFeature aclFeature : entriesAfterRestart) {
+        int before = restartRefCounter.get(aclFeature);
+        assertEquals("ReferenceCount After Restart should be doubled",
+            before * 2, aclFeature.getRefCount());
+      }
+    }
+    {
+      //restart by loading fsimage
+      cluster.getNameNodeRpc()
+          .setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
+      cluster.getNameNodeRpc().saveNamespace();
+      cluster.getNameNodeRpc()
+          .setSafeMode(SafeModeAction.SAFEMODE_LEAVE, false);
+      cluster.restartNameNode(true);
+      List<AclFeature> entriesAfterRestart = AclStorage.getUniqueAclFeatures()
+          .getEntries();
+      assertEquals("Entries before and after should be same",
+          entriesBeforeRestart, entriesAfterRestart);
+      for (AclFeature aclFeature : entriesAfterRestart) {
+        int before = restartRefCounter.get(aclFeature);
+        assertEquals("ReferenceCount After 2 Restarts should be tripled",
+            before * 3, aclFeature.getRefCount());
+      }
+    }
+  }
+
   /**
    * Creates a FileSystem for the super-user.
    *
@@ -1358,10 +1617,12 @@ private void initFileSystems() throws Exception {
    * @throws Exception if restart fails
    */
   private void restartCluster() throws Exception {
+    destroyFileSystems();
     shutdown();
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(false)
       .build();
     cluster.waitActive();
+    initFileSystems();
   }
 
   /**
@@ -1386,10 +1647,7 @@ private static void assertAclFeature(boolean expectAclFeature)
    */
   private static void assertAclFeature(Path pathToCheck,
       boolean expectAclFeature) throws IOException {
-    INode inode = cluster.getNamesystem().getFSDirectory()
-      .getNode(pathToCheck.toUri().getPath(), false);
-    assertNotNull(inode);
-    AclFeature aclFeature = inode.getAclFeature();
+    AclFeature aclFeature = getAclFeature(pathToCheck, cluster);
     if (expectAclFeature) {
       assertNotNull(aclFeature);
       // Intentionally capturing a reference to the entries, not using nested
@@ -1403,6 +1661,18 @@ private static void assertAclFeature(Path pathToCheck,
     }
   }
 
+  /**
+   * Get AclFeature for the path
+   */
+  public static AclFeature getAclFeature(Path pathToCheck,
+      MiniDFSCluster cluster) throws IOException {
+    INode inode = cluster.getNamesystem().getFSDirectory()
+        .getINode(pathToCheck.toUri().getPath(), false);
+    assertNotNull(inode);
+    AclFeature aclFeature = inode.getAclFeature();
+    return aclFeature;
+  }
+
   /**
    * Asserts the value of the FsPermission bits on the inode of the test path.
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 1fb1c1f9942b8..c8def37df1a7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -28,11 +28,11 @@
 import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -60,6 +60,7 @@
 import org.apache.hadoop.net.DNS;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Tool;
@@ -144,12 +145,11 @@ void close() {
   static void setNameNodeLoggingLevel(Level logLevel) {
     LOG.fatal("Log level = " + logLevel.toString());
     // change log level to NameNode logs
-    LogManager.getLogger(NameNode.class.getName()).setLevel(logLevel);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(logLevel);
-    LogManager.getLogger(NetworkTopology.class.getName()).setLevel(logLevel);
-    LogManager.getLogger(FSNamesystem.class.getName()).setLevel(logLevel);
-    LogManager.getLogger(LeaseManager.class.getName()).setLevel(logLevel);
-    LogManager.getLogger(Groups.class.getName()).setLevel(logLevel);
+    DFSTestUtil.setNameNodeLogLevel(logLevel);
+    GenericTestUtils.setLogLevel(LogManager.getLogger(
+            NetworkTopology.class.getName()), logLevel);
+    GenericTestUtils.setLogLevel(LogManager.getLogger(
+            Groups.class.getName()), logLevel);
   }
 
   /**
@@ -977,7 +977,7 @@ void formBlockReport() {
       // fill remaining slots with blocks that do not exist
       for(int idx = blocks.size()-1; idx >= nrBlocks; idx--)
         blocks.set(idx, new Block(blocks.size() - idx, 0, 0));
-      blockReportList = new BlockListAsLongs(blocks,null).getBlockListAsLongs();
+      blockReportList = new BlockListAsLongs(blocks).getBlockListAsLongs();
     }
 
     long[] getBlockReportList() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
index 61e7f145fb5c3..7aad37854eff6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
@@ -64,8 +64,8 @@ public static FSNamesystem getNamesystem(NameNode namenode) {
    */
   public static LocatedBlocks getBlockLocations(NameNode namenode,
       String src, long offset, long length) throws IOException {
-    return namenode.getNamesystem().getBlockLocations(
-        src, offset, length, false, true, true);
+    return namenode.getNamesystem().getBlockLocations("foo",
+        src, offset, length);
   }
   
   public static HdfsFileStatus getFileInfo(NameNode namenode, String src,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
index 301ee25ad6b85..a417c3d79d6dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.junit.After;
 import org.junit.Before;
@@ -87,21 +87,21 @@ public void testAddBlock() throws Exception {
     
     // check file1
     INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile();
-    BlockInfo[] file1Blocks = file1Node.getBlocks();
+    BlockInfoContiguous[] file1Blocks = file1Node.getBlocks();
     assertEquals(1, file1Blocks.length);
     assertEquals(BLOCKSIZE - 1, file1Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file1Blocks[0].getBlockUCState());
     
     // check file2
     INodeFile file2Node = fsdir.getINode4Write(file2.toString()).asFile();
-    BlockInfo[] file2Blocks = file2Node.getBlocks();
+    BlockInfoContiguous[] file2Blocks = file2Node.getBlocks();
     assertEquals(1, file2Blocks.length);
     assertEquals(BLOCKSIZE, file2Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file2Blocks[0].getBlockUCState());
     
     // check file3
     INodeFile file3Node = fsdir.getINode4Write(file3.toString()).asFile();
-    BlockInfo[] file3Blocks = file3Node.getBlocks();
+    BlockInfoContiguous[] file3Blocks = file3Node.getBlocks();
     assertEquals(2, file3Blocks.length);
     assertEquals(BLOCKSIZE, file3Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file3Blocks[0].getBlockUCState());
@@ -110,7 +110,7 @@ public void testAddBlock() throws Exception {
     
     // check file4
     INodeFile file4Node = fsdir.getINode4Write(file4.toString()).asFile();
-    BlockInfo[] file4Blocks = file4Node.getBlocks();
+    BlockInfoContiguous[] file4Blocks = file4Node.getBlocks();
     assertEquals(2, file4Blocks.length);
     assertEquals(BLOCKSIZE, file4Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file4Blocks[0].getBlockUCState());
@@ -141,7 +141,7 @@ public void testAddBlockUC() throws Exception {
       FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
       
       INodeFile fileNode = fsdir.getINode4Write(file1.toString()).asFile();
-      BlockInfo[] fileBlocks = fileNode.getBlocks();
+      BlockInfoContiguous[] fileBlocks = fileNode.getBlocks();
       assertEquals(2, fileBlocks.length);
       assertEquals(BLOCKSIZE, fileBlocks[0].getNumBytes());
       assertEquals(BlockUCState.COMPLETE, fileBlocks[0].getBlockUCState());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java
new file mode 100644
index 0000000000000..06dfade3f4f0c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockgroup.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestAddBlockgroup {
+
+  public static final Log LOG = LogFactory.getLog(TestAddBlockgroup.class);
+
+  private final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS +
+      HdfsConstants.NUM_PARITY_BLOCKS;
+  private final short NUM_DATANODES = GROUP_SIZE;
+
+  private static final int BLOCKSIZE = 1024;
+  private static final short REPLICATION = 3;
+
+  private MiniDFSCluster cluster;
+  private Configuration conf;
+
+  @Before
+  public void setup() throws IOException {
+    conf = new Configuration();
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
+        .build();
+    cluster.waitActive();
+    cluster.getFileSystem().setStoragePolicy(new Path("/"),
+        HdfsConstants.EC_STORAGE_POLICY_NAME);
+  }
+
+  @After
+  public void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test
+  public void testAddBlockGroup() throws Exception {
+    DistributedFileSystem fs = cluster.getFileSystem();
+    FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+
+    final Path file1 = new Path("/file1");
+    DFSTestUtil.createFile(fs, file1, BLOCKSIZE * 2, REPLICATION, 0L);
+    INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile();
+    BlockInfoContiguous[] file1Blocks = file1Node.getBlocks();
+    assertEquals(2, file1Blocks.length);
+    assertEquals(GROUP_SIZE, file1Blocks[0].numNodes());
+    assertEquals(HdfsConstants.MAX_BLOCKS_IN_GROUP,
+        file1Blocks[1].getBlockId() - file1Blocks[0].getBlockId());
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
index 872ff9c490f6d..1fbe160a4711f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
@@ -35,8 +35,8 @@
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.junit.AfterClass;
@@ -91,12 +91,12 @@ private void verifyFileBlocks(String file,
         " isUnderConstruction = " + inode.isUnderConstruction() +
         " expected to be " + isFileOpen,
         inode.isUnderConstruction() == isFileOpen);
-    BlockInfo[] blocks = inode.getBlocks();
+    BlockInfoContiguous[] blocks = inode.getBlocks();
     assertTrue("File does not have blocks: " + inode.toString(),
         blocks != null && blocks.length > 0);
     
     int idx = 0;
-    BlockInfo curBlock;
+    BlockInfoContiguous curBlock;
     // all blocks but the last two should be regular blocks
     for(; idx < blocks.length - 2; idx++) {
       curBlock = blocks[idx];
@@ -170,7 +170,7 @@ public void testGetBlockLocations() throws IOException {
       final List<LocatedBlock> blocks = lb.getLocatedBlocks();
       assertEquals(i, blocks.size());
       final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
-      assertTrue(b instanceof BlockInfoUnderConstruction);
+      assertTrue(b instanceof BlockInfoContiguousUnderConstruction);
 
       if (++i < NUM_BLOCKS) {
         // write one more block
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index 93076928f3e21..3617ee3ca3a2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -88,6 +88,7 @@
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.apache.htrace.Sampler;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -908,7 +909,7 @@ public Boolean get() {
 
     // Uncache and check each path in sequence
     RemoteIterator<CacheDirectiveEntry> entries =
-      new CacheDirectiveIterator(nnRpc, null);
+      new CacheDirectiveIterator(nnRpc, null, Sampler.NEVER);
     for (int i=0; i<numFiles; i++) {
       CacheDirectiveEntry entry = entries.next();
       nnRpc.removeCacheDirective(entry.getInfo().getId());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
index bb4689d363472..95da8387cdda0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
@@ -633,6 +633,22 @@ public boolean matches(Object argument) {
         });
   }
 
+  private void checkTempImages(NNStorage storage) throws IOException {
+    List<File> dirs = new ArrayList<File>();
+    dirs.add(storage.getStorageDir(0).getCurrentDir());
+    dirs.add(storage.getStorageDir(1).getCurrentDir());
+
+    for (File dir : dirs) {
+      File[] list = dir.listFiles();
+      for (File f : list) {
+        // Throw an exception if a temp image file is found.
+        if(f.getName().contains(NNStorage.NameNodeFile.IMAGE_NEW.getName())) {
+          throw new IOException("Found " + f);
+        }
+      }
+    }
+  }
+
   /**
    * Simulate 2NN failing to send the whole file (error type 3)
    * The length header in the HTTP transfer should prevent
@@ -694,6 +710,9 @@ private void doSendFailTest(String exceptionSubstring)
         GenericTestUtils.assertExceptionContains(exceptionSubstring, e);
       }
       Mockito.reset(faultInjector);
+      // Make sure there is no temporary files left around.
+      checkTempImages(cluster.getNameNode().getFSImage().getStorage());
+      checkTempImages(secondary.getFSImage().getStorage());
       secondary.shutdown(); // secondary namenode crash!
       secondary = null;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index d0502b3660398..b7e8c254c527a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -22,8 +22,8 @@
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.junit.Test;
@@ -54,7 +54,9 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
     // set file's parent as root and put the file to inodeMap, so
     // FSNamesystem's isFileDeleted() method will return false on this file
     if (file.getParent() == null) {
-      INodeDirectory parent = mock(INodeDirectory.class);
+      INodeDirectory mparent = mock(INodeDirectory.class);
+      INodeDirectory parent = new INodeDirectory(mparent.getId(), new byte[0],
+          mparent.getPermissionStatus(), mparent.getAccessTime());
       parent.setLocalName(new byte[0]);
       parent.addChild(file);
       file.setParent(parent);
@@ -62,7 +64,7 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
     namesystem.dir.getINodeMap().put(file);
 
     FSNamesystem namesystemSpy = spy(namesystem);
-    BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
+    BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction(
         block, (short) 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
     blockInfo.setBlockCollection(file);
     blockInfo.setGenerationStamp(genStamp);
@@ -71,8 +73,9 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
     doReturn(true).when(file).isUnderConstruction();
 
     doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
+    doReturn(blockInfo).when(file).getLastBlock();
     doReturn("").when(namesystemSpy).closeFileCommitBlocks(
-        any(INodeFile.class), any(BlockInfo.class));
+        any(INodeFile.class), any(BlockInfoContiguous.class));
     doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog();
 
     return namesystemSpy;
@@ -100,11 +103,12 @@ public void testCommitBlockSynchronization() throws IOException {
         lastBlock, genStamp, length, false, false, newTargets, null);
 
     // Simulate 'completing' the block.
-    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+    BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)
         .getStoredBlock(any(Block.class));
+    doReturn(completedBlockInfo).when(file).getLastBlock();
 
     // Repeat the call to make sure it does not throw
     namesystemSpy.commitBlockSynchronization(
@@ -171,11 +175,12 @@ public void testCommitBlockSynchronizationWithClose() throws IOException {
     namesystemSpy.commitBlockSynchronization(
         lastBlock, genStamp, length, true, false, newTargets, null);
 
-    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+    BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)
         .getStoredBlock(any(Block.class));
+    doReturn(completedBlockInfo).when(file).getLastBlock();
 
     namesystemSpy.commitBlockSynchronization(
         lastBlock, genStamp, length, true, false, newTargets, null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
index 28f5eb497b203..a9aba864e94e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
@@ -239,10 +239,10 @@ private void checkDFSAdminDecommissionStatus(
       System.setOut(oldOut);
     }
   }
+
   /**
    * Tests Decommissioning Status in DFS.
    */
-
   @Test
   public void testDecommissionStatus() throws IOException, InterruptedException {
     InetSocketAddress addr = new InetSocketAddress("localhost", cluster
@@ -351,6 +351,11 @@ public void testDecommissionStatusAfterDNRestart()
     assertTrue("the node should be DECOMMISSION_IN_PROGRESSS",
         dead.get(0).isDecommissionInProgress());
 
+    // Check DatanodeManager#getDecommissionNodes, make sure it returns
+    // the node as decommissioning, even if it's dead
+    List<DatanodeDescriptor> decomlist = dm.getDecommissioningNodes();
+    assertTrue("The node should be be decommissioning", decomlist.size() == 1);
+    
     // Delete the under-replicated file, which should let the 
     // DECOMMISSION_IN_PROGRESS node become DECOMMISSIONED
     cleanupFile(fileSys, f);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 267821f85beee..7d4eb31bd3122 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.FileNotFoundException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -226,10 +228,19 @@ public void testRenameRace() throws Exception {
   private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot)
       throws Exception {
     LOG.info("Start testing, hasSnapshot: " + hasSnapshot);
-    final String testPaths[] = {
-        "/test-file",
-        "/testdir/testdir1/test-file"
-    };
+    ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>> testList =
+        new ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>> ();
+    testList.add(
+        new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file", false));
+    testList.add(     
+        new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file1", true));
+    testList.add(
+        new AbstractMap.SimpleImmutableEntry<String, Boolean>(
+            "/testdir/testdir1/test-file", false));
+    testList.add(
+        new AbstractMap.SimpleImmutableEntry<String, Boolean>(
+            "/testdir/testdir1/test-file1", true));
+    
     final Path rootPath = new Path("/");
     final Configuration conf = new Configuration();
     // Disable permissions so that another user can recover the lease.
@@ -247,8 +258,11 @@ private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot)
 
       DistributedFileSystem fs = cluster.getFileSystem();
       int stId = 0;
-      for (String testPath : testPaths) {
-        LOG.info("test on " + testPath + " snapshot: " + hasSnapshot);
+      for(AbstractMap.SimpleImmutableEntry<String, Boolean> stest : testList) {
+        String testPath = stest.getKey();
+        Boolean mkSameDir = stest.getValue();
+        LOG.info("test on " + testPath + " mkSameDir: " + mkSameDir
+            + " snapshot: " + hasSnapshot);
         Path fPath = new Path(testPath);
         //find grandest non-root parent
         Path grandestNonRootParent = fPath;
@@ -304,7 +318,11 @@ private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot)
 
         LOG.info("Deleting recursively " + grandestNonRootParent);
         fs.delete(grandestNonRootParent, true);
-
+        if (mkSameDir && !grandestNonRootParent.toString().equals(testPath)) {
+          LOG.info("Recreate dir " + grandestNonRootParent + " testpath: "
+              + testPath);
+          fs.mkdirs(grandestNonRootParent);
+        }
         delayer.proceed();
         LOG.info("Now wait for result");
         delayer.waitForResult();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 2a8f2895a3d4f..6d8d205cb8621 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -70,7 +70,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
@@ -205,7 +205,7 @@ public void run() {
 
       for (int i = 0; i < numTransactions; i++) {
         INodeFile inode = new INodeFile(namesystem.dir.allocateNewInodeId(), null,
-            p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+            p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
         inode.toUnderConstruction("", "");
 
         editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
index ad067cfa197fd..11905bd447978 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
@@ -40,7 +40,6 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -201,8 +200,9 @@ public void testINodeXAttrsLimit() throws Exception {
     List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(2);
     newXAttrs.add(newSystemXAttr);
     newXAttrs.add(newRawXAttr);
-    List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
-        EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+    List<XAttr> xAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
+                                                     newXAttrs, EnumSet.of(
+            XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
     assertEquals(xAttrs.size(), 4);
     
     // Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
@@ -211,8 +211,9 @@ public void testINodeXAttrsLimit() throws Exception {
         setValue(new byte[]{0x34, 0x34, 0x34}).build();
     newXAttrs.set(0, newXAttr1);
     try {
-      fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
-          EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+      FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, newXAttrs,
+                                  EnumSet.of(XAttrSetFlag.CREATE,
+                                             XAttrSetFlag.REPLACE));
       fail("Setting user visible xattr on inode should fail if " +
           "reaching limit.");
     } catch (IOException e) {
@@ -275,8 +276,9 @@ public void testXAttrMultiSetRemove() throws Exception {
       for (int i = 0; i < toAdd.size(); i++) {
         LOG.info("Will add XAttr " + toAdd.get(i));
       }
-      List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
-          EnumSet.of(XAttrSetFlag.CREATE));
+      List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
+                                                          toAdd, EnumSet.of(
+              XAttrSetFlag.CREATE));
       verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
       existingXAttrs = newXAttrs;
     }
@@ -296,8 +298,9 @@ public void testXAttrMultiSetRemove() throws Exception {
       final int expectedNumToRemove = toRemove.size();
       LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
       List<XAttr> removedXAttrs = Lists.newArrayList();
-      List<XAttr> newXAttrs = fsdir.filterINodeXAttrs(existingXAttrs,
-          toRemove, removedXAttrs);
+      List<XAttr> newXAttrs = FSDirXAttrOp.filterINodeXAttrs(existingXAttrs,
+                                                             toRemove,
+                                                             removedXAttrs);
       assertEquals("Unexpected number of removed XAttrs",
           expectedNumToRemove, removedXAttrs.size());
       verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
@@ -316,8 +319,8 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
     toAdd.add(generatedXAttrs.get(2));
     toAdd.add(generatedXAttrs.get(0));
     try {
-      fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
-          .CREATE));
+      FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
+                                  EnumSet.of(XAttrSetFlag.CREATE));
       fail("Specified the same xattr to be set twice");
     } catch (IOException e) {
       GenericTestUtils.assertExceptionContains("Cannot specify the same " +
@@ -328,15 +331,15 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
     toAdd.remove(generatedXAttrs.get(0));
     existingXAttrs.add(generatedXAttrs.get(0));
     try {
-      fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
-          .CREATE));
+      FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
+                                  EnumSet.of(XAttrSetFlag.CREATE));
       fail("Set XAttr that is already set without REPLACE flag");
     } catch (IOException e) {
       GenericTestUtils.assertExceptionContains("already exists", e);
     }
     try {
-      fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
-          .REPLACE));
+      FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
+                                  EnumSet.of(XAttrSetFlag.REPLACE));
       fail("Set XAttr that does not exist without the CREATE flag");
     } catch (IOException e) {
       GenericTestUtils.assertExceptionContains("does not exist", e);
@@ -344,8 +347,9 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
 
     // Sanity test for CREATE
     toAdd.remove(generatedXAttrs.get(0));
-    List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
-        EnumSet.of(XAttrSetFlag.CREATE));
+    List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
+                                                        toAdd, EnumSet.of(
+            XAttrSetFlag.CREATE));
     assertEquals("Unexpected toAdd size", 2, toAdd.size());
     for (XAttr x : toAdd) {
       assertTrue("Did not find added XAttr " + x, newXAttrs.contains(x));
@@ -362,8 +366,8 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
           .build();
       toAdd.add(xAttr);
     }
-    newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
-        EnumSet.of(XAttrSetFlag.REPLACE));
+    newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
+                                            EnumSet.of(XAttrSetFlag.REPLACE));
     assertEquals("Unexpected number of new XAttrs", 3, newXAttrs.size());
     for (int i=0; i<3; i++) {
       assertArrayEquals("Unexpected XAttr value",
@@ -376,8 +380,9 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
     for (int i=0; i<4; i++) {
       toAdd.add(generatedXAttrs.get(i));
     }
-    newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
-        EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+    newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
+                                            EnumSet.of(XAttrSetFlag.CREATE,
+                                                       XAttrSetFlag.REPLACE));
     verifyXAttrsPresent(newXAttrs, 4);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index f21834ea295f7..c68ae046dfc2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
@@ -97,7 +97,7 @@ private void testPersistHelper(Configuration conf) throws IOException {
       INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
       assertEquals("hello".length(), file2Node.computeFileSize());
       assertTrue(file2Node.isUnderConstruction());
-      BlockInfo[] blks = file2Node.getBlocks();
+      BlockInfoContiguous[] blks = file2Node.getBlocks();
       assertEquals(1, blks.length);
       assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
       // check lease manager
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java
index 39e1165359d54..c044fb0cff1ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java
@@ -17,11 +17,16 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
 
 import java.lang.management.ManagementFactory;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
 
@@ -51,66 +56,28 @@ public void run() {
         // come from hadoop metrics framework for the class FSNamesystem.
         ObjectName mxbeanNamefsn = new ObjectName(
             "Hadoop:service=NameNode,name=FSNamesystem");
-        Integer blockCapacity = (Integer) (mbs.getAttribute(mxbeanNamefsn,
-            "BlockCapacity"));
 
         // Metrics that belong to "FSNamesystemState".
         // These are metrics that FSNamesystem registers directly with MBeanServer.
         ObjectName mxbeanNameFsns = new ObjectName(
             "Hadoop:service=NameNode,name=FSNamesystemState");
-        String FSState = (String) (mbs.getAttribute(mxbeanNameFsns,
-            "FSState"));
-        Long blocksTotal = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "BlocksTotal"));
-        Long capacityTotal = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "CapacityTotal"));
-        Long capacityRemaining = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "CapacityRemaining"));
-        Long capacityUsed = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "CapacityUsed"));
-        Long filesTotal = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "FilesTotal"));
-        Long pendingReplicationBlocks = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "PendingReplicationBlocks"));
-        Long underReplicatedBlocks = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "UnderReplicatedBlocks"));
-        Long scheduledReplicationBlocks = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "ScheduledReplicationBlocks"));
-        Integer totalLoad = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "TotalLoad"));
-        Integer numLiveDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "NumLiveDataNodes"));
-        Integer numDeadDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-           "NumDeadDataNodes"));
-        Integer numStaleDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "NumStaleDataNodes"));
-        Integer numDecomLiveDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "NumDecomLiveDataNodes"));
-        Integer numDecomDeadDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "NumDecomDeadDataNodes"));
-        Integer numDecommissioningDataNodes = (Integer) (mbs.getAttribute(mxbeanNameFsns,
-            "NumDecommissioningDataNodes"));
-        String snapshotStats = (String) (mbs.getAttribute(mxbeanNameFsns,
-            "SnapshotStats"));
-        Long MaxObjects = (Long) (mbs.getAttribute(mxbeanNameFsns,
-            "MaxObjects"));
-        Integer numStaleStorages = (Integer) (mbs.getAttribute(
-            mxbeanNameFsns, "NumStaleStorages"));
 
         // Metrics that belong to "NameNodeInfo".
         // These are metrics that FSNamesystem registers directly with MBeanServer.
         ObjectName mxbeanNameNni = new ObjectName(
             "Hadoop:service=NameNode,name=NameNodeInfo");
-        String safemode = (String) (mbs.getAttribute(mxbeanNameNni,
-            "Safemode"));
-        String liveNodes = (String) (mbs.getAttribute(mxbeanNameNni,
-            "LiveNodes"));
-        String deadNodes = (String) (mbs.getAttribute(mxbeanNameNni,
-            "DeadNodes"));
-        String decomNodes = (String) (mbs.getAttribute(mxbeanNameNni,
-            "DecomNodes"));
-        String corruptFiles = (String) (mbs.getAttribute(mxbeanNameNni,
-            "CorruptFiles"));
+
+        final Set<ObjectName> mbeans = new HashSet<ObjectName>();
+        mbeans.add(mxbeanNamefsn);
+        mbeans.add(mxbeanNameFsns);
+        mbeans.add(mxbeanNameNni);
+
+        for(ObjectName mbean : mbeans) {
+          MBeanInfo attributes = mbs.getMBeanInfo(mbean);
+          for (MBeanAttributeInfo attributeInfo : attributes.getAttributes()) {
+            mbs.getAttribute(mbean, attributeInfo.getName());
+          }
+        }
 
         succeeded = true;
       } catch (Exception e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextAcl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextAcl.java
index 51baa386723d2..f9a6889b8f4a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextAcl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextAcl.java
@@ -27,9 +27,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.BeforeClass;
 
 /**
@@ -40,9 +38,7 @@ public class TestFileContextAcl extends FSAclBaseTest {
   @BeforeClass
   public static void init() throws Exception {
     conf = new Configuration();
-    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
+    startCluster();
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
new file mode 100644
index 0000000000000..253727dae6d07
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -0,0 +1,990 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.AppendTestUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFileTruncate {
+  static {
+    GenericTestUtils.setLogLevel(NameNode.stateChangeLog, Level.ALL);
+    GenericTestUtils.setLogLevel(FSEditLogLoader.LOG, Level.ALL);
+  }
+  static final Log LOG = LogFactory.getLog(TestFileTruncate.class);
+  static final int BLOCK_SIZE = 4;
+  static final short REPLICATION = 3;
+  static final int DATANODE_NUM = 3;
+  static final int SUCCESS_ATTEMPTS = 300;
+  static final int RECOVERY_ATTEMPTS = 600;
+  static final long SLEEP = 100L;
+
+  static final long LOW_SOFTLIMIT = 100L;
+  static final long LOW_HARDLIMIT = 200L;
+  static final int SHORT_HEARTBEAT = 1;
+
+  static Configuration conf;
+  static MiniDFSCluster cluster;
+  static DistributedFileSystem fs;
+
+  @BeforeClass
+  public static void startUp() throws IOException {
+    conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, SHORT_HEARTBEAT);
+    cluster = new MiniDFSCluster.Builder(conf)
+        .format(true)
+        .numDataNodes(DATANODE_NUM)
+        .nameNodePort(NameNode.DEFAULT_PORT)
+        .waitSafeMode(true)
+        .build();
+    fs = cluster.getFileSystem();
+  }
+
+  @AfterClass
+  public static void tearDown() throws IOException {
+    if(fs != null)      fs.close();
+    if(cluster != null) cluster.shutdown();
+  }
+
+  /**
+   * Truncate files of different sizes byte by byte.
+   */
+  @Test
+  public void testBasicTruncate() throws IOException {
+    int startingFileSize = 3 * BLOCK_SIZE;
+
+    Path parent = new Path("/test");
+    fs.mkdirs(parent);
+    fs.setQuota(parent, 100, 1000);
+    byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+    for (int fileLength = startingFileSize; fileLength > 0;
+                                            fileLength -= BLOCK_SIZE - 1) {
+      for (int toTruncate = 0; toTruncate <= fileLength; toTruncate++) {
+        final Path p = new Path(parent, "testBasicTruncate" + fileLength);
+        writeContents(contents, fileLength, p);
+
+        int newLength = fileLength - toTruncate;
+        boolean isReady = fs.truncate(p, newLength);
+        LOG.info("fileLength=" + fileLength + ", newLength=" + newLength
+            + ", toTruncate=" + toTruncate + ", isReady=" + isReady);
+
+        assertEquals("File must be closed for zero truncate"
+            + " or truncating at the block boundary",
+            isReady, toTruncate == 0 || newLength % BLOCK_SIZE == 0);
+        if (!isReady) {
+          checkBlockRecovery(p);
+        }
+
+        ContentSummary cs = fs.getContentSummary(parent);
+        assertEquals("Bad disk space usage",
+            cs.getSpaceConsumed(), newLength * REPLICATION);
+        // validate the file content
+        checkFullFile(p, newLength, contents);
+      }
+    }
+    fs.delete(parent, true);
+  }
+
+  /** Truncate the same file multiple times until its size is zero. */
+  @Test
+  public void testMultipleTruncate() throws IOException {
+    Path dir = new Path("/testMultipleTruncate");
+    fs.mkdirs(dir);
+    final Path p = new Path(dir, "file");
+    final byte[] data = new byte[100 * BLOCK_SIZE];
+    DFSUtil.getRandom().nextBytes(data);
+    writeContents(data, data.length, p);
+
+    for(int n = data.length; n > 0; ) {
+      final int newLength = DFSUtil.getRandom().nextInt(n);
+      final boolean isReady = fs.truncate(p, newLength);
+      LOG.info("newLength=" + newLength + ", isReady=" + isReady);
+      assertEquals("File must be closed for truncating at the block boundary",
+          isReady, newLength % BLOCK_SIZE == 0);
+      if (!isReady) {
+        checkBlockRecovery(p);
+      }
+      checkFullFile(p, newLength, data);
+      n = newLength;
+    }
+
+    fs.delete(dir, true);
+  }
+
+  /**
+   * Truncate files and then run other operations such as
+   * rename, set replication, set permission, etc.
+   */
+  @Test
+  public void testTruncateWithOtherOperations() throws IOException {
+    Path dir = new Path("/testTruncateOtherOperations");
+    fs.mkdirs(dir);
+    final Path p = new Path(dir, "file");
+    final byte[] data = new byte[2 * BLOCK_SIZE];
+
+    DFSUtil.getRandom().nextBytes(data);
+    writeContents(data, data.length, p);
+
+    final int newLength = data.length - 1;
+    boolean isReady = fs.truncate(p, newLength);
+    assertFalse(isReady);
+
+    fs.setReplication(p, (short)(REPLICATION - 1));
+    fs.setPermission(p, FsPermission.createImmutable((short)0444));
+
+    final Path q = new Path(dir, "newFile");
+    fs.rename(p, q);
+
+    checkBlockRecovery(q);
+    checkFullFile(q, newLength, data);
+
+    cluster.restartNameNode();
+    checkFullFile(q, newLength, data);
+
+    fs.delete(dir, true);
+  }
+
+  @Test
+  public void testSnapshotWithAppendTruncate() throws IOException {
+    testSnapshotWithAppendTruncate(0, 1, 2);
+    testSnapshotWithAppendTruncate(0, 2, 1);
+    testSnapshotWithAppendTruncate(1, 0, 2);
+    testSnapshotWithAppendTruncate(1, 2, 0);
+    testSnapshotWithAppendTruncate(2, 0, 1);
+    testSnapshotWithAppendTruncate(2, 1, 0);
+  }
+
+  /**
+   * Create three snapshots with appended and truncated file.
+   * Delete snapshots in the specified order and verify that
+   * remaining snapshots are still readable.
+   */
+  void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
+    FSDirectory fsDir = cluster.getNamesystem().getFSDirectory();
+    Path parent = new Path("/test");
+    fs.mkdirs(parent);
+    fs.setQuota(parent, 100, 1000);
+    fs.allowSnapshot(parent);
+    String truncateFile = "testSnapshotWithAppendTruncate";
+    final Path src = new Path(parent, truncateFile);
+    int[] length = new int[4];
+    length[0] = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+    DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L);
+    Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock();
+    Path[] snapshotFiles = new Path[4];
+
+    // Diskspace consumed should be 10 bytes * 3. [blk 1,2,3]
+    ContentSummary contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(30L));
+
+    // Add file to snapshot and append
+    String[] ss = new String[] {"ss0", "ss1", "ss2", "ss3"};
+    Path snapshotDir = fs.createSnapshot(parent, ss[0]);
+    snapshotFiles[0] = new Path(snapshotDir, truncateFile);
+    length[1] = length[2] = length[0] + BLOCK_SIZE + 1;
+    DFSTestUtil.appendFile(fs, src, BLOCK_SIZE + 1);
+    Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock()
+                                         .getBlock().getLocalBlock();
+
+    // Diskspace consumed should be 15 bytes * 3. [blk 1,2,3,4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(45L));
+
+    // Create another snapshot without changes
+    snapshotDir = fs.createSnapshot(parent, ss[1]);
+    snapshotFiles[1] = new Path(snapshotDir, truncateFile);
+
+    // Create another snapshot and append
+    snapshotDir = fs.createSnapshot(parent, ss[2]);
+    snapshotFiles[2] = new Path(snapshotDir, truncateFile);
+    DFSTestUtil.appendFile(fs, src, BLOCK_SIZE -1 + BLOCK_SIZE / 2);
+    Block appendedBlk = getLocatedBlocks(src).getLastLocatedBlock()
+                                             .getBlock().getLocalBlock();
+
+    // Diskspace consumed should be 20 bytes * 3. [blk 1,2,3,4,5]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(60L));
+
+    // Truncate to block boundary
+    int newLength = length[0] + BLOCK_SIZE / 2;
+    boolean isReady = fs.truncate(src, newLength);
+    assertTrue("Recovery is not expected.", isReady);
+    assertFileLength(snapshotFiles[2], length[2]);
+    assertFileLength(snapshotFiles[1], length[1]);
+    assertFileLength(snapshotFiles[0], length[0]);
+    assertBlockNotPresent(appendedBlk);
+
+    // Diskspace consumed should be 16 bytes * 3. [blk 1,2,3 SS:4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+    // Truncate full block again
+    newLength = length[0] - BLOCK_SIZE / 2;
+    isReady = fs.truncate(src, newLength);
+    assertTrue("Recovery is not expected.", isReady);
+    assertFileLength(snapshotFiles[2], length[2]);
+    assertFileLength(snapshotFiles[1], length[1]);
+    assertFileLength(snapshotFiles[0], length[0]);
+
+    // Diskspace consumed should be 16 bytes * 3. [blk 1,2 SS:3,4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+    // Truncate half of the last block
+    newLength -= BLOCK_SIZE / 2;
+    isReady = fs.truncate(src, newLength);
+    assertFalse("Recovery is expected.", isReady);
+    checkBlockRecovery(src);
+    assertFileLength(snapshotFiles[2], length[2]);
+    assertFileLength(snapshotFiles[1], length[1]);
+    assertFileLength(snapshotFiles[0], length[0]);
+    Block replacedBlk = getLocatedBlocks(src).getLastLocatedBlock()
+        .getBlock().getLocalBlock();
+
+    // Diskspace consumed should be 16 bytes * 3. [blk 1,6 SS:2,3,4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(54L));
+
+    snapshotDir = fs.createSnapshot(parent, ss[3]);
+    snapshotFiles[3] = new Path(snapshotDir, truncateFile);
+    length[3] = newLength;
+
+    // Delete file. Should still be able to read snapshots
+    int numINodes = fsDir.getInodeMapSize();
+    isReady = fs.delete(src, false);
+    assertTrue("Delete failed.", isReady);
+    assertFileLength(snapshotFiles[3], length[3]);
+    assertFileLength(snapshotFiles[2], length[2]);
+    assertFileLength(snapshotFiles[1], length[1]);
+    assertFileLength(snapshotFiles[0], length[0]);
+    assertEquals("Number of INodes should not change",
+        numINodes, fsDir.getInodeMapSize());
+
+    fs.deleteSnapshot(parent, ss[3]);
+
+    assertBlockExists(firstBlk);
+    assertBlockExists(lastBlk);
+    assertBlockNotPresent(replacedBlk);
+
+    // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+    // delete snapshots in the specified order
+    fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
+    assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
+    assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+    assertBlockExists(firstBlk);
+    assertBlockExists(lastBlk);
+    assertEquals("Number of INodes should not change",
+        numINodes, fsDir.getInodeMapSize());
+
+    // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(48L));
+
+    fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+    assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+    assertBlockExists(firstBlk);
+    contentSummary = fs.getContentSummary(parent);
+    if(fs.exists(snapshotFiles[0])) {
+      // Diskspace consumed should be 0 bytes * 3. [SS:1,2,3]
+      assertBlockNotPresent(lastBlk);
+      assertThat(contentSummary.getSpaceConsumed(), is(36L));
+    } else {
+      // Diskspace consumed should be 48 bytes * 3. [SS:1,2,3,4]
+      assertThat(contentSummary.getSpaceConsumed(), is(48L));
+    }
+    assertEquals("Number of INodes should not change",
+        numINodes, fsDir .getInodeMapSize());
+
+    fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+    assertBlockNotPresent(firstBlk);
+    assertBlockNotPresent(lastBlk);
+
+    // Diskspace consumed should be 0 bytes * 3. []
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(0L));
+    assertNotEquals("Number of INodes should change",
+        numINodes, fsDir.getInodeMapSize());
+  }
+
+  /**
+   * Create three snapshots with file truncated 3 times.
+   * Delete snapshots in the specified order and verify that
+   * remaining snapshots are still readable.
+   */
+  @Test
+  public void testSnapshotWithTruncates() throws IOException {
+    testSnapshotWithTruncates(0, 1, 2);
+    testSnapshotWithTruncates(0, 2, 1);
+    testSnapshotWithTruncates(1, 0, 2);
+    testSnapshotWithTruncates(1, 2, 0);
+    testSnapshotWithTruncates(2, 0, 1);
+    testSnapshotWithTruncates(2, 1, 0);
+  }
+
+  void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
+    Path parent = new Path("/test");
+    fs.mkdirs(parent);
+    fs.setQuota(parent, 100, 1000);
+    fs.allowSnapshot(parent);
+    String truncateFile = "testSnapshotWithTruncates";
+    final Path src = new Path(parent, truncateFile);
+    int[] length = new int[3];
+    length[0] = 3 * BLOCK_SIZE;
+    DFSTestUtil.createFile(fs, src, 64, length[0], BLOCK_SIZE, REPLICATION, 0L);
+    Block firstBlk = getLocatedBlocks(src).get(0).getBlock().getLocalBlock();
+    Block lastBlk = getLocatedBlocks(src).getLastLocatedBlock()
+                                         .getBlock().getLocalBlock();
+    Path[] snapshotFiles = new Path[3];
+
+    // Diskspace consumed should be 12 bytes * 3. [blk 1,2,3]
+    ContentSummary contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(36L));
+
+    // Add file to snapshot and append
+    String[] ss = new String[] {"ss0", "ss1", "ss2"};
+    Path snapshotDir = fs.createSnapshot(parent, ss[0]);
+    snapshotFiles[0] = new Path(snapshotDir, truncateFile);
+    length[1] = 2 * BLOCK_SIZE;
+    boolean isReady = fs.truncate(src, 2 * BLOCK_SIZE);
+    assertTrue("Recovery is not expected.", isReady);
+
+    // Diskspace consumed should be 12 bytes * 3. [blk 1,2 SS:3]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(36L));
+    snapshotDir = fs.createSnapshot(parent, ss[1]);
+    snapshotFiles[1] = new Path(snapshotDir, truncateFile);
+
+    // Create another snapshot with truncate
+    length[2] = BLOCK_SIZE + BLOCK_SIZE / 2;
+    isReady = fs.truncate(src, BLOCK_SIZE + BLOCK_SIZE / 2);
+    assertFalse("Recovery is expected.", isReady);
+    checkBlockRecovery(src);
+    snapshotDir = fs.createSnapshot(parent, ss[2]);
+    snapshotFiles[2] = new Path(snapshotDir, truncateFile);
+    assertFileLength(snapshotFiles[0], length[0]);
+    assertBlockExists(lastBlk);
+
+    // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(42L));
+
+    fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
+    assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
+    assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+    assertFileLength(src, length[2]);
+    assertBlockExists(firstBlk);
+
+    contentSummary = fs.getContentSummary(parent);
+    if(fs.exists(snapshotFiles[0])) {
+      // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+      assertThat(contentSummary.getSpaceConsumed(), is(42L));
+      assertBlockExists(lastBlk);
+    } else {
+      // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2]
+      assertThat(contentSummary.getSpaceConsumed(), is(30L));
+      assertBlockNotPresent(lastBlk);
+    }
+
+    fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+    assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
+    assertFileLength(src, length[2]);
+    assertBlockExists(firstBlk);
+
+    contentSummary = fs.getContentSummary(parent);
+    if(fs.exists(snapshotFiles[0])) {
+      // Diskspace consumed should be 14 bytes * 3. [blk 1,4 SS:2,3]
+      assertThat(contentSummary.getSpaceConsumed(), is(42L));
+      assertBlockExists(lastBlk);
+    } else if(fs.exists(snapshotFiles[1])) {
+      // Diskspace consumed should be 10 bytes * 3. [blk 1,4 SS:2]
+      assertThat(contentSummary.getSpaceConsumed(), is(30L));
+      assertBlockNotPresent(lastBlk);
+    } else {
+      // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:]
+      assertThat(contentSummary.getSpaceConsumed(), is(18L));
+      assertBlockNotPresent(lastBlk);
+    }
+
+    fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+    assertFileLength(src, length[2]);
+    assertBlockExists(firstBlk);
+
+    // Diskspace consumed should be 6 bytes * 3. [blk 1,4 SS:]
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(18L));
+    assertThat(contentSummary.getLength(), is(6L));
+
+    fs.delete(src, false);
+    assertBlockNotPresent(firstBlk);
+
+    // Diskspace consumed should be 0 bytes * 3. []
+    contentSummary = fs.getContentSummary(parent);
+    assertThat(contentSummary.getSpaceConsumed(), is(0L));
+  }
+
+  /**
+   * Failure / recovery test for truncate.
+   * In this failure the DNs fail to recover the blocks and the NN triggers
+   * lease recovery.
+   * File stays in RecoveryInProgress until DataNodes report recovery.
+   */
+  @Test
+  public void testTruncateFailure() throws IOException {
+    int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+    int toTruncate = 1;
+
+    byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+    final Path dir = new Path("/dir");
+    final Path p = new Path(dir, "testTruncateFailure");
+    {
+      FSDataOutputStream out = fs.create(p, false, BLOCK_SIZE, REPLICATION,
+          BLOCK_SIZE);
+      out.write(contents, 0, startingFileSize);
+      try {
+        fs.truncate(p, 0);
+        fail("Truncate must fail on open file.");
+      } catch (IOException expected) {
+        GenericTestUtils.assertExceptionContains(
+            "Failed to TRUNCATE_FILE", expected);
+      } finally {
+        out.close();
+      }
+    }
+
+    {
+      FSDataOutputStream out = fs.append(p);
+      try {
+        fs.truncate(p, 0);
+        fail("Truncate must fail for append.");
+      } catch (IOException expected) {
+        GenericTestUtils.assertExceptionContains(
+            "Failed to TRUNCATE_FILE", expected);
+      } finally {
+        out.close();
+      }
+    }
+
+    try {
+      fs.truncate(p, -1);
+      fail("Truncate must fail for a negative new length.");
+    } catch (HadoopIllegalArgumentException expected) {
+      GenericTestUtils.assertExceptionContains(
+          "Cannot truncate to a negative file size", expected);
+    }
+
+    try {
+      fs.truncate(p, startingFileSize + 1);
+      fail("Truncate must fail for a larger new length.");
+    } catch (Exception expected) {
+      GenericTestUtils.assertExceptionContains(
+          "Cannot truncate to a larger file size", expected);
+    }
+
+    try {
+      fs.truncate(dir, 0);
+      fail("Truncate must fail for a directory.");
+    } catch (Exception expected) {
+      GenericTestUtils.assertExceptionContains(
+          "Path is not a file", expected);
+    }
+
+    try {
+      fs.truncate(new Path(dir, "non-existing"), 0);
+      fail("Truncate must fail for a non-existing file.");
+    } catch (Exception expected) {
+      GenericTestUtils.assertExceptionContains(
+          "File does not exist", expected);
+    }
+
+    
+    fs.setPermission(p, FsPermission.createImmutable((short)0664));
+    {
+      final UserGroupInformation fooUgi = 
+          UserGroupInformation.createUserForTesting("foo", new String[]{"foo"});
+      try {
+        final FileSystem foofs = DFSTestUtil.getFileSystemAs(fooUgi, conf);
+        foofs.truncate(p, 0);
+        fail("Truncate must fail for no WRITE permission.");
+      } catch (Exception expected) {
+        GenericTestUtils.assertExceptionContains(
+            "Permission denied", expected);
+      }
+    }
+
+    cluster.shutdownDataNodes();
+    NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
+        .setLeasePeriod(LOW_SOFTLIMIT, LOW_HARDLIMIT);
+
+    int newLength = startingFileSize - toTruncate;
+    boolean isReady = fs.truncate(p, newLength);
+    assertThat("truncate should have triggered block recovery.",
+        isReady, is(false));
+
+    {
+      try {
+        fs.truncate(p, 0);
+        fail("Truncate must fail since a trancate is already in pregress.");
+      } catch (IOException expected) {
+        GenericTestUtils.assertExceptionContains(
+            "Failed to TRUNCATE_FILE", expected);
+      }
+    }
+
+    boolean recoveryTriggered = false;
+    for(int i = 0; i < RECOVERY_ATTEMPTS; i++) {
+      String leaseHolder =
+          NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(),
+          p.toUri().getPath());
+      if(leaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER)) {
+        recoveryTriggered = true;
+        break;
+      }
+      try { Thread.sleep(SLEEP); } catch (InterruptedException ignored) {}
+    }
+    assertThat("lease recovery should have occurred in ~" +
+        SLEEP * RECOVERY_ATTEMPTS + " ms.", recoveryTriggered, is(true));
+    cluster.startDataNodes(conf, DATANODE_NUM, true,
+        StartupOption.REGULAR, null);
+    cluster.waitActive();
+
+    checkBlockRecovery(p);
+
+    NameNodeAdapter.getLeaseManager(cluster.getNamesystem())
+        .setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,
+            HdfsConstants.LEASE_HARDLIMIT_PERIOD);
+
+    checkFullFile(p, newLength, contents);
+    fs.delete(p, false);
+  }
+
+  /**
+   * EditLogOp load test for Truncate.
+   */
+  @Test
+  public void testTruncateEditLogLoad() throws IOException {
+    // purge previously accumulated edits
+    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    fs.saveNamespace();
+    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+    int startingFileSize = 2 * BLOCK_SIZE + BLOCK_SIZE / 2;
+    int toTruncate = 1;
+    final String s = "/testTruncateEditLogLoad";
+    final Path p = new Path(s);
+    byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+    writeContents(contents, startingFileSize, p);
+
+    int newLength = startingFileSize - toTruncate;
+    boolean isReady = fs.truncate(p, newLength);
+    assertThat("truncate should have triggered block recovery.",
+        isReady, is(false));
+
+    cluster.restartNameNode();
+
+    String holder = UserGroupInformation.getCurrentUser().getUserName();
+    cluster.getNamesystem().recoverLease(s, holder, "");
+
+    checkBlockRecovery(p);
+    checkFullFile(p, newLength, contents);
+    fs.delete(p, false);
+  }
+
+  /**
+   * Upgrade, RollBack, and restart test for Truncate.
+   */
+  @Test
+  public void testUpgradeAndRestart() throws IOException {
+    Path parent = new Path("/test");
+    fs.mkdirs(parent);
+    fs.setQuota(parent, 100, 1000);
+    fs.allowSnapshot(parent);
+    String truncateFile = "testUpgrade";
+    final Path p = new Path(parent, truncateFile);
+    int startingFileSize = 2 * BLOCK_SIZE;
+    int toTruncate = 1;
+    byte[] contents = AppendTestUtil.initBuffer(startingFileSize);
+    writeContents(contents, startingFileSize, p);
+
+    Path snapshotDir = fs.createSnapshot(parent, "ss0");
+    Path snapshotFile = new Path(snapshotDir, truncateFile);
+
+    int newLengthBeforeUpgrade = startingFileSize - toTruncate;
+    boolean isReady = fs.truncate(p, newLengthBeforeUpgrade);
+    assertThat("truncate should have triggered block recovery.",
+        isReady, is(false));
+
+    checkBlockRecovery(p);
+
+    checkFullFile(p, newLengthBeforeUpgrade, contents);
+    assertFileLength(snapshotFile, startingFileSize);
+    long totalBlockBefore = cluster.getNamesystem().getBlocksTotal();
+
+    restartCluster(StartupOption.UPGRADE);
+
+    assertThat("SafeMode should be OFF",
+        cluster.getNamesystem().isInSafeMode(), is(false));
+    assertThat("NameNode should be performing upgrade.",
+        cluster.getNamesystem().isUpgradeFinalized(), is(false));
+    FileStatus fileStatus = fs.getFileStatus(p);
+    assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade));
+
+    int newLengthAfterUpgrade = newLengthBeforeUpgrade - toTruncate;
+    Block oldBlk = getLocatedBlocks(p).getLastLocatedBlock()
+        .getBlock().getLocalBlock();
+    isReady = fs.truncate(p, newLengthAfterUpgrade);
+    assertThat("truncate should have triggered block recovery.",
+        isReady, is(false));
+    fileStatus = fs.getFileStatus(p);
+    assertThat(fileStatus.getLen(), is((long) newLengthAfterUpgrade));
+    assertThat("Should copy on truncate during upgrade",
+        getLocatedBlocks(p).getLastLocatedBlock().getBlock()
+        .getLocalBlock().getBlockId(), is(not(equalTo(oldBlk.getBlockId()))));
+
+    checkBlockRecovery(p);
+
+    checkFullFile(p, newLengthAfterUpgrade, contents);
+    assertThat("Total block count should be unchanged from copy-on-truncate",
+        cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+
+    restartCluster(StartupOption.ROLLBACK);
+
+    assertThat("File does not exist " + p, fs.exists(p), is(true));
+    fileStatus = fs.getFileStatus(p);
+    assertThat(fileStatus.getLen(), is((long) newLengthBeforeUpgrade));
+    checkFullFile(p, newLengthBeforeUpgrade, contents);
+    assertThat("Total block count should be unchanged from rolling back",
+        cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+
+    restartCluster(StartupOption.REGULAR);
+    assertThat("Total block count should be unchanged from start-up",
+        cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+    checkFullFile(p, newLengthBeforeUpgrade, contents);
+    assertFileLength(snapshotFile, startingFileSize);
+
+    // empty edits and restart
+    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    fs.saveNamespace();
+    cluster.restartNameNode(true);
+    assertThat("Total block count should be unchanged from start-up",
+        cluster.getNamesystem().getBlocksTotal(), is(totalBlockBefore));
+    checkFullFile(p, newLengthBeforeUpgrade, contents);
+    assertFileLength(snapshotFile, startingFileSize);
+
+    fs.deleteSnapshot(parent, "ss0");
+    fs.delete(parent, true);
+    assertThat("File " + p + " shouldn't exist", fs.exists(p), is(false));
+  }
+
+  /**
+   * Check truncate recovery.
+   */
+  @Test
+  public void testTruncateRecovery() throws IOException {
+    FSNamesystem fsn = cluster.getNamesystem();
+    String client = "client";
+    String clientMachine = "clientMachine";
+    Path parent = new Path("/test");
+    String src = "/test/testTruncateRecovery";
+    Path srcPath = new Path(src);
+
+    byte[] contents = AppendTestUtil.initBuffer(BLOCK_SIZE);
+    writeContents(contents, BLOCK_SIZE, srcPath);
+
+    INodesInPath iip = fsn.getFSDirectory().getINodesInPath4Write(src, true);
+    INodeFile file = iip.getLastINode().asFile();
+    long initialGenStamp = file.getLastBlock().getGenerationStamp();
+    // Test that prepareFileForTruncate sets up in-place truncate.
+    fsn.writeLock();
+    try {
+      Block oldBlock = file.getLastBlock();
+      Block truncateBlock =
+          fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+      // In-place truncate uses old block id with new genStamp.
+      assertThat(truncateBlock.getBlockId(),
+          is(equalTo(oldBlock.getBlockId())));
+      assertThat(truncateBlock.getNumBytes(),
+          is(oldBlock.getNumBytes()));
+      assertThat(truncateBlock.getGenerationStamp(),
+          is(fsn.getBlockIdManager().getGenerationStampV2()));
+      assertThat(file.getLastBlock().getBlockUCState(),
+          is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
+      long blockRecoveryId = ((BlockInfoContiguousUnderConstruction) file.getLastBlock())
+          .getBlockRecoveryId();
+      assertThat(blockRecoveryId, is(initialGenStamp + 1));
+      fsn.getEditLog().logTruncate(
+          src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock);
+    } finally {
+      fsn.writeUnlock();
+    }
+
+    // Re-create file and ensure we are ready to copy on truncate
+    writeContents(contents, BLOCK_SIZE, srcPath);
+    fs.allowSnapshot(parent);
+    fs.createSnapshot(parent, "ss0");
+    iip = fsn.getFSDirectory().getINodesInPath(src, true);
+    file = iip.getLastINode().asFile();
+    file.recordModification(iip.getLatestSnapshotId(), true);
+    assertThat(file.isBlockInLatestSnapshot(file.getLastBlock()), is(true));
+    initialGenStamp = file.getLastBlock().getGenerationStamp();
+    // Test that prepareFileForTruncate sets up copy-on-write truncate
+    fsn.writeLock();
+    try {
+      Block oldBlock = file.getLastBlock();
+      Block truncateBlock =
+          fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+      // Copy-on-write truncate makes new block with new id and genStamp
+      assertThat(truncateBlock.getBlockId(),
+          is(not(equalTo(oldBlock.getBlockId()))));
+      assertThat(truncateBlock.getNumBytes() < oldBlock.getNumBytes(),
+          is(true));
+      assertThat(truncateBlock.getGenerationStamp(),
+          is(fsn.getBlockIdManager().getGenerationStampV2()));
+      assertThat(file.getLastBlock().getBlockUCState(),
+          is(HdfsServerConstants.BlockUCState.UNDER_RECOVERY));
+      long blockRecoveryId = ((BlockInfoContiguousUnderConstruction) file.getLastBlock())
+          .getBlockRecoveryId();
+      assertThat(blockRecoveryId, is(initialGenStamp + 1));
+      fsn.getEditLog().logTruncate(
+          src, client, clientMachine, BLOCK_SIZE-1, Time.now(), truncateBlock);
+    } finally {
+      fsn.writeUnlock();
+    }
+    checkBlockRecovery(srcPath);
+    fs.deleteSnapshot(parent, "ss0");
+    fs.delete(parent, true);
+  }
+
+  @Test
+  public void testTruncateShellCommand() throws Exception {
+    final Path parent = new Path("/test");
+    final Path src = new Path("/test/testTruncateShellCommand");
+    final int oldLength = 2*BLOCK_SIZE + 1;
+    final int newLength = BLOCK_SIZE + 1;
+
+    String[] argv =
+        new String[]{"-truncate", String.valueOf(newLength), src.toString()};
+    runTruncateShellCommand(src, oldLength, argv);
+
+    // wait for block recovery
+    checkBlockRecovery(src);
+    assertThat(fs.getFileStatus(src).getLen(), is((long) newLength));
+    fs.delete(parent, true);
+  }
+
+  @Test
+  public void testTruncateShellCommandOnBlockBoundary() throws Exception {
+    final Path parent = new Path("/test");
+    final Path src = new Path("/test/testTruncateShellCommandOnBoundary");
+    final int oldLength = 2 * BLOCK_SIZE;
+    final int newLength = BLOCK_SIZE;
+
+    String[] argv =
+        new String[]{"-truncate", String.valueOf(newLength), src.toString()};
+    runTruncateShellCommand(src, oldLength, argv);
+
+    // shouldn't need to wait for block recovery
+    assertThat(fs.getFileStatus(src).getLen(), is((long) newLength));
+    fs.delete(parent, true);
+  }
+
+  @Test
+  public void testTruncateShellCommandWithWaitOption() throws Exception {
+    final Path parent = new Path("/test");
+    final Path src = new Path("/test/testTruncateShellCommandWithWaitOption");
+    final int oldLength = 2 * BLOCK_SIZE + 1;
+    final int newLength = BLOCK_SIZE + 1;
+
+    String[] argv = new String[]{"-truncate", "-w", String.valueOf(newLength),
+        src.toString()};
+    runTruncateShellCommand(src, oldLength, argv);
+
+    // shouldn't need to wait for block recovery
+    assertThat(fs.getFileStatus(src).getLen(), is((long) newLength));
+    fs.delete(parent, true);
+  }
+
+  private void runTruncateShellCommand(Path src, int oldLength,
+                                       String[] shellOpts) throws Exception {
+    // create file and write data
+    writeContents(AppendTestUtil.initBuffer(oldLength), oldLength, src);
+    assertThat(fs.getFileStatus(src).getLen(), is((long)oldLength));
+
+    // truncate file using shell
+    FsShell shell = null;
+    try {
+      shell = new FsShell(conf);
+      assertThat(ToolRunner.run(shell, shellOpts), is(0));
+    } finally {
+      if(shell != null) {
+        shell.close();
+      }
+    }
+  }
+
+  @Test
+  public void testTruncate4Symlink() throws IOException {
+    final int fileLength = 3 * BLOCK_SIZE;
+
+    final Path parent = new Path("/test");
+    fs.mkdirs(parent);
+    final byte[] contents = AppendTestUtil.initBuffer(fileLength);
+    final Path file = new Path(parent, "testTruncate4Symlink");
+    writeContents(contents, fileLength, file);
+
+    final Path link = new Path(parent, "link");
+    fs.createSymlink(file, link, false);
+
+    final int newLength = fileLength/3;
+    boolean isReady = fs.truncate(link, newLength);
+
+    assertTrue("Recovery is not expected.", isReady);
+
+    FileStatus fileStatus = fs.getFileStatus(file);
+    assertThat(fileStatus.getLen(), is((long) newLength));
+
+    ContentSummary cs = fs.getContentSummary(parent);
+    assertEquals("Bad disk space usage",
+        cs.getSpaceConsumed(), newLength * REPLICATION);
+    // validate the file content
+    checkFullFile(file, newLength, contents);
+
+    fs.delete(parent, true);
+  }
+
+  static void writeContents(byte[] contents, int fileLength, Path p)
+      throws IOException {
+    FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION,
+        BLOCK_SIZE);
+    out.write(contents, 0, fileLength);
+    out.close();
+  }
+
+  static void checkBlockRecovery(Path p) throws IOException {
+    checkBlockRecovery(p, fs);
+  }
+
+  public static void checkBlockRecovery(Path p, DistributedFileSystem dfs)
+      throws IOException {
+    boolean success = false;
+    for(int i = 0; i < SUCCESS_ATTEMPTS; i++) {
+      LocatedBlocks blocks = getLocatedBlocks(p, dfs);
+      boolean noLastBlock = blocks.getLastLocatedBlock() == null;
+      if(!blocks.isUnderConstruction() &&
+          (noLastBlock || blocks.isLastBlockComplete())) {
+        success = true;
+        break;
+      }
+      try { Thread.sleep(SLEEP); } catch (InterruptedException ignored) {}
+    }
+    assertThat("inode should complete in ~" + SLEEP * SUCCESS_ATTEMPTS + " ms.",
+        success, is(true));
+  }
+
+  static LocatedBlocks getLocatedBlocks(Path src) throws IOException {
+    return getLocatedBlocks(src, fs);
+  }
+
+  static LocatedBlocks getLocatedBlocks(Path src, DistributedFileSystem dfs)
+      throws IOException {
+    return dfs.getClient().getLocatedBlocks(src.toString(), 0, Long.MAX_VALUE);
+  }
+
+  static void assertBlockExists(Block blk) {
+    assertNotNull("BlocksMap does not contain block: " + blk,
+        cluster.getNamesystem().getStoredBlock(blk));
+  }
+
+  static void assertBlockNotPresent(Block blk) {
+    assertNull("BlocksMap should not contain block: " + blk,
+        cluster.getNamesystem().getStoredBlock(blk));
+  }
+
+  static void assertFileLength(Path file, long length) throws IOException {
+    byte[] data = DFSTestUtil.readFileBuffer(fs, file);
+    assertEquals("Wrong data size in snapshot.", length, data.length);
+  }
+
+  static void checkFullFile(Path p, int newLength, byte[] contents)
+      throws IOException {
+    AppendTestUtil.checkFullFile(fs, p, newLength, contents, p.toString());
+  }
+
+  static void restartCluster(StartupOption o)
+      throws IOException {
+    cluster.shutdown();
+    if(StartupOption.ROLLBACK == o)
+      NameNode.doRollback(conf, false);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM)
+        .format(false)
+        .nameNodePort(NameNode.DEFAULT_PORT)
+        .startupOption(o==StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
+        .dnStartupOption(o!=StartupOption.ROLLBACK ? StartupOption.REGULAR : o)
+        .build();
+    fs = cluster.getFileSystem();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index ef7de0d1c57f9..5574a30816e17 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -65,7 +65,7 @@
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -345,7 +345,7 @@ public void testFsckMove() throws Exception {
         totalMissingBlocks += ctFile.getTotalMissingBlocks();
       }
       for (CorruptedTestFile ctFile : ctFiles) {
-        ctFile.removeBlocks();
+        ctFile.removeBlocks(cluster);
       }
       // Wait for fsck to discover all the missing blocks
       while (true) {
@@ -432,14 +432,15 @@ private byte[] cacheInitialContents() throws IOException {
       return content;
     }
     
-    public void removeBlocks() throws AccessControlException,
-        FileNotFoundException, UnresolvedLinkException, IOException {
+    public void removeBlocks(MiniDFSCluster cluster)
+        throws AccessControlException, FileNotFoundException,
+        UnresolvedLinkException, IOException {
       for (int corruptIdx : blocksToCorrupt) {
         // Corrupt a block by deleting it
         ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(
             name, blockSize * corruptIdx, Long.MAX_VALUE).get(0).getBlock();
         for (int i = 0; i < numDataNodes; i++) {
-          File blockFile = MiniDFSCluster.getBlockFile(i, block);
+          File blockFile = cluster.getBlockFile(i, block);
           if(blockFile != null && blockFile.exists()) {
             assertTrue(blockFile.delete());
           }
@@ -517,7 +518,7 @@ public void testFsckMoveAndDelete() throws Exception {
       ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(
           corruptFileName, 0, Long.MAX_VALUE).get(0).getBlock();
       for (int i=0; i<4; i++) {
-        File blockFile = MiniDFSCluster.getBlockFile(i, block);
+        File blockFile = cluster.getBlockFile(i, block);
         if(blockFile != null && blockFile.exists()) {
           assertTrue(blockFile.delete());
         }
@@ -647,7 +648,7 @@ public void testCorruptBlock() throws Exception {
     assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
     
     // corrupt replicas
-    File blockFile = MiniDFSCluster.getBlockFile(0, block);
+    File blockFile = cluster.getBlockFile(0, block);
     if (blockFile != null && blockFile.exists()) {
       RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
       FileChannel channel = raFile.getChannel();
@@ -711,9 +712,9 @@ public void testFsckError() throws Exception {
       DFSTestUtil.waitReplication(fs, filePath, (short)1);
       
       // intentionally corrupt NN data structure
-      INodeFile node = (INodeFile)cluster.getNamesystem().dir.getNode(
-          fileName, true);
-      final BlockInfo[] blocks = node.getBlocks(); 
+      INodeFile node = (INodeFile) cluster.getNamesystem().dir.getINode
+          (fileName, true);
+      final BlockInfoContiguous[] blocks = node.getBlocks();
       assertEquals(blocks.length, 1);
       blocks[0].setNumBytes(-1L);  // set the block length to be negative
       
@@ -996,9 +997,9 @@ public void testFsckFileNotFound() throws Exception {
     DatanodeManager dnManager = mock(DatanodeManager.class);
     
     when(namenode.getNamesystem()).thenReturn(fsName);
-    when(fsName.getBlockLocations(anyString(), anyLong(), anyLong(),
-        anyBoolean(), anyBoolean(), anyBoolean())).
-        thenThrow(new FileNotFoundException()) ;
+    when(fsName.getBlockLocations(
+        anyString(), anyLong(), anyLong(), anyBoolean(), anyBoolean()))
+        .thenThrow(new FileNotFoundException());
     when(fsName.getBlockManager()).thenReturn(blockManager);
     when(blockManager.getDatanodeManager()).thenReturn(dnManager);
 
@@ -1303,7 +1304,7 @@ public void testBlockIdCKCorruption() throws Exception {
 
       // corrupt replicas
       block = DFSTestUtil.getFirstBlock(dfs, path);
-      File blockFile = MiniDFSCluster.getBlockFile(0, block);
+      File blockFile = cluster.getBlockFile(0, block);
       if (blockFile != null && blockFile.exists()) {
         RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
         FileChannel channel = raFile.getChannel();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java
new file mode 100644
index 0000000000000..f4cb6249950de
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test fsck with multiple NameNodes
+ */
+public class TestFsckWithMultipleNameNodes {
+  static final Log LOG = LogFactory.getLog(TestFsckWithMultipleNameNodes.class);
+  {
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+  }
+
+  
+  private static final String FILE_NAME = "/tmp.txt";
+  private static final Path FILE_PATH = new Path(FILE_NAME);
+  
+  private static final Random RANDOM = new Random();
+
+  static {
+    TestBalancer.initTestSetup();
+  }
+
+  /** Common objects used in various methods. */
+  private static class Suite {
+    final MiniDFSCluster cluster;
+    final ClientProtocol[] clients;
+    final short replication;
+    
+    Suite(MiniDFSCluster cluster, final int nNameNodes, final int nDataNodes)
+        throws IOException {
+      this.cluster = cluster;
+      clients = new ClientProtocol[nNameNodes];
+      for(int i = 0; i < nNameNodes; i++) {
+        clients[i] = cluster.getNameNode(i).getRpcServer();
+      }
+      replication = (short)Math.max(1, nDataNodes - 1);
+    }
+
+    /** create a file with a length of <code>fileLen</code> */
+    private void createFile(int index, long len
+        ) throws IOException, InterruptedException, TimeoutException {
+      final FileSystem fs = cluster.getFileSystem(index);
+      DFSTestUtil.createFile(fs, FILE_PATH, len, replication, RANDOM.nextLong());
+      DFSTestUtil.waitReplication(fs, FILE_PATH, replication);
+    }
+
+  }
+
+  private static Configuration createConf() {
+    final Configuration conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 1L);
+    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
+    return conf;
+  }
+
+  private void runTest(final int nNameNodes, final int nDataNodes,
+      Configuration conf) throws Exception {
+    LOG.info("nNameNodes=" + nNameNodes + ", nDataNodes=" + nDataNodes);
+
+    LOG.info("RUN_TEST -1");
+    final MiniDFSCluster cluster = new MiniDFSCluster
+        .Builder(conf)
+        .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(nNameNodes))
+        .numDataNodes(nDataNodes)
+        .build();
+    LOG.info("RUN_TEST 0");
+    DFSTestUtil.setFederatedConfiguration(cluster, conf);
+
+    try {
+      cluster.waitActive();
+      LOG.info("RUN_TEST 1");
+      final Suite s = new Suite(cluster, nNameNodes, nDataNodes);
+      for(int i = 0; i < nNameNodes; i++) {
+        s.createFile(i, 1024);
+      }
+
+      LOG.info("RUN_TEST 2");
+      final String[] urls = new String[nNameNodes];
+      for(int i = 0; i < urls.length; i++) {
+        urls[i] = cluster.getFileSystem(i).getUri() + FILE_NAME;
+        LOG.info("urls[" + i + "]=" + urls[i]);
+        final String result = TestFsck.runFsck(conf, 0, false, urls[i]);
+        LOG.info("result=" + result);
+        Assert.assertTrue(result.contains("Status: HEALTHY"));
+      }
+    } finally {
+      cluster.shutdown();
+    }
+    LOG.info("RUN_TEST 6");
+  }
+  
+  /** Test a cluster with even distribution, 
+   * then a new empty node is added to the cluster
+   */
+  @Test
+  public void testFsck() throws Exception {
+    final Configuration conf = createConf();
+    runTest(3, 1, conf);
+  }
+
+}
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHDFSConcat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHDFSConcat.java
index 6d1f4521f8bb0..ddf5a3e23d83b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHDFSConcat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHDFSConcat.java
@@ -40,9 +40,12 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -99,7 +102,7 @@ public void testConcat() throws IOException, InterruptedException {
     HdfsFileStatus fStatus;
     FSDataInputStream stm;
     
-    String trg = new String("/trg");
+    String trg = "/trg";
     Path trgPath = new Path(trg);
     DFSTestUtil.createFile(dfs, trgPath, fileLen, REPL_FACTOR, 1);
     fStatus  = nn.getFileInfo(trg);
@@ -112,7 +115,7 @@ public void testConcat() throws IOException, InterruptedException {
     long [] lens = new long [numFiles];
     
     
-    int i = 0;
+    int i;
     for(i=0; i<files.length; i++) {
       files[i] = new Path("/file"+i);
       Path path = files[i];
@@ -385,6 +388,75 @@ public void testIllegalArg() throws IOException {
     } catch (Exception e) {
       // exspected
     }
- 
+  }
+
+  /**
+   * make sure we update the quota correctly after concat
+   */
+  @Test
+  public void testConcatWithQuotaDecrease() throws IOException {
+    final short srcRepl = 3; // note this is different with REPL_FACTOR
+    final int srcNum = 10;
+    final Path foo = new Path("/foo");
+    final Path[] srcs = new Path[srcNum];
+    final Path target = new Path(foo, "target");
+    DFSTestUtil.createFile(dfs, target, blockSize, REPL_FACTOR, 0L);
+
+    dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+
+    for (int i = 0; i < srcNum; i++) {
+      srcs[i] = new Path(foo, "src" + i);
+      DFSTestUtil.createFile(dfs, srcs[i], blockSize * 2, srcRepl, 0L);
+    }
+
+    ContentSummary summary = dfs.getContentSummary(foo);
+    Assert.assertEquals(11, summary.getFileCount());
+    Assert.assertEquals(blockSize * REPL_FACTOR +
+            blockSize * 2 * srcRepl * srcNum, summary.getSpaceConsumed());
+
+    dfs.concat(target, srcs);
+    summary = dfs.getContentSummary(foo);
+    Assert.assertEquals(1, summary.getFileCount());
+    Assert.assertEquals(
+        blockSize * REPL_FACTOR + blockSize * 2 * REPL_FACTOR * srcNum,
+        summary.getSpaceConsumed());
+  }
+
+  @Test
+  public void testConcatWithQuotaIncrease() throws IOException {
+    final short repl = 3;
+    final int srcNum = 10;
+    final Path foo = new Path("/foo");
+    final Path bar = new Path(foo, "bar");
+    final Path[] srcs = new Path[srcNum];
+    final Path target = new Path(bar, "target");
+    DFSTestUtil.createFile(dfs, target, blockSize, repl, 0L);
+
+    final long dsQuota = blockSize * repl + blockSize * srcNum * REPL_FACTOR;
+    dfs.setQuota(foo, Long.MAX_VALUE - 1, dsQuota);
+
+    for (int i = 0; i < srcNum; i++) {
+      srcs[i] = new Path(bar, "src" + i);
+      DFSTestUtil.createFile(dfs, srcs[i], blockSize, REPL_FACTOR, 0L);
+    }
+
+    ContentSummary summary = dfs.getContentSummary(bar);
+    Assert.assertEquals(11, summary.getFileCount());
+    Assert.assertEquals(dsQuota, summary.getSpaceConsumed());
+
+    try {
+      dfs.concat(target, srcs);
+      fail("QuotaExceededException expected");
+    } catch (RemoteException e) {
+      Assert.assertTrue(
+          e.unwrapRemoteException() instanceof QuotaExceededException);
+    }
+
+    dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+    dfs.concat(target, srcs);
+    summary = dfs.getContentSummary(bar);
+    Assert.assertEquals(1, summary.getFileCount());
+    Assert.assertEquals(blockSize * repl * (srcNum + 1),
+        summary.getSpaceConsumed());
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index f79277acd7d8f..61d2b3e00360e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -59,7 +59,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
@@ -285,7 +285,7 @@ private INodeFile[] createINodeFiles(int nCount, String fileNamePrefix) {
       iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
           preferredBlockSize);
       iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
-      BlockInfo newblock = new BlockInfo(replication);
+      BlockInfoContiguous newblock = new BlockInfoContiguous(replication);
       iNodes[i].addBlock(newblock);
     }
     
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index 9b454ea066229..2f114a78c6ac1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -62,9 +62,11 @@ public void testRemoveLeaseWithPrefixPath() throws Exception {
    */
   @Test (timeout=1000)
   public void testCheckLeaseNotInfiniteLoop() {
+    FSDirectory dir = Mockito.mock(FSDirectory.class);
     FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
     Mockito.when(fsn.isRunning()).thenReturn(true);
     Mockito.when(fsn.hasWriteLock()).thenReturn(true);
+    Mockito.when(fsn.getFSDirectory()).thenReturn(dir);
     LeaseManager lm = new LeaseManager(fsn);
 
     //Make sure the leases we are going to add exceed the hard limit
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
index 7e36acb48c35a..92ea111495ebe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
@@ -28,7 +28,6 @@
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,13 +39,11 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.TestFileCorruption;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
 
 /**
  * This class tests the listCorruptFileBlocks API.
@@ -57,7 +54,7 @@
  * blocks/files are also returned.
  */
 public class TestListCorruptFileBlocks {
-  static final Log LOG = NameNode.stateChangeLog;
+  static final Logger LOG = NameNode.stateChangeLog;
 
   /** check if nn.getCorruptFiles() returns a file that has corrupted blocks */
   @Test (timeout=300000)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeAcl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeAcl.java
index 7fe45594e24b3..6a36c9879b117 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeAcl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeAcl.java
@@ -18,8 +18,6 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.BeforeClass;
 
 /**
@@ -31,8 +29,6 @@ public class TestNameNodeAcl extends FSAclBaseTest {
   @BeforeClass
   public static void init() throws Exception {
     conf = new Configuration();
-    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
+    startCluster();
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 03ade90b4159b..c649621db73a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -26,9 +26,12 @@
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
+import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
 import org.apache.hadoop.util.VersionInfo;
+import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.Test;
 import org.mortbay.util.ajax.JSON;
 
@@ -38,10 +41,15 @@
 import java.lang.management.ManagementFactory;
 import java.net.URI;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op;
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -257,4 +265,112 @@ public void testLastContactTime() throws Exception {
       }
     }
   }
+
+  @Test(timeout=120000)
+  @SuppressWarnings("unchecked")
+  public void testTopUsers() throws Exception {
+    final Configuration conf = new Configuration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+      cluster.waitActive();
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxbeanNameFsns = new ObjectName(
+          "Hadoop:service=NameNode,name=FSNamesystemState");
+      FileSystem fs = cluster.getFileSystem();
+      final Path path = new Path("/");
+      final int NUM_OPS = 10;
+      for (int i=0; i< NUM_OPS; i++) {
+        fs.listStatus(path);
+        fs.setTimes(path, 0, 1);
+      }
+      String topUsers =
+          (String) (mbs.getAttribute(mxbeanNameFsns, "TopUserOpCounts"));
+      ObjectMapper mapper = new ObjectMapper();
+      Map<String, Object> map = mapper.readValue(topUsers, Map.class);
+      assertTrue("Could not find map key timestamp", 
+          map.containsKey("timestamp"));
+      assertTrue("Could not find map key windows", map.containsKey("windows"));
+      List<Map<String, List<Map<String, Object>>>> windows =
+          (List<Map<String, List<Map<String, Object>>>>) map.get("windows");
+      assertEquals("Unexpected num windows", 3, windows.size());
+      for (Map<String, List<Map<String, Object>>> window : windows) {
+        final List<Map<String, Object>> ops = window.get("ops");
+        assertEquals("Unexpected num ops", 3, ops.size());
+        for (Map<String, Object> op: ops) {
+          final long count = Long.parseLong(op.get("totalCount").toString());
+          final String opType = op.get("opType").toString();
+          final int expected;
+          if (opType.equals(TopConf.ALL_CMDS)) {
+            expected = 2*NUM_OPS;
+          } else {
+            expected = NUM_OPS;
+          }
+          assertEquals("Unexpected total count", expected, count);
+        }
+      }
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
+  @Test(timeout=120000)
+  public void testTopUsersDisabled() throws Exception {
+    final Configuration conf = new Configuration();
+    // Disable nntop
+    conf.setBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, false);
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+      cluster.waitActive();
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxbeanNameFsns = new ObjectName(
+          "Hadoop:service=NameNode,name=FSNamesystemState");
+      FileSystem fs = cluster.getFileSystem();
+      final Path path = new Path("/");
+      final int NUM_OPS = 10;
+      for (int i=0; i< NUM_OPS; i++) {
+        fs.listStatus(path);
+        fs.setTimes(path, 0, 1);
+      }
+      String topUsers =
+          (String) (mbs.getAttribute(mxbeanNameFsns, "TopUserOpCounts"));
+      assertNull("Did not expect to find TopUserOpCounts bean!", topUsers);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
+  @Test(timeout=120000)
+  public void testTopUsersNoPeriods() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.setBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, true);
+    conf.set(DFSConfigKeys.NNTOP_WINDOWS_MINUTES_KEY, "");
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+      cluster.waitActive();
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxbeanNameFsns = new ObjectName(
+          "Hadoop:service=NameNode,name=FSNamesystemState");
+      FileSystem fs = cluster.getFileSystem();
+      final Path path = new Path("/");
+      final int NUM_OPS = 10;
+      for (int i=0; i< NUM_OPS; i++) {
+        fs.listStatus(path);
+        fs.setTimes(path, 0, 1);
+      }
+      String topUsers =
+          (String) (mbs.getAttribute(mxbeanNameFsns, "TopUserOpCounts"));
+      assertNotNull("Expected TopUserOpCounts bean!", topUsers);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
index b9e62e3f6e9a1..2e6b4a336df56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
@@ -232,14 +232,18 @@ public void testAppend() throws Exception {
     
     // Retried append requests succeed
     newCall();
-    LastBlockWithStatus b = nnRpc.append(src, "holder");
-    Assert.assertEquals(b, nnRpc.append(src, "holder"));
-    Assert.assertEquals(b, nnRpc.append(src, "holder"));
+    LastBlockWithStatus b = nnRpc.append(src, "holder",
+        new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
+    Assert.assertEquals(b, nnRpc.append(src, "holder",
+        new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND))));
+    Assert.assertEquals(b, nnRpc.append(src, "holder",
+        new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND))));
     
     // non-retried call fails
     newCall();
     try {
-      nnRpc.append(src, "holder");
+      nnRpc.append(src, "holder",
+          new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
       Assert.fail("testAppend - expected exception is not thrown");
     } catch (Exception e) {
       // Expected
@@ -409,7 +413,7 @@ public void testRetryCacheRebuild() throws Exception {
 
     LightWeightCache<CacheEntry, CacheEntry> cacheSet = 
         (LightWeightCache<CacheEntry, CacheEntry>) namesystem.getRetryCache().getCacheSet();
-    assertEquals(23, cacheSet.size());
+    assertEquals(25, cacheSet.size());
     
     Map<CacheEntry, CacheEntry> oldEntries = 
         new HashMap<CacheEntry, CacheEntry>();
@@ -428,7 +432,7 @@ public void testRetryCacheRebuild() throws Exception {
     assertTrue(namesystem.hasRetryCache());
     cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) namesystem
         .getRetryCache().getCacheSet();
-    assertEquals(23, cacheSet.size());
+    assertEquals(25, cacheSet.size());
     iter = cacheSet.iterator();
     while (iter.hasNext()) {
       CacheEntry entry = iter.next();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProcessCorruptBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProcessCorruptBlocks.java
index abb2337a5ddf2..168ebb988ac68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProcessCorruptBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProcessCorruptBlocks.java
@@ -267,14 +267,14 @@ private void corruptBlock(MiniDFSCluster cluster, FileSystem fs, final Path file
     // corrupt the block on datanode dnIndex
     // the indexes change once the nodes are restarted.
     // But the datadirectory will not change
-    assertTrue(MiniDFSCluster.corruptReplica(dnIndex, block));
+    assertTrue(cluster.corruptReplica(dnIndex, block));
 
     DataNodeProperties dnProps = cluster.stopDataNode(0);
 
     // Each datanode has multiple data dirs, check each
     for (int dirIndex = 0; dirIndex < 2; dirIndex++) {
       final String bpid = cluster.getNamesystem().getBlockPoolId();
-      File storageDir = MiniDFSCluster.getStorageDir(dnIndex, dirIndex);
+      File storageDir = cluster.getStorageDir(dnIndex, dirIndex);
       File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
       File scanLogFile = new File(dataDir, "dncp_block_verification.log.curr");
       if (scanLogFile.exists()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
index d1a23779a7d68..e416e00e1dace 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
@@ -18,11 +18,11 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.FileNotFoundException;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -104,19 +104,18 @@ public void testAllowSnapshot() throws Exception {
     }
   }
   
-  static Snapshot getSnapshot(INodesInPath inodesInPath, String name) {
+  static Snapshot getSnapshot(INodesInPath inodesInPath, String name,
+      int index) {
     if (name == null) {
       return null;
     }
-    final int i = inodesInPath.getSnapshotRootIndex() - 1;
-    final INode inode = inodesInPath.getINodes()[i];
+    final INode inode = inodesInPath.getINode(index - 1);
     return inode.asDirectory().getSnapshot(DFSUtil.string2Bytes(name));
   }
 
   static void assertSnapshot(INodesInPath inodesInPath, boolean isSnapshot,
       final Snapshot snapshot, int index) {
     assertEquals(isSnapshot, inodesInPath.isSnapshot());
-    assertEquals(index, inodesInPath.getSnapshotRootIndex());
     assertEquals(Snapshot.getSnapshotId(isSnapshot ? snapshot : null),
         inodesInPath.getPathSnapshotId());
     if (!isSnapshot) {
@@ -124,7 +123,7 @@ static void assertSnapshot(INodesInPath inodesInPath, boolean isSnapshot,
           inodesInPath.getLatestSnapshotId());
     }
     if (isSnapshot && index >= 0) {
-      assertEquals(Snapshot.Root.class, inodesInPath.getINodes()[index].getClass());
+      assertEquals(Snapshot.Root.class, inodesInPath.getINode(index).getClass());
     }
   }
 
@@ -141,39 +140,27 @@ public void testNonSnapshotPathINodes() throws Exception {
     // Get the inodes by resolving the path of a normal file
     String[] names = INode.getPathNames(file1.toString());
     byte[][] components = INode.getPathComponents(names);
-    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] inodes = nodesInPath.getINodes();
+    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // The number of inodes should be equal to components.length
-    assertEquals(inodes.length, components.length);
+    assertEquals(nodesInPath.length(), components.length);
     // The returned nodesInPath should be non-snapshot
     assertSnapshot(nodesInPath, false, null, -1);
 
     // The last INode should be associated with file1
     assertTrue("file1=" + file1 + ", nodesInPath=" + nodesInPath,
-        inodes[components.length - 1] != null);
-    assertEquals(inodes[components.length - 1].getFullPathName(),
+        nodesInPath.getINode(components.length - 1) != null);
+    assertEquals(nodesInPath.getINode(components.length - 1).getFullPathName(),
         file1.toString());
-    assertEquals(inodes[components.length - 2].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 2).getFullPathName(),
         sub1.toString());
-    assertEquals(inodes[components.length - 3].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 3).getFullPathName(),
         dir.toString());
     
-    // Call getExistingPathINodes and request only one INode. This is used
-    // when identifying the INode for a given path.
-    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 1, false);
-    inodes = nodesInPath.getINodes();
-    assertEquals(inodes.length, 1);
+    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false);
+    assertEquals(nodesInPath.length(), components.length);
     assertSnapshot(nodesInPath, false, null, -1);
-    assertEquals(inodes[0].getFullPathName(), file1.toString());
-    
-    // Call getExistingPathINodes and request 2 INodes. This is usually used
-    // when identifying the parent INode of a given path.
-    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 2, false);
-    inodes = nodesInPath.getINodes();
-    assertEquals(inodes.length, 2);
-    assertSnapshot(nodesInPath, false, null, -1);
-    assertEquals(inodes[1].getFullPathName(), file1.toString());
-    assertEquals(inodes[0].getFullPathName(), sub1.toString());
+    assertEquals(nodesInPath.getLastINode().getFullPathName(), file1.toString());
   }
   
   /** 
@@ -190,54 +177,41 @@ public void testSnapshotPathINodes() throws Exception {
     String snapshotPath = sub1.toString() + "/.snapshot/s1/file1";
     String[] names = INode.getPathNames(snapshotPath);
     byte[][] components = INode.getPathComponents(names);
-    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] inodes = nodesInPath.getINodes();
+    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // Length of inodes should be (components.length - 1), since we will ignore
     // ".snapshot" 
-    assertEquals(inodes.length, components.length - 1);
+    assertEquals(nodesInPath.length(), components.length - 1);
     // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s1, file1}
-    final Snapshot snapshot = getSnapshot(nodesInPath, "s1");
+    final Snapshot snapshot = getSnapshot(nodesInPath, "s1", 3);
     assertSnapshot(nodesInPath, true, snapshot, 3);
     // Check the INode for file1 (snapshot file)
-    INode snapshotFileNode = inodes[inodes.length - 1]; 
+    INode snapshotFileNode = nodesInPath.getLastINode();
     assertINodeFile(snapshotFileNode, file1);
     assertTrue(snapshotFileNode.getParent().isWithSnapshot());
     
     // Call getExistingPathINodes and request only one INode.
-    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 1, false);
-    inodes = nodesInPath.getINodes();
-    assertEquals(inodes.length, 1);
-    // The snapshotroot (s1) is not included in inodes. Thus the
-    // snapshotRootIndex should be -1.
-    assertSnapshot(nodesInPath, true, snapshot, -1);
+    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false);
+    assertEquals(nodesInPath.length(), components.length - 1);
+    assertSnapshot(nodesInPath, true, snapshot, 3);
     // Check the INode for file1 (snapshot file)
-    assertINodeFile(inodes[inodes.length - 1], file1);
-    
-    // Call getExistingPathINodes and request 2 INodes.
-    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 2, false);
-    inodes = nodesInPath.getINodes();
-    assertEquals(inodes.length, 2);
-    // There should be two INodes in inodes: s1 and snapshot of file1. Thus the
-    // SnapshotRootIndex should be 0.
-    assertSnapshot(nodesInPath, true, snapshot, 0);
-    assertINodeFile(inodes[inodes.length - 1], file1);
-    
+    assertINodeFile(nodesInPath.getLastINode(), file1);
+
     // Resolve the path "/TestSnapshot/sub1/.snapshot"  
     String dotSnapshotPath = sub1.toString() + "/.snapshot";
     names = INode.getPathNames(dotSnapshotPath);
     components = INode.getPathComponents(names);
-    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    inodes = nodesInPath.getINodes();
-    // The number of INodes returned should be components.length - 1 since we
-    // will ignore ".snapshot"
-    assertEquals(inodes.length, components.length - 1);
+    nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, false);
+    // The number of INodes returned should still be components.length
+    // since we put a null in the inode array for ".snapshot"
+    assertEquals(nodesInPath.length(), components.length);
 
     // No SnapshotRoot dir is included in the resolved inodes  
     assertSnapshot(nodesInPath, true, snapshot, -1);
-    // The last INode should be the INode for sub1
-    final INode last = inodes[inodes.length - 1];
-    assertEquals(last.getFullPathName(), sub1.toString());
-    assertFalse(last instanceof INodeFile);
+    // The last INode should be null, the last but 1 should be sub1
+    assertNull(nodesInPath.getLastINode());
+    assertEquals(nodesInPath.getINode(-2).getFullPathName(), sub1.toString());
+    assertTrue(nodesInPath.getINode(-2).isDirectory());
     
     String[] invalidPathComponent = {"invalidDir", "foo", ".snapshot", "bar"};
     Path invalidPath = new Path(invalidPathComponent[0]);
@@ -274,17 +248,17 @@ public void testSnapshotPathINodesAfterDeletion() throws Exception {
       String snapshotPath = sub1.toString() + "/.snapshot/s2/file1";
       String[] names = INode.getPathNames(snapshotPath);
       byte[][] components = INode.getPathComponents(names);
-      INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-      INode[] inodes = nodesInPath.getINodes();
+      INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+          components, false);
       // Length of inodes should be (components.length - 1), since we will ignore
       // ".snapshot" 
-      assertEquals(inodes.length, components.length - 1);
+      assertEquals(nodesInPath.length(), components.length - 1);
       // SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s2, file1}
-      snapshot = getSnapshot(nodesInPath, "s2");
+      snapshot = getSnapshot(nodesInPath, "s2", 3);
       assertSnapshot(nodesInPath, true, snapshot, 3);
   
       // Check the INode for file1 (snapshot file)
-      final INode inode = inodes[inodes.length - 1];
+      final INode inode = nodesInPath.getLastINode();
       assertEquals(file1.getName(), inode.getLocalName());
       assertTrue(inode.asFile().isWithSnapshot());
     }
@@ -292,26 +266,36 @@ public void testSnapshotPathINodesAfterDeletion() throws Exception {
     // Check the INodes for path /TestSnapshot/sub1/file1
     String[] names = INode.getPathNames(file1.toString());
     byte[][] components = INode.getPathComponents(names);
-    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] inodes = nodesInPath.getINodes();
+    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // The length of inodes should be equal to components.length
-    assertEquals(inodes.length, components.length);
+    assertEquals(nodesInPath.length(), components.length);
     // The number of non-null elements should be components.length - 1 since
     // file1 has been deleted
-    assertEquals(nodesInPath.getNumNonNull(), components.length - 1);
+    assertEquals(getNumNonNull(nodesInPath), components.length - 1);
     // The returned nodesInPath should be non-snapshot
     assertSnapshot(nodesInPath, false, snapshot, -1);
     // The last INode should be null, and the one before should be associated
     // with sub1
-    assertNull(inodes[components.length - 1]);
-    assertEquals(inodes[components.length - 2].getFullPathName(),
+    assertNull(nodesInPath.getINode(components.length - 1));
+    assertEquals(nodesInPath.getINode(components.length - 2).getFullPathName(),
         sub1.toString());
-    assertEquals(inodes[components.length - 3].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 3).getFullPathName(),
         dir.toString());
     hdfs.deleteSnapshot(sub1, "s2");
     hdfs.disallowSnapshot(sub1);
   }
 
+  private int getNumNonNull(INodesInPath iip) {
+    List<INode> inodes = iip.getReadOnlyINodes();
+    for (int i = inodes.size() - 1; i >= 0; i--) {
+      if (inodes.get(i) != null) {
+        return i+1;
+      }
+    }
+    return 0;
+  }
+
   /**
    * for snapshot file while adding a new file after snapshot.
    */
@@ -332,40 +316,40 @@ public void testSnapshotPathINodesWithAddedFile() throws Exception {
       String snapshotPath = sub1.toString() + "/.snapshot/s4/file3";
       String[] names = INode.getPathNames(snapshotPath);
       byte[][] components = INode.getPathComponents(names);
-      INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-      INode[] inodes = nodesInPath.getINodes();
+      INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+          components, false);
       // Length of inodes should be (components.length - 1), since we will ignore
       // ".snapshot" 
-      assertEquals(inodes.length, components.length - 1);
+      assertEquals(nodesInPath.length(), components.length - 1);
       // The number of non-null inodes should be components.length - 2, since
       // snapshot of file3 does not exist
-      assertEquals(nodesInPath.getNumNonNull(), components.length - 2);
-      s4 = getSnapshot(nodesInPath, "s4");
+      assertEquals(getNumNonNull(nodesInPath), components.length - 2);
+      s4 = getSnapshot(nodesInPath, "s4", 3);
 
       // SnapshotRootIndex should still be 3: {root, Testsnapshot, sub1, s4, null}
       assertSnapshot(nodesInPath, true, s4, 3);
   
       // Check the last INode in inodes, which should be null
-      assertNull(inodes[inodes.length - 1]);
+      assertNull(nodesInPath.getINode(nodesInPath.length() - 1));
     }
 
     // Check the inodes for /TestSnapshot/sub1/file3
     String[] names = INode.getPathNames(file3.toString());
     byte[][] components = INode.getPathComponents(names);
-    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] inodes = nodesInPath.getINodes();
+    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // The number of inodes should be equal to components.length
-    assertEquals(inodes.length, components.length);
+    assertEquals(nodesInPath.length(), components.length);
 
     // The returned nodesInPath should be non-snapshot
     assertSnapshot(nodesInPath, false, s4, -1);
 
     // The last INode should be associated with file3
-    assertEquals(inodes[components.length - 1].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 1).getFullPathName(),
         file3.toString());
-    assertEquals(inodes[components.length - 2].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 2).getFullPathName(),
         sub1.toString());
-    assertEquals(inodes[components.length - 3].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 3).getFullPathName(),
         dir.toString());
     hdfs.deleteSnapshot(sub1, "s4");
     hdfs.disallowSnapshot(sub1);
@@ -379,16 +363,17 @@ public void testSnapshotPathINodesAfterModification() throws Exception {
     // First check the INode for /TestSnapshot/sub1/file1
     String[] names = INode.getPathNames(file1.toString());
     byte[][] components = INode.getPathComponents(names);
-    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] inodes = nodesInPath.getINodes();
+    INodesInPath nodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // The number of inodes should be equal to components.length
-    assertEquals(inodes.length, components.length);
+    assertEquals(nodesInPath.length(), components.length);
 
     // The last INode should be associated with file1
-    assertEquals(inodes[components.length - 1].getFullPathName(),
+    assertEquals(nodesInPath.getINode(components.length - 1).getFullPathName(),
         file1.toString());
     // record the modification time of the inode
-    final long modTime = inodes[inodes.length - 1].getModificationTime();
+    final long modTime = nodesInPath.getINode(nodesInPath.length() - 1)
+        .getModificationTime();
     
     // Create a snapshot for the dir, and check the inodes for the path
     // pointing to a snapshot file
@@ -402,15 +387,15 @@ public void testSnapshotPathINodesAfterModification() throws Exception {
     String snapshotPath = sub1.toString() + "/.snapshot/s3/file1";
     names = INode.getPathNames(snapshotPath);
     components = INode.getPathComponents(names);
-    INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
-    INode[] ssInodes = ssNodesInPath.getINodes();
+    INodesInPath ssNodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     // Length of ssInodes should be (components.length - 1), since we will
     // ignore ".snapshot" 
-    assertEquals(ssInodes.length, components.length - 1);
-    final Snapshot s3 = getSnapshot(ssNodesInPath, "s3");
+    assertEquals(ssNodesInPath.length(), components.length - 1);
+    final Snapshot s3 = getSnapshot(ssNodesInPath, "s3", 3);
     assertSnapshot(ssNodesInPath, true, s3, 3);
     // Check the INode for snapshot of file1
-    INode snapshotFileNode = ssInodes[ssInodes.length - 1]; 
+    INode snapshotFileNode = ssNodesInPath.getLastINode();
     assertEquals(snapshotFileNode.getLocalName(), file1.getName());
     assertTrue(snapshotFileNode.asFile().isWithSnapshot());
     // The modification time of the snapshot INode should be the same with the
@@ -421,16 +406,17 @@ public void testSnapshotPathINodesAfterModification() throws Exception {
     // Check the INode for /TestSnapshot/sub1/file1 again
     names = INode.getPathNames(file1.toString());
     components = INode.getPathComponents(names);
-    INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir, components);
+    INodesInPath newNodesInPath = INodesInPath.resolve(fsdir.rootDir,
+        components, false);
     assertSnapshot(newNodesInPath, false, s3, -1);
-    INode[] newInodes = newNodesInPath.getINodes();
     // The number of inodes should be equal to components.length
-    assertEquals(newInodes.length, components.length);
+    assertEquals(newNodesInPath.length(), components.length);
     // The last INode should be associated with file1
     final int last = components.length - 1;
-    assertEquals(newInodes[last].getFullPathName(), file1.toString());
+    assertEquals(newNodesInPath.getINode(last).getFullPathName(),
+        file1.toString());
     // The modification time of the INode for file3 should have been changed
-    Assert.assertFalse(modTime == newInodes[last].getModificationTime());
+    Assert.assertFalse(modTime == newNodesInPath.getINode(last).getModificationTime());
     hdfs.deleteSnapshot(sub1, "s3");
     hdfs.disallowSnapshot(sub1);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 75d5b70d5e306..f625bb4df921a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -25,9 +25,10 @@
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -50,7 +51,6 @@
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -65,9 +65,6 @@
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 
-import com.google.common.base.Supplier;
-import com.google.common.collect.Lists;
-
 
 public class TestDNFencing {
   
@@ -82,9 +79,7 @@ public class TestDNFencing {
   private FileSystem fs;
 
   static {
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
   
   @Before
@@ -165,7 +160,12 @@ public void testDnFencing() throws Exception {
     
     banner("Metadata after nodes have all block-reported");
     doMetasave(nn2);
-    
+
+    // Force a rescan of postponedMisreplicatedBlocks.
+    BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
+    BlockManagerTestUtil.checkHeartbeat(nn2BM);
+    BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);
+
     // The blocks should no longer be postponed.
     assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
     
@@ -251,7 +251,12 @@ public void testNNClearsCommandsOnFailoverAfterStartup()
     
     banner("Metadata after nodes have all block-reported");
     doMetasave(nn2);
-    
+
+    // Force a rescan of postponedMisreplicatedBlocks.
+    BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
+    BlockManagerTestUtil.checkHeartbeat(nn2BM);
+    BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);
+
     // The block should no longer be postponed.
     assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
     
@@ -347,6 +352,11 @@ public void testNNClearsCommandsOnFailoverWithReplChanges()
     banner("Metadata after nodes have all block-reported");
     doMetasave(nn2);
     
+    // Force a rescan of postponedMisreplicatedBlocks.
+    BlockManager nn2BM = nn2.getNamesystem().getBlockManager();
+    BlockManagerTestUtil.checkHeartbeat(nn2BM);
+    BlockManagerTestUtil.rescanPostponedMisreplicatedBlocks(nn2BM);
+
     // The block should no longer be postponed.
     assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
     
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
index 93830c1d28738..e7cba7501191c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java
@@ -109,6 +109,10 @@ public void testFencingStress() throws Exception {
     HAStressTestHarness harness = new HAStressTestHarness();
     harness.conf.setInt(
         DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
+    harness.conf.setInt(
+        DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+    harness.conf.setInt(
+        DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
 
     final MiniDFSCluster cluster = harness.startCluster();
     try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
index 9aa01221b3e6d..e0aad02228f22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
@@ -19,21 +19,33 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.AppendTestUtil;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.hdfs.server.namenode.TestFileTruncate;
 import org.apache.hadoop.hdfs.tools.DFSck;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 public class TestHAAppend {
+  static final int COUNT = 5;
 
+  static FSDataOutputStream createAndHflush(FileSystem fs, Path file,
+      byte[] data, int length) throws IOException{
+    FSDataOutputStream out = fs.create(file, false, 4096, (short)3, 1024);
+    out.write(data, 0, length);
+    out.hflush();
+    return out;
+  }
+  
   /**
    * Test to verify the processing of PendingDataNodeMessageQueue in case of
    * append. One block will marked as corrupt if the OP_ADD, OP_UPDATE_BLOCKS
@@ -58,22 +70,37 @@ public void testMultipleAppendsDuringCatchupTailing() throws Exception {
       fs = HATestUtil.configureFailoverFs(cluster, conf);
 
       Path fileToAppend = new Path("/FileToAppend");
+      Path fileToTruncate = new Path("/FileToTruncate");
+      
+      final byte[] data = new byte[1 << 16];
+      DFSUtil.getRandom().nextBytes(data);
+      final int[] appendPos = AppendTestUtil.randomFilePartition(
+          data.length, COUNT);
+      final int[] truncatePos = AppendTestUtil.randomFilePartition(
+          data.length, 1);
 
       // Create file, write some data, and hflush so that the first
       // block is in the edit log prior to roll.
-      FSDataOutputStream out = fs.create(fileToAppend);
-      out.writeBytes("/data");
-      out.hflush();
+      FSDataOutputStream out = createAndHflush(
+          fs, fileToAppend, data, appendPos[0]);
+
+      FSDataOutputStream out4Truncate = createAndHflush(
+          fs, fileToTruncate, data, data.length);
       
       // Let the StandbyNode catch the creation of the file. 
       cluster.getNameNode(0).getRpcServer().rollEditLog();
       cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
       out.close();
+      out4Truncate.close();
 
       // Append and re-close a few time, so that many block entries are queued.
-      for (int i = 0; i < 5; i++) {
-        DFSTestUtil.appendFile(fs, fileToAppend, "data");
+      for (int i = 0; i < COUNT; i++) {
+        int end = i < COUNT - 1? appendPos[i + 1]: data.length;
+        out = fs.append(fileToAppend);
+        out.write(data, appendPos[i], end - appendPos[i]);
+        out.close();
       }
+      boolean isTruncateReady = fs.truncate(fileToTruncate, truncatePos[0]);
 
       // Ensure that blocks have been reported to the SBN ahead of the edits
       // arriving.
@@ -90,6 +117,16 @@ public void testMultipleAppendsDuringCatchupTailing() throws Exception {
       
       assertEquals("CorruptBlocks should be empty.", 0, cluster.getNameNode(1)
           .getNamesystem().getCorruptReplicaBlocks());
+
+      AppendTestUtil.checkFullFile(fs, fileToAppend, data.length, data,
+          fileToAppend.toString());
+
+      if (!isTruncateReady) {
+        TestFileTruncate.checkBlockRecovery(fileToTruncate,
+            cluster.getFileSystem(1));
+      }
+      AppendTestUtil.checkFullFile(fs, fileToTruncate, truncatePos[0], data,
+          fileToTruncate.toString());
     } finally {
       if (null != cluster) {
         cluster.shutdown();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index 25c023f7ed5ab..c5aad9c1bc4c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
@@ -84,9 +84,8 @@ public class TestHASafeMode {
   private MiniDFSCluster cluster;
   
   static {
-    ((Log4JLogger)LogFactory.getLog(FSImage.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
   }
   
   @Before
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
index 37c7df9a95684..4d4fed63aa610 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
@@ -67,7 +67,7 @@ public void testChangedStorageId() throws IOException, URISyntaxException,
       // Change the gen stamp of the block on datanode to go back in time (gen
       // stamps start at 1000)
       ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
-      assertTrue(MiniDFSCluster.changeGenStampOfBlock(0, block, 900));
+      assertTrue(cluster.changeGenStampOfBlock(0, block, 900));
       
       // Stop the DN so the replica with the changed gen stamp will be reported
       // when this DN starts up.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
index 08c652553e9af..18d6dfca8945b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
@@ -28,7 +28,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -42,17 +41,14 @@
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.retry.RetryInvocationHandler;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
@@ -70,12 +66,9 @@
  */
 public class TestPipelinesFailover {
   static {
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(
-        "org.apache.hadoop.io.retry.RetryInvocationHandler")).getLogger().setLevel(Level.ALL);
-
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(LogFactory.getLog(RetryInvocationHandler
+            .class), Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
   
   protected static final Log LOG = LogFactory.getLog(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index 3739bd9f52ffa..c0d320c2f8ce0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -71,7 +71,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
@@ -163,7 +163,7 @@ public void testRetryCacheOnStandbyNN() throws Exception {
     FSNamesystem fsn0 = cluster.getNamesystem(0);
     LightWeightCache<CacheEntry, CacheEntry> cacheSet = 
         (LightWeightCache<CacheEntry, CacheEntry>) fsn0.getRetryCache().getCacheSet();
-    assertEquals(23, cacheSet.size());
+    assertEquals(25, cacheSet.size());
     
     Map<CacheEntry, CacheEntry> oldEntries = 
         new HashMap<CacheEntry, CacheEntry>();
@@ -184,7 +184,7 @@ public void testRetryCacheOnStandbyNN() throws Exception {
     FSNamesystem fsn1 = cluster.getNamesystem(1);
     cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) fsn1
         .getRetryCache().getCacheSet();
-    assertEquals(23, cacheSet.size());
+    assertEquals(25, cacheSet.size());
     iter = cacheSet.iterator();
     while (iter.hasNext()) {
       CacheEntry entry = iter.next();
@@ -438,7 +438,8 @@ void prepare() throws Exception {
 
     @Override
     void invoke() throws Exception {
-      lbk = client.getNamenode().append(fileName, client.getClientName());
+      lbk = client.getNamenode().append(fileName, client.getClientName(),
+          new EnumSetWritable<>(EnumSet.of(CreateFlag.APPEND)));
     }
     
     // check if the inode of the file is under construction
@@ -701,7 +702,8 @@ void prepare() throws Exception {
       final Path filePath = new Path(file);
       DFSTestUtil.createFile(dfs, filePath, BlockSize, DataNodes, 0);
       // append to the file and leave the last block under construction
-      out = this.client.append(file, BlockSize, null, null);
+      out = this.client.append(file, BlockSize, EnumSet.of(CreateFlag.APPEND),
+          null, null);
       byte[] appendContent = new byte[100];
       new Random().nextBytes(appendContent);
       out.write(appendContent);
@@ -741,8 +743,8 @@ void invoke() throws Exception {
     boolean checkNamenodeBeforeReturn() throws Exception {
       INodeFile fileNode = cluster.getNamesystem(0).getFSDirectory()
           .getINode4Write(file).asFile();
-      BlockInfoUnderConstruction blkUC = 
-          (BlockInfoUnderConstruction) (fileNode.getBlocks())[1];
+      BlockInfoContiguousUnderConstruction blkUC =
+          (BlockInfoContiguousUnderConstruction) (fileNode.getBlocks())[1];
       int datanodeNum = blkUC.getExpectedStorageLocations().length;
       for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) {
         Thread.sleep(1000);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
index d78d43b6059e9..9042f8a86654f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyBlockManagement.java
@@ -17,13 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -31,22 +26,13 @@
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
-import org.junit.Assert;
 import org.junit.Test;
 
-import com.google.common.base.Supplier;
+import static org.junit.Assert.assertEquals;
 
 /**
  * Makes sure that standby doesn't do the unnecessary block management such as
@@ -60,9 +46,7 @@ public class TestStandbyBlockManagement {
   private static final Path TEST_FILE_PATH = new Path(TEST_FILE);
 
   static {
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   @Test(timeout=60000)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
index 1d75c30e3da61..33af0e2197909 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
@@ -466,7 +466,7 @@ public CompressionOutputStream createOutputStream(OutputStream out)
         throws IOException {
       CompressionOutputStream ret = super.createOutputStream(out);
       CompressionOutputStream spy = Mockito.spy(ret);
-      Mockito.doAnswer(new GenericTestUtils.SleepAnswer(2))
+      Mockito.doAnswer(new GenericTestUtils.SleepAnswer(5))
         .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
       return spy;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java
index ecd52437b0fc8..622ed94861e10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java
@@ -23,7 +23,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -35,10 +34,8 @@
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -60,9 +57,7 @@ public class TestStandbyIsHot {
   private static final Path TEST_FILE_PATH = new Path(TEST_FILE);
 
   static {
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)LogFactory.getLog(BlockManager.class)).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   @Test(timeout=60000)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index c028a4a31a891..6c378226e6802 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -47,10 +47,7 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
-import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
 import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
-import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.MetricsSource;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -58,7 +55,6 @@
 import org.apache.hadoop.util.Time;
 import org.apache.log4j.Level;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -93,11 +89,6 @@ public class TestNameNodeMetrics {
     CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true);
     ((Log4JLogger)LogFactory.getLog(MetricsAsserts.class))
       .getLogger().setLevel(Level.DEBUG);
-    /**
-     * need it to test {@link #testTopAuditLogger}
-     */
-    CONF.set(DFS_NAMENODE_AUDIT_LOGGERS_KEY,
-        TopAuditLogger.class.getName());
   }
   
   private MiniDFSCluster cluster;
@@ -112,7 +103,6 @@ private static Path getTestPath(String fileName) {
   
   @Before
   public void setUp() throws Exception {
-    TopMetrics.reset();//reset the static init done by prev test
     cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT).build();
     cluster.waitActive();
     namesystem = cluster.getNamesystem();
@@ -465,53 +455,4 @@ public void testSyncAndBlockReportMetric() throws Exception {
     assertQuantileGauges("Syncs1s", rb);
     assertQuantileGauges("BlockReport1s", rb);
   }
-
-  /**
-   * Test whether {@link TopMetrics} is registered with metrics system
-   * @throws Exception
-   */
-  @Test
-  public void testTopMetrics() throws Exception {
-    final String testUser = "NNTopTestUser";
-    final String testOp = "NNTopTestOp";
-    final String metricName =
-        RollingWindowManager.createMetricName(testOp, testUser);
-    TopMetrics.getInstance().report(testUser, testOp);
-    final String regName = TopConf.TOP_METRICS_REGISTRATION_NAME;
-    MetricsRecordBuilder rb = getMetrics(regName);
-    assertGauge(metricName, 1L, rb);
-  }
-
-  /**
-   * Test whether {@link TopAuditLogger} is registered as an audit logger
-   * @throws Exception
-   */
-  @Test
-  public void testTopAuditLogger() throws Exception {
-    //note: the top audit logger should already be set in conf
-    //issue one command, any command is fine
-    FileSystem fs = cluster.getFileSystem();
-    long time = System.currentTimeMillis();
-    fs.setTimes(new Path("/"), time, time);
-    //the command should be reflected in the total count of all users
-    final String testUser = TopConf.ALL_USERS;
-    final String testOp = TopConf.CMD_TOTAL;
-    final String metricName =
-        RollingWindowManager.createMetricName(testOp, testUser);
-    final String regName = TopConf.TOP_METRICS_REGISTRATION_NAME;
-    MetricsRecordBuilder rb = getMetrics(regName);
-    assertGaugeGreaterThan(metricName, 1L, rb);
-  }
-
-  /**
-   * Assert a long gauge metric greater than
-   * @param name  of the metric
-   * @param expected  minimum expected value of the metric
-   * @param rb  the record builder mock used to getMetrics
-   */
-  public static void assertGaugeGreaterThan(String name, long expected,
-                                 MetricsRecordBuilder rb) {
-    Assert.assertTrue("Bad value for metric " + name,
-        expected <= MetricsAsserts.getLongGauge(name, rb));
-  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
index f94d3eb169ba9..11b19f3af2c47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
@@ -34,7 +34,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -44,11 +43,11 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage;
-import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
+import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -62,7 +61,7 @@
 import org.apache.hadoop.ipc.ProtobufRpcEngine.Server;
 import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Level;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 
 /**
@@ -79,28 +78,24 @@ public static void disableLogs() {
         "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService",
     };
     for(String n : lognames) {
-      setLevel2OFF(LogFactory.getLog(n));
+      GenericTestUtils.disableLog(LogFactory.getLog(n));
     }
     
-    setLevel2OFF(LogFactory.getLog(UserGroupInformation.class));
-    setLevel2OFF(LogFactory.getLog(BlockManager.class));
-    setLevel2OFF(LogFactory.getLog(FSNamesystem.class));
-    setLevel2OFF(LogFactory.getLog(DirectoryScanner.class));
-    setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class));
+    GenericTestUtils.disableLog(LogFactory.getLog(UserGroupInformation.class));
+    GenericTestUtils.disableLog(LogFactory.getLog(BlockManager.class));
+    GenericTestUtils.disableLog(LogFactory.getLog(FSNamesystem.class));
+    GenericTestUtils.disableLog(LogFactory.getLog(DirectoryScanner.class));
+    GenericTestUtils.disableLog(LogFactory.getLog(MetricsSystemImpl.class));
     
-    setLevel2OFF(DataBlockScanner.LOG);
-    setLevel2OFF(HttpServer2.LOG);
-    setLevel2OFF(DataNode.LOG);
-    setLevel2OFF(BlockPoolSliceStorage.LOG);
-    setLevel2OFF(LeaseManager.LOG);
-    setLevel2OFF(NameNode.stateChangeLog);
-    setLevel2OFF(NameNode.blockStateChangeLog);
-    setLevel2OFF(DFSClient.LOG);
-    setLevel2OFF(Server.LOG);
-  }
-
-  static void setLevel2OFF(Object log) {
-    ((Log4JLogger)log).getLogger().setLevel(Level.OFF);
+    GenericTestUtils.disableLog(BlockScanner.LOG);
+    GenericTestUtils.disableLog(HttpServer2.LOG);
+    GenericTestUtils.disableLog(DataNode.LOG);
+    GenericTestUtils.disableLog(BlockPoolSliceStorage.LOG);
+    GenericTestUtils.disableLog(LeaseManager.LOG);
+    GenericTestUtils.disableLog(NameNode.stateChangeLog);
+    GenericTestUtils.disableLog(NameNode.blockStateChangeLog);
+    GenericTestUtils.disableLog(DFSClient.LOG);
+    GenericTestUtils.disableLog(Server.LOG);
   }
 
   private SnapshotTestHelper() {
@@ -181,8 +176,8 @@ public static void checkSnapshotCreation(DistributedFileSystem hdfs,
    * 
    * Specific information for different types of INode: 
    * {@link INodeDirectory}:childrenSize 
-   * {@link INodeFile}: fileSize, block list. Check {@link BlockInfo#toString()} 
-   * and {@link BlockInfoUnderConstruction#toString()} for detailed information.
+   * {@link INodeFile}: fileSize, block list. Check {@link BlockInfoContiguous#toString()}
+   * and {@link BlockInfoContiguousUnderConstruction#toString()} for detailed information.
    * {@link FileWithSnapshot}: next link
    * </pre>
    * @see INode#dumpTreeRecursively()
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
index 3be1d36ca516d..ed8738085d3ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
@@ -23,6 +23,7 @@
 import static org.apache.hadoop.fs.permission.FsAction.*;
 import static org.junit.Assert.*;
 
+import java.io.IOException;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
@@ -37,15 +38,16 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.namenode.AclFeature;
+import org.apache.hadoop.hdfs.server.namenode.AclStorage;
 import org.apache.hadoop.hdfs.server.namenode.AclTestHelpers;
+import org.apache.hadoop.hdfs.server.namenode.FSAclBaseTest;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
-
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -608,7 +610,6 @@ public void testChangeAclExceedsQuota() throws Exception {
 
     aclSpec = Lists.newArrayList(
       aclEntry(ACCESS, USER, "bruce", READ));
-    exception.expect(NSQuotaExceededException.class);
     hdfs.modifyAclEntries(filePath, aclSpec);
   }
 
@@ -644,7 +645,6 @@ public void testRemoveAclExceedsQuota() throws Exception {
 
     aclSpec = Lists.newArrayList(
       aclEntry(ACCESS, USER, "bruce", READ));
-    exception.expect(NSQuotaExceededException.class);
     hdfs.removeAcl(filePath);
   }
 
@@ -657,6 +657,190 @@ public void testGetAclStatusDotSnapshotPath() throws Exception {
     assertArrayEquals(new AclEntry[] { }, returned);
   }
 
+  @Test
+  public void testDeDuplication() throws Exception {
+    int startSize = AclStorage.getUniqueAclFeatures().getUniqueElementsSize();
+    // unique default AclEntries for this test
+    List<AclEntry> aclSpec = Lists.newArrayList(
+        aclEntry(ACCESS, USER, "testdeduplicateuser", ALL),
+        aclEntry(ACCESS, GROUP, "testdeduplicategroup", ALL));
+    hdfs.mkdirs(path);
+    hdfs.modifyAclEntries(path, aclSpec);
+    assertEquals("One more ACL feature should be unique", startSize + 1,
+        AclStorage.getUniqueAclFeatures().getUniqueElementsSize());
+    Path subdir = new Path(path, "sub-dir");
+    hdfs.mkdirs(subdir);
+    Path file = new Path(path, "file");
+    hdfs.create(file).close();
+    AclFeature aclFeature;
+    {
+      // create the snapshot with root directory having ACLs should refer to
+      // same ACLFeature without incrementing the reference count
+      aclFeature = FSAclBaseTest.getAclFeature(path, cluster);
+      assertEquals("Reference count should be one before snapshot", 1,
+          aclFeature.getRefCount());
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      AclFeature snapshotAclFeature = FSAclBaseTest.getAclFeature(snapshotPath,
+          cluster);
+      assertSame(aclFeature, snapshotAclFeature);
+      assertEquals("Reference count should be increased", 2,
+          snapshotAclFeature.getRefCount());
+    }
+    {
+      // deleting the snapshot with root directory having ACLs should not alter
+      // the reference count of the ACLFeature
+      deleteSnapshotWithAclAndVerify(aclFeature, path, startSize);
+    }
+    {
+      hdfs.modifyAclEntries(subdir, aclSpec);
+      aclFeature = FSAclBaseTest.getAclFeature(subdir, cluster);
+      assertEquals("Reference count should be 1", 1, aclFeature.getRefCount());
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      Path subdirInSnapshot = new Path(snapshotPath, "sub-dir");
+      AclFeature snapshotAcl = FSAclBaseTest.getAclFeature(subdirInSnapshot,
+          cluster);
+      assertSame(aclFeature, snapshotAcl);
+      assertEquals("Reference count should remain same", 1,
+          aclFeature.getRefCount());
+
+      // Delete the snapshot with sub-directory containing the ACLs should not
+      // alter the reference count for AclFeature
+      deleteSnapshotWithAclAndVerify(aclFeature, subdir, startSize);
+    }
+    {
+      hdfs.modifyAclEntries(file, aclSpec);
+      aclFeature = FSAclBaseTest.getAclFeature(file, cluster);
+      assertEquals("Reference count should be 1", 1, aclFeature.getRefCount());
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      Path fileInSnapshot = new Path(snapshotPath, file.getName());
+      AclFeature snapshotAcl = FSAclBaseTest.getAclFeature(fileInSnapshot,
+          cluster);
+      assertSame(aclFeature, snapshotAcl);
+      assertEquals("Reference count should remain same", 1,
+          aclFeature.getRefCount());
+
+      // Delete the snapshot with contained file having ACLs should not
+      // alter the reference count for AclFeature
+      deleteSnapshotWithAclAndVerify(aclFeature, file, startSize);
+    }
+    {
+      // Modifying the ACLs of root directory of the snapshot should refer new
+      // AclFeature. And old AclFeature should be referenced by snapshot
+      hdfs.modifyAclEntries(path, aclSpec);
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      AclFeature snapshotAcl = FSAclBaseTest.getAclFeature(snapshotPath,
+          cluster);
+      aclFeature = FSAclBaseTest.getAclFeature(path, cluster);
+      assertEquals("Before modification same ACL should be referenced twice", 2,
+          aclFeature.getRefCount());
+      List<AclEntry> newAcl = Lists.newArrayList(aclEntry(ACCESS, USER,
+          "testNewUser", ALL));
+      hdfs.modifyAclEntries(path, newAcl);
+      aclFeature = FSAclBaseTest.getAclFeature(path, cluster);
+      AclFeature snapshotAclPostModification = FSAclBaseTest.getAclFeature(
+          snapshotPath, cluster);
+      assertSame(snapshotAcl, snapshotAclPostModification);
+      assertNotSame(aclFeature, snapshotAclPostModification);
+      assertEquals("Old ACL feature reference count should be same", 1,
+          snapshotAcl.getRefCount());
+      assertEquals("New ACL feature reference should be used", 1,
+          aclFeature.getRefCount());
+      deleteSnapshotWithAclAndVerify(aclFeature, path, startSize);
+    }
+    {
+      // Modifying the ACLs of sub directory of the snapshot root should refer
+      // new AclFeature. And old AclFeature should be referenced by snapshot
+      hdfs.modifyAclEntries(subdir, aclSpec);
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      Path subdirInSnapshot = new Path(snapshotPath, "sub-dir");
+      AclFeature snapshotAclFeature = FSAclBaseTest.getAclFeature(
+          subdirInSnapshot, cluster);
+      List<AclEntry> newAcl = Lists.newArrayList(aclEntry(ACCESS, USER,
+          "testNewUser", ALL));
+      hdfs.modifyAclEntries(subdir, newAcl);
+      aclFeature = FSAclBaseTest.getAclFeature(subdir, cluster);
+      assertNotSame(aclFeature, snapshotAclFeature);
+      assertEquals("Reference count should remain same", 1,
+          snapshotAclFeature.getRefCount());
+      assertEquals("New AclFeature should be used", 1, aclFeature.getRefCount());
+
+      deleteSnapshotWithAclAndVerify(aclFeature, subdir, startSize);
+    }
+    {
+      // Modifying the ACLs of file inside the snapshot root should refer new
+      // AclFeature. And old AclFeature should be referenced by snapshot
+      hdfs.modifyAclEntries(file, aclSpec);
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      Path fileInSnapshot = new Path(snapshotPath, file.getName());
+      AclFeature snapshotAclFeature = FSAclBaseTest.getAclFeature(
+          fileInSnapshot, cluster);
+      List<AclEntry> newAcl = Lists.newArrayList(aclEntry(ACCESS, USER,
+          "testNewUser", ALL));
+      hdfs.modifyAclEntries(file, newAcl);
+      aclFeature = FSAclBaseTest.getAclFeature(file, cluster);
+      assertNotSame(aclFeature, snapshotAclFeature);
+      assertEquals("Reference count should remain same", 1,
+          snapshotAclFeature.getRefCount());
+      deleteSnapshotWithAclAndVerify(aclFeature, file, startSize);
+    }
+    {
+      // deleting the original directory containing dirs and files with ACLs
+      // with snapshot
+      hdfs.delete(path, true);
+      Path dir = new Path(subdir, "dir");
+      hdfs.mkdirs(dir);
+      hdfs.modifyAclEntries(dir, aclSpec);
+      file = new Path(subdir, "file");
+      hdfs.create(file).close();
+      aclSpec.add(aclEntry(ACCESS, USER, "testNewUser", ALL));
+      hdfs.modifyAclEntries(file, aclSpec);
+      AclFeature fileAcl = FSAclBaseTest.getAclFeature(file, cluster);
+      AclFeature dirAcl = FSAclBaseTest.getAclFeature(dir, cluster);
+      Path snapshotPath = SnapshotTestHelper.createSnapshot(hdfs, path,
+          snapshotName);
+      Path dirInSnapshot = new Path(snapshotPath, "sub-dir/dir");
+      AclFeature snapshotDirAclFeature = FSAclBaseTest.getAclFeature(
+          dirInSnapshot, cluster);
+      Path fileInSnapshot = new Path(snapshotPath, "sub-dir/file");
+      AclFeature snapshotFileAclFeature = FSAclBaseTest.getAclFeature(
+          fileInSnapshot, cluster);
+      assertSame(fileAcl, snapshotFileAclFeature);
+      assertSame(dirAcl, snapshotDirAclFeature);
+      hdfs.delete(subdir, true);
+      assertEquals(
+          "Original ACLs references should be maintained for snapshot", 1,
+          snapshotFileAclFeature.getRefCount());
+      assertEquals(
+          "Original ACLs references should be maintained for snapshot", 1,
+          snapshotDirAclFeature.getRefCount());
+      hdfs.deleteSnapshot(path, snapshotName);
+      assertEquals("ACLs should be deleted from snapshot", startSize, AclStorage
+          .getUniqueAclFeatures().getUniqueElementsSize());
+    }
+  }
+
+  private void deleteSnapshotWithAclAndVerify(AclFeature aclFeature,
+      Path pathToCheckAcl, int totalAclFeatures) throws IOException {
+    hdfs.deleteSnapshot(path, snapshotName);
+    AclFeature afterDeleteAclFeature = FSAclBaseTest.getAclFeature(
+        pathToCheckAcl, cluster);
+    assertSame(aclFeature, afterDeleteAclFeature);
+    assertEquals("Reference count should remain same"
+        + " even after deletion of snapshot", 1,
+        afterDeleteAclFeature.getRefCount());
+
+    hdfs.removeAcl(pathToCheckAcl);
+    assertEquals("Reference count should be 0", 0, aclFeature.getRefCount());
+    assertEquals("Unique ACL features should remain same", totalAclFeatures,
+        AclStorage.getUniqueAclFeatures().getUniqueElementsSize());
+  }
+
   /**
    * Asserts that permission is denied to the given fs/user for the given
    * directory.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java
index 6b879b47d0a70..d565dad0db8a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java
@@ -34,13 +34,11 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
 import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
-import org.apache.hadoop.ipc.RemoteException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -239,7 +237,7 @@ public void testSnapshotLimit() throws Exception {
   }
 
   @Test (timeout=300000)
-  public void testSnapshotWithQuota() throws Exception {
+  public void testSnapshotName() throws Exception {
     final String dirStr = "/testSnapshotWithQuota/dir";
     final Path dir = new Path(dirStr);
     hdfs.mkdirs(dir, new FsPermission((short)0777));
@@ -266,43 +264,6 @@ public void testSnapshotWithQuota() throws Exception {
       Assert.assertEquals(HdfsConstants.DOT_SNAPSHOT_DIR, parent.getName());
       Assert.assertEquals(dir, parent.getParent());
     }
-    final Path f2 = new Path(foo, "f2");
-    DFSTestUtil.createFile(hdfs, f2, BLOCKSIZE, REPLICATION, SEED);
-    
-    try {
-      // normal create file should fail with quota
-      final Path f3 = new Path(foo, "f3");
-      DFSTestUtil.createFile(hdfs, f3, BLOCKSIZE, REPLICATION, SEED);
-      Assert.fail();
-    } catch(NSQuotaExceededException e) {
-      SnapshotTestHelper.LOG.info("The exception is expected.", e);
-    }
-
-    try {
-      // createSnapshot should fail with quota
-      hdfs.createSnapshot(dir);
-      Assert.fail();
-    } catch(NSQuotaExceededException e) {
-      SnapshotTestHelper.LOG.info("The exception is expected.", e);
-    }
-
-    try {
-      // setPermission f1 should fail with quote since it cannot add diff.
-      hdfs.setPermission(f1, new FsPermission((short)0));
-      Assert.fail();
-    } catch(RemoteException e) {
-      Assert.assertSame(NSQuotaExceededException.class,
-          e.unwrapRemoteException().getClass());
-      SnapshotTestHelper.LOG.info("The exception is expected.", e);
-    }
-
-    // setPermission f2 since it was created after the snapshot
-    hdfs.setPermission(f2, new FsPermission((short)0));
-
-    // increase quota and retry the commands.
-    hdfs.setQuota(dir, NS_QUOTA + 2, HdfsConstants.QUOTA_DONT_SET);
-    hdfs.createSnapshot(dir, "s1");
-    hdfs.setPermission(foo, new FsPermission((short)0444));
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index 62041e8e3bcda..ba318dee63963 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
-import org.apache.hadoop.security.AccessControlException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -169,7 +168,7 @@ public void testOpenFilesWithMultipleSnapshotsWithoutCheckpoint()
   }
 
   private void doTestMultipleSnapshots(boolean saveNamespace)
-      throws IOException, AccessControlException {
+      throws IOException {
     Path path = new Path("/test");
     doWriteAndAbort(fs, path);
     fs.createSnapshot(path, "s2");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index 62f08418e23ee..207a4e8ca2772 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -50,7 +50,7 @@
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
@@ -62,6 +62,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
 import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
+import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
 import org.apache.hadoop.hdfs.server.namenode.Quota;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
@@ -73,10 +74,11 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
 
 /** Testing rename with snapshots. */
 public class TestRenameWithSnapshots {
-  {
+  static {
     SnapshotTestHelper.disableLogs();
   }
   private static final Log LOG = LogFactory.getLog(TestRenameWithSnapshots.class);
@@ -1197,7 +1199,7 @@ public void testRenameDirAndDeleteSnapshot_2() throws Exception {
     restartClusterAndCheckImage(true);
     // make sure the whole referred subtree has been destroyed
     Quota.Counts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();  
-    assertEquals(4, q.get(Quota.NAMESPACE));
+    assertEquals(3, q.get(Quota.NAMESPACE));
     assertEquals(0, q.get(Quota.DISKSPACE));
     
     hdfs.deleteSnapshot(sdir1, "s1");
@@ -1558,29 +1560,34 @@ public void testRenameUndo_5() throws Exception {
     SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
     SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
     
-    // set ns quota of dir2 to 5, so the current remaining is 2 (already has
-    // dir2, subdir2, and s2)
-    hdfs.setQuota(dir2, 5, Long.MAX_VALUE - 1);
+    // set ns quota of dir2 to 4, so the current remaining is 2 (already has
+    // dir2, and subdir2)
+    hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
     
     final Path foo2 = new Path(subdir2, foo.getName());
+    FSDirectory fsdir2 = Mockito.spy(fsdir);
+    Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
+        .addLastINode((INodesInPath) Mockito.anyObject(),
+            (INode) Mockito.anyObject(), Mockito.anyBoolean());
+    Whitebox.setInternalState(fsn, "dir", fsdir2);
     // rename /test/dir1/foo to /test/dir2/subdir2/foo. 
-    // FSDirectory#verifyQuota4Rename will pass since foo/bar only be counted 
-    // as 2 in NS quota. However, the rename operation will fail when adding
-    // foo to subdir2, since we will create a snapshot diff for subdir2. 
+    // FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
+    // However, the rename operation will fail since we let addLastINode throw
+    // NSQuotaExceededException
     boolean rename = hdfs.rename(foo, foo2);
     assertFalse(rename);
     
     // check the undo
     assertTrue(hdfs.exists(foo));
     assertTrue(hdfs.exists(bar));
-    INodeDirectory dir1Node = fsdir.getINode4Write(dir1.toString())
+    INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
         .asDirectory();
     List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
         .getChildrenList(Snapshot.CURRENT_STATE_ID));
     assertEquals(1, childrenList.size());
     INode fooNode = childrenList.get(0);
     assertTrue(fooNode.asDirectory().isWithSnapshot());
-    INode barNode = fsdir.getINode4Write(bar.toString());
+    INode barNode = fsdir2.getINode4Write(bar.toString());
     assertTrue(barNode.getClass() == INodeFile.class);
     assertSame(fooNode, barNode.getParent());
     List<DirectoryDiff> diffList = dir1Node
@@ -1591,17 +1598,17 @@ public void testRenameUndo_5() throws Exception {
     assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
     
     // check dir2
-    INodeDirectory dir2Node = fsdir.getINode4Write(dir2.toString()).asDirectory();
+    INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
     assertTrue(dir2Node.isSnapshottable());
     Quota.Counts counts = dir2Node.computeQuotaUsage();
-    assertEquals(3, counts.get(Quota.NAMESPACE));
+    assertEquals(2, counts.get(Quota.NAMESPACE));
     assertEquals(0, counts.get(Quota.DISKSPACE));
     childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
         .getChildrenList(Snapshot.CURRENT_STATE_ID));
     assertEquals(1, childrenList.size());
     INode subdir2Node = childrenList.get(0);
     assertSame(dir2Node, subdir2Node.getParent());
-    assertSame(subdir2Node, fsdir.getINode4Write(subdir2.toString()));
+    assertSame(subdir2Node, fsdir2.getINode4Write(subdir2.toString()));
     diffList = dir2Node.getDiffs().asList();
     assertEquals(1, diffList.size());
     diff = diffList.get(0);
@@ -1628,27 +1635,28 @@ public void testRenameUndo_6() throws Exception {
     SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
     SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
     
-    // set ns quota of dir2 to 4, so the current remaining is 0 (already has
-    // dir2, sub_dir2, subsub_dir2, and s2)
+    // set ns quota of dir2 to 4, so the current remaining is 1 (already has
+    // dir2, sub_dir2, and subsub_dir2)
     hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
-    
+    FSDirectory fsdir2 = Mockito.spy(fsdir);
+    Mockito.doThrow(new RuntimeException("fake exception")).when(fsdir2)
+        .removeLastINode((INodesInPath) Mockito.anyObject());
+    Whitebox.setInternalState(fsn, "dir", fsdir2);
     // rename /test/dir1/foo to /test/dir2/sub_dir2/subsub_dir2. 
     // FSDirectory#verifyQuota4Rename will pass since foo only be counted 
     // as 1 in NS quota. However, the rename operation will fail when removing
-    // subsub_dir2 since this step tries to add a snapshot diff in sub_dir2.
+    // subsub_dir2.
     try {
       hdfs.rename(foo, subsub_dir2, Rename.OVERWRITE);
       fail("Expect QuotaExceedException");
-    } catch (QuotaExceededException e) {
-      String msg = "Failed to record modification for snapshot: "
-          + "The NameSpace quota (directories and files)"
-          + " is exceeded: quota=4 file count=5"; 
+    } catch (Exception e) {
+      String msg = "fake exception";
       GenericTestUtils.assertExceptionContains(msg, e);
     }
     
     // check the undo
     assertTrue(hdfs.exists(foo));
-    INodeDirectory dir1Node = fsdir.getINode4Write(dir1.toString())
+    INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
         .asDirectory();
     List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
         .getChildrenList(Snapshot.CURRENT_STATE_ID));
@@ -1664,19 +1672,18 @@ public void testRenameUndo_6() throws Exception {
     assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
     
     // check dir2
-    INodeDirectory dir2Node = fsdir.getINode4Write(dir2.toString()).asDirectory();
+    INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
     assertTrue(dir2Node.isSnapshottable());
     Quota.Counts counts = dir2Node.computeQuotaUsage();
-    assertEquals(4, counts.get(Quota.NAMESPACE));
+    assertEquals(3, counts.get(Quota.NAMESPACE));
     assertEquals(0, counts.get(Quota.DISKSPACE));
     childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
         .getChildrenList(Snapshot.CURRENT_STATE_ID));
     assertEquals(1, childrenList.size());
     INode subdir2Node = childrenList.get(0);
-    assertTrue(subdir2Node.asDirectory().isWithSnapshot());
     assertSame(dir2Node, subdir2Node.getParent());
-    assertSame(subdir2Node, fsdir.getINode4Write(sub_dir2.toString()));
-    INode subsubdir2Node = fsdir.getINode4Write(subsub_dir2.toString());
+    assertSame(subdir2Node, fsdir2.getINode4Write(sub_dir2.toString()));
+    INode subsubdir2Node = fsdir2.getINode4Write(subsub_dir2.toString());
     assertTrue(subsubdir2Node.getClass() == INodeDirectory.class);
     assertSame(subdir2Node, subsubdir2Node.getParent());
     
@@ -1685,9 +1692,6 @@ public void testRenameUndo_6() throws Exception {
     diff = diffList.get(0);
     assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
     assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
-    
-    diffList = subdir2Node.asDirectory().getDiffs().asList();
-    assertEquals(0, diffList.size());
   }
   
   /**
@@ -1787,7 +1791,7 @@ public void testRenameExceedQuota() throws Exception {
     INode dir2Node = fsdir.getINode4Write(dir2.toString());
     assertTrue(dir2Node.asDirectory().isSnapshottable());
     Quota.Counts counts = dir2Node.computeQuotaUsage();
-    assertEquals(7, counts.get(Quota.NAMESPACE));
+    assertEquals(4, counts.get(Quota.NAMESPACE));
     assertEquals(BLOCKSIZE * REPL * 2, counts.get(Quota.DISKSPACE));
   }
   
@@ -1955,11 +1959,11 @@ public void testRenameDirAndDeleteSnapshot_3() throws Exception {
     final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString())
         .asDirectory();
     Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();  
-    assertEquals(4, q1.get(Quota.NAMESPACE));
+    assertEquals(3, q1.get(Quota.NAMESPACE));
     final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
         .asDirectory();
     Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();  
-    assertEquals(2, q2.get(Quota.NAMESPACE));
+    assertEquals(1, q2.get(Quota.NAMESPACE));
     
     final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
         foo.getName());
@@ -2025,11 +2029,11 @@ public void testRenameDirAndDeleteSnapshot_4() throws Exception {
         .asDirectory();
     // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3)
     Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();  
-    assertEquals(9, q1.get(Quota.NAMESPACE));
+    assertEquals(7, q1.get(Quota.NAMESPACE));
     final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
         .asDirectory();
     Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();  
-    assertEquals(2, q2.get(Quota.NAMESPACE));
+    assertEquals(1, q2.get(Quota.NAMESPACE));
     
     final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
         foo.getName());
@@ -2066,10 +2070,10 @@ public void testRenameDirAndDeleteSnapshot_4() throws Exception {
   
   /**
    * This test demonstrates that 
-   * {@link INodeDirectory#removeChild(INode, Snapshot)}
+   * {@link INodeDirectory#removeChild}
    * and 
-   * {@link INodeDirectory#addChild(INode, boolean, Snapshot)}
-   * should use {@link INode#isInLatestSnapshot(Snapshot)} to check if the 
+   * {@link INodeDirectory#addChild}
+   * should use {@link INode#isInLatestSnapshot} to check if the
    * added/removed child should be recorded in snapshots.
    */
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
index 12fba733818de..b20e2ad4de515 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
@@ -25,15 +25,15 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.PrintStream;
 import java.io.RandomAccessFile;
-import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Random;
 
+import org.apache.commons.io.output.NullOutputStream;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -57,6 +57,7 @@
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper.TestDirectoryTree.Node;
 import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
@@ -256,8 +257,7 @@ public void testOfflineImageViewer() throws Exception {
         FSImageTestUtil.getFSImage(
         cluster.getNameNode()).getStorage().getStorageDir(0));
     assertNotNull("Didn't generate or can't find fsimage", originalFsimage);
-    StringWriter output = new StringWriter();
-    PrintWriter o = new PrintWriter(output);
+    PrintStream o = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
     PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
     v.visit(new RandomAccessFile(originalFsimage, "r"));
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
index c7b6b7ff0f7b6..c6c8dadac0bfc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -108,14 +108,14 @@ static INodeFile assertBlockCollection(String path, int numBlocks,
      final FSDirectory dir, final BlockManager blkManager) throws Exception {
     final INodeFile file = INodeFile.valueOf(dir.getINode(path), path);
     assertEquals(numBlocks, file.getBlocks().length);
-    for(BlockInfo b : file.getBlocks()) {
+    for(BlockInfoContiguous b : file.getBlocks()) {
       assertBlockCollection(blkManager, file, b);
     }
     return file;
   }
 
   static void assertBlockCollection(final BlockManager blkManager,
-      final INodeFile file, final BlockInfo b) { 
+      final INodeFile file, final BlockInfoContiguous b) {
     Assert.assertSame(b, blkManager.getStoredBlock(b));
     Assert.assertSame(file, blkManager.getBlockCollection(b));
     Assert.assertSame(file, b.getBlockCollection());
@@ -146,10 +146,10 @@ public void testDeletionWithSnapshots() throws Exception {
     {
       final INodeFile f2 = assertBlockCollection(file2.toString(), 3, fsdir,
           blockmanager);
-      BlockInfo[] blocks = f2.getBlocks();
+      BlockInfoContiguous[] blocks = f2.getBlocks();
       hdfs.delete(sub2, true);
       // The INode should have been removed from the blocksMap
-      for(BlockInfo b : blocks) {
+      for(BlockInfoContiguous b : blocks) {
         assertNull(blockmanager.getBlockCollection(b));
       }
     }
@@ -177,7 +177,7 @@ public void testDeletionWithSnapshots() throws Exception {
     // Check the block information for file0
     final INodeFile f0 = assertBlockCollection(file0.toString(), 4, fsdir,
         blockmanager);
-    BlockInfo[] blocks0 = f0.getBlocks();
+    BlockInfoContiguous[] blocks0 = f0.getBlocks();
     
     // Also check the block information for snapshot of file0
     Path snapshotFile0 = SnapshotTestHelper.getSnapshotPath(sub1, "s0",
@@ -187,7 +187,7 @@ public void testDeletionWithSnapshots() throws Exception {
     // Delete file0
     hdfs.delete(file0, true);
     // Make sure the blocks of file0 is still in blocksMap
-    for(BlockInfo b : blocks0) {
+    for(BlockInfoContiguous b : blocks0) {
       assertNotNull(blockmanager.getBlockCollection(b));
     }
     assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -201,7 +201,7 @@ public void testDeletionWithSnapshots() throws Exception {
     hdfs.deleteSnapshot(sub1, "s1");
 
     // Make sure the first block of file0 is still in blocksMap
-    for(BlockInfo b : blocks0) {
+    for(BlockInfoContiguous b : blocks0) {
       assertNotNull(blockmanager.getBlockCollection(b));
     }
     assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -293,7 +293,7 @@ public void testDeletionWithZeroSizeBlock() throws Exception {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfo[] blks = barNode.getBlocks();
+    BlockInfoContiguous[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     assertEquals(BLOCKSIZE, blks[0].getNumBytes());
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
@@ -331,7 +331,7 @@ public void testDeletionWithZeroSizeBlock2() throws Exception {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfo[] blks = barNode.getBlocks();
+    BlockInfoContiguous[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
     cluster.getNameNodeRpc()
@@ -370,7 +370,7 @@ public void testDeletionWithZeroSizeBlock3() throws Exception {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfo[] blks = barNode.getBlocks();
+    BlockInfoContiguous[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
     cluster.getNameNodeRpc()
@@ -421,7 +421,7 @@ public void testDeletionOfLaterBlocksWithZeroSizeFirstBlock() throws Exception {
     out.write(testData);
     out.close();
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfo[] blks = barNode.getBlocks();
+    BlockInfoContiguous[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     assertEquals(testData.length, blks[0].getNumBytes());
     
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index 1450a7d2321da..bc4ec90be5f70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -42,7 +42,7 @@
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -260,12 +260,12 @@ public void testDeleteCurrentFileDirectory() throws Exception {
     DFSTestUtil.createFile(hdfs, tempFile, BLOCKSIZE, REPLICATION, seed);
     final INodeFile temp = TestSnapshotBlocksMap.assertBlockCollection(
         tempFile.toString(), 1, fsdir, blockmanager);
-    BlockInfo[] blocks = temp.getBlocks();
+    BlockInfoContiguous[] blocks = temp.getBlocks();
     hdfs.delete(tempDir, true);
     // check dir's quota usage
-    checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 3);
+    checkQuotaUsageComputation(dir, 8, BLOCKSIZE * REPLICATION * 3);
     // check blocks of tempFile
-    for (BlockInfo b : blocks) {
+    for (BlockInfoContiguous b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -279,7 +279,7 @@ public void testDeleteCurrentFileDirectory() throws Exception {
     // create snapshot s1
     SnapshotTestHelper.createSnapshot(hdfs, dir, "s1");
     // check dir's quota usage
-    checkQuotaUsageComputation(dir, 14L, BLOCKSIZE * REPLICATION * 4);
+    checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
     
     // get two snapshots for later use
     Snapshot snapshot0 = fsdir.getINode(dir.toString()).asDirectory()
@@ -295,7 +295,7 @@ public void testDeleteCurrentFileDirectory() throws Exception {
     hdfs.delete(noChangeDirParent, true);
     // while deletion, we add a diff for metaChangeFile2 as its snapshot copy
     // for s1, we also add diffs for both sub and noChangeDirParent
-    checkQuotaUsageComputation(dir, 17L, BLOCKSIZE * REPLICATION * 4);
+    checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
     
     // check the snapshot copy of noChangeDir 
     Path snapshotNoChangeDir = SnapshotTestHelper.getSnapshotPath(dir, "s1",
@@ -337,12 +337,12 @@ public void testDeleteCurrentFileDirectory() throws Exception {
     final INodeFile newFileNode = TestSnapshotBlocksMap.assertBlockCollection(
         newFile.toString(), 1, fsdir, blockmanager);
     blocks = newFileNode.getBlocks();
-    checkQuotaUsageComputation(dir, 18L, BLOCKSIZE * REPLICATION * 5);
+    checkQuotaUsageComputation(dir, 10L, BLOCKSIZE * REPLICATION * 5);
     hdfs.delete(sub, true);
     // while deletion, we add diff for subsub and metaChangeFile1, and remove
     // newFile
-    checkQuotaUsageComputation(dir, 19L, BLOCKSIZE * REPLICATION * 4);
-    for (BlockInfo b : blocks) {
+    checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
+    for (BlockInfoContiguous b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -426,13 +426,13 @@ public void testDeleteEarliestSnapshot1() throws Exception {
     // create snapshot s1 for sub
     SnapshotTestHelper.createSnapshot(hdfs, sub, snapshotName);
     // check quota usage computation
-    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 2);
+    checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
     // delete s1
     hdfs.deleteSnapshot(sub, snapshotName);
     checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
     // now we can create a snapshot with the same name
     hdfs.createSnapshot(sub, snapshotName);
-    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 2);
+    checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
     
     // create a new file under sub
     Path newFile = new Path(sub, "newFile");
@@ -440,14 +440,14 @@ public void testDeleteEarliestSnapshot1() throws Exception {
     // create another snapshot s2
     String snapshotName2 = "s2";
     hdfs.createSnapshot(sub, snapshotName2);
-    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * REPLICATION * 3);
+    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 3);
     // Get the filestatus of sub under snapshot s2
     Path ss = SnapshotTestHelper
         .getSnapshotPath(sub, snapshotName2, "newFile");
     FileStatus statusBeforeDeletion = hdfs.getFileStatus(ss);
     // delete s1
     hdfs.deleteSnapshot(sub, snapshotName);
-    checkQuotaUsageComputation(sub, 5, BLOCKSIZE * REPLICATION * 3);
+    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 3);
     FileStatus statusAfterDeletion = hdfs.getFileStatus(ss);
     System.out.println("Before deletion: " + statusBeforeDeletion.toString()
         + "\n" + "After deletion: " + statusAfterDeletion.toString());
@@ -479,33 +479,33 @@ public void testDeleteEarliestSnapshot2() throws Exception {
     
     final INodeFile toDeleteFileNode = TestSnapshotBlocksMap
         .assertBlockCollection(toDeleteFile.toString(), 1, fsdir, blockmanager);
-    BlockInfo[] blocks = toDeleteFileNode.getBlocks();
+    BlockInfoContiguous[] blocks = toDeleteFileNode.getBlocks();
     
     // create snapshot s0 on dir
     SnapshotTestHelper.createSnapshot(hdfs, dir, "s0");
-    checkQuotaUsageComputation(dir, 8, 3 * BLOCKSIZE * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
     
     // delete /TestSnapshot/sub/noChangeDir/metaChangeDir/toDeleteFile
     hdfs.delete(toDeleteFile, true);
     // the deletion adds diff of toDeleteFile and metaChangeDir
-    checkQuotaUsageComputation(dir, 10, 3 * BLOCKSIZE * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
     // change metadata of /TestSnapshot/sub/noChangeDir/metaChangeDir and
     // /TestSnapshot/sub/noChangeDir/metaChangeFile
     hdfs.setReplication(metaChangeFile, REPLICATION_1);
     hdfs.setOwner(metaChangeDir, "unknown", "unknown");
-    checkQuotaUsageComputation(dir, 11, 3 * BLOCKSIZE * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
     
     // create snapshot s1 on dir
     hdfs.createSnapshot(dir, "s1");
-    checkQuotaUsageComputation(dir, 12, 3 * BLOCKSIZE * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
     
     // delete snapshot s0
     hdfs.deleteSnapshot(dir, "s0");
     // namespace: remove toDeleteFile and its diff, metaChangeFile's diff, 
     // metaChangeDir's diff, dir's diff. diskspace: remove toDeleteFile, and 
     // metaChangeFile's replication factor decreases
-    checkQuotaUsageComputation(dir, 7, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
-    for (BlockInfo b : blocks) {
+    checkQuotaUsageComputation(dir, 6, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
+    for (BlockInfoContiguous b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -679,21 +679,21 @@ public void testCombineSnapshotDiff3() throws Exception {
     // create another snapshot
     SnapshotTestHelper.createSnapshot(hdfs, dir, "s2");
     
-    checkQuotaUsageComputation(dir, 11, BLOCKSIZE * 2 * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 2 * REPLICATION);
     
     // delete subsubdir and subDir2
     hdfs.delete(subsubDir, true);
     hdfs.delete(subDir2, true);
     
     // add diff of s2 to subDir1, subsubDir, and subDir2
-    checkQuotaUsageComputation(dir, 14, BLOCKSIZE * 2 * REPLICATION);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 2 * REPLICATION);
     
     // delete snapshot s2
     hdfs.deleteSnapshot(dir, "s2");
     
     // delete s2 diff in dir, subDir2, and subsubDir. Delete newFile, newDir,
     // and newFile2. Rename s2 diff to s1 for subDir1 
-    checkQuotaUsageComputation(dir, 8, 0);
+    checkQuotaUsageComputation(dir, 4, 0);
     // Check rename of snapshot diff in subDir1
     Path subdir1_s1 = SnapshotTestHelper.getSnapshotPath(dir, "s1",
         subDir1.getName());
@@ -714,7 +714,6 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
       int dirNodeNum) throws Exception {
     Path modDir = modDirStr.isEmpty() ? snapshotRoot : new Path(snapshotRoot,
         modDirStr);
-    final int delta = modDirStr.isEmpty() ? 0 : 1;
     Path file10 = new Path(modDir, "file10");
     Path file11 = new Path(modDir, "file11");
     Path file12 = new Path(modDir, "file12");
@@ -728,72 +727,59 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
 
     // create snapshot s1 for snapshotRoot
     SnapshotTestHelper.createSnapshot(hdfs, snapshotRoot, "s1");
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 5, 8 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 8 * BLOCKSIZE);
     
     // delete file11
     hdfs.delete(file11, true);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6 + delta,
-        8 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 8 * BLOCKSIZE);
     
     // modify file12
     hdfs.setReplication(file12, REPLICATION);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7 + delta,
-        9 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 9 * BLOCKSIZE);
     
     // modify file13
     hdfs.setReplication(file13, REPLICATION);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 8 + delta,
-        10 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 10 * BLOCKSIZE);
     
     // create file14
     DFSTestUtil.createFile(hdfs, file14, BLOCKSIZE, REPLICATION, seed);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 9 + delta,
-        13 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 5, 13 * BLOCKSIZE);
     
     // create file15
     DFSTestUtil.createFile(hdfs, file15, BLOCKSIZE, REPLICATION, seed);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 10 + delta,
-        16 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 16 * BLOCKSIZE);
     
     // create snapshot s2 for snapshotRoot
     hdfs.createSnapshot(snapshotRoot, "s2");
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 11 + delta,
-        16 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 16 * BLOCKSIZE);
     
     // create file11 again: (0, d) + (c, 0)
     DFSTestUtil.createFile(hdfs, file11, BLOCKSIZE, REPLICATION, seed);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 12 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // delete file12
     hdfs.delete(file12, true);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 13 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // modify file13
     hdfs.setReplication(file13, (short) (REPLICATION - 2));
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 14 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // delete file14: (c, 0) + (0, d)
     hdfs.delete(file14, true);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 15 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // modify file15
     hdfs.setReplication(file15, REPLICATION_1);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 16 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // create snapshot s3 for snapshotRoot
     hdfs.createSnapshot(snapshotRoot, "s3");
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 17 + delta * 2,
-        19 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
     
     // modify file10, to check if the posterior diff was set correctly
     hdfs.setReplication(file10, REPLICATION);
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 18 + delta * 2,
-        20 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 20 * BLOCKSIZE);
     
     Path file10_s1 = SnapshotTestHelper.getSnapshotPath(snapshotRoot, "s1",
         modDirStr + "file10");
@@ -813,14 +799,13 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
     FileStatus statusBeforeDeletion13 = hdfs.getFileStatus(file13_s1);
     INodeFile file14Node = TestSnapshotBlocksMap.assertBlockCollection(
         file14_s2.toString(), 1, fsdir, blockmanager);
-    BlockInfo[] blocks_14 = file14Node.getBlocks();
+    BlockInfoContiguous[] blocks_14 = file14Node.getBlocks();
     TestSnapshotBlocksMap.assertBlockCollection(file15_s2.toString(), 1, fsdir,
         blockmanager);
     
     // delete s2, in which process we need to combine the diff in s2 to s1
     hdfs.deleteSnapshot(snapshotRoot, "s2");
-    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 12 + delta,
-        14 * BLOCKSIZE);
+    checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 14 * BLOCKSIZE);
     
     // check the correctness of s1
     FileStatus statusAfterDeletion10 = hdfs.getFileStatus(file10_s1);
@@ -851,7 +836,7 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
         modDirStr + "file15");
     assertFalse(hdfs.exists(file14_s1));
     assertFalse(hdfs.exists(file15_s1));
-    for (BlockInfo b : blocks_14) {
+    for (BlockInfoContiguous b : blocks_14) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -873,23 +858,23 @@ public void testDeleteSnapshotWithDirModification() throws Exception {
     
     // create snapshot s1 for sub1, and change the metadata of sub1
     SnapshotTestHelper.createSnapshot(hdfs, sub, "s1");
-    checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     hdfs.setOwner(sub, "user2", "group2");
-    checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     
     // create snapshot s2 for sub1, but do not modify sub1 afterwards
     hdfs.createSnapshot(sub, "s2");
-    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     
     // create snapshot s3 for sub1, and change the metadata of sub1
     hdfs.createSnapshot(sub, "s3");
-    checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     hdfs.setOwner(sub, "user3", "group3");
-    checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     
     // delete snapshot s3
     hdfs.deleteSnapshot(sub, "s3");
-    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     
     // check sub1's metadata in snapshot s2
     FileStatus statusOfS2 = hdfs.getFileStatus(new Path(sub,
@@ -899,7 +884,7 @@ public void testDeleteSnapshotWithDirModification() throws Exception {
     
     // delete snapshot s2
     hdfs.deleteSnapshot(sub, "s2");
-    checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3);
+    checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
     
     // check sub1's metadata in snapshot s1
     FileStatus statusOfS1 = hdfs.getFileStatus(new Path(sub,
@@ -957,34 +942,34 @@ public void testRenameSnapshotDiff() throws Exception {
     
     // create snapshot s0 on sub
     SnapshotTestHelper.createSnapshot(hdfs, sub, "s0");
-    checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 6);
+    checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 6);
     // make some changes on both sub and subsub
     final Path subFile1 = new Path(sub, "file1");
     final Path subsubFile1 = new Path(subsub, "file1");
     DFSTestUtil.createFile(hdfs, subFile1, BLOCKSIZE, REPLICATION_1, seed);
     DFSTestUtil.createFile(hdfs, subsubFile1, BLOCKSIZE, REPLICATION, seed);
-    checkQuotaUsageComputation(sub, 8, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     // create snapshot s1 on sub
     SnapshotTestHelper.createSnapshot(hdfs, sub, "s1");
-    checkQuotaUsageComputation(sub, 9, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     // create snapshot s2 on dir
     SnapshotTestHelper.createSnapshot(hdfs, dir, "s2");
-    checkQuotaUsageComputation(dir, 11, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(sub, 9, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     // make changes on subsub and subsubFile1
     hdfs.setOwner(subsub, "unknown", "unknown");
     hdfs.setReplication(subsubFile1, REPLICATION_1);
-    checkQuotaUsageComputation(dir, 13, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(sub, 11, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     // make changes on sub
     hdfs.delete(subFile1, true);
-    checkQuotaUsageComputation(new Path("/"), 16, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(dir, 15, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(sub, 13, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(new Path("/"), 8, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     Path subsubSnapshotCopy = SnapshotTestHelper.getSnapshotPath(dir, "s2",
         sub.getName() + Path.SEPARATOR + subsub.getName());
@@ -1003,9 +988,9 @@ public void testRenameSnapshotDiff() throws Exception {
     
     // delete snapshot s2
     hdfs.deleteSnapshot(dir, "s2");
-    checkQuotaUsageComputation(new Path("/"), 14, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(dir, 13, BLOCKSIZE * 11);
-    checkQuotaUsageComputation(sub, 12, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(new Path("/"), 8, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
+    checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
     
     // no snapshot copy for s2
     try {
@@ -1122,4 +1107,31 @@ public void testHANNRestartAfterSnapshotDeletion() throws Exception {
     // wait till the cluster becomes active
     cluster.waitClusterUp();
   }
+
+  @Test
+  public void testCorrectNumberOfBlocksAfterRestart() throws IOException {
+    final Path foo = new Path("/foo");
+    final Path bar = new Path(foo, "bar");
+    final Path file = new Path(foo, "file");
+    final String snapshotName = "ss0";
+
+    DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, seed);
+    hdfs.mkdirs(bar);
+    hdfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+    hdfs.setQuota(bar, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+    hdfs.allowSnapshot(foo);
+
+    hdfs.createSnapshot(foo, snapshotName);
+    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    hdfs.saveNamespace();
+
+    hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+    hdfs.deleteSnapshot(foo, snapshotName);
+    hdfs.delete(bar, true);
+    hdfs.delete(foo, true);
+
+    long numberOfBlocks = cluster.getNamesystem().getBlocksTotal();
+    cluster.restartNameNode(0);
+    assertEquals(numberOfBlocks, cluster.getNamesystem().getBlocksTotal());
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
index e1ca26393b92c..5264cb71b0bc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java
@@ -40,7 +40,7 @@
 /**
  * This class tests the replication handling/calculation of snapshots. In
  * particular, {@link INodeFile#getFileReplication()} and
- * {@link INodeFileWithSnapshot#getBlockReplication()} are tested to make sure
+ * {@link INodeFile#getBlockReplication()} are tested to make sure
  * the number of replication is calculated correctly with/without snapshots.
  */
 public class TestSnapshotReplication {
@@ -82,7 +82,7 @@ public void tearDown() throws Exception {
    * Check the replication of a given file. We test both
    * {@link INodeFile#getFileReplication()} and
    * {@link INodeFile#getBlockReplication()}.
-   * 
+   *
    * @param file The given file
    * @param replication The expected replication number
    * @param blockReplication The expected replication number for the block
@@ -132,8 +132,7 @@ INodeFile getINodeFile(Path p) throws Exception {
    *          as their expected replication number stored in their corresponding
    *          INodes
    * @param expectedBlockRep
-   *          The expected replication number that should be returned by
-   *          {@link INodeFileWithSnapshot#getBlockReplication()} of all the INodes
+   *          The expected replication number
    * @throws Exception
    */
   private void checkSnapshotFileReplication(Path currentFile,
@@ -143,8 +142,8 @@ private void checkSnapshotFileReplication(Path currentFile,
     assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
     // Then check replication for every snapshot
     for (Path ss : snapshotRepMap.keySet()) {
-      final INodesInPath iip = fsdir.getLastINodeInPath(ss.toString());
-      final INodeFile ssInode = (INodeFile)iip.getLastINode();
+      final INodesInPath iip = fsdir.getINodesInPath(ss.toString(), true);
+      final INodeFile ssInode = iip.getLastINode().asFile();
       // The replication number derived from the
       // INodeFileWithLink#getBlockReplication should always == expectedBlockRep
       assertEquals(expectedBlockRep, ssInode.getBlockReplication());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
index 28277c51932b1..4b957bf9365d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
@@ -343,77 +343,6 @@ public void testRemoveXAttrSnapshotPath() throws Exception {
     hdfs.removeXAttr(snapshotPath, name1);
   }
 
-  /**
-   * Assert exception of setting xattr when exceeding quota.
-   */
-  @Test
-  public void testSetXAttrExceedsQuota() throws Exception {
-    Path filePath = new Path(path, "file1");
-    Path fileSnapshotPath = new Path(snapshotPath, "file1");
-    FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
-    hdfs.allowSnapshot(path);
-    hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
-    FileSystem.create(hdfs, filePath,
-        FsPermission.createImmutable((short) 0600)).close();
-    hdfs.setXAttr(filePath, name1, value1);
-
-    hdfs.createSnapshot(path, snapshotName);
-
-    byte[] value = hdfs.getXAttr(filePath, name1);
-    Assert.assertArrayEquals(value, value1);
-
-    value = hdfs.getXAttr(fileSnapshotPath, name1);
-    Assert.assertArrayEquals(value, value1);
-
-    exception.expect(NSQuotaExceededException.class);
-    hdfs.setXAttr(filePath, name2, value2);
-  }
-
-
-  /**
-   * Test that an exception is thrown when adding an XAttr Feature to
-   * a snapshotted path
-   */
-  @Test
-  public void testSetXAttrAfterSnapshotExceedsQuota() throws Exception {
-    Path filePath = new Path(path, "file1");
-    FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
-    hdfs.allowSnapshot(path);
-    hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
-    FileSystem.create(hdfs, filePath,
-        FsPermission.createImmutable((short) 0600)).close();
-    hdfs.createSnapshot(path, snapshotName);
-    // This adds an XAttr feature, which can throw an exception
-    exception.expect(NSQuotaExceededException.class);
-    hdfs.setXAttr(filePath, name1, value1);
-  }
-
-  /**
-   * Assert exception of removing xattr when exceeding quota.
-   */
-  @Test
-  public void testRemoveXAttrExceedsQuota() throws Exception {
-    Path filePath = new Path(path, "file1");
-    Path fileSnapshotPath = new Path(snapshotPath, "file1");
-    FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
-    hdfs.allowSnapshot(path);
-    hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
-    FileSystem.create(hdfs, filePath,
-        FsPermission.createImmutable((short) 0600)).close();
-    hdfs.setXAttr(filePath, name1, value1);
-
-    hdfs.createSnapshot(path, snapshotName);
-
-    byte[] value = hdfs.getXAttr(filePath, name1);
-    Assert.assertArrayEquals(value, value1);
-
-    value = hdfs.getXAttr(fileSnapshotPath, name1);
-    Assert.assertArrayEquals(value, value1);
-
-    exception.expect(NSQuotaExceededException.class);
-    hdfs.removeXAttr(filePath, name1);
-  }
-
   /**
    * Test that users can copy a snapshot while preserving its xattrs.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/top/window/TestRollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/top/window/TestRollingWindowManager.java
index de2171443b975..494ed08dbeab0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/top/window/TestRollingWindowManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/top/window/TestRollingWindowManager.java
@@ -17,16 +17,19 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.top.window;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 import org.junit.Before;
 import org.junit.Test;
 
-import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.MetricValueMap;
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op;
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow;
+import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.User;
+import static org.junit.Assert.assertEquals;
 
 public class TestRollingWindowManager {
 
@@ -61,33 +64,39 @@ public void testTops() {
     for (int i = 0; i < users.length; i++)
       manager.recordMetric(time, "close", users[i], i + 1);
     time++;
-    MetricValueMap tops = manager.snapshot(time);
+    TopWindow tops = manager.snapshot(time);
 
-    assertEquals("The number of returned top metrics is invalid",
-        2 * (N_TOP_USERS + 1), tops.size());
-    int userIndex = users.length - 2;
-    String metricName = RollingWindowManager.createMetricName("open",
-        users[userIndex]);
-    boolean includes = tops.containsKey(metricName);
-    assertTrue("The order of entries in top metrics is wrong", includes);
-    assertEquals("The reported value by top is different from recorded one",
-        (userIndex + 1) * 2, ((Long) tops.get(metricName)).longValue());
+    assertEquals("Unexpected number of ops", 2, tops.getOps().size());
+    for (Op op : tops.getOps()) {
+      final List<User> topUsers = op.getTopUsers();
+      assertEquals("Unexpected number of users", N_TOP_USERS, topUsers.size());
+      if (op.getOpType() == "open") {
+        for (int i = 0; i < topUsers.size(); i++) {
+          User user = topUsers.get(i);
+          assertEquals("Unexpected count for user " + user.getUser(),
+              (users.length-i)*2, user.getCount());
+        }
+        // Closed form of sum(range(2,42,2))
+        assertEquals("Unexpected total count for op", 
+            (2+(users.length*2))*(users.length/2),
+            op.getTotalCount());
+      }
+    }
 
     // move the window forward not to see the "open" results
     time += WINDOW_LEN_MS - 2;
-    // top should not include only "close" results
     tops = manager.snapshot(time);
-    assertEquals("The number of returned top metrics is invalid",
-        N_TOP_USERS + 1, tops.size());
-    includes = tops.containsKey(metricName);
-    assertFalse("After rolling, the top list still includes the stale metrics",
-        includes);
-
-    metricName = RollingWindowManager.createMetricName("close",
-        users[userIndex]);
-    includes = tops.containsKey(metricName);
-    assertTrue("The order of entries in top metrics is wrong", includes);
-    assertEquals("The reported value by top is different from recorded one",
-        (userIndex + 1), ((Long) tops.get(metricName)).longValue());
+    assertEquals("Unexpected number of ops", 1, tops.getOps().size());
+    final Op op = tops.getOps().get(0);
+    assertEquals("Should only see close ops", "close", op.getOpType());
+    final List<User> topUsers = op.getTopUsers();
+    for (int i = 0; i < topUsers.size(); i++) {
+      User user = topUsers.get(i);
+      assertEquals("Unexpected count for user " + user.getUser(),
+          (users.length-i), user.getCount());
+    }
+    // Closed form of sum(range(1,21))
+    assertEquals("Unexpected total count for op",
+        (1 + users.length) * (users.length / 2), op.getTotalCount());
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
index c1169dc3b19ea..077361c1cfb80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
@@ -22,11 +22,11 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -34,8 +34,6 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
@@ -52,9 +50,7 @@
 public class TestWebHdfsDataLocality {
   static final Log LOG = LogFactory.getLog(TestWebHdfsDataLocality.class);
   {
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
   
   private static final String RACK0 = "/rack0";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
index 44eb79a1ad6e3..6e381c15039d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
@@ -466,7 +466,7 @@ public void testHandleTruncatedBlockFile() throws IOException {
             "waitReplication: " + e);
       }
       ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
-      File dataFile = MiniDFSCluster.getBlockFile(0, block);
+      File dataFile = cluster.getBlockFile(0, block);
       cluster.shutdown();
       cluster = null;
       RandomAccessFile raf = null;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
similarity index 67%
rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
index d80356a880381..cb249e6cc0645 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestStoragePolicyCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
@@ -15,12 +15,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hdfs;
+package org.apache.hadoop.hdfs.tools;
 
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.junit.After;
@@ -28,7 +32,7 @@
 import org.junit.Test;
 
 /**
- * Test storage policy related DFSAdmin commands
+ * Test StoragePolicyAdmin commands
  */
 public class TestStoragePolicyCommands {
   private static final short REPL = 1;
@@ -37,7 +41,7 @@ public class TestStoragePolicyCommands {
   private static Configuration conf;
   private static MiniDFSCluster cluster;
   private static DistributedFileSystem fs;
-  
+
   @Before
   public void clusterSetUp() throws IOException {
     conf = new HdfsConfiguration();
@@ -48,10 +52,10 @@ public void clusterSetUp() throws IOException {
 
   @After
   public void clusterShutdown() throws IOException{
-    if(fs != null){
+    if(fs != null) {
       fs.close();
     }
-    if(cluster != null){
+    if(cluster != null) {
       cluster.shutdown();
     }
   }
@@ -62,27 +66,28 @@ public void testSetAndGetStoragePolicy() throws Exception {
     final Path bar = new Path(foo, "bar");
     DFSTestUtil.createFile(fs, bar, SIZE, REPL, 0);
 
-    DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo", 0,
-        "The storage policy of " + foo.toString() + " is unspecified", conf);
-    DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo/bar", 0,
-        "The storage policy of " + bar.toString() + " is unspecified", conf);
+    final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
+    DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+        "The storage policy of " + foo.toString() + " is unspecified");
+    DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+        "The storage policy of " + bar.toString() + " is unspecified");
 
-    DFSTestUtil.DFSAdminRun("-setStoragePolicy /foo WARM", 0,
-        "Set storage policy WARM on " + foo.toString(), conf);
-    DFSTestUtil.DFSAdminRun("-setStoragePolicy /foo/bar COLD", 0,
-        "Set storage policy COLD on " + bar.toString(), conf);
-    DFSTestUtil.DFSAdminRun("-setStoragePolicy /fooz WARM", -1,
-        "File/Directory does not exist: /fooz", conf);
+    DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0,
+        "Set storage policy WARM on " + foo.toString());
+    DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD",
+        0, "Set storage policy COLD on " + bar.toString());
+    DFSTestUtil.toolRun(admin, "-setStoragePolicy -path /fooz -policy WARM",
+        2, "File/Directory does not exist: /fooz");
 
     final BlockStoragePolicySuite suite = BlockStoragePolicySuite
         .createDefaultSuite();
     final BlockStoragePolicy warm = suite.getPolicy("WARM");
     final BlockStoragePolicy cold = suite.getPolicy("COLD");
-    DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo", 0,
-        "The storage policy of " + foo.toString() + ":\n" + warm, conf);
-    DFSTestUtil.DFSAdminRun("-getStoragePolicy /foo/bar", 0,
-        "The storage policy of " + bar.toString() + ":\n" + cold, conf);
-    DFSTestUtil.DFSAdminRun("-getStoragePolicy /fooz", -1,
-        "File/Directory does not exist: /fooz", conf);
+    DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo", 0,
+        "The storage policy of " + foo.toString() + ":\n" + warm);
+    DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /foo/bar", 0,
+        "The storage policy of " + bar.toString() + ":\n" + cold);
+    DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /fooz", 2,
+        "File/Directory does not exist: /fooz");
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 36b5201bdc8f1..a8d1d54450145 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -20,22 +20,26 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.io.StringReader;
-import java.io.StringWriter;
 import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -43,6 +47,7 @@
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
 
+import org.apache.commons.io.output.NullOutputStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -50,6 +55,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -186,10 +192,10 @@ private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
   @Test(expected = IOException.class)
   public void testTruncatedFSImage() throws IOException {
     File truncatedFile = folder.newFile();
-    StringWriter output = new StringWriter();
+    PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
     copyPartOfFile(originalFsimage, truncatedFile);
-    new FileDistributionCalculator(new Configuration(), 0, 0, new PrintWriter(
-        output)).visit(new RandomAccessFile(truncatedFile, "r"));
+    new FileDistributionCalculator(new Configuration(), 0, 0, output)
+        .visit(new RandomAccessFile(truncatedFile, "r"));
   }
 
   private void copyPartOfFile(File src, File dest) throws IOException {
@@ -208,20 +214,21 @@ private void copyPartOfFile(File src, File dest) throws IOException {
 
   @Test
   public void testFileDistributionCalculator() throws IOException {
-    StringWriter output = new StringWriter();
-    PrintWriter o = new PrintWriter(output);
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    PrintStream o = new PrintStream(output);
     new FileDistributionCalculator(new Configuration(), 0, 0, o)
         .visit(new RandomAccessFile(originalFsimage, "r"));
     o.close();
 
+    String outputString = output.toString();
     Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
-    Matcher matcher = p.matcher(output.getBuffer());
+    Matcher matcher = p.matcher(outputString);
     assertTrue(matcher.find() && matcher.groupCount() == 1);
     int totalFiles = Integer.parseInt(matcher.group(1));
     assertEquals(NUM_DIRS * FILES_PER_DIR, totalFiles);
 
     p = Pattern.compile("totalDirectories = (\\d+)\n");
-    matcher = p.matcher(output.getBuffer());
+    matcher = p.matcher(outputString);
     assertTrue(matcher.find() && matcher.groupCount() == 1);
     int totalDirs = Integer.parseInt(matcher.group(1));
     // totalDirs includes root directory, empty directory, and xattr directory
@@ -236,7 +243,7 @@ public int compare(FileStatus first, FileStatus second) {
       }
     });
     p = Pattern.compile("maxFileSize = (\\d+)\n");
-    matcher = p.matcher(output.getBuffer());
+    matcher = p.matcher(output.toString("UTF-8"));
     assertTrue(matcher.find() && matcher.groupCount() == 1);
     assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
   }
@@ -252,13 +259,13 @@ public void testFileDistributionCalculatorWithOptions() throws Exception {
   @Test
   public void testPBImageXmlWriter() throws IOException, SAXException,
       ParserConfigurationException {
-    StringWriter output = new StringWriter();
-    PrintWriter o = new PrintWriter(output);
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+    PrintStream o = new PrintStream(output);
     PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
     v.visit(new RandomAccessFile(originalFsimage, "r"));
     SAXParserFactory spf = SAXParserFactory.newInstance();
     SAXParser parser = spf.newSAXParser();
-    final String xml = output.getBuffer().toString();
+    final String xml = output.toString();
     parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
   }
 
@@ -298,7 +305,7 @@ public void testWebImageViewer() throws Exception {
       verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND, url);
 
       // LISTSTATUS operation to a invalid prefix
-      url = new URL("http://localhost:" + port + "/webhdfs/v1?op=LISTSTATUS");
+      url = new URL("http://localhost:" + port + "/foo");
       verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND, url);
 
       // GETFILESTATUS operation
@@ -327,6 +334,51 @@ public void testWebImageViewer() throws Exception {
     }
   }
 
+  @Test
+  public void testPBDelimitedWriter() throws IOException, InterruptedException {
+    testPBDelimitedWriter("");  // Test in memory db.
+    testPBDelimitedWriter(
+        new FileSystemTestHelper().getTestRootDir() + "/delimited.db");
+  }
+
+  private void testPBDelimitedWriter(String db)
+      throws IOException, InterruptedException {
+    final String DELIMITER = "\t";
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+    try (PrintStream o = new PrintStream(output)) {
+      PBImageDelimitedTextWriter v =
+          new PBImageDelimitedTextWriter(o, DELIMITER, db);
+      v.visit(new RandomAccessFile(originalFsimage, "r"));
+    }
+
+    Set<String> fileNames = new HashSet<>();
+    try (
+        ByteArrayInputStream input =
+            new ByteArrayInputStream(output.toByteArray());
+        BufferedReader reader =
+            new BufferedReader(new InputStreamReader(input))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        System.out.println(line);
+        String[] fields = line.split(DELIMITER);
+        assertEquals(12, fields.length);
+        fileNames.add(fields[0]);
+      }
+    }
+
+    // writtenFiles does not contain root directory and "invalid XML char" dir.
+    for (Iterator<String> it = fileNames.iterator(); it.hasNext(); ) {
+      String filename = it.next();
+      if (filename.startsWith("/dirContainingInvalidXMLChar")) {
+        it.remove();
+      } else if (filename.equals("/")) {
+        it.remove();
+      }
+    }
+    assertEquals(writtenFiles.keySet(), fileNames);
+  }
+
   private static void compareFile(FileStatus expected, FileStatus status) {
     assertEquals(expected.getAccessTime(), status.getAccessTime());
     assertEquals(expected.getBlockSize(), status.getBlockSize());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java
deleted file mode 100644
index a1e49ccee28f1..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestChunkedArrayList.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.util;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-
-import org.junit.Test;
-
-import com.google.common.base.Stopwatch;
-
-public class TestChunkedArrayList {
-
-  @Test
-  public void testBasics() {
-    final int N_ELEMS = 100000;
-    ChunkedArrayList<Integer> l = new ChunkedArrayList<Integer>();
-    assertTrue(l.isEmpty());
-    // Insert a bunch of elements.
-    for (int i = 0; i < N_ELEMS; i++) {
-      l.add(i);
-    }
-    assertFalse(l.isEmpty());
-    assertEquals(N_ELEMS, l.size());
-
-    // Check that it got chunked.
-    assertTrue(l.getNumChunks() > 10);
-    assertEquals(8192, l.getMaxChunkSize());
-  }
-  
-  @Test
-  public void testIterator() {
-    ChunkedArrayList<Integer> l = new ChunkedArrayList<Integer>();
-    for (int i = 0; i < 30000; i++) {
-      l.add(i);
-    }
-    
-    int i = 0;
-    for (int fromList : l) {
-      assertEquals(i, fromList);
-      i++;
-    }
-  }
-  
-  @Test
-  public void testPerformance() {
-    String obj = "hello world";
-    
-    final int numElems = 1000000;
-    final int numTrials = 5;
-    
-    for (int trial = 0; trial < numTrials; trial++) {
-      System.gc();
-      {
-        ArrayList<String> arrayList = new ArrayList<String>();
-        Stopwatch sw = new Stopwatch();
-        sw.start();
-        for (int i = 0; i < numElems; i++) {
-          arrayList.add(obj);
-        }
-        System.out.println("       ArrayList " + sw.elapsedMillis());
-      }
-      
-      // test ChunkedArrayList
-      System.gc();
-      {
-        ChunkedArrayList<String> chunkedList = new ChunkedArrayList<String>();
-        Stopwatch sw = new Stopwatch();
-        sw.start();
-        for (int i = 0; i < numElems; i++) {
-          chunkedList.add(obj);
-        }
-        System.out.println("ChunkedArrayList " + sw.elapsedMillis());
-      }
-    }
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
index b4216f04c4d6c..80369fd0efc1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
@@ -17,21 +17,31 @@
  */
 package org.apache.hadoop.hdfs.web;
 
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doReturn;
+
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.security.PrivilegedExceptionAction;
 
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FSMainOperationsBaseTest;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.AppendTestUtil;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.web.resources.ExceptionHandler;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.log4j.Level;
@@ -128,6 +138,74 @@ public void testConcat() throws Exception {
     Assert.assertEquals(1024*4, fileStatus.getLen());
   }
 
+  @Test
+  public void testTruncate() throws Exception {
+    final short repl = 3;
+    final int blockSize = 1024;
+    final int numOfBlocks = 2;
+    Path dir = getTestRootPath(fSys, "test/hadoop");
+    Path file = getTestRootPath(fSys, "test/hadoop/file");
+
+    final byte[] data = getFileData(numOfBlocks, blockSize);
+    createFile(fSys, file, data, blockSize, repl);
+
+    final int newLength = blockSize;
+
+    boolean isReady = fSys.truncate(file, newLength);
+
+    Assert.assertTrue("Recovery is not expected.", isReady);
+
+    FileStatus fileStatus = fSys.getFileStatus(file);
+    Assert.assertEquals(fileStatus.getLen(), newLength);
+    AppendTestUtil.checkFullFile(fSys, file, newLength, data, file.toString());
+
+    ContentSummary cs = fSys.getContentSummary(dir);
+    Assert.assertEquals("Bad disk space usage", cs.getSpaceConsumed(),
+        newLength * repl);
+    Assert.assertTrue("Deleted", fSys.delete(dir, true));
+  }
+
+  // Test that WebHdfsFileSystem.jsonParse() closes the connection's input
+  // stream.
+  // Closing the inputstream in jsonParse will allow WebHDFS to reuse
+  // connections to the namenode rather than needing to always open new ones.
+  boolean closedInputStream = false;
+  @Test
+  public void testJsonParseClosesInputStream() throws Exception {
+    final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fileSystem;
+    Path file = getTestRootPath(fSys, "test/hadoop/file");
+    createFile(file);
+    final HttpOpParam.Op op = GetOpParam.Op.GETHOMEDIRECTORY;
+    final URL url = webhdfs.toUrl(op, file);
+    final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod(op.getType().toString());
+    conn.connect();
+
+    InputStream myIn = new InputStream(){
+      private HttpURLConnection localConn = conn;
+      @Override
+      public void close() throws IOException {
+        closedInputStream = true;
+        localConn.getInputStream().close();
+      }
+      @Override
+      public int read() throws IOException {
+        return localConn.getInputStream().read();
+      }
+    };
+    final HttpURLConnection spyConn = spy(conn);
+    doReturn(myIn).when(spyConn).getInputStream();
+
+    try {
+      Assert.assertFalse(closedInputStream);
+      WebHdfsFileSystem.jsonParse(spyConn, false);
+      Assert.assertTrue(closedInputStream);
+    } catch(IOException ioe) {
+      junit.framework.TestCase.fail();
+    }
+    conn.disconnect();
+  }
+
   @Override
   @Test
   public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSAcl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSAcl.java
index 2b14fe119bea0..6b44b26e41580 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSAcl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSAcl.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hdfs.web;
 
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.FSAclBaseTest;
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -34,9 +32,7 @@ public class TestWebHDFSAcl extends FSAclBaseTest {
   @BeforeClass
   public static void init() throws Exception {
     conf = WebHdfsTestUtil.createConf();
-    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
+    startCluster();
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java
index 335c346cee136..11abd2c070bd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsWithMultipleNameNodes.java
@@ -21,7 +21,6 @@
 import java.net.URI;
 
 import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -29,14 +28,12 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -51,11 +48,9 @@ public class TestWebHdfsWithMultipleNameNodes {
 
   static private void setLogLevel() {
     ((Log4JLogger)LOG).getLogger().setLevel(Level.ALL);
-    ((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);
+    GenericTestUtils.setLogLevel(NamenodeWebHdfsMethods.LOG, Level.ALL);
 
-    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.OFF);
-    ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
+    DFSTestUtil.setNameNodeLogLevel(Level.ALL);
   }
 
   private static final Configuration conf = new HdfsConfiguration();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
index 77860ba1f3814..7b3568d271ffb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
@@ -72,12 +72,12 @@ public void testCreateAndDestroySpanReceiver() throws Exception {
       Assert.assertEquals("ret:0, Added trace span receiver 1 with " +
           "configuration local-file-span-receiver.path = " + tracePath + NEWLINE,
           runTraceCommand(trace, "-add", "-host", getHostPortForNN(cluster),
-              "-class", "org.htrace.impl.LocalFileSpanReceiver",
+              "-class", "org.apache.htrace.impl.LocalFileSpanReceiver",
               "-Clocal-file-span-receiver.path=" + tracePath));
       String list =
           runTraceCommand(trace, "-list", "-host", getHostPortForNN(cluster));
       Assert.assertTrue(list.startsWith("ret:0"));
-      Assert.assertTrue(list.contains("1   org.htrace.impl.LocalFileSpanReceiver"));
+      Assert.assertTrue(list.contains("1   org.apache.htrace.impl.LocalFileSpanReceiver"));
       Assert.assertEquals("ret:0, Removed trace span receiver 1" + NEWLINE,
           runTraceCommand(trace, "-remove", "1", "-host",
               getHostPortForNN(cluster)));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
index 0f96c6f599719..0bbd5b4c52233 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracing.java
@@ -25,12 +25,12 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.htrace.HTraceConfiguration;
-import org.htrace.Sampler;
-import org.htrace.Span;
-import org.htrace.SpanReceiver;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.HTraceConfiguration;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Span;
+import org.apache.htrace.SpanReceiver;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
@@ -101,8 +101,6 @@ public void testWriteTraceHooks() throws Exception {
     Assert.assertNotNull(s);
     long spanStart = s.getStartTimeMillis();
     long spanEnd = s.getStopTimeMillis();
-    Assert.assertTrue(spanStart - startTime < 100);
-    Assert.assertTrue(spanEnd - endTime < 100);
 
     // There should only be one trace id as it should all be homed in the
     // top trace.
@@ -272,7 +270,7 @@ public Boolean get() {
    */
   public static class SetSpanReceiver implements SpanReceiver {
 
-    public void configure(HTraceConfiguration conf) {
+    public SetSpanReceiver(HTraceConfiguration conf) {
     }
 
     public void receiveSpan(Span span) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
index 800cc6bb82047..298155865c5c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTracingShortCircuitLocalRead.java
@@ -30,10 +30,10 @@
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.util.NativeCodeLoader;
-import org.htrace.Sampler;
-import org.htrace.Span;
-import org.htrace.Trace;
-import org.htrace.TraceScope;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index 08607ebb520b4..da8c190866eeb 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 5e1d7186ed434..2a1b549dda6d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <EDITS>
-  <EDITS_VERSION>-60</EDITS_VERSION>
+  <EDITS_VERSION>-62</EDITS_VERSION>
   <RECORD>
     <OPCODE>OP_START_LOG_SEGMENT</OPCODE>
     <DATA>
@@ -13,8 +13,8 @@
       <TXID>2</TXID>
       <DELEGATION_KEY>
         <KEY_ID>1</KEY_ID>
-        <EXPIRY_DATE>1412805665311</EXPIRY_DATE>
-        <KEY>c1cad1109e33ae77</KEY>
+        <EXPIRY_DATE>1422569009939</EXPIRY_DATE>
+        <KEY>907cb34000041937</KEY>
       </DELEGATION_KEY>
     </DATA>
   </RECORD>
@@ -24,8 +24,8 @@
       <TXID>3</TXID>
       <DELEGATION_KEY>
         <KEY_ID>2</KEY_ID>
-        <EXPIRY_DATE>1412805665314</EXPIRY_DATE>
-        <KEY>0632068587d6574c</KEY>
+        <EXPIRY_DATE>1422569009941</EXPIRY_DATE>
+        <KEY>178fa1bd83474b43</KEY>
       </DELEGATION_KEY>
     </DATA>
   </RECORD>
@@ -37,19 +37,19 @@
       <INODEID>16386</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114467969</MTIME>
-      <ATIME>1412114467969</ATIME>
+      <MTIME>1421877810832</MTIME>
+      <ATIME>1421877810832</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>13</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>6</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -60,60 +60,93 @@
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468019</MTIME>
-      <ATIME>1412114467969</ATIME>
+      <MTIME>1421877810888</MTIME>
+      <ATIME>1421877810832</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
       <OVERWRITE>false</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
     </DATA>
   </RECORD>
   <RECORD>
-    <OPCODE>OP_SET_STORAGE_POLICY</OPCODE>
+    <OPCODE>OP_APPEND</OPCODE>
     <DATA>
       <TXID>6</TXID>
       <PATH>/file_create</PATH>
-      <POLICYID>12</POLICYID>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+      <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+      <NEWBLOCK>false</NEWBLOCK>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>8</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
-    <OPCODE>OP_RENAME_OLD</OPCODE>
+    <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
       <TXID>7</TXID>
       <LENGTH>0</LENGTH>
+      <INODEID>0</INODEID>
+      <PATH>/file_create</PATH>
+      <REPLICATION>1</REPLICATION>
+      <MTIME>1421877810899</MTIME>
+      <ATIME>1421877810832</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME></CLIENT_NAME>
+      <CLIENT_MACHINE></CLIENT_MACHINE>
+      <OVERWRITE>false</OVERWRITE>
+      <PERMISSION_STATUS>
+        <USERNAME>jing</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_STORAGE_POLICY</OPCODE>
+    <DATA>
+      <TXID>8</TXID>
+      <PATH>/file_create</PATH>
+      <POLICYID>7</POLICYID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_RENAME_OLD</OPCODE>
+    <DATA>
+      <TXID>9</TXID>
+      <LENGTH>0</LENGTH>
       <SRC>/file_create</SRC>
       <DST>/file_moved</DST>
-      <TIMESTAMP>1412114468027</TIMESTAMP>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>16</RPC_CALLID>
+      <TIMESTAMP>1421877810907</TIMESTAMP>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>11</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_DELETE</OPCODE>
     <DATA>
-      <TXID>8</TXID>
+      <TXID>10</TXID>
       <LENGTH>0</LENGTH>
       <PATH>/file_moved</PATH>
-      <TIMESTAMP>1412114468034</TIMESTAMP>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>17</RPC_CALLID>
+      <TIMESTAMP>1421877810915</TIMESTAMP>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>12</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_MKDIR</OPCODE>
     <DATA>
-      <TXID>9</TXID>
+      <TXID>11</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16387</INODEID>
       <PATH>/directory_mkdir</PATH>
-      <TIMESTAMP>1412114468041</TIMESTAMP>
+      <TIMESTAMP>1421877810923</TIMESTAMP>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>493</MODE>
       </PERMISSION_STATUS>
@@ -122,94 +155,94 @@
   <RECORD>
     <OPCODE>OP_ALLOW_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>10</TXID>
+      <TXID>12</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_DISALLOW_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>11</TXID>
+      <TXID>13</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ALLOW_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>12</TXID>
+      <TXID>14</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_CREATE_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>13</TXID>
+      <TXID>15</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>22</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>17</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_RENAME_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>14</TXID>
+      <TXID>16</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
       <SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>23</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>18</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_DELETE_SNAPSHOT</OPCODE>
     <DATA>
-      <TXID>15</TXID>
+      <TXID>17</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>24</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>19</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>16</TXID>
+      <TXID>18</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16388</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468073</MTIME>
-      <ATIME>1412114468073</ATIME>
+      <MTIME>1421877810946</MTIME>
+      <ATIME>1421877810946</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>25</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>20</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>17</TXID>
+      <TXID>19</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468075</MTIME>
-      <ATIME>1412114468073</ATIME>
+      <MTIME>1421877810948</MTIME>
+      <ATIME>1421877810946</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
       <OVERWRITE>false</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
@@ -218,7 +251,7 @@
   <RECORD>
     <OPCODE>OP_SET_REPLICATION</OPCODE>
     <DATA>
-      <TXID>18</TXID>
+      <TXID>20</TXID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
     </DATA>
@@ -226,7 +259,7 @@
   <RECORD>
     <OPCODE>OP_SET_PERMISSIONS</OPCODE>
     <DATA>
-      <TXID>19</TXID>
+      <TXID>21</TXID>
       <SRC>/file_create</SRC>
       <MODE>511</MODE>
     </DATA>
@@ -234,7 +267,7 @@
   <RECORD>
     <OPCODE>OP_SET_OWNER</OPCODE>
     <DATA>
-      <TXID>20</TXID>
+      <TXID>22</TXID>
       <SRC>/file_create</SRC>
       <USERNAME>newOwner</USERNAME>
     </DATA>
@@ -242,7 +275,7 @@
   <RECORD>
     <OPCODE>OP_TIMES</OPCODE>
     <DATA>
-      <TXID>21</TXID>
+      <TXID>23</TXID>
       <LENGTH>0</LENGTH>
       <PATH>/file_create</PATH>
       <MTIME>1285195527000</MTIME>
@@ -252,7 +285,7 @@
   <RECORD>
     <OPCODE>OP_SET_QUOTA</OPCODE>
     <DATA>
-      <TXID>22</TXID>
+      <TXID>24</TXID>
       <SRC>/directory_mkdir</SRC>
       <NSQUOTA>1000</NSQUOTA>
       <DSQUOTA>-1</DSQUOTA>
@@ -261,57 +294,57 @@
   <RECORD>
     <OPCODE>OP_RENAME</OPCODE>
     <DATA>
-      <TXID>23</TXID>
+      <TXID>25</TXID>
       <LENGTH>0</LENGTH>
       <SRC>/file_create</SRC>
       <DST>/file_moved</DST>
-      <TIMESTAMP>1412114468093</TIMESTAMP>
+      <TIMESTAMP>1421877810968</TIMESTAMP>
       <OPTIONS>NONE</OPTIONS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>32</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>27</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>24</TXID>
+      <TXID>26</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16389</INODEID>
       <PATH>/file_concat_target</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468097</MTIME>
-      <ATIME>1412114468097</ATIME>
+      <MTIME>1421877810972</MTIME>
+      <ATIME>1421877810972</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>34</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>29</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>25</TXID>
+      <TXID>27</TXID>
       <BLOCK_ID>1073741825</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>26</TXID>
+      <TXID>28</TXID>
       <GENSTAMPV2>1001</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>27</TXID>
+      <TXID>29</TXID>
       <PATH>/file_concat_target</PATH>
       <BLOCK>
         <BLOCK_ID>1073741825</BLOCK_ID>
@@ -325,21 +358,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>28</TXID>
+      <TXID>30</TXID>
       <BLOCK_ID>1073741826</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>29</TXID>
+      <TXID>31</TXID>
       <GENSTAMPV2>1002</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>30</TXID>
+      <TXID>32</TXID>
       <PATH>/file_concat_target</PATH>
       <BLOCK>
         <BLOCK_ID>1073741825</BLOCK_ID>
@@ -358,21 +391,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>31</TXID>
+      <TXID>33</TXID>
       <BLOCK_ID>1073741827</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>32</TXID>
+      <TXID>34</TXID>
       <GENSTAMPV2>1003</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>33</TXID>
+      <TXID>35</TXID>
       <PATH>/file_concat_target</PATH>
       <BLOCK>
         <BLOCK_ID>1073741826</BLOCK_ID>
@@ -391,13 +424,13 @@
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>34</TXID>
+      <TXID>36</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_concat_target</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468349</MTIME>
-      <ATIME>1412114468097</ATIME>
+      <MTIME>1421877811083</MTIME>
+      <ATIME>1421877810972</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
@@ -418,7 +451,7 @@
         <GENSTAMP>1003</GENSTAMP>
       </BLOCK>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
@@ -427,44 +460,44 @@
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>35</TXID>
+      <TXID>37</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16390</INODEID>
       <PATH>/file_concat_0</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468351</MTIME>
-      <ATIME>1412114468351</ATIME>
+      <MTIME>1421877811086</MTIME>
+      <ATIME>1421877811086</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>47</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>39</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>36</TXID>
+      <TXID>38</TXID>
       <BLOCK_ID>1073741828</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>37</TXID>
+      <TXID>39</TXID>
       <GENSTAMPV2>1004</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>38</TXID>
+      <TXID>40</TXID>
       <PATH>/file_concat_0</PATH>
       <BLOCK>
         <BLOCK_ID>1073741828</BLOCK_ID>
@@ -478,21 +511,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>39</TXID>
+      <TXID>41</TXID>
       <BLOCK_ID>1073741829</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>40</TXID>
+      <TXID>42</TXID>
       <GENSTAMPV2>1005</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>41</TXID>
+      <TXID>43</TXID>
       <PATH>/file_concat_0</PATH>
       <BLOCK>
         <BLOCK_ID>1073741828</BLOCK_ID>
@@ -511,21 +544,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>42</TXID>
+      <TXID>44</TXID>
       <BLOCK_ID>1073741830</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>43</TXID>
+      <TXID>45</TXID>
       <GENSTAMPV2>1006</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>44</TXID>
+      <TXID>46</TXID>
       <PATH>/file_concat_0</PATH>
       <BLOCK>
         <BLOCK_ID>1073741829</BLOCK_ID>
@@ -544,13 +577,13 @@
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>45</TXID>
+      <TXID>47</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_concat_0</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468370</MTIME>
-      <ATIME>1412114468351</ATIME>
+      <MTIME>1421877811108</MTIME>
+      <ATIME>1421877811086</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
@@ -571,7 +604,7 @@
         <GENSTAMP>1006</GENSTAMP>
       </BLOCK>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
@@ -580,44 +613,44 @@
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>46</TXID>
+      <TXID>48</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16391</INODEID>
       <PATH>/file_concat_1</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468373</MTIME>
-      <ATIME>1412114468373</ATIME>
+      <MTIME>1421877811110</MTIME>
+      <ATIME>1421877811110</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>59</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>48</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>47</TXID>
+      <TXID>49</TXID>
       <BLOCK_ID>1073741831</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>48</TXID>
+      <TXID>50</TXID>
       <GENSTAMPV2>1007</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>49</TXID>
+      <TXID>51</TXID>
       <PATH>/file_concat_1</PATH>
       <BLOCK>
         <BLOCK_ID>1073741831</BLOCK_ID>
@@ -631,21 +664,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>50</TXID>
+      <TXID>52</TXID>
       <BLOCK_ID>1073741832</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>51</TXID>
+      <TXID>53</TXID>
       <GENSTAMPV2>1008</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>52</TXID>
+      <TXID>54</TXID>
       <PATH>/file_concat_1</PATH>
       <BLOCK>
         <BLOCK_ID>1073741831</BLOCK_ID>
@@ -664,21 +697,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>53</TXID>
+      <TXID>55</TXID>
       <BLOCK_ID>1073741833</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>54</TXID>
+      <TXID>56</TXID>
       <GENSTAMPV2>1009</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>55</TXID>
+      <TXID>57</TXID>
       <PATH>/file_concat_1</PATH>
       <BLOCK>
         <BLOCK_ID>1073741832</BLOCK_ID>
@@ -697,13 +730,13 @@
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>56</TXID>
+      <TXID>58</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_concat_1</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468392</MTIME>
-      <ATIME>1412114468373</ATIME>
+      <MTIME>1421877811131</MTIME>
+      <ATIME>1421877811110</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
@@ -724,7 +757,7 @@
         <GENSTAMP>1009</GENSTAMP>
       </BLOCK>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
@@ -733,83 +766,209 @@
   <RECORD>
     <OPCODE>OP_CONCAT_DELETE</OPCODE>
     <DATA>
-      <TXID>57</TXID>
+      <TXID>59</TXID>
       <LENGTH>0</LENGTH>
       <TRG>/file_concat_target</TRG>
-      <TIMESTAMP>1412114468395</TIMESTAMP>
+      <TIMESTAMP>1421877811134</TIMESTAMP>
       <SOURCES>
         <SOURCE1>/file_concat_0</SOURCE1>
         <SOURCE2>/file_concat_1</SOURCE2>
       </SOURCES>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>70</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>56</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
-    <OPCODE>OP_SYMLINK</OPCODE>
+    <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>58</TXID>
+      <TXID>60</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16392</INODEID>
+      <PATH>/file_create</PATH>
+      <REPLICATION>1</REPLICATION>
+      <MTIME>1421877811137</MTIME>
+      <ATIME>1421877811137</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+      <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+      <OVERWRITE>true</OVERWRITE>
+      <PERMISSION_STATUS>
+        <USERNAME>jing</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>58</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+    <DATA>
+      <TXID>61</TXID>
+      <BLOCK_ID>1073741834</BLOCK_ID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+    <DATA>
+      <TXID>62</TXID>
+      <GENSTAMPV2>1010</GENSTAMPV2>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD_BLOCK</OPCODE>
+    <DATA>
+      <TXID>63</TXID>
+      <PATH>/file_create</PATH>
+      <BLOCK>
+        <BLOCK_ID>1073741834</BLOCK_ID>
+        <NUM_BYTES>0</NUM_BYTES>
+        <GENSTAMP>1010</GENSTAMP>
+      </BLOCK>
+      <RPC_CLIENTID></RPC_CLIENTID>
+      <RPC_CALLID>-2</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+    <DATA>
+      <TXID>64</TXID>
+      <BLOCK_ID>1073741835</BLOCK_ID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+    <DATA>
+      <TXID>65</TXID>
+      <GENSTAMPV2>1011</GENSTAMPV2>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD_BLOCK</OPCODE>
+    <DATA>
+      <TXID>66</TXID>
+      <PATH>/file_create</PATH>
+      <BLOCK>
+        <BLOCK_ID>1073741834</BLOCK_ID>
+        <NUM_BYTES>512</NUM_BYTES>
+        <GENSTAMP>1010</GENSTAMP>
+      </BLOCK>
+      <BLOCK>
+        <BLOCK_ID>1073741835</BLOCK_ID>
+        <NUM_BYTES>0</NUM_BYTES>
+        <GENSTAMP>1011</GENSTAMP>
+      </BLOCK>
+      <RPC_CLIENTID></RPC_CLIENTID>
+      <RPC_CALLID>-2</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_CLOSE</OPCODE>
+    <DATA>
+      <TXID>67</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>0</INODEID>
+      <PATH>/file_create</PATH>
+      <REPLICATION>1</REPLICATION>
+      <MTIME>1421877811152</MTIME>
+      <ATIME>1421877811137</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME></CLIENT_NAME>
+      <CLIENT_MACHINE></CLIENT_MACHINE>
+      <OVERWRITE>false</OVERWRITE>
+      <BLOCK>
+        <BLOCK_ID>1073741834</BLOCK_ID>
+        <NUM_BYTES>512</NUM_BYTES>
+        <GENSTAMP>1010</GENSTAMP>
+      </BLOCK>
+      <BLOCK>
+        <BLOCK_ID>1073741835</BLOCK_ID>
+        <NUM_BYTES>512</NUM_BYTES>
+        <GENSTAMP>1011</GENSTAMP>
+      </BLOCK>
+      <PERMISSION_STATUS>
+        <USERNAME>jing</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_TRUNCATE</OPCODE>
+    <DATA>
+      <TXID>68</TXID>
+      <SRC>/file_create</SRC>
+      <CLIENTNAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENTNAME>
+      <CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
+      <NEWLENGTH>512</NEWLENGTH>
+      <TIMESTAMP>1421877811154</TIMESTAMP>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SYMLINK</OPCODE>
+    <DATA>
+      <TXID>69</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>16393</INODEID>
       <PATH>/file_symlink</PATH>
       <VALUE>/file_concat_target</VALUE>
-      <MTIME>1412114468398</MTIME>
-      <ATIME>1412114468398</ATIME>
+      <MTIME>1421877811160</MTIME>
+      <ATIME>1421877811160</ATIME>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>511</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>71</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>65</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>59</TXID>
+      <TXID>70</TXID>
       <LENGTH>0</LENGTH>
-      <INODEID>16393</INODEID>
+      <INODEID>16394</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114468401</MTIME>
-      <ATIME>1412114468401</ATIME>
+      <MTIME>1421877811163</MTIME>
+      <ATIME>1421877811163</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>72</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>66</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>60</TXID>
-      <BLOCK_ID>1073741834</BLOCK_ID>
+      <TXID>71</TXID>
+      <BLOCK_ID>1073741836</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>61</TXID>
-      <GENSTAMPV2>1010</GENSTAMPV2>
+      <TXID>72</TXID>
+      <GENSTAMPV2>1012</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_BLOCK</OPCODE>
     <DATA>
-      <TXID>62</TXID>
+      <TXID>73</TXID>
       <PATH>/hard-lease-recovery-test</PATH>
       <BLOCK>
-        <BLOCK_ID>1073741834</BLOCK_ID>
+        <BLOCK_ID>1073741836</BLOCK_ID>
         <NUM_BYTES>0</NUM_BYTES>
-        <GENSTAMP>1010</GENSTAMP>
+        <GENSTAMP>1012</GENSTAMP>
       </BLOCK>
       <RPC_CLIENTID></RPC_CLIENTID>
       <RPC_CALLID>-2</RPC_CALLID>
@@ -818,12 +977,12 @@
   <RECORD>
     <OPCODE>OP_UPDATE_BLOCKS</OPCODE>
     <DATA>
-      <TXID>63</TXID>
+      <TXID>74</TXID>
       <PATH>/hard-lease-recovery-test</PATH>
       <BLOCK>
-        <BLOCK_ID>1073741834</BLOCK_ID>
+        <BLOCK_ID>1073741836</BLOCK_ID>
         <NUM_BYTES>11</NUM_BYTES>
-        <GENSTAMP>1010</GENSTAMP>
+        <GENSTAMP>1012</GENSTAMP>
       </BLOCK>
       <RPC_CLIENTID></RPC_CLIENTID>
       <RPC_CALLID>-2</RPC_CALLID>
@@ -832,15 +991,15 @@
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>64</TXID>
-      <GENSTAMPV2>1011</GENSTAMPV2>
+      <TXID>75</TXID>
+      <GENSTAMPV2>1013</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REASSIGN_LEASE</OPCODE>
     <DATA>
-      <TXID>65</TXID>
-      <LEASEHOLDER>DFSClient_NONMAPREDUCE_1474796918_1</LEASEHOLDER>
+      <TXID>76</TXID>
+      <LEASEHOLDER>DFSClient_NONMAPREDUCE_-986598042_1</LEASEHOLDER>
       <PATH>/hard-lease-recovery-test</PATH>
       <NEWHOLDER>HDFS_NameNode</NEWHOLDER>
     </DATA>
@@ -848,24 +1007,24 @@
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>66</TXID>
+      <TXID>77</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1412114470807</MTIME>
-      <ATIME>1412114468401</ATIME>
+      <MTIME>1421877813736</MTIME>
+      <ATIME>1421877811163</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME></CLIENT_NAME>
       <CLIENT_MACHINE></CLIENT_MACHINE>
       <OVERWRITE>false</OVERWRITE>
       <BLOCK>
-        <BLOCK_ID>1073741834</BLOCK_ID>
+        <BLOCK_ID>1073741836</BLOCK_ID>
         <NUM_BYTES>11</NUM_BYTES>
-        <GENSTAMP>1011</GENSTAMP>
+        <GENSTAMP>1013</GENSTAMP>
       </BLOCK>
       <PERMISSION_STATUS>
-        <USERNAME>aagarwal</USERNAME>
+        <USERNAME>jing</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
@@ -874,72 +1033,72 @@
   <RECORD>
     <OPCODE>OP_ADD_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>67</TXID>
+      <TXID>78</TXID>
       <POOLNAME>pool1</POOLNAME>
-      <OWNERNAME>aagarwal</OWNERNAME>
+      <OWNERNAME>jing</OWNERNAME>
       <GROUPNAME>staff</GROUPNAME>
       <MODE>493</MODE>
       <LIMIT>9223372036854775807</LIMIT>
       <MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>79</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>73</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>68</TXID>
+      <TXID>79</TXID>
       <POOLNAME>pool1</POOLNAME>
       <LIMIT>99</LIMIT>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>80</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>74</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>69</TXID>
+      <TXID>80</TXID>
       <ID>1</ID>
       <PATH>/path</PATH>
       <REPLICATION>1</REPLICATION>
       <POOL>pool1</POOL>
-      <EXPIRATION>2305844421328165416</EXPIRATION>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>81</RPC_CALLID>
+      <EXPIRATION>2305844431091508160</EXPIRATION>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>75</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>70</TXID>
+      <TXID>81</TXID>
       <ID>1</ID>
       <REPLICATION>2</REPLICATION>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>82</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>76</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>71</TXID>
+      <TXID>82</TXID>
       <ID>1</ID>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>83</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>77</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>72</TXID>
+      <TXID>83</TXID>
       <POOLNAME>pool1</POOLNAME>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>84</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>78</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_ACL</OPCODE>
     <DATA>
-      <TXID>73</TXID>
+      <TXID>84</TXID>
       <SRC>/file_concat_target</SRC>
       <ENTRY>
         <SCOPE>ACCESS</SCOPE>
@@ -972,62 +1131,62 @@
   <RECORD>
     <OPCODE>OP_SET_XATTR</OPCODE>
     <DATA>
-      <TXID>74</TXID>
+      <TXID>85</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a1</NAME>
         <VALUE>0x313233</VALUE>
       </XATTR>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>86</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>80</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_XATTR</OPCODE>
     <DATA>
-      <TXID>75</TXID>
+      <TXID>86</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a2</NAME>
         <VALUE>0x373839</VALUE>
       </XATTR>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>87</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>81</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_XATTR</OPCODE>
     <DATA>
-      <TXID>76</TXID>
+      <TXID>87</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a2</NAME>
       </XATTR>
-      <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
-      <RPC_CALLID>88</RPC_CALLID>
+      <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+      <RPC_CALLID>82</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
     <DATA>
-      <TXID>77</TXID>
-      <STARTTIME>1412114471510</STARTTIME>
+      <TXID>88</TXID>
+      <STARTTIME>1421877814254</STARTTIME>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
     <DATA>
-      <TXID>78</TXID>
-      <FINALIZETIME>1412114471510</FINALIZETIME>
+      <TXID>89</TXID>
+      <FINALIZETIME>1421877814254</FINALIZETIME>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_END_LOG_SEGMENT</OPCODE>
     <DATA>
-      <TXID>79</TXID>
+      <TXID>90</TXID>
     </DATA>
   </RECORD>
 </EDITS>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-24-datanode-dir.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-24-datanode-dir.tgz
index 9f666fed090ec..032bf0d3d6638 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-24-datanode-dir.tgz and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-24-datanode-dir.tgz differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLI.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLI.xml
index 21031ad2d2039..82a580926a1a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLI.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLI.xml
@@ -317,6 +317,59 @@
         </comparator>
       </comparators>
     </test>
+    <test>
+      <description>setfacl : Add minimal default ACL</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /dir1</command>
+        <command>-fs NAMENODE -setfacl -m default:user::rwx /dir1</command>
+        <command>-fs NAMENODE -getfacl /dir1</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -R /dir1</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output># file: /dir1</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output># owner: USERNAME</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output># group: supergroup</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>user::rwx</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>group::r-x</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>other::r-x</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>default:user::rwx</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>default:group::r-x</expected-output>
+        </comparator>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>default:other::r-x</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpAcrossOutputComparator</type>
+          <expected-output>.*(?!default\:mask)*</expected-output>
+        </comparator>
+      </comparators>
+    </test>
     <test>
       <description>setfacl : try adding default ACL to file</description>
       <test-commands>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
index ebbf773b1333b..89c93e26b1c4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
@@ -238,13 +238,35 @@
     </test>
 
     <test>
-      <description>Test failure of renaming a non-EZ file from an EZ</description>
+        <description>Test failure of renaming an EZ file into a non-EZ</description>
+        <test-commands>
+            <command>-fs NAMENODE -mkdir /src</command>
+            <command>-fs NAMENODE -mkdir /dst</command>
+            <command>-fs NAMENODE -ls /</command>-
+            <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+            <command>-fs NAMENODE -touchz /src/foo</command>
+            <command>-fs NAMENODE -mv /src/foo /dst</command>-
+        </test-commands>
+        <cleanup-commands>
+            <command>-fs NAMENODE -rm /src/foo</command>
+            <command>-fs NAMENODE -rmdir /src</command>
+            <command>-fs NAMENODE -rmdir /dst</command>
+        </cleanup-commands>
+        <comparators>
+            <comparator>
+                <type>SubstringComparator</type>
+                <expected-output>/src/foo can't be moved from an encryption zone.</expected-output>
+            </comparator>
+        </comparators>
+    </test>
+
+    <test>
+      <description>Test success of renaming an EZ root</description>
       <test-commands>
         <command>-fs NAMENODE -mkdir /src</command>
-        <command>-fs NAMENODE -mkdir /dst</command>
-        <command>-fs NAMENODE -ls /</command>-
         <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
         <command>-fs NAMENODE -mv /src /dst</command>-
+        <command>-fs NAMENODE -ls /</command>-
       </test-commands>
       <cleanup-commands>
         <command>-fs NAMENODE -rmdir /src</command>
@@ -253,7 +275,7 @@
       <comparators>
         <comparator>
           <type>SubstringComparator</type>
-          <expected-output>/src can't be moved from an encryption zone</expected-output>
+          <expected-output>/dst</expected-output>
         </comparator>
       </comparators>
     </test>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index c86b06d3445dc..6c71b6ee1f8a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -8699,6 +8699,48 @@
       </comparators>
     </test>
 
+    <test> <!-- TESTED -->
+      <description>count: file using absolute path showing header record</description>
+      <test-commands>
+        <command>-fs NAMENODE -touchz /file1</command>
+        <command>-fs NAMENODE -count -v /file1</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm /file1</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test> <!-- TESTED -->
+      <description>count: file using absolute path with -q option and showing header record</description>
+      <test-commands>
+        <command>-fs NAMENODE -touchz /file1</command>
+        <command>-fs NAMENODE -count -q -v /file1</command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm /file1</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*QUOTA       REM_QUOTA     SPACE_QUOTA REM_SPACE_QUOTA    DIR_COUNT   FILE_COUNT       CONTENT_SIZE PATHNAME</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
     <!-- Tests for chmod -->
     <test> <!-- TESTED -->
       <description>chmod: change permission(octal mode) of file in absolute path</description>
@@ -15521,7 +15563,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-setSpaceQuota &lt;quota&gt; &lt;dirname&gt;...&lt;dirname&gt;: Set the disk space quota &lt;quota&gt; for each directory &lt;dirName&gt;.( )*</expected-output>
+          <expected-output>^-setSpaceQuota &lt;quota&gt; &lt;dirname&gt;...&lt;dirname&gt; -storageType &lt;storagetype&gt;: Set the disk space quota &lt;quota&gt; for each directory &lt;dirName&gt;.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -15535,6 +15577,14 @@
           <type>RegexpComparator</type>
           <expected-output>^( |\t)*2. user is not an administrator, or( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*3. the directory does not exist or is a file.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*The storage type specific quota is set when -storageType option is specified.( )*</expected-output>
+        </comparator>
       </comparators>
     </test>
 
@@ -15548,7 +15598,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-clrSpaceQuota &lt;dirname&gt;...&lt;dirname&gt;: Clear the disk space quota for each directory &lt;dirName&gt;.( )*</expected-output>
+          <expected-output>^-clrSpaceQuota &lt;dirname&gt;...&lt;dirname&gt; -storageType &lt;storagetype&gt;: Clear the disk space quota for each directory &lt;dirName&gt;.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -15566,6 +15616,10 @@
           <type>RegexpComparator</type>
           <expected-output>^( |\t)*It does not fault if the directory has no quota.( )*</expected-output>
         </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*The storage type specific quota is cleared when -storageType option is specified.( )*</expected-output>
+        </comparator>
       </comparators>
     </test>
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.tgz
new file mode 100644
index 0000000000000..30b03246f955a
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.tgz differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.txt b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.txt
new file mode 100644
index 0000000000000..6b0e1eacdf323
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22FixesStorageIDs.txt
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout
+# upgrade test.
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+
+/f01    4021661486
+/f02    4021661486
+/f03    4021661486
+/f04    4021661486
+overallCRC	3193029345
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.tgz
new file mode 100644
index 0000000000000..74c1649abcaf0
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.tgz differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.txt b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.txt
new file mode 100644
index 0000000000000..6b0e1eacdf323
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom22via26FixesStorageIDs.txt
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout
+# upgrade test.
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+
+/f01    4021661486
+/f02    4021661486
+/f03    4021661486
+/f04    4021661486
+overallCRC	3193029345
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.tgz
new file mode 100644
index 0000000000000..69fbaf61ac814
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.tgz differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.txt b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.txt
new file mode 100644
index 0000000000000..6b0e1eacdf323
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testUpgradeFrom26PreservesStorageIDs.txt
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Similar to hadoop-dfs-dir.txt, except this is used for a datanode layout
+# upgrade test.
+# Uncomment the following line to produce checksum info for a new DFS image.
+#printChecksums
+
+/f01    4021661486
+/f02    4021661486
+/f03    4021661486
+/f04    4021661486
+overallCRC	3193029345
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 3f34acd679b8e..583c6c166cacf 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -3,6 +3,8 @@ Hadoop MapReduce Change Log
 Trunk (Unreleased)
 
   INCOMPATIBLE CHANGES
+    MAPREDUCE-5785. Derive heap size or mapreduce.*.memory.mb automatically.
+    (Gera Shegalov and Karthik Kambatla via gera)
 
   NEW FEATURES
 
@@ -94,6 +96,9 @@ Trunk (Unreleased)
 
   BUG FIXES
 
+    MAPREDUCE-6191. Improve clearing stale state of Java serialization
+                    testcase.  (Sam Liu via Eric Yang)
+
     MAPREDUCE-5714. Removed forceful JVM exit in shutDownJob.  
                      (Jinghui Wang via Eric Yang)
 
@@ -235,17 +240,58 @@ Release 2.7.0 - UNRELEASED
 
   IMPROVEMENTS
 
+    MAPREDUCE-6149. Document override log4j.properties in MR job.
+    (Junping Du via harsh)
+
+    MAPREDUCE-6194. Bubble up final exception in failures during creation
+    of output collectors (Varun Saxena via harsh)
+
+    MAPREDUCE-5420. Remove mapreduce.task.tmp.dir from mapred-default.xml
+    (James Carman via harsh)
+
     MAPREDUCE-5932. Provide an option to use a dedicated reduce-side shuffle
     log (Gera Shegalov via jlowe)
 
+    MAPREDUCE-6046. Change the class name for logs in RMCommunicator
+    (Sahil Takiar via devaraj)
+
+    HADOOP-11032. Replace use of Guava's Stopwatch with Hadoop's StopWatch
+    (ozawa)
+
+    MAPREDUCE-6173. Document the configuration of deploying MR over distributed
+    cache with enabling wired encryption at the same time. 
+    (Junping Du via xgong)
+
+    MAPREDUCE-6141. History server leveldb recovery store (jlowe)
+
+    MAPREDUCE-6150. Update document of Rumen (Masatake Iwasaki via aw)
+
+    MAPREDUCE-6151. Update document of GridMix (Masatake Iwasaki via aw)
+
+    MAPREDUCE-6143. add configuration for mapreduce speculative execution in
+    MR2 (zxu via rkanter)
+
+    MAPREDUCE-5800. Use Job#getInstance instead of deprecated constructors
+    (aajisaka)
+
+    MAPREDUCE-6227. DFSIO for truncate. (shv via yliu)
+
   OPTIMIZATIONS
 
     MAPREDUCE-6169. MergeQueue should release reference to the current item 
     from key and value at the end of the iteration to save memory. 
     (Zhihai Xu via kasha)
 
+    MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw)
+
   BUG FIXES
 
+    MAPREDUCE-6210. Use getApplicationAttemptId() instead of getApplicationId()
+    for logging AttemptId in RMContainerAllocator.java (Leitao Guo via aajisaka)
+
+    MAPREDUCE-6177. Minor typo in the EncryptedShuffle document about
+    ssl-client.xml (Yangping Wu via harsh)
+
     MAPREDUCE-5918. LineRecordReader can return the same decompressor to
     CodecPool multiple times (Sergey Murylev via raviprak)
 
@@ -264,6 +310,42 @@ Release 2.7.0 - UNRELEASED
     MAPREDUCE-6160. Potential NullPointerException in MRClientProtocol
     interface implementation. (Rohith via jlowe)
 
+    MAPREDUCE-4879. TeraOutputFormat may overwrite an existing output
+    directory. (gera)
+
+    MAPREDUCE-6166. Reducers do not validate checksum of map outputs when
+    fetching directly to disk. (Eric Payne via gera)
+
+    MAPREDUCE-6045. need close the DataInputStream after open it in
+    TestMapReduce.java (zxu via rkanter)
+
+    MAPREDUCE-6199. AbstractCounters are not reset completely on
+    deserialization (adhoot via rkanter)
+
+    MAPREDUCE-6206. TestAggregatedTransferRate fails on non-US systems (Jens
+    Rabe via jlowe)
+
+    MAPREDUCE-3283. mapred classpath CLI does not display the complete classpath
+    (Varun Saxena via cnauroth)
+
+    MAPREDUCE-6230. Fixed RMContainerAllocator to update the new AMRMToken
+    service name properly. (Jason Lowe via jianhe)
+
+    MAPREDUCE-6231. Grep example job is not working on a fully-distributed
+    cluster. (aajisaka)
+
+    MAPREDUCE-6243. Fix findbugs warnings in hadoop-rumen. (Masatake Iwasaki
+    via aajisaka)
+
+    MAPREDUCE-5988. Fix dead links to the javadocs in mapreduce project.
+    (aajisaka)
+
+    MAPREDUCE-6186. Redundant call to requireJob() while displaying the conf
+    page (Rohit Agarwal via jlowe)
+
+    MAPREDUCE-6233. org.apache.hadoop.mapreduce.TestLargeSort.testLargeSort
+    failed in trunk (zxu via rkanter)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 667777a4c84a8..6d0c781fc9c89 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -78,9 +78,7 @@ case ${COMMAND} in
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
   ;;
   classpath)
-    hadoop_finalize
-    echo "${CLASSPATH}"
-    exit 0
+    hadoop_do_classpath_subcommand "$@" 
   ;;
   distcp)
     CLASS=org.apache.hadoop.tools.DistCp
@@ -95,7 +93,7 @@ case ${COMMAND} in
     hadoop_debug "Appending HADOOP_JOB_HISTORYSERVER_OPTS onto HADOOP_OPTS"
     HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}"
     if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then
-      JAVA_HEAP_MAX="-Xmx${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}m"
+      HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}"
     fi
     HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
   ;;
@@ -135,6 +133,8 @@ case ${COMMAND} in
   ;;
 esac
 
+hadoop_verify_user "${COMMAND}"
+
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 
@@ -147,7 +147,6 @@ if [[  "${HADOOP_DAEMON_MODE}" != "default" ]]; then
   HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
 fi
 
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
 hadoop_finalize
 
 if [[ -n "${supportdaemonization}" ]]; then
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
index 39baf4eadff63..fe6e9e6cfedef 100644
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -34,28 +34,22 @@ function hadoop_subproject_init
   # used interchangeable from here on out
   # ...
   # this should get deprecated at some point.
-  HADOOP_LOG_DIR="${HADOOP_MAPRED_LOG_DIR:-$HADOOP_LOG_DIR}"
-  HADOOP_MAPRED_LOG_DIR="${HADOOP_LOG_DIR}"
-  
-  HADOOP_LOGFILE="${HADOOP_MAPRED_LOGFILE:-$HADOOP_LOGFILE}"
-  HADOOP_MAPRED_LOGFILE="${HADOOP_LOGFILE}"
-  
-  HADOOP_NICENESS="${HADOOP_MAPRED_NICENESS:-$HADOOP_NICENESS}"
-  HADOOP_MAPRED_NICENESS="${HADOOP_NICENESS}"
-  
-  HADOOP_STOP_TIMEOUT="${HADOOP_MAPRED_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}"
-  HADOOP_MAPRED_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
+
+  hadoop_deprecate_envvar HADOOP_MAPRED_LOG_DIR HADOOP_LOG_DIR
+
+  hadoop_deprecate_envvar HADOOP_MAPRED_LOGFILE HADOOP_LOGFILE
   
-  HADOOP_PID_DIR="${HADOOP_MAPRED_PID_DIR:-$HADOOP_PID_DIR}"
-  HADOOP_MAPRED_PID_DIR="${HADOOP_PID_DIR}"
+  hadoop_deprecate_envvar HADOOP_MAPRED_NICENESS HADOOP_NICENESS
   
-  HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
-  HADOOP_MAPRED_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
+  hadoop_deprecate_envvar HADOOP_MAPRED_STOP_TIMEOUT HADOOP_STOP_TIMEOUT
   
+  hadoop_deprecate_envvar HADOOP_MAPRED_PID_DIR HADOOP_PID_DIR
+
+  hadoop_deprecate_envvar HADOOP_MAPRED_ROOT_LOGGER HADOOP_ROOT_LOGGER
+
   HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}"
-  
-  HADOOP_IDENT_STRING="${HADOOP_MAPRED_IDENT_STRING:-$HADOOP_IDENT_STRING}"
-  HADOOP_MAPRED_IDENT_STRING="${HADOOP_IDENT_STRING}"
+
+  hadoop_deprecate_envvar HADOOP_MAPRED_IDENT_STRING HADOOP_IDENT_STRING
 }
 
 if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
diff --git a/hadoop-mapreduce-project/bin/mapred.cmd b/hadoop-mapreduce-project/bin/mapred.cmd
index bb59c03085a60..4085599419d2f 100644
--- a/hadoop-mapreduce-project/bin/mapred.cmd
+++ b/hadoop-mapreduce-project/bin/mapred.cmd
@@ -95,6 +95,14 @@ if "%1" == "--loglevel" (
   @rem add modules to CLASSPATH
   set CLASSPATH=%CLASSPATH%;%HADOOP_MAPRED_HOME%\modules\*
 
+  if %mapred-command% == classpath (
+    if not defined mapred-command-arguments (
+      @rem No need to bother starting up a JVM for this simple case.
+      @echo %CLASSPATH%
+      exit /b
+    )
+  )
+
   call :%mapred-command% %mapred-command-arguments%
   set java_arguments=%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %mapred-command-arguments%
   call %JAVA% %java_arguments%
@@ -103,7 +111,7 @@ goto :eof
 
 
 :classpath
-  @echo %CLASSPATH%
+  set CLASS=org.apache.hadoop.util.Classpath
   goto :eof
 
 :job
diff --git a/hadoop-mapreduce-project/conf/mapred-env.sh b/hadoop-mapreduce-project/conf/mapred-env.sh
index 8a4b372932cdd..bbe4a4980cd6d 100644
--- a/hadoop-mapreduce-project/conf/mapred-env.sh
+++ b/hadoop-mapreduce-project/conf/mapred-env.sh
@@ -24,52 +24,22 @@
 ## MAPRED_xyz > HADOOP_xyz > hard-coded defaults
 ##
 
-###
-# Generic settings for MapReduce
-###
-
-#Override the log4j settings for all MR apps
-# Java property: hadoop.root.logger
-# export MAPRED_ROOT_LOGGER="INFO,console"
-
-# Override Hadoop's log directory & file
-# Java property: hadoop.log.dir
-# export HADOOP_MAPRED_LOG_DIR=""
-
-# Override Hadoop's pid directory
-# export HADOOP_MAPRED_PID_DIR=
-
-# Override Hadoop's identity string. $USER by default.
-# This is used in writing log and pid files, so keep that in mind!
-# Java property: hadoop.id.str
-# export HADOOP_MAPRED_IDENT_STRING=$USER
-
-# Override Hadoop's process priority
-# Note that sub-processes will also run at this level!
-# export HADOOP_MAPRED_NICENESS=0
-
 ###
 # Job History Server specific parameters
 ###
 
-# Specify the max heapsize for the Job History Server using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either
-# MAPRED_OPTS, HADOOP_OPTS, and/or HADOOP_JOB_HISTORYSERVER_OPTS.
-# If not specified, the default value will be picked from either HADOOP_HEAPSIZE
-# or the built-in default.
-#
-#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+# Specify the max heapsize for the JobHistoryServer.  If no units are
+# given, it will be assumed to be in MB.
+# This value will be overridden by an Xmx setting specified in HADOOP_OPTS,
+# and/or HADOOP_JOB_HISTORYSERVER_OPTS.
+# Default is the same as HADOOP_HEAPSIZE_MAX.
+#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=
 
-# Specify the JVM options to be used when starting the ResourceManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
+# Specify the JVM options to be used when starting the HistoryServer.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
 #export HADOOP_JOB_HISTORYSERVER_OPTS=
 
 # Specify the log4j settings for the JobHistoryServer
 # Java property: hadoop.root.logger
 #export HADOOP_JHS_LOGGER=INFO,RFA
-
-
-
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
index 817b3a566840c..936dc5a00d1d4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.TaskLog.LogName;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -99,36 +100,7 @@ public static void setVMEnv(Map<String, String> environment,
   }
 
   private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) {
-    String userClasspath = "";
-    String adminClasspath = "";
-    if (isMapTask) {
-      userClasspath = 
-          jobConf.get(
-              JobConf.MAPRED_MAP_TASK_JAVA_OPTS, 
-              jobConf.get(
-                  JobConf.MAPRED_TASK_JAVA_OPTS, 
-                  JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)
-          );
-      adminClasspath = 
-          jobConf.get(
-              MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
-              MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-    } else {
-      userClasspath =
-          jobConf.get(
-              JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, 
-              jobConf.get(
-                  JobConf.MAPRED_TASK_JAVA_OPTS,
-                  JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)
-              );
-      adminClasspath =
-          jobConf.get(
-              MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
-              MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
-    }
-    
-    // Add admin classpath first so it can be overridden by user.
-    return adminClasspath + " " + userClasspath;
+    return jobConf.getTaskJavaOpts(isMapTask ? TaskType.MAP : TaskType.REDUCE);
   }
 
   public static List<String> getVMCommand(
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 45ddb9eb60a4b..97de8fae2ebb0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -870,8 +870,7 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId,
         TaskAttemptStartedEvent tase = (TaskAttemptStartedEvent) event;
         tEvent.addEventInfo("TASK_TYPE", tase.getTaskType().toString());
         tEvent.addEventInfo("TASK_ATTEMPT_ID",
-            tase.getTaskAttemptId().toString() == null ?
-            "" : tase.getTaskAttemptId().toString());
+            tase.getTaskAttemptId().toString());
         tEvent.addEventInfo("START_TIME", tase.getStartTime());
         tEvent.addEventInfo("HTTP_PORT", tase.getHttpPort());
         tEvent.addEventInfo("TRACKER_NAME", tase.getTrackerName());
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index dfc6a3f5a1277..f4b434b9c6d2a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -563,19 +563,8 @@ public TaskAttemptImpl(TaskId taskId, int i,
     stateMachine = stateMachineFactory.make(this);
   }
 
-  private int getMemoryRequired(Configuration conf, TaskType taskType) {
-    int memory = 1024;
-    if (taskType == TaskType.MAP)  {
-      memory =
-          conf.getInt(MRJobConfig.MAP_MEMORY_MB,
-              MRJobConfig.DEFAULT_MAP_MEMORY_MB);
-    } else if (taskType == TaskType.REDUCE) {
-      memory =
-          conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
-              MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
-    }
-    
-    return memory;
+  private int getMemoryRequired(JobConf conf, TaskType taskType) {
+    return conf.getMemoryRequired(TypeConverter.fromYarn(taskType));
   }
 
   private int getCpuRequired(Configuration conf, TaskType taskType) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
index 6c58a683d1f27..5d4fa12ead2a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
@@ -67,7 +67,7 @@
  */
 public abstract class RMCommunicator extends AbstractService
     implements RMHeartbeatHandler {
-  private static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
+  private static final Log LOG = LogFactory.getLog(RMCommunicator.class);
   private int rmPollInterval;//millis
   protected ApplicationId applicationId;
   private final AtomicBoolean stopped;
@@ -76,7 +76,6 @@ public abstract class RMCommunicator extends AbstractService
   protected EventHandler eventHandler;
   protected ApplicationMasterProtocol scheduler;
   private final ClientService clientService;
-  protected int lastResponseID;
   private Resource maxContainerCapability;
   protected Map<ApplicationAccessType, String> applicationACLs;
   private volatile long lastHeartbeatTime;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index e0689971dc6eb..1acfeec163b5e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -75,6 +75,7 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.client.api.NMTokenCache;
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
@@ -684,7 +685,7 @@ private List<Container> getResources() throws Exception {
         JobEventType.JOB_AM_REBOOT));
       throw new YarnRuntimeException(
         "Resource Manager doesn't recognize AttemptId: "
-            + this.getContext().getApplicationID(), e);
+            + this.getContext().getApplicationAttemptId(), e);
     } catch (ApplicationMasterNotRegisteredException e) {
       LOG.info("ApplicationMaster is out of sync with ResourceManager,"
           + " hence resync and send outstanding requests.");
@@ -783,10 +784,8 @@ private void updateAMRMToken(Token token) throws IOException {
           .getIdentifier().array(), token.getPassword().array(), new Text(
           token.getKind()), new Text(token.getService()));
     UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
-    if (UserGroupInformation.isSecurityEnabled()) {
-      currentUGI = UserGroupInformation.getLoginUser();
-    }
     currentUGI.addToken(amrmToken);
+    amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
   }
 
   @VisibleForTesting
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
index 392a51aebd692..07a49af375fad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.util.Clock;
 
+import com.google.common.annotations.VisibleForTesting;
 
 public class DefaultSpeculator extends AbstractService implements
     Speculator {
@@ -62,12 +63,11 @@ public class DefaultSpeculator extends AbstractService implements
   private static final long NOT_RUNNING = Long.MIN_VALUE + 4;
   private static final long TOO_LATE_TO_SPECULATE = Long.MIN_VALUE + 5;
 
-  private static final long SOONEST_RETRY_AFTER_NO_SPECULATE = 1000L * 1L;
-  private static final long SOONEST_RETRY_AFTER_SPECULATE = 1000L * 15L;
-
-  private static final double PROPORTION_RUNNING_TASKS_SPECULATABLE = 0.1;
-  private static final double PROPORTION_TOTAL_TASKS_SPECULATABLE = 0.01;
-  private static final int  MINIMUM_ALLOWED_SPECULATIVE_TASKS = 10;
+  private long soonestRetryAfterNoSpeculate;
+  private long soonestRetryAfterSpeculate;
+  private double proportionRunningTasksSpeculatable;
+  private double proportionTotalTasksSpeculatable;
+  private int  minimumAllowedSpeculativeTasks;
 
   private static final Log LOG = LogFactory.getLog(DefaultSpeculator.class);
 
@@ -163,6 +163,21 @@ public DefaultSpeculator(Configuration conf, AppContext context, Clock clock) {
     this.estimator = estimator;
     this.clock = clock;
     this.eventHandler = context.getEventHandler();
+    this.soonestRetryAfterNoSpeculate =
+        conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
+                MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
+    this.soonestRetryAfterSpeculate =
+        conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
+                MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
+    this.proportionRunningTasksSpeculatable =
+        conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
+                MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
+    this.proportionTotalTasksSpeculatable =
+        conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
+                MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
+    this.minimumAllowedSpeculativeTasks =
+        conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
+                MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
   }
 
 /*   *************************************************************    */
@@ -182,8 +197,8 @@ public void run() {
                 try {
                   int speculations = computeSpeculations();
                   long mininumRecomp
-                      = speculations > 0 ? SOONEST_RETRY_AFTER_SPECULATE
-                                         : SOONEST_RETRY_AFTER_NO_SPECULATE;
+                      = speculations > 0 ? soonestRetryAfterSpeculate
+                                         : soonestRetryAfterNoSpeculate;
 
                   long wait = Math.max(mininumRecomp,
                         clock.getTime() - backgroundRunStartTime);
@@ -497,8 +512,8 @@ private int maybeScheduleASpeculation(TaskType type) {
       Map<TaskId, Task> tasks = job.getTasks(type);
 
       int numberAllowedSpeculativeTasks
-          = (int) Math.max(MINIMUM_ALLOWED_SPECULATIVE_TASKS,
-                           PROPORTION_TOTAL_TASKS_SPECULATABLE * tasks.size());
+          = (int) Math.max(minimumAllowedSpeculativeTasks,
+              proportionTotalTasksSpeculatable * tasks.size());
 
       TaskId bestTaskID = null;
       long bestSpeculationValue = -1L;
@@ -523,7 +538,7 @@ private int maybeScheduleASpeculation(TaskType type) {
       }
       numberAllowedSpeculativeTasks
           = (int) Math.max(numberAllowedSpeculativeTasks,
-                           PROPORTION_RUNNING_TASKS_SPECULATABLE * numberRunningTasks);
+              proportionRunningTasksSpeculatable * numberRunningTasks);
 
       // If we found a speculation target, fire it off
       if (bestTaskID != null
@@ -583,4 +598,29 @@ public void resetHeartBeatTime(long lastHeartBeatTime) {
       this.lastHeartBeatTime = lastHeartBeatTime;
     }
   }
+
+  @VisibleForTesting
+  public long getSoonestRetryAfterNoSpeculate() {
+    return soonestRetryAfterNoSpeculate;
+  }
+
+  @VisibleForTesting
+  public long getSoonestRetryAfterSpeculate() {
+    return soonestRetryAfterSpeculate;
+  }
+
+  @VisibleForTesting
+  public double getProportionRunningTasksSpeculatable() {
+    return proportionRunningTasksSpeculatable;
+  }
+
+  @VisibleForTesting
+  public double getProportionTotalTasksSpeculatable() {
+    return proportionTotalTasksSpeculatable;
+  }
+
+  @VisibleForTesting
+  public int getMinimumAllowedSpeculativeTasks() {
+    return minimumAllowedSpeculativeTasks;
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
index d7929cc8e385a..53f21db12d1f2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
@@ -313,7 +313,6 @@ protected Class<? extends View> confPage() {
    * Render the /conf page
    */
   public void conf() {
-    requireJob();
     try {
       requireJob();
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java
index 0edeb16834973..4eed7e36600db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java
@@ -83,6 +83,6 @@ public class NavBlock extends HtmlBlock {
         li().a("/conf", "Configuration")._().
         li().a("/logs", "Local logs")._().
         li().a("/stacks", "Server stacks")._().
-        li().a("/metrics", "Server metrics")._()._()._();
+        li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index a64f1d6a1ab11..de35d840b9473 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public void testTimelineEventHandling() throws Exception {
     long currentTime = System.currentTimeMillis();
     try {
       yarnCluster = new MiniYARNCluster(
-            TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
+            TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
       yarnCluster.init(conf);
       yarnCluster.start();
       jheh.start();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index c25cf5060e90d..fe0f34152be8c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.mapred.TaskCompletionEvent;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
@@ -137,7 +138,22 @@ public class TestRuntimeEstimators {
 
     estimator.contextualize(conf, myAppContext);
 
+    conf.setLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE, 500L);
+    conf.setLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE, 5000L);
+    conf.setDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS, 0.1);
+    conf.setDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS, 0.001);
+    conf.setInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS, 5);
     speculator = new DefaultSpeculator(conf, myAppContext, estimator, clock);
+    Assert.assertEquals("wrong SPECULATIVE_RETRY_AFTER_NO_SPECULATE value",
+        500L, speculator.getSoonestRetryAfterNoSpeculate());
+    Assert.assertEquals("wrong SPECULATIVE_RETRY_AFTER_SPECULATE value",
+        5000L, speculator.getSoonestRetryAfterSpeculate());
+    Assert.assertEquals(speculator.getProportionRunningTasksSpeculatable(),
+        0.1, 0.00001);
+    Assert.assertEquals(speculator.getProportionTotalTasksSpeculatable(),
+        0.001, 0.00001);
+    Assert.assertEquals("wrong SPECULATIVE_MINIMUM_ALLOWED_TASKS value",
+        5, speculator.getMinimumAllowedSpeculativeTasks());
 
     dispatcher.register(Speculator.EventType.class, speculator);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index b1e9cf070b43a..57573cc169f76 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -18,8 +18,10 @@
 
 package org.apache.hadoop.mapreduce.v2.app.job.impl;
 
+import java.util.ArrayList;
 import java.util.Map;
 
+import org.apache.hadoop.mapreduce.TaskType;
 import org.junit.Assert;
 
 import org.apache.commons.logging.Log;
@@ -56,8 +58,8 @@ public void testCommandLine() throws Exception {
     Assert.assertEquals(
       "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
       " -Djava.net.preferIPv4Stack=true" +
-      " -Dhadoop.metrics.log.level=WARN" +
-      "  -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
+      " -Dhadoop.metrics.log.level=WARN " +
+      "  -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
       " -Dlog4j.configuration=container-log4j.properties" +
       " -Dyarn.app.container.log.dir=<LOG_DIR>" +
       " -Dyarn.app.container.log.filesize=0" +
@@ -67,7 +69,7 @@ public void testCommandLine() throws Exception {
       " attempt_0_0000_m_000000_0" +
       " 0" +
       " 1><LOG_DIR>/stdout" +
-      " 2><LOG_DIR>/stderr ]", app.myCommandLine);
+      " 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
     
     Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
       app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
@@ -119,8 +121,8 @@ private void testReduceCommandLine(Configuration conf)
     Assert.assertEquals(
         "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
             " -Djava.net.preferIPv4Stack=true" +
-            " -Dhadoop.metrics.log.level=WARN" +
-            "  -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
+            " -Dhadoop.metrics.log.level=WARN " +
+            "  -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
             " -Dlog4j.configuration=container-log4j.properties" +
             " -Dyarn.app.container.log.dir=<LOG_DIR>" +
             " -Dyarn.app.container.log.filesize=0" +
@@ -134,7 +136,7 @@ private void testReduceCommandLine(Configuration conf)
             " attempt_0_0000_r_000000_0" +
             " 0" +
             " 1><LOG_DIR>/stdout" +
-            " 2><LOG_DIR>/stderr ]", app.myCommandLine);
+            " 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
 
     Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job",
         app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER"));
@@ -161,8 +163,8 @@ public void testCommandLineWithLog4JConifg() throws Exception {
     Assert.assertEquals(
       "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" +
       " -Djava.net.preferIPv4Stack=true" +
-      " -Dhadoop.metrics.log.level=WARN" +
-      "  -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
+      " -Dhadoop.metrics.log.level=WARN " +
+      "  -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
       " -Dlog4j.configuration=" + testLogPropertieFile +
       " -Dyarn.app.container.log.dir=<LOG_DIR>" +
       " -Dyarn.app.container.log.filesize=0" +
@@ -172,12 +174,81 @@ public void testCommandLineWithLog4JConifg() throws Exception {
       " attempt_0_0000_m_000000_0" +
       " 0" +
       " 1><LOG_DIR>/stdout" +
-      " 2><LOG_DIR>/stderr ]", app.myCommandLine);
+      " 2><LOG_DIR>/stderr ]", app.launchCmdList.get(0));
+  }
+
+  @Test
+  public void testAutoHeapSizes() throws Exception {
+    // Don't specify heap size or memory-mb
+    testAutoHeapSize(-1, -1, null);
+
+    // Don't specify heap size
+    testAutoHeapSize(512, 768, null);
+    testAutoHeapSize(100, 768, null);
+    testAutoHeapSize(512, 100, null);
+    // Specify heap size
+    testAutoHeapSize(512, 768, "-Xmx100m");
+    testAutoHeapSize(512, 768, "-Xmx500m");
+
+    // Specify heap size but not the memory
+    testAutoHeapSize(-1, -1, "-Xmx100m");
+    testAutoHeapSize(-1, -1, "-Xmx500m");
+  }
+
+  private void testAutoHeapSize(int mapMb, int redMb, String xmxArg)
+      throws Exception {
+    JobConf conf = new JobConf();
+    float heapRatio = conf.getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO,
+        MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO);
+
+    // Verify map and reduce java opts are not set by default
+    Assert.assertNull("Default map java opts!",
+        conf.get(MRJobConfig.MAP_JAVA_OPTS));
+    Assert.assertNull("Default reduce java opts!",
+        conf.get(MRJobConfig.REDUCE_JAVA_OPTS));
+    // Set the memory-mbs and java-opts
+    if (mapMb > 0) {
+      conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMb);
+    } else {
+      mapMb = conf.getMemoryRequired(TaskType.MAP);
+    }
+
+    if (redMb > 0) {
+      conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, redMb);
+    } else {
+      redMb = conf.getMemoryRequired(TaskType.REDUCE);
+    }
+    if (xmxArg != null) {
+      conf.set(MRJobConfig.MAP_JAVA_OPTS, xmxArg);
+      conf.set(MRJobConfig.REDUCE_JAVA_OPTS, xmxArg);
+    }
+
+    // Submit job to let unspecified fields be picked up
+    MyMRApp app = new MyMRApp(1, 1, true, this.getClass().getName(), true);
+    Job job = app.submit(conf);
+    app.waitForState(job, JobState.SUCCEEDED);
+    app.verifyCompleted();
+
+    // Go through the tasks and verify the values are as expected
+    for (String cmd : app.launchCmdList) {
+      final boolean isMap = cmd.contains("_m_");
+      int heapMb;
+      if (xmxArg == null) {
+        heapMb = (int)(Math.ceil((isMap ? mapMb : redMb) * heapRatio));
+      } else {
+        final String javaOpts = conf.get(isMap
+            ? MRJobConfig.MAP_JAVA_OPTS
+            : MRJobConfig.REDUCE_JAVA_OPTS);
+        heapMb = JobConf.parseMaximumHeapSizeMB(javaOpts);
+      }
+      Assert.assertEquals("Incorrect heapsize in the command opts",
+          heapMb, JobConf.parseMaximumHeapSizeMB(cmd));
+    }
   }
 
   private static final class MyMRApp extends MRApp {
 
-    private String myCommandLine;
+    private ArrayList<String> launchCmdList = new ArrayList<>();
     private Map<String, String> cmdEnvironment;
 
     public MyMRApp(int maps, int reduces, boolean autoComplete,
@@ -196,7 +267,7 @@ public void handle(ContainerLauncherEvent event) {
                 launchEvent.getContainerLaunchContext();
             String cmdString = launchContext.getCommands().toString();
             LOG.info("launchContext " + cmdString);
-            myCommandLine = cmdString;
+            launchCmdList.add(cmdString);
             cmdEnvironment = launchContext.getEnvironment();
           }
           super.handle(event);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 364267020819f..4759693a9164e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -31,6 +31,7 @@
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -45,6 +46,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -75,7 +77,9 @@
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -110,6 +114,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ControlledClock;
@@ -2295,6 +2300,93 @@ public void testRMContainerAllocatorResendsRequestsOnRMRestart()
 
   }
 
+  @Test(timeout=60000)
+  public void testAMRMTokenUpdate() throws Exception {
+    LOG.info("Running testAMRMTokenUpdate");
+
+    final String rmAddr = "somermaddress:1234";
+    final Configuration conf = new YarnConfiguration();
+    conf.setLong(
+      YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
+    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
+    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+
+    final MyResourceManager rm = new MyResourceManager(conf);
+    rm.start();
+    AMRMTokenSecretManager secretMgr =
+        rm.getRMContext().getAMRMTokenSecretManager();
+    DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+        .getDispatcher();
+
+    // Submit the application
+    RMApp app = rm.submitApp(1024);
+    dispatcher.await();
+
+    MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+    amNodeManager.nodeHeartbeat(true);
+    dispatcher.await();
+
+    final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+        .getAppAttemptId();
+    final ApplicationId appId = app.getApplicationId();
+    rm.sendAMLaunched(appAttemptId);
+    dispatcher.await();
+
+    JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+    final Job mockJob = mock(Job.class);
+    when(mockJob.getReport()).thenReturn(
+        MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
+            0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
+
+    final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
+        .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
+    Assert.assertNotNull("app should have a token", oldToken);
+    UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
+        "someuser", new String[0]);
+    Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
+        new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
+          @Override
+          public Token<AMRMTokenIdentifier> run() throws Exception {
+            MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
+                appAttemptId, mockJob);
+
+            // Keep heartbeating until RM thinks the token has been updated
+            Token<AMRMTokenIdentifier> currentToken = oldToken;
+            long startTime = Time.monotonicNow();
+            while (currentToken == oldToken) {
+              if (Time.monotonicNow() - startTime > 20000) {
+                Assert.fail("Took to long to see AMRM token change");
+              }
+              Thread.sleep(100);
+              allocator.schedule();
+              currentToken = rm.getRMContext().getRMApps().get(appId)
+                  .getRMAppAttempt(appAttemptId).getAMRMToken();
+            }
+
+            return currentToken;
+          }
+        });
+
+    // verify there is only one AMRM token in the UGI and it matches the
+    // updated token from the RM
+    int tokenCount = 0;
+    Token<? extends TokenIdentifier> ugiToken = null;
+    for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
+      if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+        ugiToken = token;
+        ++tokenCount;
+      }
+    }
+
+    Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
+    Assert.assertArrayEquals("token identifier not updated",
+        newToken.getIdentifier(), ugiToken.getIdentifier());
+    Assert.assertArrayEquals("token password not updated",
+        newToken.getPassword(), ugiToken.getPassword());
+    Assert.assertEquals("AMRM token service not updated",
+        new Text(rmAddr), ugiToken.getService());
+  }
+
   public static void main(String[] args) throws Exception {
     TestRMContainerAllocator t = new TestRMContainerAllocator();
     t.testSimple();
@@ -2304,6 +2396,7 @@ public static void main(String[] args) throws Exception {
     t.testReportedAppProgressWithOnlyMaps();
     t.testBlackListedNodes();
     t.testCompletedTasksRecalculateSchedule();
+    t.testAMRMTokenUpdate();
   }
 
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/package-info.java
index cb534304a578b..827b272c2848d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/package-info.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/package-info.java
@@ -15,6 +15,4 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
 package org.apache.hadoop.mapreduce.v2.api.protocolrecords;
-import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index e5a49b5e1bed8..f7cba9f8069e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -197,6 +197,13 @@ public class JHAdminConfig {
   public static final String MR_HS_FS_STATE_STORE_URI =
       MR_HISTORY_PREFIX + "recovery.store.fs.uri";
 
+  /**
+   * The local path where server state will be stored when
+   * HistoryServerLeveldbStateStoreService is configured as the state store
+   */
+  public static final String MR_HS_LEVELDB_STATE_STORE_PATH =
+      MR_HISTORY_PREFIX + "recovery.store.leveldb.path";
+
   /** Whether to use fixed ports with the minicluster. */
   public static final String MR_HISTORY_MINICLUSTER_FIXED_PORTS = MR_HISTORY_PREFIX
        + "minicluster.fixed.ports";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 0ae56717ab988..5e45b49e9ecec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -28,6 +28,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -45,9 +46,9 @@
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.StringUtils;
 
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.Iterables;
 
 /** 
@@ -223,7 +224,7 @@ protected FileStatus[] listStatus(JobConf job) throws IOException {
             org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS,
             org.apache.hadoop.mapreduce.lib.input.FileInputFormat.DEFAULT_LIST_STATUS_NUM_THREADS);
     
-    Stopwatch sw = new Stopwatch().start();
+    StopWatch sw = new StopWatch().start();
     if (numThreads == 1) {
       List<FileStatus> locatedFiles = singleThreadedListStatus(job, dirs, inputFilter, recursive); 
       result = locatedFiles.toArray(new FileStatus[locatedFiles.size()]);
@@ -242,7 +243,8 @@ protected FileStatus[] listStatus(JobConf job) throws IOException {
 
     sw.stop();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Time taken to get FileStatuses: " + sw.elapsedMillis());
+      LOG.debug("Time taken to get FileStatuses: "
+          + sw.now(TimeUnit.MILLISECONDS));
     }
     LOG.info("Total input paths to process : " + result.length);
     return result;
@@ -309,7 +311,7 @@ protected FileSplit makeSplit(Path file, long start, long length,
    * they're too big.*/ 
   public InputSplit[] getSplits(JobConf job, int numSplits)
     throws IOException {
-    Stopwatch sw = new Stopwatch().start();
+    StopWatch sw = new StopWatch().start();
     FileStatus[] files = listStatus(job);
     
     // Save the number of input files for metrics/loadgen
@@ -371,7 +373,7 @@ public InputSplit[] getSplits(JobConf job, int numSplits)
     sw.stop();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Total # of splits generated by getSplits: " + splits.size()
-          + ", TimeTaken: " + sw.elapsedMillis());
+          + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
     }
     return splits.toArray(new FileSplit[splits.size()]);
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 03f1160e9fe4a..98a643f112d09 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -20,8 +20,10 @@
 
 
 import java.io.IOException;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -44,6 +46,7 @@
 import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.util.ConfigUtil;
 import org.apache.hadoop.security.Credentials;
@@ -114,6 +117,8 @@
 public class JobConf extends Configuration {
 
   private static final Log LOG = LogFactory.getLog(JobConf.class);
+  private static final Pattern JAVA_OPTS_XMX_PATTERN =
+          Pattern.compile(".*(?:^|\\s)-Xmx(\\d+)([gGmMkK]?)(?:$|\\s).*");
 
   static{
     ConfigUtil.loadResources();
@@ -247,9 +252,9 @@ public class JobConf extends Configuration {
    */
   public static final String MAPRED_REDUCE_TASK_JAVA_OPTS = 
     JobContext.REDUCE_JAVA_OPTS;
-  
-  public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m";
-  
+
+  public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "";
+
   /**
    * @deprecated
    * Configuration key to set the maximum virtual memory available to the child
@@ -2022,7 +2027,123 @@ private void checkAndWarnDeprecation() {
       LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT));
     }
   }
-  
+
+  private String getConfiguredTaskJavaOpts(TaskType taskType) {
+    String userClasspath = "";
+    String adminClasspath = "";
+    if (taskType == TaskType.MAP) {
+      userClasspath = get(MAPRED_MAP_TASK_JAVA_OPTS,
+          get(MAPRED_TASK_JAVA_OPTS, DEFAULT_MAPRED_TASK_JAVA_OPTS));
+      adminClasspath = get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
+          MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+    } else {
+      userClasspath = get(MAPRED_REDUCE_TASK_JAVA_OPTS,
+          get(MAPRED_TASK_JAVA_OPTS, DEFAULT_MAPRED_TASK_JAVA_OPTS));
+      adminClasspath = get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
+          MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS);
+    }
+
+    return adminClasspath + " " + userClasspath;
+  }
+
+  @Private
+  public String getTaskJavaOpts(TaskType taskType) {
+    String javaOpts = getConfiguredTaskJavaOpts(taskType);
+
+    if (!javaOpts.contains("-Xmx")) {
+      float heapRatio = getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO,
+          MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO);
+
+      if (heapRatio > 1.0f || heapRatio < 0) {
+        LOG.warn("Invalid value for " + MRJobConfig.HEAP_MEMORY_MB_RATIO
+            + ", using the default.");
+        heapRatio = MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO;
+      }
+
+      int taskContainerMb = getMemoryRequired(taskType);
+      int taskHeapSize = (int)Math.ceil(taskContainerMb * heapRatio);
+
+      String xmxArg = String.format("-Xmx%dm", taskHeapSize);
+      LOG.info("Task java-opts do not specify heap size. Setting task attempt" +
+          " jvm max heap size to " + xmxArg);
+
+      javaOpts += " " + xmxArg;
+    }
+
+    return javaOpts;
+  }
+
+  /**
+   * Parse the Maximum heap size from the java opts as specified by the -Xmx option
+   * Format: -Xmx<size>[g|G|m|M|k|K]
+   * @param javaOpts String to parse to read maximum heap size
+   * @return Maximum heap size in MB or -1 if not specified
+   */
+  @Private
+  @VisibleForTesting
+  public static int parseMaximumHeapSizeMB(String javaOpts) {
+    // Find the last matching -Xmx following word boundaries
+    Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts);
+    if (m.matches()) {
+      int size = Integer.parseInt(m.group(1));
+      if (size <= 0) {
+        return -1;
+      }
+      if (m.group(2).isEmpty()) {
+        // -Xmx specified in bytes
+        return size / (1024 * 1024);
+      }
+      char unit = m.group(2).charAt(0);
+      switch (unit) {
+        case 'g':
+        case 'G':
+          // -Xmx specified in GB
+          return size * 1024;
+        case 'm':
+        case 'M':
+          // -Xmx specified in MB
+          return size;
+        case 'k':
+        case 'K':
+          // -Xmx specified in KB
+          return size / 1024;
+      }
+    }
+    // -Xmx not specified
+    return -1;
+  }
+
+  private int getMemoryRequiredHelper(
+      String configName, int defaultValue, int heapSize, float heapRatio) {
+    int memory = getInt(configName, -1);
+    if (memory <= 0) {
+      if (heapSize > 0) {
+        memory = (int) Math.ceil(heapSize / heapRatio);
+        LOG.info("Figured value for " + configName + " from javaOpts");
+      } else {
+        memory = defaultValue;
+      }
+    }
+
+    return memory;
+  }
+
+  @Private
+  public int getMemoryRequired(TaskType taskType) {
+    int memory = 1024;
+    int heapSize = parseMaximumHeapSizeMB(getConfiguredTaskJavaOpts(taskType));
+    float heapRatio = getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO,
+        MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO);
+    if (taskType == TaskType.MAP) {
+      return getMemoryRequiredHelper(MRJobConfig.MAP_MEMORY_MB,
+          MRJobConfig.DEFAULT_MAP_MEMORY_MB, heapSize, heapRatio);
+    } else if (taskType == TaskType.REDUCE) {
+      return getMemoryRequiredHelper(MRJobConfig.REDUCE_MEMORY_MB,
+          MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, heapSize, heapRatio);
+    } else {
+      return memory;
+    }
+  }
 
 }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 75b414196d8c4..1a4901b257d1a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -387,6 +387,7 @@ private <T> T getSplitDetails(Path file, long offset)
     Class<?>[] collectorClasses = job.getClasses(
       JobContext.MAP_OUTPUT_COLLECTOR_CLASS_ATTR, MapOutputBuffer.class);
     int remainingCollectors = collectorClasses.length;
+    Exception lastException = null;
     for (Class clazz : collectorClasses) {
       try {
         if (!MapOutputCollector.class.isAssignableFrom(clazz)) {
@@ -406,10 +407,12 @@ private <T> T getSplitDetails(Path file, long offset)
         if (--remainingCollectors > 0) {
           msg += " (" + remainingCollectors + " more collector(s) to try)";
         }
+        lastException = e;
         LOG.warn(msg, e);
       }
     }
-    throw new IOException("Unable to initialize any output collector");
+    throw new IOException("Initialization of all the collectors failed. " +
+      "Error in last collector was :" + lastException.getMessage(), lastException);
   }
 
   @SuppressWarnings("unchecked")
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 3a4c513f3f16b..5274438782c13 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -34,6 +34,7 @@
 
 import javax.crypto.SecretKey;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -624,8 +625,9 @@ public class TaskReporter
      * Using AtomicBoolean since we need an atomic read & reset method. 
      */  
     private AtomicBoolean progressFlag = new AtomicBoolean(false);
-    
-    TaskReporter(Progress taskProgress,
+
+    @VisibleForTesting
+    public TaskReporter(Progress taskProgress,
                  TaskUmbilicalProtocol umbilical) {
       this.umbilical = umbilical;
       this.taskProgress = taskProgress;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/jobcontrol/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/jobcontrol/Job.java
index dba17b96c0750..e79ab87c28137 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/jobcontrol/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/jobcontrol/Job.java
@@ -51,7 +51,7 @@ public class Job extends ControlledJob {
    */
   @SuppressWarnings("unchecked")
   public Job(JobConf jobConf, ArrayList<?> dependingJobs) throws IOException {
-    super(new org.apache.hadoop.mapreduce.Job(jobConf), 
+    super(org.apache.hadoop.mapreduce.Job.getInstance(jobConf),
           (List<ControlledJob>) dependingJobs);
   }
 
@@ -93,7 +93,7 @@ public synchronized JobConf getJobConf() {
    */
   public synchronized void setJobConf(JobConf jobConf) {
     try {
-      super.setJob(new org.apache.hadoop.mapreduce.Job(jobConf));
+      super.setJob(org.apache.hadoop.mapreduce.Job.getInstance(jobConf));
     } catch (IOException ioe) { 
       LOG.info("Exception" + ioe);
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
index b9297f851bb90..da1db70fd3e5b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
@@ -73,7 +73,7 @@ public CombineFileInputFormat() {
   public InputSplit[] getSplits(JobConf job, int numSplits) 
     throws IOException {
     List<org.apache.hadoop.mapreduce.InputSplit> newStyleSplits =
-      super.getSplits(new Job(job));
+      super.getSplits(Job.getInstance(job));
     InputSplit[] ret = new InputSplit[newStyleSplits.size()];
     for(int pos = 0; pos < newStyleSplits.size(); ++pos) {
       org.apache.hadoop.mapreduce.lib.input.CombineFileSplit newStyleSplit = 
@@ -129,7 +129,7 @@ public org.apache.hadoop.mapreduce.RecordReader<K, V> createRecordReader(
    * @throws IOException if zero items.
    */
   protected FileStatus[] listStatus(JobConf job) throws IOException {
-    List<FileStatus> result = super.listStatus(new Job(job));
+    List<FileStatus> result = super.listStatus(Job.getInstance(job));
     return result.toArray(new FileStatus[result.size()]);
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/InputSampler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/InputSampler.java
index a55abe646ccae..d378bcac79aac 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/InputSampler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/InputSampler.java
@@ -46,7 +46,7 @@ public InputSampler(JobConf conf) {
 
   public static <K,V> void writePartitionFile(JobConf job, Sampler<K,V> sampler)
       throws IOException, ClassNotFoundException, InterruptedException {
-    writePartitionFile(new Job(job), sampler);
+    writePartitionFile(Job.getInstance(job), sampler);
   }
   /**
    * Interface to sample using an {@link org.apache.hadoop.mapred.InputFormat}.
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/db/DBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/db/DBInputFormat.java
index 9b32530558eb5..2715705407ef6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/db/DBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/db/DBInputFormat.java
@@ -177,7 +177,7 @@ public RecordReader<LongWritable, T> getRecordReader(InputSplit split,
   /** {@inheritDoc} */
   public InputSplit[] getSplits(JobConf job, int chunks) throws IOException {
     List<org.apache.hadoop.mapreduce.InputSplit> newSplits = 
-      super.getSplits(new Job(job));
+      super.getSplits(Job.getInstance(job));
     InputSplit[] ret = new InputSplit[newSplits.size()];
     int i = 0;
     for (org.apache.hadoop.mapreduce.InputSplit s : newSplits) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index cfc34377d41ac..493ca5f6ea5e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -120,7 +120,7 @@ public static enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL }
    */
   @Deprecated
   public Job() throws IOException {
-    this(new Configuration());
+    this(new JobConf(new Configuration()));
   }
 
   /**
@@ -136,7 +136,7 @@ public Job(Configuration conf) throws IOException {
    */
   @Deprecated
   public Job(Configuration conf, String jobName) throws IOException {
-    this(conf);
+    this(new JobConf(conf));
     setJobName(jobName);
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 230361ca80e0e..d06b075e2f478 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -86,12 +86,41 @@ public interface MRJobConfig {
 
   public static final String SKIP_OUTDIR = "mapreduce.job.skip.outdir";
 
+  // SPECULATIVE_SLOWNODE_THRESHOLD is obsolete and will be deleted in the future
+  @Deprecated
   public static final String SPECULATIVE_SLOWNODE_THRESHOLD = "mapreduce.job.speculative.slownodethreshold";
 
   public static final String SPECULATIVE_SLOWTASK_THRESHOLD = "mapreduce.job.speculative.slowtaskthreshold";
 
+  // SPECULATIVECAP is obsolete and will be deleted in the future
+  @Deprecated
   public static final String SPECULATIVECAP = "mapreduce.job.speculative.speculativecap";
 
+  public static final String SPECULATIVECAP_RUNNING_TASKS =
+      "mapreduce.job.speculative.speculative-cap-running-tasks";
+  public static final double DEFAULT_SPECULATIVECAP_RUNNING_TASKS =
+      0.1;
+
+  public static final String SPECULATIVECAP_TOTAL_TASKS =
+      "mapreduce.job.speculative.speculative-cap-total-tasks";
+  public static final double DEFAULT_SPECULATIVECAP_TOTAL_TASKS =
+      0.01;
+
+  public static final String SPECULATIVE_MINIMUM_ALLOWED_TASKS =
+      "mapreduce.job.speculative.minimum-allowed-tasks";
+  public static final int DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS =
+      10;
+
+  public static final String SPECULATIVE_RETRY_AFTER_NO_SPECULATE =
+      "mapreduce.job.speculative.retry-after-no-speculate";
+  public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE =
+      1000L;
+
+  public static final String SPECULATIVE_RETRY_AFTER_SPECULATE =
+      "mapreduce.job.speculative.retry-after-speculate";
+  public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE =
+      15000L;
+
   public static final String JOB_LOCAL_DIR = "mapreduce.job.local.dir";
 
   public static final String OUTPUT_KEY_CLASS = "mapreduce.job.output.key.class";
@@ -166,8 +195,6 @@ public interface MRJobConfig {
 
   public static final String PRESERVE_FILES_PATTERN = "mapreduce.task.files.preserve.filepattern";
 
-  public static final String TASK_TEMP_DIR = "mapreduce.task.tmp.dir";
-
   public static final String TASK_DEBUGOUT_LINES = "mapreduce.task.debugout.lines";
 
   public static final String RECORDS_BEFORE_PROGRESS = "mapreduce.task.merge.progress.records";
@@ -812,6 +839,11 @@ public interface MRJobConfig {
   public static final String TASK_PREEMPTION =
       "mapreduce.job.preemption";
 
+  public static final String HEAP_MEMORY_MB_RATIO =
+      "mapreduce.job.heap.memory-mb.ratio";
+
+  public static final float DEFAULT_HEAP_MEMORY_MB_RATIO = 0.8f;
+
   public static final String MR_ENCRYPTED_INTERMEDIATE_DATA =
       "mapreduce.job.encrypted-intermediate-data";
   public static final boolean DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA = false;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
index 401bbb2502cbd..dd81ebb55f9b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
@@ -307,6 +307,10 @@ public synchronized void readFields(DataInput in) throws IOException {
       fgroups.put(group.getName(), group);
     }
     int numGroups = WritableUtils.readVInt(in);
+    if (!groups.isEmpty()) {
+      groups.clear();
+      limits.reset();
+    }
     while (numGroups-- > 0) {
       limits.checkGroups(groups.size() + 1);
       G group = groupFactory.newGenericGroup(
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
index 3821694b2fb17..9546c8d7632c6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
@@ -124,8 +124,15 @@ public synchronized LimitExceededException violation() {
     return firstViolation;
   }
 
+  // This allows initialization of global settings and not for an instance
   public static synchronized void reset(Configuration conf) {
     isInited = false;
     init(conf);
   }
+
+  // This allows resetting of an instance to allow reuse
+  public synchronized void reset() {
+    totalCounters = 0;
+    firstViolation = null;
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/package-info.java
index 56d57d010443c..8f6df2c14f747 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/package-info.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/package-info.java
@@ -22,9 +22,7 @@
  *
  * cf. MAPREDUCE-901 for rationales.
  */
-@InterfaceAudience.Private
 @InterfaceStability.Evolving
 package org.apache.hadoop.mapreduce.counters;
 
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
index 9242c82d4e8c8..d8833da52558c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
@@ -164,7 +164,7 @@ public static Job createValueAggregatorJob(Configuration conf, String args[])
       conf.set(MRJobConfig.JAR, userJarFile);
     }
 
-    Job theJob = new Job(conf);
+    Job theJob = Job.getInstance(conf);
     if (userJarFile == null) {
       theJob.setJarByClass(ValueAggregator.class);
     } 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/DelegatingInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/DelegatingInputFormat.java
index 964419c805746..35f18056001ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/DelegatingInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/DelegatingInputFormat.java
@@ -53,7 +53,7 @@ public class DelegatingInputFormat<K, V> extends InputFormat<K, V> {
   public List<InputSplit> getSplits(JobContext job) 
       throws IOException, InterruptedException {
     Configuration conf = job.getConfiguration();
-    Job jobCopy =new Job(conf);
+    Job jobCopy = Job.getInstance(conf);
     List<InputSplit> splits = new ArrayList<InputSplit>();
     Map<Path, InputFormat> formatMap = 
       MultipleInputs.getInputFormatMap(job);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index 56fb9fcdf1152..a3ffe019c8f9b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import java.util.concurrent.TimeUnit;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,9 +44,9 @@
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StopWatch;
 import org.apache.hadoop.util.StringUtils;
 
-import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 
 /** 
@@ -259,7 +260,7 @@ protected List<FileStatus> listStatus(JobContext job
 
     int numThreads = job.getConfiguration().getInt(LIST_STATUS_NUM_THREADS,
         DEFAULT_LIST_STATUS_NUM_THREADS);
-    Stopwatch sw = new Stopwatch().start();
+    StopWatch sw = new StopWatch().start();
     if (numThreads == 1) {
       result = singleThreadedListStatus(job, dirs, inputFilter, recursive);
     } else {
@@ -276,7 +277,8 @@ protected List<FileStatus> listStatus(JobContext job
     
     sw.stop();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Time taken to get FileStatuses: " + sw.elapsedMillis());
+      LOG.debug("Time taken to get FileStatuses: "
+          + sw.now(TimeUnit.MILLISECONDS));
     }
     LOG.info("Total input paths to process : " + result.size()); 
     return result;
@@ -376,7 +378,7 @@ protected FileSplit makeSplit(Path file, long start, long length,
    * @throws IOException
    */
   public List<InputSplit> getSplits(JobContext job) throws IOException {
-    Stopwatch sw = new Stopwatch().start();
+    StopWatch sw = new StopWatch().start();
     long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
     long maxSize = getMaxSplitSize(job);
 
@@ -427,7 +429,7 @@ public List<InputSplit> getSplits(JobContext job) throws IOException {
     sw.stop();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Total # of splits generated by getSplits: " + splits.size()
-          + ", TimeTaken: " + sw.elapsedMillis());
+          + ", TimeTaken: " + sw.now(TimeUnit.MILLISECONDS));
     }
     return splits;
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/ControlledJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/ControlledJob.java
index f16092ecce7f3..4d5be68dc4a70 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/ControlledJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/ControlledJob.java
@@ -84,7 +84,7 @@ public ControlledJob(Job job, List<ControlledJob> dependingJobs)
    * @throws IOException
    */
   public ControlledJob(Configuration conf) throws IOException {
-    this(new Job(conf), null);
+    this(Job.getInstance(conf), null);
   }
 	
   @Override
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/Parser.java
index 275272b3c7c93..c557e14136622 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/Parser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/join/Parser.java
@@ -323,7 +323,7 @@ public void parse(List<Token> ll, Configuration conf) throws IOException {
     }
 
     private Configuration getConf(Configuration jconf) throws IOException {
-      Job job = new Job(jconf);
+      Job job = Job.getInstance(jconf);
       FileInputFormat.setInputPaths(job, indir);
       return job.getConfiguration();
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
index 7974b78fb8901..24baa596b31ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
@@ -503,7 +503,7 @@ private TaskAttemptContext getContext(String nameOutput) throws IOException {
     
     // The following trick leverages the instantiation of a record writer via
     // the job thus supporting arbitrary output formats.
-    Job job = new Job(context.getConfiguration());
+    Job job = Job.getInstance(context.getConfiguration());
     job.setOutputFormatClass(getNamedOutputFormatClass(context, nameOutput));
     job.setOutputKeyClass(getNamedOutputKeyClass(context, nameOutput));
     job.setOutputValueClass(getNamedOutputValueClass(context, nameOutput));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java
index e709a2823aa17..4668f49cc236a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java
@@ -348,7 +348,7 @@ public static <K,V> void writePartitionFile(Job job, Sampler<K,V> sampler)
    * Configures a JobConf instance and calls {@link #writePartitionFile}.
    */
   public int run(String[] args) throws Exception {
-    Job job = new Job(getConf());
+    Job job = Job.getInstance(getConf());
     ArrayList<String> otherArgs = new ArrayList<String>();
     Sampler<K,V> sampler = null;
     for(int i=0; i < args.length; ++i) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
index 632abdfc064ab..f2619d7790513 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
@@ -83,7 +83,7 @@ public void setConf(Configuration conf) {
         ? FileSystem.getLocal(conf)     // assume in DistributedCache
         : partFile.getFileSystem(conf);
 
-      Job job = new Job(conf);
+      Job job = Job.getInstance(conf);
       Class<K> keyClass = (Class<K>)job.getMapOutputKeyClass();
       K[] splitPoints = readPartitions(fs, partFile, keyClass, conf);
       if (splitPoints.length != job.getNumReduceTasks() - 1) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java
index 6e0e92bd4df06..8275fd0eba3c1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -32,6 +33,7 @@
 
 import org.apache.hadoop.io.IOUtils;
 
+import org.apache.hadoop.mapred.IFileInputStream;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.MapOutputFile;
@@ -52,6 +54,7 @@ class OnDiskMapOutput<K, V> extends MapOutput<K, V> {
   private final MergeManagerImpl<K, V> merger;
   private final OutputStream disk; 
   private long compressedSize;
+  private final Configuration conf;
 
   public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId,
                          MergeManagerImpl<K,V> merger, long size,
@@ -60,7 +63,7 @@ public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId,
                          int fetcher, boolean primaryMapOutput)
       throws IOException {
     this(mapId, reduceId, merger, size, conf, mapOutputFile, fetcher,
-        primaryMapOutput, FileSystem.getLocal(conf),
+        primaryMapOutput, FileSystem.getLocal(conf).getRaw(),
         mapOutputFile.getInputFileForWrite(mapId.getTaskID(), size));
   }
 
@@ -77,6 +80,7 @@ public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId,
     this.outputPath = outputPath;
     tmpOutputPath = getTempPath(outputPath, fetcher);
     disk = CryptoUtils.wrapIfNecessary(conf, fs.create(tmpOutputPath));
+    this.conf = conf;
   }
 
   @VisibleForTesting
@@ -89,13 +93,14 @@ public void shuffle(MapHost host, InputStream input,
                       long compressedLength, long decompressedLength,
                       ShuffleClientMetrics metrics,
                       Reporter reporter) throws IOException {
+    input = new IFileInputStream(input, compressedLength, conf);
     // Copy data to local-disk
     long bytesLeft = compressedLength;
     try {
       final int BYTES_TO_READ = 64 * 1024;
       byte[] buf = new byte[BYTES_TO_READ];
       while (bytesLeft > 0) {
-        int n = input.read(buf, 0, (int) Math.min(bytesLeft, BYTES_TO_READ));
+        int n = ((IFileInputStream)input).readWithChecksum(buf, 0, (int) Math.min(bytesLeft, BYTES_TO_READ));
         if (n < 0) {
           throw new IOException("read past end of stream reading " + 
                                 getMapId());
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
index 8c7952bc87fc0..bbbf99c4beb11 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
@@ -214,12 +214,10 @@ private static void addDeprecatedKeys()  {
         MRJobConfig.SKIP_RECORDS),
       new DeprecationDelta("mapred.skip.out.dir",
         MRJobConfig.SKIP_OUTDIR),
-      new DeprecationDelta("mapred.speculative.execution.slowNodeThreshold",
-        MRJobConfig.SPECULATIVE_SLOWNODE_THRESHOLD),
       new DeprecationDelta("mapred.speculative.execution.slowTaskThreshold",
         MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD),
       new DeprecationDelta("mapred.speculative.execution.speculativeCap",
-        MRJobConfig.SPECULATIVECAP),
+        MRJobConfig.SPECULATIVECAP_RUNNING_TASKS),
       new DeprecationDelta("job.local.dir",
         MRJobConfig.JOB_LOCAL_DIR),
       new DeprecationDelta("mapreduce.inputformat.class",
@@ -274,8 +272,6 @@ private static void addDeprecatedKeys()  {
         MRJobConfig.PRESERVE_FAILED_TASK_FILES),
       new DeprecationDelta("keep.task.files.pattern",
         MRJobConfig.PRESERVE_FILES_PATTERN),
-      new DeprecationDelta("mapred.child.tmp",
-        MRJobConfig.TASK_TEMP_DIR),
       new DeprecationDelta("mapred.debug.out.lines",
         MRJobConfig.TASK_DEBUGOUT_LINES),
       new DeprecationDelta("mapred.merge.recordsBeforeProgress",
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 43ddb1311d5a4..6e8067976f49d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -208,9 +208,11 @@
 
 <property>
   <name>mapreduce.map.memory.mb</name>
-  <value>1024</value>
+  <value>-1</value>
   <description>The amount of memory to request from the scheduler for each
-  map task.
+    map task. If this is not specified or is non-positive, it is inferred from
+    mapreduce.map.java.opts and mapreduce.job.heap.memory-mb.ratio.
+    If java-opts are also not specified, we set it to 1024.
   </description>
 </property>
 
@@ -224,9 +226,11 @@
 
 <property>
   <name>mapreduce.reduce.memory.mb</name>
-  <value>1024</value>
+  <value>-1</value>
   <description>The amount of memory to request from the scheduler for each
-  reduce task.
+    reduce task. If this is not specified or is non-positive, it is inferred
+    from mapreduce.reduce.java.opts and mapreduce.job.heap.memory-mb.ratio.
+    If java-opts are also not specified, we set it to 1024.
   </description>
 </property>
 
@@ -240,7 +244,7 @@
 
 <property>
   <name>mapred.child.java.opts</name>
-  <value>-Xmx200m</value>
+  <value></value>
   <description>Java opts for the task processes.
   The following symbol, if present, will be interpolated: @taskid@ is replaced 
   by current TaskID. Any other occurrences of '@' will go unchanged.
@@ -251,7 +255,10 @@
   Usage of -Djava.library.path can cause programs to no longer function if
   hadoop native libraries are used. These values should instead be set as part 
   of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and 
-  mapreduce.reduce.env config settings. 
+  mapreduce.reduce.env config settings.
+
+  If -Xmx is not set, it is inferred from mapreduce.{map|reduce}.memory.mb and
+  mapreduce.job.heap.memory-mb.ratio.
   </description>
 </property>
 
@@ -260,7 +267,9 @@
   <name>mapreduce.map.java.opts</name>
   <value></value>
   <description>Java opts only for the child processes that are maps. If set,
-  this will be used instead of mapred.child.java.opts.
+  this will be used instead of mapred.child.java.opts. If -Xmx is not set,
+  it is inferred from mapreduce.map.memory.mb and
+  mapreduce.job.heap.memory-mb.ratio.
   </description>
 </property>
 -->
@@ -270,7 +279,9 @@
   <name>mapreduce.reduce.java.opts</name>
   <value></value>
   <description>Java opts only for the child processes that are reduces. If set,
-  this will be used instead of mapred.child.java.opts.
+  this will be used instead of mapred.child.java.opts. If -Xmx is not set,
+  it is inferred from mapreduce.reduce.memory.mb and
+  mapreduce.job.heap.memory-mb.ratio.
   </description>
 </property>
 -->
@@ -319,23 +330,13 @@
   </description>
 </property>
 
-<property>
-  <name>mapreduce.task.tmp.dir</name>
-  <value>./tmp</value>
-  <description> To set the value of tmp directory for map and reduce tasks.
-  If the value is an absolute path, it is directly assigned. Otherwise, it is
-  prepended with task's working directory. The java tasks are executed with
-  option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and
-  streaming are set with environment variable,
-   TMPDIR='the absolute path of the tmp dir'
-  </description>
-</property>
-
 <property>
   <name>mapreduce.map.log.level</name>
   <value>INFO</value>
   <description>The logging level for the map task. The allowed levels are:
   OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  The setting here could be overridden if "mapreduce.job.log4j-properties-file"
+  is set.
   </description>
 </property>
 
@@ -344,6 +345,8 @@
   <value>INFO</value>
   <description>The logging level for the reduce task. The allowed levels are:
   OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  The setting here could be overridden if "mapreduce.job.log4j-properties-file"
+  is set.
   </description>
 </property>
 
@@ -487,13 +490,42 @@
   <description>If true, then multiple instances of some reduce tasks 
                may be executed in parallel.</description>
 </property>
+
 <property>
-  <name>mapreduce.job.speculative.speculativecap</name>
+  <name>mapreduce.job.speculative.speculative-cap-running-tasks</name>
   <value>0.1</value>
   <description>The max percent (0-1) of running tasks that
   can be speculatively re-executed at any time.</description>
 </property>
 
+<property>
+  <name>mapreduce.job.speculative.speculative-cap-total-tasks</name>
+  <value>0.01</value>
+  <description>The max percent (0-1) of all tasks that
+  can be speculatively re-executed at any time.</description>
+</property>
+
+<property>
+  <name>mapreduce.job.speculative.minimum-allowed-tasks</name>
+  <value>10</value>
+  <description>The minimum allowed tasks that
+  can be speculatively re-executed at any time.</description>
+</property>
+
+<property>
+  <name>mapreduce.job.speculative.retry-after-no-speculate</name>
+  <value>1000</value>
+  <description>The waiting time(ms) to do next round of speculation
+  if there is no task speculated in this round.</description>
+</property>
+
+<property>
+  <name>mapreduce.job.speculative.retry-after-speculate</name>
+  <value>15000</value>
+  <description>The waiting time(ms) to do next round of speculation
+  if there are tasks speculated in this round.</description>
+</property>
+
 <property>
   <name>mapreduce.job.map.output.collector.class</name>
   <value>org.apache.hadoop.mapred.MapTask$MapOutputBuffer</value>
@@ -513,16 +545,6 @@
   </description>
 </property>
 
-<property>
-  <name>mapreduce.job.speculative.slownodethreshold</name>
-  <value>1.0</value>
-  <description>The number of standard deviations by which a Task 
-  Tracker's average map and reduce progress-rates (finishTime-dispatchTime)
-  must be lower than the average of all successful map/reduce task's for
-  the NodeManager to be considered too slow to give a speculative task to.
-  </description>
-</property>
-
 <property>
   <name>mapreduce.job.ubertask.enable</name>
   <value>false</value>
@@ -1175,6 +1197,17 @@
   </description>
 </property>
 
+  <property>
+    <name>mapreduce.job.log4j-properties-file</name>
+    <value></value>
+    <description>Used to override the default settings of log4j in container-log4j.properties
+    for NodeManager. Like container-log4j.properties, it requires certain
+    framework appenders properly defined in this overriden file. The file on the
+    path will be added to distributed cache and classpath. If no-scheme is given
+    in the path, it defaults to point to a log4j file on the local FS.
+    </description>
+  </property>
+
 <property>
   <name>mapreduce.job.end-notification.max.retry.interval</name>
   <value>5000</value>
@@ -1375,10 +1408,23 @@
    <value></value>
   <description>Used to override the default definition of the system classes for
     the job classloader. The system classes are a comma-separated list of
-    classes that should be loaded from the system classpath, not the
-    user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending
-    in '.' (period) are treated as package names, and names starting with a '-'
-    are treated as negative matches.
+    patterns that indicate whether to load a class from the system classpath,
+    instead from the user-supplied JARs, when mapreduce.job.classloader is
+    enabled.
+
+    A positive pattern is defined as:
+        1. A single class name 'C' that matches 'C' and transitively all nested
+            classes 'C$*' defined in C;
+        2. A package name ending with a '.' (e.g., "com.example.") that matches
+            all classes from that package.
+    A negative pattern is defined by a '-' in front of a positive pattern
+    (e.g., "-com.example.").
+
+    A class is considered a system class if and only if it matches one of the
+    positive patterns and none of the negative ones. More formally:
+    A class is a member of the inclusion set I if it matches one of the positive
+    patterns. A class is a member of the exclusion set E if it matches one of
+    the negative patterns. The set of system classes S = I \ E.
   </description>
 </property>
 
@@ -1541,6 +1587,14 @@
   storage class.</description>
 </property>
 
+<property>
+  <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
+  <value>${hadoop.tmp.dir}/mapred/history/recoverystore</value>
+  <description>The URI where history server state will be stored if
+  HistoryServerLeveldbSystemStateStoreService is configured as the recovery
+  storage class.</description>
+</property>
+
 <property>
   <name>mapreduce.jobhistory.http.policy</name>
   <value>HTTP_ONLY</value>
@@ -1551,4 +1605,15 @@
     - HTTPS_ONLY : Service is provided only on https
   </description>
 </property>
+
+<property>
+  <name>mapreduce.job.heap.memory-mb.ratio</name>
+  <value>0.8</value>
+  <description>The ratio of heap-size to container-size. If no -Xmx is
+    specified, it is calculated as
+    (mapreduce.{map|reduce}.memory.mb * mapreduce.heap.memory-mb.ratio).
+    If -Xmx is specified but not mapreduce.{map|reduce}.memory.mb, it is
+    calculated as (heapSize / mapreduce.heap.memory-mb.ratio).
+  </description>
+</property>
 </configuration>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
index 9cb74038d13d8..2195e103a82d0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
@@ -116,3 +116,36 @@ Hadoop MapReduce Next Generation - Distributed Cache Deploy
   change to something like the following:
 
     <<<$HADOOP_CONF_DIR,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/lib/*,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*>>>
+
+** NOTE: 
+
+  If shuffle encryption is also enabled in the cluster, then we could meet the problem that MR job get failed with exception like below: 
+  
++---+
+2014-10-10 02:17:16,600 WARN [fetcher#1] org.apache.hadoop.mapreduce.task.reduce.Fetcher: Failed to connect to junpingdu-centos5-3.cs1cloud.internal:13562 with 1 map outputs
+javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
+    at com.sun.net.ssl.internal.ssl.Alerts.getSSLException(Alerts.java:174)
+    at com.sun.net.ssl.internal.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1731)
+    at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:241)
+    at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:235)
+    at com.sun.net.ssl.internal.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1206)
+    at com.sun.net.ssl.internal.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:136)
+    at com.sun.net.ssl.internal.ssl.Handshaker.processLoop(Handshaker.java:593)
+    at com.sun.net.ssl.internal.ssl.Handshaker.process_record(Handshaker.java:529)
+    at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:925)
+    at com.sun.net.ssl.internal.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1170)
+    at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1197)
+    at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1181)
+    at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:434)
+    at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:81)
+    at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:61)
+    at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:584)
+    at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1193)
+    at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:379)
+    at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:318)
+    at org.apache.hadoop.mapreduce.task.reduce.Fetcher.verifyConnection(Fetcher.java:427)
+....
+
++---+
+
+  This is because MR client (deployed from HDFS) cannot access ssl-client.xml in local FS under directory of $HADOOP_CONF_DIR. To fix the problem, we can add the directory with ssl-client.xml to the classpath of MR which is specified in "mapreduce.application.classpath" as mentioned above. To avoid MR application being affected by other local configurations, it is better to create a dedicated directory for putting ssl-client.xml, e.g. a sub-directory under $HADOOP_CONF_DIR, like: $HADOOP_CONF_DIR/security.
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
index da412df7877d0..1761ad89a17c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
@@ -50,9 +50,9 @@ Hadoop MapReduce Next Generation - Encrypted Shuffle
 *--------------------------------------+---------------------+-----------------+
 | <<<hadoop.ssl.keystores.factory.class>>> | <<<org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory>>> | The KeyStoresFactory implementation to use |
 *--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.server.conf>>>         | <<<ss-server.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
+| <<<hadoop.ssl.server.conf>>>         | <<<ssl-server.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
 *--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.client.conf>>>         | <<<ss-client.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
+| <<<hadoop.ssl.client.conf>>>         | <<<ssl-client.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
 *--------------------------------------+---------------------+-----------------+
 | <<<hadoop.ssl.enabled.protocols>>>   | <<<TLSv1>>>         | The supported SSL protocols (JDK6 can use <<TLSv1>>, JDK7+ can use <<TLSv1,TLSv1.1,TLSv1.2>>) |
 *--------------------------------------+---------------------+-----------------+
@@ -202,7 +202,7 @@ Hadoop MapReduce Next Generation - Encrypted Shuffle
 
 ** <<<ssl-client.xml>>> (Reducer/Fetcher) Configuration:
 
-  The mapred user should own the <<ssl-server.xml>> file and it should have
+  The mapred user should own the <<ssl-client.xml>> file and it should have
   default permissions.
 
 *---------------------------------------------+---------------------+-----------------+
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
index a7a87c9ed0c7b..4c94e59ef9cc5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
@@ -106,6 +106,27 @@ public void testBzip2SplitEndsAtCRThenLF() throws IOException {
     testSplitRecords("blockEndingInCRThenLF.txt.bz2", 136498);
   }
 
+  //This test ensures record reader doesn't lose records when it starts
+  //exactly at the starting byte of a bz2 compressed block
+  @Test
+  public void testBzip2SplitStartAtBlockMarker() throws IOException {
+    //136504 in blockEndingInCR.txt.bz2 is the byte at which the bz2 block ends
+    //In the following test cases record readers should iterate over all the records
+    //and should not miss any record.
+
+    //Start next split at just the start of the block.
+    testSplitRecords("blockEndingInCR.txt.bz2", 136504);
+
+    //Start next split a byte forward in next block.
+    testSplitRecords("blockEndingInCR.txt.bz2", 136505);
+
+    //Start next split 3 bytes forward in next block.
+    testSplitRecords("blockEndingInCR.txt.bz2", 136508);
+
+    //Start next split 10 bytes from behind the end marker.
+    testSplitRecords("blockEndingInCR.txt.bz2", 136494);
+  }
+
   // Use the LineRecordReader to read records from the file
   public ArrayList<String> readRecords(URL testFileUrl, int splitSize)
       throws IOException {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
index 7736c4854ff1a..929c0ae335207 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
@@ -24,6 +24,7 @@
 
 import java.net.HttpURLConnection;
 
+import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.MapOutputFile;
@@ -54,6 +55,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.IFileInputStream;
 import org.apache.hadoop.mapred.IFileOutputStream;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
@@ -88,6 +90,7 @@ public class TestFetcher {
   final MapHost host = new MapHost("localhost", "http://localhost:8080/");
   final TaskAttemptID map1ID = TaskAttemptID.forName("attempt_0_1_m_1_1");
   final TaskAttemptID map2ID = TaskAttemptID.forName("attempt_0_1_m_2_1");
+  FileSystem fs = null;
 
   @Rule public TestName name = new TestName();
 
@@ -118,8 +121,11 @@ public void setup() {
   }
 
   @After
-  public void teardown() {
+  public void teardown() throws IllegalArgumentException, IOException {
     LOG.info("<<<< " + name.getMethodName());
+    if (fs != null) {
+      fs.delete(new Path(name.getMethodName()),true);
+    }
   }
   
   @Test
@@ -432,6 +438,70 @@ public void testCopyFromHostExtraBytes() throws Exception {
     verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID));
   }
 
+  @Test
+  public void testCorruptedIFile() throws Exception {
+    final int fetcher = 7;
+    Path onDiskMapOutputPath = new Path(name.getMethodName() + "/foo");
+    Path shuffledToDisk =
+        OnDiskMapOutput.getTempPath(onDiskMapOutputPath, fetcher);
+    fs = FileSystem.getLocal(job).getRaw();
+    MapOutputFile mof = mock(MapOutputFile.class);
+    OnDiskMapOutput<Text,Text> odmo = new OnDiskMapOutput<Text,Text>(map1ID,
+        id, mm, 100L, job, mof, fetcher, true, fs, onDiskMapOutputPath);
+
+    String mapData = "MAPDATA12345678901234567890";
+
+    ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 14, 10, 1);
+    ByteArrayOutputStream bout = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(bout);
+    IFileOutputStream ios = new IFileOutputStream(dos);
+    header.write(dos);
+
+    int headerSize = dos.size();
+    try {
+      ios.write(mapData.getBytes());
+    } finally {
+      ios.close();
+    }
+
+    int dataSize = bout.size() - headerSize;
+
+    // Ensure that the OnDiskMapOutput shuffler can successfully read the data.
+    MapHost host = new MapHost("TestHost", "http://test/url");
+    ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
+    try {
+      // Read past the shuffle header.
+      bin.read(new byte[headerSize], 0, headerSize);
+      odmo.shuffle(host, bin, dataSize, dataSize, metrics, Reporter.NULL);
+    } finally {
+      bin.close();
+    }
+
+    // Now corrupt the IFile data.
+    byte[] corrupted = bout.toByteArray();
+    corrupted[headerSize + (dataSize / 2)] = 0x0;
+
+    try {
+      bin = new ByteArrayInputStream(corrupted);
+      // Read past the shuffle header.
+      bin.read(new byte[headerSize], 0, headerSize);
+      odmo.shuffle(host, bin, dataSize, dataSize, metrics, Reporter.NULL);
+      fail("OnDiskMapOutput.shuffle didn't detect the corrupted map partition file");
+    } catch(ChecksumException e) {
+      LOG.info("The expected checksum exception was thrown.", e);
+    } finally {
+      bin.close();
+    }
+
+    // Ensure that the shuffled file can be read.
+    IFileInputStream iFin = new IFileInputStream(fs.open(shuffledToDisk), dataSize, job);
+    try {
+      iFin.read(new byte[dataSize], 0, dataSize);
+    } finally {
+      iFin.close();
+    }
+  }
+
   @Test(timeout=10000)
   public void testInterruptInMemory() throws Exception {
     final int FETCHER = 2;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java
index 905fd443fa939..6ac23200fb23e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java
@@ -77,7 +77,7 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
         0.0f);
     Assert.assertTrue(scheduler.waitUntilDone(1));
   }
-  
+
   @SuppressWarnings("rawtypes")
   @Test
   public <K, V> void TestAggregatedTransferRate() throws Exception {
@@ -102,7 +102,7 @@ public <K, V> void TestAggregatedTransferRate() throws Exception {
     Task mockTask = mock(Task.class);
     @SuppressWarnings("unchecked")
     MapOutput<K, V> output = mock(MapOutput.class);
-     
+
     ShuffleConsumerPlugin.Context<K, V> context =
         new ShuffleConsumerPlugin.Context<K, V>(mockTaskAttemptID, job, mockFileSystem,
                                                      mockUmbilical, mockLocalDirAllocator,
@@ -125,56 +125,51 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null,
         null, progress, context.getShuffledMapsCounter(),
         context.getReduceShuffleBytes(), context.getFailedShuffleCounter());
-    TaskAttemptID attemptID0 = new TaskAttemptID( 
+    TaskAttemptID attemptID0 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
         new JobID("test",0), TaskType.MAP, 0), 0);
-     
+
     //adding the 1st interval, 40MB from 60s to 100s
     long bytes = (long)40 * 1024 * 1024;
     scheduler.copySucceeded(attemptID0, new MapHost(null, null), bytes, 60000, 100000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000000_0 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(1 of 10 at 1.00 MB/s)", progress.toString());
-     
-    TaskAttemptID attemptID1 = new TaskAttemptID( 
+    Assert.assertEquals(copyMessage(1, 1, 1), progress.toString());
+
+    TaskAttemptID attemptID1 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
         new JobID("test",0), TaskType.MAP, 1), 1);
-     
+
     //adding the 2nd interval before the 1st interval, 50MB from 0s to 50s
     bytes = (long)50 * 1024 * 1024;
     scheduler.copySucceeded(attemptID1, new MapHost(null, null), bytes, 0, 50000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000001_1 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(2 of 10 at 1.00 MB/s)", progress.toString());
-     
-    TaskAttemptID attemptID2 = new TaskAttemptID( 
+    Assert.assertEquals(copyMessage(2, 1, 1), progress.toString());
+
+    TaskAttemptID attemptID2 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
         new JobID("test",0), TaskType.MAP, 2), 2);
-         
+
     //adding the 3rd interval overlapping with the 1st and the 2nd interval
     //110MB from 25s to 80s
     bytes = (long)110 * 1024 * 1024;
     scheduler.copySucceeded(attemptID2, new MapHost(null, null), bytes, 25000, 80000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000002_2 succeeded at 2.00 MB/s)"
-        + " Aggregated copy rate(3 of 10 at 2.00 MB/s)", progress.toString());
-     
-    TaskAttemptID attemptID3 = new TaskAttemptID( 
+    Assert.assertEquals(copyMessage(3, 2, 2), progress.toString());
+
+    TaskAttemptID attemptID3 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
         new JobID("test",0), TaskType.MAP, 3), 3);
-         
+
     //adding the 4th interval just after the 2nd interval, 100MB from 100s to 300s
     bytes = (long)100 * 1024 * 1024;
     scheduler.copySucceeded(attemptID3, new MapHost(null, null), bytes, 100000, 300000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000003_3 succeeded at 0.50 MB/s)"
-        + " Aggregated copy rate(4 of 10 at 1.00 MB/s)", progress.toString());
-     
-    TaskAttemptID attemptID4 = new TaskAttemptID( 
+    Assert.assertEquals(copyMessage(4, 0.5, 1), progress.toString());
+
+    TaskAttemptID attemptID4 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
         new JobID("test",0), TaskType.MAP, 4), 4);
-         
+
     //adding the 5th interval between after 4th, 50MB from 350s to 400s
     bytes = (long)50 * 1024 * 1024;
     scheduler.copySucceeded(attemptID4, new MapHost(null, null), bytes, 350000, 400000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000004_4 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(5 of 10 at 1.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(5, 1, 1), progress.toString());
 
 
     TaskAttemptID attemptID5 = new TaskAttemptID(
@@ -183,8 +178,7 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     //adding the 6th interval between after 5th, 50MB from 450s to 500s
     bytes = (long)50 * 1024 * 1024;
     scheduler.copySucceeded(attemptID5, new MapHost(null, null), bytes, 450000, 500000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000005_5 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(6 of 10 at 1.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(6, 1, 1), progress.toString());
 
     TaskAttemptID attemptID6 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
@@ -192,8 +186,7 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     //adding the 7th interval between after 5th and 6th interval, 20MB from 320s to 340s
     bytes = (long)20 * 1024 * 1024;
     scheduler.copySucceeded(attemptID6, new MapHost(null, null), bytes, 320000, 340000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000006_6 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(7 of 10 at 1.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(7, 1, 1), progress.toString());
 
     TaskAttemptID attemptID7 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
@@ -201,8 +194,7 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     //adding the 8th interval overlapping with 4th, 5th, and 7th 30MB from 290s to 350s
     bytes = (long)30 * 1024 * 1024;
     scheduler.copySucceeded(attemptID7, new MapHost(null, null), bytes, 290000, 350000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000007_7 succeeded at 0.50 MB/s)"
-        + " Aggregated copy rate(8 of 10 at 1.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(8, 0.5, 1), progress.toString());
 
     TaskAttemptID attemptID8 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
@@ -210,8 +202,7 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     //adding the 9th interval overlapping with 5th and 6th, 50MB from 400s to 450s
     bytes = (long)50 * 1024 * 1024;
     scheduler.copySucceeded(attemptID8, new MapHost(null, null), bytes, 400000, 450000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000008_8 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(9 of 10 at 1.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(9, 1, 1), progress.toString());
 
     TaskAttemptID attemptID9 = new TaskAttemptID(
         new org.apache.hadoop.mapred.TaskID(
@@ -219,8 +210,13 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) {
     //adding the 10th interval overlapping with all intervals, 500MB from 0s to 500s
     bytes = (long)500 * 1024 * 1024;
     scheduler.copySucceeded(attemptID9, new MapHost(null, null), bytes, 0, 500000, output);
-    Assert.assertEquals("copy task(attempt_test_0000_m_000009_9 succeeded at 1.00 MB/s)"
-        + " Aggregated copy rate(10 of 10 at 2.00 MB/s)", progress.toString());
+    Assert.assertEquals(copyMessage(10, 1, 2), progress.toString());
+  }
 
+  private static String copyMessage(int attemptNo, double rate1, double rate2) {
+    int attemptZero = attemptNo - 1;
+    return String.format("copy task(attempt_test_0000_m_%06d_%d succeeded at %1.2f MB/s)"
+            + " Aggregated copy rate(%d of 10 at %1.2f MB/s)", attemptZero
+            , attemptZero, rate1, attemptNo, rate2);
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index adeb9fa621e60..fa8162b7c4a59 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -63,6 +63,10 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.fusesource.leveldbjni</groupId>
+      <artifactId>leveldbjni-all</artifactId>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index f53f18896d94d..65f8a4febfc98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -263,6 +263,10 @@ public Collection<HistoryFileInfo> values() {
     public HistoryFileInfo get(JobId jobId) {
       return cache.get(jobId);
     }
+
+    public boolean isFull() {
+      return cache.size() >= maxSize;
+    }
   }
 
   /**
@@ -668,6 +672,10 @@ void initExisting() throws IOException {
     for (FileStatus fs : timestampedDirList) {
       // TODO Could verify the correct format for these directories.
       addDirectoryToSerialNumberIndex(fs.getPath());
+    }
+    for (int i= timestampedDirList.size() - 1;
+        i >= 0 && !jobListCache.isFull(); i--) {
+      FileStatus fs = timestampedDirList.get(i); 
       addDirectoryToJobListCache(fs.getPath());
     }
   }
@@ -848,7 +856,7 @@ public void run() {
             }
           });
         }
-      } else if (old != null && !old.isMovePending()) {
+      } else if (!old.isMovePending()) {
         //This is a duplicate so just delete it
         if (LOG.isDebugEnabled()) {
           LOG.debug("Duplicate: deleting");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
new file mode 100644
index 0000000000000..16366b1ff85d1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
@@ -0,0 +1,379 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs;
+
+import static org.fusesource.leveldbjni.JniDBFactory.asString;
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.Logger;
+import org.iq80.leveldb.Options;
+
+public class HistoryServerLeveldbStateStoreService extends
+    HistoryServerStateStoreService {
+
+  private static final String DB_NAME = "mr-jhs-state";
+  private static final String DB_SCHEMA_VERSION_KEY = "jhs-schema-version";
+  private static final String TOKEN_MASTER_KEY_KEY_PREFIX = "tokens/key_";
+  private static final String TOKEN_STATE_KEY_PREFIX = "tokens/token_";
+
+  private static final Version CURRENT_VERSION_INFO =
+      Version.newInstance(1, 0);
+
+  private DB db;
+
+  public static final Log LOG =
+      LogFactory.getLog(HistoryServerLeveldbStateStoreService.class);
+
+  @Override
+  protected void initStorage(Configuration conf) throws IOException {
+  }
+
+  @Override
+  protected void startStorage() throws IOException {
+    Path storeRoot = createStorageDir(getConfig());
+    Options options = new Options();
+    options.createIfMissing(false);
+    options.logger(new LeveldbLogger());
+    LOG.info("Using state database at " + storeRoot + " for recovery");
+    File dbfile = new File(storeRoot.toString());
+    try {
+      db = JniDBFactory.factory.open(dbfile, options);
+    } catch (NativeDB.DBException e) {
+      if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+        LOG.info("Creating state database at " + dbfile);
+        options.createIfMissing(true);
+        try {
+          db = JniDBFactory.factory.open(dbfile, options);
+          // store version
+          storeVersion();
+        } catch (DBException dbErr) {
+          throw new IOException(dbErr.getMessage(), dbErr);
+        }
+      } else {
+        throw e;
+      }
+    }
+    checkVersion();
+  }
+
+  @Override
+  protected void closeStorage() throws IOException {
+    if (db != null) {
+      db.close();
+      db = null;
+    }
+  }
+
+  @Override
+  public HistoryServerState loadState() throws IOException {
+    HistoryServerState state = new HistoryServerState();
+    int numKeys = loadTokenMasterKeys(state);
+    LOG.info("Recovered " + numKeys + " token master keys");
+    int numTokens = loadTokens(state);
+    LOG.info("Recovered " + numTokens + " tokens");
+    return state;
+  }
+
+  private int loadTokenMasterKeys(HistoryServerState state)
+      throws IOException {
+    int numKeys = 0;
+    LeveldbIterator iter = null;
+    try {
+      iter = new LeveldbIterator(db);
+      iter.seek(bytes(TOKEN_MASTER_KEY_KEY_PREFIX));
+      while (iter.hasNext()) {
+        Entry<byte[],byte[]> entry = iter.next();
+        String key = asString(entry.getKey());
+        if (!key.startsWith(TOKEN_MASTER_KEY_KEY_PREFIX)) {
+          break;
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Loading master key from " + key);
+        }
+        try {
+          loadTokenMasterKey(state, entry.getValue());
+        } catch (IOException e) {
+          throw new IOException("Error loading token master key from " + key,
+              e);
+        }
+        ++numKeys;
+      }
+    } catch (DBException e) {
+      throw new IOException(e);
+    } finally {
+      if (iter != null) {
+        iter.close();
+      }
+    }
+    return numKeys;
+  }
+
+  private void loadTokenMasterKey(HistoryServerState state, byte[] data)
+      throws IOException {
+    DelegationKey key = new DelegationKey();
+    DataInputStream in =
+        new DataInputStream(new ByteArrayInputStream(data));
+    try {
+      key.readFields(in);
+    } finally {
+      IOUtils.cleanup(LOG, in);
+    }
+    state.tokenMasterKeyState.add(key);
+  }
+
+  private int loadTokens(HistoryServerState state) throws IOException {
+    int numTokens = 0;
+    LeveldbIterator iter = null;
+    try {
+      iter = new LeveldbIterator(db);
+      iter.seek(bytes(TOKEN_STATE_KEY_PREFIX));
+      while (iter.hasNext()) {
+        Entry<byte[],byte[]> entry = iter.next();
+        String key = asString(entry.getKey());
+        if (!key.startsWith(TOKEN_STATE_KEY_PREFIX)) {
+          break;
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Loading token from " + key);
+        }
+        try {
+          loadToken(state, entry.getValue());
+        } catch (IOException e) {
+          throw new IOException("Error loading token state from " + key, e);
+        }
+        ++numTokens;
+      }
+    } catch (DBException e) {
+      throw new IOException(e);
+    } finally {
+      if (iter != null) {
+        iter.close();
+      }
+    }
+    return numTokens;
+  }
+
+  private void loadToken(HistoryServerState state, byte[] data)
+      throws IOException {
+    MRDelegationTokenIdentifier tokenId = new MRDelegationTokenIdentifier();
+    long renewDate;
+    DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+    try {
+      tokenId.readFields(in);
+      renewDate = in.readLong();
+    } finally {
+      IOUtils.cleanup(LOG, in);
+    }
+    state.tokenState.put(tokenId, renewDate);
+  }
+
+  @Override
+  public void storeToken(MRDelegationTokenIdentifier tokenId, Long renewDate)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing token " + tokenId.getSequenceNumber());
+    }
+
+    ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+    DataOutputStream dataStream = new DataOutputStream(memStream);
+    try {
+      tokenId.write(dataStream);
+      dataStream.writeLong(renewDate);
+      dataStream.close();
+      dataStream = null;
+    } finally {
+      IOUtils.cleanup(LOG, dataStream);
+    }
+
+    String dbKey = getTokenDatabaseKey(tokenId);
+    try {
+      db.put(bytes(dbKey), memStream.toByteArray());
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void updateToken(MRDelegationTokenIdentifier tokenId, Long renewDate)
+      throws IOException {
+    storeToken(tokenId, renewDate);
+  }
+
+  @Override
+  public void removeToken(MRDelegationTokenIdentifier tokenId)
+      throws IOException {
+    String dbKey = getTokenDatabaseKey(tokenId);
+    try {
+      db.delete(bytes(dbKey));
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private String getTokenDatabaseKey(MRDelegationTokenIdentifier tokenId) {
+    return TOKEN_STATE_KEY_PREFIX + tokenId.getSequenceNumber();
+  }
+
+  @Override
+  public void storeTokenMasterKey(DelegationKey masterKey)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Storing master key " + masterKey.getKeyId());
+    }
+
+    ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+    DataOutputStream dataStream = new DataOutputStream(memStream);
+    try {
+      masterKey.write(dataStream);
+      dataStream.close();
+      dataStream = null;
+    } finally {
+      IOUtils.cleanup(LOG, dataStream);
+    }
+
+    String dbKey = getTokenMasterKeyDatabaseKey(masterKey);
+    try {
+      db.put(bytes(dbKey), memStream.toByteArray());
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void removeTokenMasterKey(DelegationKey masterKey)
+      throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Removing master key " + masterKey.getKeyId());
+    }
+
+    String dbKey = getTokenMasterKeyDatabaseKey(masterKey);
+    try {
+      db.delete(bytes(dbKey));
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private String getTokenMasterKeyDatabaseKey(DelegationKey masterKey) {
+    return TOKEN_MASTER_KEY_KEY_PREFIX + masterKey.getKeyId();
+  }
+
+  private Path createStorageDir(Configuration conf) throws IOException {
+    String confPath = conf.get(JHAdminConfig.MR_HS_LEVELDB_STATE_STORE_PATH);
+    if (confPath == null) {
+      throw new IOException("No store location directory configured in " +
+          JHAdminConfig.MR_HS_LEVELDB_STATE_STORE_PATH);
+    }
+    Path root = new Path(confPath, DB_NAME);
+    FileSystem fs = FileSystem.getLocal(conf);
+    fs.mkdirs(root, new FsPermission((short)0700));
+    return root;
+  }
+
+  Version loadVersion() throws IOException {
+    byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
+    // if version is not stored previously, treat it as 1.0.
+    if (data == null || data.length == 0) {
+      return Version.newInstance(1, 0);
+    }
+    Version version =
+        new VersionPBImpl(VersionProto.parseFrom(data));
+    return version;
+  }
+
+  private void storeVersion() throws IOException {
+    dbStoreVersion(CURRENT_VERSION_INFO);
+  }
+
+  void dbStoreVersion(Version state) throws IOException {
+    String key = DB_SCHEMA_VERSION_KEY;
+    byte[] data =
+        ((VersionPBImpl) state).getProto().toByteArray();
+    try {
+      db.put(bytes(key), data);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  Version getCurrentVersion() {
+    return CURRENT_VERSION_INFO;
+  }
+
+  /**
+   * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
+   * 2) Any incompatible change of state-store is a major upgrade, and any
+   *    compatible change of state-store is a minor upgrade.
+   * 3) Within a minor upgrade, say 1.1 to 1.2:
+   *    overwrite the version info and proceed as normal.
+   * 4) Within a major upgrade, say 1.2 to 2.0:
+   *    throw exception and indicate user to use a separate upgrade tool to
+   *    upgrade state or remove incompatible old state.
+   */
+  private void checkVersion() throws IOException {
+    Version loadedVersion = loadVersion();
+    LOG.info("Loaded state version info " + loadedVersion);
+    if (loadedVersion.equals(getCurrentVersion())) {
+      return;
+    }
+    if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+      LOG.info("Storing state version info " + getCurrentVersion());
+      storeVersion();
+    } else {
+      throw new IOException(
+        "Incompatible version for state: expecting state version "
+            + getCurrentVersion() + ", but loading version " + loadedVersion);
+    }
+  }
+
+  private static class LeveldbLogger implements Logger {
+    private static final Log LOG = LogFactory.getLog(LeveldbLogger.class);
+
+    @Override
+    public void log(String message) {
+      LOG.info(message);
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/package-info.java
index 73c8ab59f2cdb..00b93c9a2b19a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/package-info.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/package-info.java
@@ -15,6 +15,4 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
 package org.apache.hadoop.mapreduce.v2.hs;
-import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
index 231e362a5aea4..7e49d520e7ff7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
@@ -70,6 +70,6 @@ public class HsNavBlock extends HtmlBlock {
           li().a("/conf", "Configuration")._().
           li().a("/logs", "Local logs")._().
           li().a("/stacks", "Server stacks")._().
-          li().a("/metrics", "Server metrics")._()._()._();
+          li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/resources/css/site.css b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerLeveldbStateStoreService.java
new file mode 100644
index 0000000000000..2af2f84302c5d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryServerLeveldbStateStoreService.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestHistoryServerLeveldbStateStoreService {
+
+  private static final File testDir = new File(
+      System.getProperty("test.build.data",
+          System.getProperty("java.io.tmpdir")),
+      "TestHistoryServerLeveldbSystemStateStoreService");
+
+  private Configuration conf;
+
+  @Before
+  public void setup() {
+    FileUtil.fullyDelete(testDir);
+    testDir.mkdirs();
+    conf = new Configuration();
+    conf.setBoolean(JHAdminConfig.MR_HS_RECOVERY_ENABLE, true);
+    conf.setClass(JHAdminConfig.MR_HS_STATE_STORE,
+        HistoryServerLeveldbStateStoreService.class,
+        HistoryServerStateStoreService.class);
+    conf.set(JHAdminConfig.MR_HS_LEVELDB_STATE_STORE_PATH,
+        testDir.getAbsoluteFile().toString());
+  }
+
+  @After
+  public void cleanup() {
+    FileUtil.fullyDelete(testDir);
+  }
+
+  private HistoryServerStateStoreService createAndStartStore()
+      throws IOException {
+    HistoryServerStateStoreService store =
+        HistoryServerStateStoreServiceFactory.getStore(conf);
+    assertTrue("Factory did not create a leveldb store",
+        store instanceof HistoryServerLeveldbStateStoreService);
+    store.init(conf);
+    store.start();
+    return store;
+  }
+
+  @Test
+  public void testCheckVersion() throws IOException {
+    HistoryServerLeveldbStateStoreService store =
+        new HistoryServerLeveldbStateStoreService();
+    store.init(conf);
+    store.start();
+
+    // default version
+    Version defaultVersion = store.getCurrentVersion();
+    assertEquals(defaultVersion, store.loadVersion());
+
+    // compatible version
+    Version compatibleVersion =
+        Version.newInstance(defaultVersion.getMajorVersion(),
+          defaultVersion.getMinorVersion() + 2);
+    store.dbStoreVersion(compatibleVersion);
+    assertEquals(compatibleVersion, store.loadVersion());
+    store.close();
+    store = new HistoryServerLeveldbStateStoreService();
+    store.init(conf);
+    store.start();
+
+    // overwrite the compatible version
+    assertEquals(defaultVersion, store.loadVersion());
+
+    // incompatible version
+    Version incompatibleVersion =
+      Version.newInstance(defaultVersion.getMajorVersion() + 1,
+          defaultVersion.getMinorVersion());
+    store.dbStoreVersion(incompatibleVersion);
+    store.close();
+    store = new HistoryServerLeveldbStateStoreService();
+    try {
+      store.init(conf);
+      store.start();
+      fail("Incompatible version, should have thrown before here.");
+    } catch (ServiceStateException e) {
+      assertTrue("Exception message mismatch",
+        e.getMessage().contains("Incompatible version for state:"));
+    }
+    store.close();
+  }
+
+  @Test
+  public void testTokenStore() throws IOException {
+    HistoryServerStateStoreService store = createAndStartStore();
+
+    // verify initially the store is empty
+    HistoryServerState state = store.loadState();
+    assertTrue("token state not empty", state.tokenState.isEmpty());
+    assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty());
+
+    // store a key and some tokens
+    final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
+    final MRDelegationTokenIdentifier token1 =
+        new MRDelegationTokenIdentifier(new Text("tokenOwner1"),
+            new Text("tokenRenewer1"), new Text("tokenUser1"));
+    token1.setSequenceNumber(1);
+    final Long tokenDate1 = 1L;
+    final MRDelegationTokenIdentifier token2 =
+        new MRDelegationTokenIdentifier(new Text("tokenOwner2"),
+            new Text("tokenRenewer2"), new Text("tokenUser2"));
+    token2.setSequenceNumber(12345678);
+    final Long tokenDate2 = 87654321L;
+
+    store.storeTokenMasterKey(key1);
+    store.storeToken(token1, tokenDate1);
+    store.storeToken(token2, tokenDate2);
+    store.close();
+
+    // verify the key and tokens can be recovered
+    store = createAndStartStore();
+    state = store.loadState();
+    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
+    assertTrue("missing token 1", state.tokenState.containsKey(token1));
+    assertEquals("incorrect token 1 date", tokenDate1,
+        state.tokenState.get(token1));
+    assertTrue("missing token 2", state.tokenState.containsKey(token2));
+    assertEquals("incorrect token 2 date", tokenDate2,
+        state.tokenState.get(token2));
+    assertEquals("incorrect master key count", 1,
+        state.tokenMasterKeyState.size());
+    assertTrue("missing master key 1",
+        state.tokenMasterKeyState.contains(key1));
+
+    // store some more keys and tokens, remove the previous key and one
+    // of the tokens, and renew a previous token
+    final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
+    final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
+    final MRDelegationTokenIdentifier token3 =
+        new MRDelegationTokenIdentifier(new Text("tokenOwner3"),
+            new Text("tokenRenewer3"), new Text("tokenUser3"));
+    token3.setSequenceNumber(12345679);
+    final Long tokenDate3 = 87654321L;
+
+    store.removeToken(token1);
+    store.storeTokenMasterKey(key2);
+    final Long newTokenDate2 = 975318642L;
+    store.updateToken(token2, newTokenDate2);
+    store.removeTokenMasterKey(key1);
+    store.storeTokenMasterKey(key3);
+    store.storeToken(token3, tokenDate3);
+    store.close();
+
+    // verify the new keys and tokens are recovered, the removed key and
+    // token are no longer present, and the renewed token has the updated
+    // expiration date
+    store = createAndStartStore();
+    state = store.loadState();
+    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
+    assertFalse("token 1 not removed", state.tokenState.containsKey(token1));
+    assertTrue("missing token 2", state.tokenState.containsKey(token2));
+    assertEquals("incorrect token 2 date", newTokenDate2,
+        state.tokenState.get(token2));
+    assertTrue("missing token 3", state.tokenState.containsKey(token3));
+    assertEquals("incorrect token 3 date", tokenDate3,
+        state.tokenState.get(token3));
+    assertEquals("incorrect master key count", 2,
+        state.tokenMasterKeyState.size());
+    assertFalse("master key 1 not removed",
+        state.tokenMasterKeyState.contains(key1));
+    assertTrue("missing master key 2",
+        state.tokenMasterKeyState.contains(key2));
+    assertTrue("missing master key 3",
+        state.tokenMasterKeyState.contains(key3));
+    store.close();
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
index bd9c9c57241d2..608b8abf1a6c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
@@ -113,7 +113,6 @@
 <property><!--Loaded from job.xml--><name>hadoop.proxyuser.user.groups</name><value>users</value></property>
 <property><!--Loaded from job.xml--><name>dfs.namenode.name.dir.restore</name><value>false</value></property>
 <property><!--Loaded from job.xml--><name>io.seqfile.lazydecompress</name><value>true</value></property>
-<property><!--Loaded from job.xml--><name>dfs.https.enable</name><value>false</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.reduce.merge.inmem.threshold</name><value>1000</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.input.fileinputformat.split.minsize</name><value>0</value></property>
 <property><!--Loaded from job.xml--><name>dfs.replication</name><value>3</value></property>
@@ -166,7 +165,6 @@
 <property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.attempts</name><value>5</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.jobhistory.max-age-ms</name><value>10000000000</value></property>
 <property><!--Loaded from job.xml--><name>yarn.resourcemanager.zookeeper-store.session.timeout-ms</name><value>60000</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.task.tmp.dir</name><value>./tmp</value></property>
 <property><!--Loaded from job.xml--><name>dfs.default.chunk.view.size</name><value>32768</value></property>
 <property><!--Loaded from job.xml--><name>kfs.bytes-per-checksum</name><value>512</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.reduce.memory.mb</name><value>512</value></property>
@@ -210,7 +208,6 @@
 <property><!--Loaded from job.xml--><name>mapreduce.job.dir</name><value>/tmp/hadoop-yarn/staging/user/.staging/job_1329348432655_0001</value></property>
 <property><!--Loaded from job.xml--><name>io.map.index.skip</name><value>0</value></property>
 <property><!--Loaded from job.xml--><name>net.topology.node.switch.mapping.impl</name><value>org.apache.hadoop.net.ScriptBasedMapping</value></property>
-<property><!--Loaded from job.xml--><name>dfs.namenode.logging.level</name><value>info</value></property>
 <property><!--Loaded from job.xml--><name>fs.s3.maxRetries</name><value>4</value></property>
 <property><!--Loaded from job.xml--><name>s3native.client-write-packet-size</name><value>65536</value></property>
 <property><!--Loaded from job.xml--><name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name><value>1000</value></property>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/RandomTextWriterJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/RandomTextWriterJob.java
index 8d357cb3f7f64..2d341e9fda732 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/RandomTextWriterJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/RandomTextWriterJob.java
@@ -65,7 +65,7 @@ public Job createJob(Configuration conf) throws IOException {
     }
     conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
 
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
 
     job.setJarByClass(RandomTextWriterJob.class);
     job.setJobName("random-text-writer");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index 78f1ffa6841a9..d9cd07ba051d5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -31,7 +31,6 @@
 import java.util.Date;
 import java.util.Random;
 import java.util.StringTokenizer;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -92,13 +91,13 @@ public class TestDFSIO implements Tool {
   private static final String BASE_FILE_NAME = "test_io_";
   private static final String DEFAULT_RES_FILE_NAME = "TestDFSIO_results.log";
   private static final long MEGA = ByteMultiple.MB.value();
-  private static final int DEFAULT_NR_BYTES = 1;
+  private static final int DEFAULT_NR_BYTES = 128;
   private static final int DEFAULT_NR_FILES = 4;
   private static final String USAGE =
                     "Usage: " + TestDFSIO.class.getSimpleName() +
                     " [genericOptions]" +
                     " -read [-random | -backward | -skip [-skipSize Size]] |" +
-                    " -write | -append | -clean" +
+                    " -write | -append | -truncate | -clean" +
                     " [-compression codecClassName]" +
                     " [-nrFiles N]" +
                     " [-size Size[B|KB|MB|GB|TB]]" +
@@ -120,7 +119,8 @@ private static enum TestType {
     TEST_TYPE_APPEND("append"),
     TEST_TYPE_READ_RANDOM("random read"),
     TEST_TYPE_READ_BACKWARD("backward read"),
-    TEST_TYPE_READ_SKIP("skip read");
+    TEST_TYPE_READ_SKIP("skip read"),
+    TEST_TYPE_TRUNCATE("truncate");
 
     private String type;
 
@@ -191,6 +191,9 @@ private static Path getAppendDir(Configuration conf) {
   private static Path getRandomReadDir(Configuration conf) {
     return new Path(getBaseDir(conf), "io_random_read");
   }
+  private static Path getTruncateDir(Configuration conf) {
+    return new Path(getBaseDir(conf), "io_truncate");
+  }
   private static Path getDataDir(Configuration conf) {
     return new Path(getBaseDir(conf), "io_data");
   }
@@ -201,6 +204,7 @@ private static Path getDataDir(Configuration conf) {
   @BeforeClass
   public static void beforeClass() throws Exception {
     bench = new TestDFSIO();
+    bench.getConf().setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
     cluster = new MiniDFSCluster.Builder(bench.getConf())
                                 .numDataNodes(2)
                                 .format(true)
@@ -277,6 +281,16 @@ public void testAppend() throws Exception {
     bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
   }
 
+  @Test (timeout = 60000)
+  public void testTruncate() throws Exception {
+    FileSystem fs = cluster.getFileSystem();
+    bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
+    long tStart = System.currentTimeMillis();
+    bench.truncateTest(fs);
+    long execTime = System.currentTimeMillis() - tStart;
+    bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
+  }
+
   @SuppressWarnings("deprecation")
   private void createControlFile(FileSystem fs,
                                   long nrBytes, // in bytes
@@ -299,9 +313,9 @@ private void createControlFile(FileSystem fs,
       } catch(Exception e) {
         throw new IOException(e.getLocalizedMessage());
       } finally {
-    	if (writer != null)
+        if (writer != null)
           writer.close();
-    	writer = null;
+        writer = null;
       }
     }
     LOG.info("created control files for: "+nrFiles+" files");
@@ -611,6 +625,51 @@ private void randomReadTest(FileSystem fs) throws IOException {
     runIOTest(RandomReadMapper.class, readDir);
   }
 
+  /**
+   * Truncate mapper class.
+   * The mapper truncates given file to the newLength, specified by -size.
+   */
+  public static class TruncateMapper extends IOStatMapper {
+    private static final long DELAY = 100L;
+
+    private Path filePath;
+    private long fileSize;
+
+    @Override // IOMapperBase
+    public Closeable getIOStream(String name) throws IOException {
+      filePath = new Path(getDataDir(getConf()), name);
+      fileSize = fs.getFileStatus(filePath).getLen();
+      return null;
+    }
+
+    @Override // IOMapperBase
+    public Long doIO(Reporter reporter, 
+                       String name, 
+                       long newLength // in bytes
+                     ) throws IOException {
+      boolean isClosed = fs.truncate(filePath, newLength);
+      reporter.setStatus("truncating " + name + " to newLength " + 
+          newLength  + " ::host = " + hostName);
+      for(int i = 0; !isClosed; i++) {
+        try {
+          Thread.sleep(DELAY);
+        } catch (InterruptedException ignored) {}
+        FileStatus status = fs.getFileStatus(filePath);
+        assert status != null : "status is null";
+        isClosed = (status.getLen() == newLength);
+        reporter.setStatus("truncate recover for " + name + " to newLength " + 
+            newLength + " attempt " + i + " ::host = " + hostName);
+      }
+      return Long.valueOf(fileSize - newLength);
+    }
+  }
+
+  private void truncateTest(FileSystem fs) throws IOException {
+    Path TruncateDir = getTruncateDir(config);
+    fs.delete(TruncateDir, true);
+    runIOTest(TruncateMapper.class, TruncateDir);
+  }
+
   private void sequentialTest(FileSystem fs, 
                               TestType testType, 
                               long fileSize, // in bytes
@@ -632,6 +691,9 @@ private void sequentialTest(FileSystem fs,
     case TEST_TYPE_READ_SKIP:
       ioer = new RandomReadMapper();
       break;
+    case TEST_TYPE_TRUNCATE:
+      ioer = new TruncateMapper();
+      break;
     default:
       return;
     }
@@ -665,7 +727,7 @@ public int run(String[] args) throws IOException {
     String resFileName = DEFAULT_RES_FILE_NAME;
     String compressionClass = null;
     boolean isSequential = false;
-    String version = TestDFSIO.class.getSimpleName() + ".1.7";
+    String version = TestDFSIO.class.getSimpleName() + ".1.8";
 
     LOG.info(version);
     if (args.length == 0) {
@@ -689,6 +751,8 @@ public int run(String[] args) throws IOException {
       } else if (args[i].equalsIgnoreCase("-skip")) {
         if (testType != TestType.TEST_TYPE_READ) return -1;
         testType = TestType.TEST_TYPE_READ_SKIP;
+      } else if (args[i].equalsIgnoreCase("-truncate")) {
+        testType = TestType.TEST_TYPE_TRUNCATE;
       } else if (args[i].equalsIgnoreCase("-clean")) {
         testType = TestType.TEST_TYPE_CLEANUP;
       } else if (args[i].toLowerCase().startsWith("-seq")) {
@@ -762,6 +826,11 @@ else if(testType == TestType.TEST_TYPE_READ_SKIP && skipSize == 0)
     case TEST_TYPE_READ_BACKWARD:
     case TEST_TYPE_READ_SKIP:
       randomReadTest(fs);
+      break;
+    case TEST_TYPE_TRUNCATE:
+      truncateTest(fs);
+      break;
+   default:
     }
     long execTime = System.currentTimeMillis() - tStart;
   
@@ -797,7 +866,7 @@ static float toMB(long bytes) {
     return ((float)bytes)/MEGA;
   }
 
-  private void analyzeResult(	FileSystem fs,
+  private void analyzeResult( FileSystem fs,
                               TestType testType,
                               long execTime,
                               String resFileName
@@ -870,13 +939,17 @@ private Path getReduceFilePath(TestType testType) {
     case TEST_TYPE_READ_BACKWARD:
     case TEST_TYPE_READ_SKIP:
       return new Path(getRandomReadDir(config), "part-00000");
+    case TEST_TYPE_TRUNCATE:
+      return new Path(getTruncateDir(config), "part-00000");
+    default:
     }
     return null;
   }
 
   private void analyzeResult(FileSystem fs, TestType testType, long execTime)
       throws IOException {
-    analyzeResult(fs, testType, execTime, DEFAULT_RES_FILE_NAME);
+    String dir = System.getProperty("test.build.dir", "target/test-dir");
+    analyzeResult(fs, testType, execTime, dir + "/" + DEFAULT_RES_FILE_NAME);
   }
 
   private void cleanup(FileSystem fs)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
new file mode 100644
index 0000000000000..c47d97134607d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.loadGenerator;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.UnknownHostException;
+import java.util.EnumSet;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+
+/** The load generator is a tool for testing NameNode behavior under
+ * different client loads.
+ * The main code is in HadoopCommon, @LoadGenerator. This class, LoadGeneratorMR
+ * lets you run that LoadGenerator as a MapReduce job.
+ * 
+ * The synopsis of the command is
+ * java LoadGeneratorMR
+ *   -mr <numMapJobs> <outputDir> : results in outputDir/Results
+ *   the rest of the args are the same as the original LoadGenerator.
+ *
+ */
+public class LoadGeneratorMR extends LoadGenerator {
+  public static final Log LOG = LogFactory.getLog(LoadGenerator.class);
+  private static int numMapTasks = 1;
+  private String mrOutDir;
+  
+  final private static String USAGE_CMD = "java LoadGeneratorMR\n";
+  final private static String USAGE = USAGE_CMD
+		  + "-mr <numMapJobs> <outputDir> [MUST be first 3 args] \n" + USAGE_ARGS ;
+  
+  // Constant "keys" used to communicate between map and reduce
+  final private static Text OPEN_EXECTIME = new Text("OpenExecutionTime");
+  final private static Text NUMOPS_OPEN = new Text("NumOpsOpen");
+  final private static Text LIST_EXECTIME = new Text("ListExecutionTime");
+  final private static Text NUMOPS_LIST = new Text("NumOpsList");
+  final private static Text DELETE_EXECTIME = new Text("DeletionExecutionTime");
+  final private static Text NUMOPS_DELETE = new Text("NumOpsDelete");
+  final private static Text CREATE_EXECTIME = new Text("CreateExecutionTime");
+  final private static Text NUMOPS_CREATE = new Text("NumOpsCreate");
+  final private static Text WRITE_CLOSE_EXECTIME = new Text("WriteCloseExecutionTime");
+  final private static Text NUMOPS_WRITE_CLOSE = new Text("NumOpsWriteClose");
+  final private static Text ELAPSED_TIME = new Text("ElapsedTime");
+  final private static Text TOTALOPS = new Text("TotalOps");
+  
+  // Config keys to pass args from Main to the Job
+  final private static String LG_ROOT = "LG.root";
+  final private static String LG_SCRIPTFILE = "LG.scriptFile";
+  final private static String LG_MAXDELAYBETWEENOPS = "LG.maxDelayBetweenOps";
+  final private static String LG_NUMOFTHREADS = "LG.numOfThreads";
+  final private static String LG_READPR = "LG.readPr";
+  final private static String LG_WRITEPR = "LG.writePr";
+  final private static String LG_SEED = "LG.r";
+  final private static String LG_NUMMAPTASKS = "LG.numMapTasks";
+  final private static String LG_ELAPSEDTIME = "LG.elapsedTime";
+  final private static String LG_STARTTIME = "LG.startTime";
+  final private static String LG_FLAGFILE = "LG.flagFile";
+
+
+  /** Constructor */
+  public LoadGeneratorMR() throws IOException, UnknownHostException {
+	super();
+  }
+  
+  public LoadGeneratorMR(Configuration conf) throws IOException, UnknownHostException {
+    this();
+    setConf(conf);
+  } 
+  
+  /** Main function called by tool runner.
+   * It first initializes data by parsing the command line arguments.
+   * It then calls the loadGenerator
+   */
+  @Override
+  public int run(String[] args) throws Exception {
+    int exitCode = parseArgsMR(args);
+    if (exitCode != 0) {
+      return exitCode;
+    }
+    System.out.println("Running LoadGeneratorMR against fileSystem: " + 
+    FileContext.getFileContext().getDefaultFileSystem().getUri());
+
+    return submitAsMapReduce(); // reducer will print the results
+  }
+
+
+  /** 
+   * Parse the command line arguments and initialize the data.
+   * Only parse the first arg: -mr <numMapTasks> <mrOutDir> (MUST be first three Args)
+   * The rest are parsed by the Parent LoadGenerator
+   **/
+ 
+	private int parseArgsMR(String[] args) throws IOException {
+	  try {
+		if (args.length >= 3 && args[0].equals("-mr")) {
+		  numMapTasks = Integer.parseInt(args[1]);
+		  mrOutDir = args[2];
+		  if (mrOutDir.startsWith("-")) {
+			System.err.println("Missing output file parameter, instead got: "
+							+ mrOutDir);
+			System.err.println(USAGE);
+			return -1;
+		  }
+		} else {
+		  System.err.println(USAGE);
+		  ToolRunner.printGenericCommandUsage(System.err);
+		  return -1;
+		}
+		String[] strippedArgs = new String[args.length - 3];
+		for (int i = 0; i < strippedArgs.length; i++) {
+		  strippedArgs[i] = args[i + 3];
+		}
+		super.parseArgs(true, strippedArgs); // Parse normal LoadGenerator args
+	  } catch (NumberFormatException e) {
+		System.err.println("Illegal parameter: " + e.getLocalizedMessage());
+		System.err.println(USAGE);
+		return -1;
+	  }
+	  return 0;
+	}
+
+  /** Main program
+   * 
+   * @param args command line arguments
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(new Configuration(), new LoadGeneratorMR(), args);
+    System.exit(res);
+  }
+
+  
+  // The following methods are only used when LoadGenerator is run a MR job
+  /**
+   * Based on args we submit the LoadGenerator as MR job.
+   * Number of MapTasks is numMapTasks
+   * @return exitCode for job submission
+   */
+  private int submitAsMapReduce() {
+    
+    System.out.println("Running as a MapReduce job with " + 
+        numMapTasks + " mapTasks;  Output to file " + mrOutDir);
+
+
+    Configuration conf = new Configuration(getConf());
+    
+    // First set all the args of LoadGenerator as Conf vars to pass to MR tasks
+
+    conf.set(LG_ROOT , root.toString());
+    conf.setInt(LG_MAXDELAYBETWEENOPS, maxDelayBetweenOps);
+    conf.setInt(LG_NUMOFTHREADS, numOfThreads);
+    conf.set(LG_READPR, readProbs[0]+""); //Pass Double as string
+    conf.set(LG_WRITEPR, writeProbs[0]+""); //Pass Double as string
+    conf.setLong(LG_SEED, seed); //No idea what this is
+    conf.setInt(LG_NUMMAPTASKS, numMapTasks);
+    if (scriptFile == null && durations[0] <=0) {
+      System.err.println("When run as a MapReduce job, elapsed Time or ScriptFile must be specified");
+      System.exit(-1);
+    }
+    conf.setLong(LG_ELAPSEDTIME, durations[0]);
+    conf.setLong(LG_STARTTIME, startTime); 
+    if (scriptFile != null) {
+      conf.set(LG_SCRIPTFILE , scriptFile);
+    }
+    conf.set(LG_FLAGFILE, flagFile.toString());
+    
+    // Now set the necessary conf variables that apply to run MR itself.
+    JobConf jobConf = new JobConf(conf, LoadGenerator.class);
+    jobConf.setJobName("NNLoadGeneratorViaMR");
+    jobConf.setNumMapTasks(numMapTasks);
+    jobConf.setNumReduceTasks(1); // 1 reducer to collect the results
+
+    jobConf.setOutputKeyClass(Text.class);
+    jobConf.setOutputValueClass(IntWritable.class);
+
+    jobConf.setMapperClass(MapperThatRunsNNLoadGenerator.class);
+    jobConf.setReducerClass(ReducerThatCollectsLGdata.class);
+
+    jobConf.setInputFormat(DummyInputFormat.class);
+    jobConf.setOutputFormat(TextOutputFormat.class);
+    
+    // Explicitly set number of max map attempts to 1.
+    jobConf.setMaxMapAttempts(1);
+    // Explicitly turn off speculative execution
+    jobConf.setSpeculativeExecution(false);
+
+    // This mapReduce job has no input but has output
+    FileOutputFormat.setOutputPath(jobConf, new Path(mrOutDir));
+
+    try {
+      JobClient.runJob(jobConf);
+    } catch (IOException e) {
+      System.err.println("Failed to run job: " + e.getMessage());
+      return -1;
+    }
+    return 0;
+    
+  }
+
+  
+  // Each split is empty
+  public static class EmptySplit implements InputSplit {
+    public void write(DataOutput out) throws IOException {}
+    public void readFields(DataInput in) throws IOException {}
+    public long getLength() {return 0L;}
+    public String[] getLocations() {return new String[0];}
+  }
+
+  // Dummy Input format to send 1 record - number of spits is numMapTasks
+  public static class DummyInputFormat extends Configured implements
+      InputFormat<LongWritable, Text> {
+
+    public InputSplit[] getSplits(JobConf conf, int numSplits) {
+      numSplits = conf.getInt("LG.numMapTasks", 1);
+      InputSplit[] ret = new InputSplit[numSplits];
+      for (int i = 0; i < numSplits; ++i) {
+        ret[i] = new EmptySplit();
+      }
+      return ret;
+    }
+
+    public RecordReader<LongWritable, Text> getRecordReader(
+        InputSplit ignored, JobConf conf, Reporter reporter) throws IOException {
+
+      return new RecordReader<LongWritable, Text>() {
+
+        boolean sentOneRecord = false;
+
+        public boolean next(LongWritable key, Text value)
+            throws IOException {
+          key.set(1);
+          value.set("dummy");
+          if (sentOneRecord == false) { // first call
+            sentOneRecord = true;
+            return true;
+          }
+          return false; // we have sent one record - we are done
+        }
+
+        public LongWritable createKey() {
+          return new LongWritable();
+        }
+        public Text createValue() {
+          return new Text();
+        }
+        public long getPos() throws IOException {
+          return 1;
+        }
+        public void close() throws IOException {
+        }
+        public float getProgress() throws IOException {
+          return 1;
+        }
+      };
+    }
+  }
+
+  public static class MapperThatRunsNNLoadGenerator extends MapReduceBase
+      implements Mapper<LongWritable, Text, Text, IntWritable> {
+
+    private JobConf jobConf;
+
+    @Override
+    public void configure(JobConf job) {
+      this.jobConf = job;
+      getArgsFromConfiguration(jobConf);
+    }
+
+    private class ProgressThread extends Thread {
+
+      boolean keepGoing; // while this is true, thread runs.
+      private Reporter reporter;
+
+      public ProgressThread(final Reporter r) {
+        this.reporter = r;
+        this.keepGoing = true;
+      }
+
+      public void run() {
+        while (keepGoing) {
+          if (!ProgressThread.interrupted()) {
+            try {
+              sleep(30 * 1000);
+            } catch (InterruptedException e) {
+            }
+          }
+          reporter.progress();
+        }
+      }
+    }
+
+    public void map(LongWritable key, Text value,
+        OutputCollector<Text, IntWritable> output, Reporter reporter)
+        throws IOException {
+      ProgressThread progressThread = new ProgressThread(reporter);
+      progressThread.start();
+      try {
+        new LoadGenerator(jobConf).generateLoadOnNN();
+        System.out
+            .println("Finished generating load on NN, sending results to the reducer");
+        printResults(System.out);
+        progressThread.keepGoing = false;
+        progressThread.join();
+
+        // Send results to Reducer
+        output.collect(OPEN_EXECTIME,
+            new IntWritable((int) executionTime[OPEN]));
+        output.collect(NUMOPS_OPEN, new IntWritable((int) numOfOps[OPEN]));
+
+        output.collect(LIST_EXECTIME,
+            new IntWritable((int) executionTime[LIST]));
+        output.collect(NUMOPS_LIST, new IntWritable((int) numOfOps[LIST]));
+
+        output.collect(DELETE_EXECTIME, new IntWritable(
+            (int) executionTime[DELETE]));
+        output.collect(NUMOPS_DELETE, new IntWritable((int) numOfOps[DELETE]));
+
+        output.collect(CREATE_EXECTIME, new IntWritable(
+            (int) executionTime[CREATE]));
+        output.collect(NUMOPS_CREATE, new IntWritable((int) numOfOps[CREATE]));
+
+        output.collect(WRITE_CLOSE_EXECTIME, new IntWritable(
+            (int) executionTime[WRITE_CLOSE]));
+        output.collect(NUMOPS_WRITE_CLOSE, new IntWritable(
+            (int) numOfOps[WRITE_CLOSE]));
+
+        output.collect(TOTALOPS, new IntWritable((int) totalOps));
+        output.collect(ELAPSED_TIME, new IntWritable((int) totalTime));
+
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+
+    public void getArgsFromConfiguration(Configuration conf) {
+
+      maxDelayBetweenOps = conf.getInt(LG_MAXDELAYBETWEENOPS,
+          maxDelayBetweenOps);
+      numOfThreads = conf.getInt(LG_NUMOFTHREADS, numOfThreads);
+      readProbs[0] = Double.parseDouble(conf.get(LG_READPR, readProbs[0] + ""));
+      writeProbs[0] = Double.parseDouble(conf.get(LG_WRITEPR, writeProbs[0]
+          + ""));
+      seed = conf.getLong(LG_SEED, seed);
+      numMapTasks = conf.getInt(LG_NUMMAPTASKS, numMapTasks);
+      root = new Path(conf.get(LG_ROOT, root.toString()));
+      durations[0] = conf.getLong(LG_ELAPSEDTIME, 0);
+      startTime = conf.getLong(LG_STARTTIME, 0);
+      scriptFile = conf.get(LG_SCRIPTFILE, null);
+      flagFile = new Path(conf.get(LG_FLAGFILE, FLAGFILE_DEFAULT));
+      if (durations[0] > 0 && scriptFile != null) {
+        System.err.println("Cannot specify both ElapsedTime and ScriptFile, exiting");
+        System.exit(-1);
+      }
+     
+      try {
+        if (scriptFile != null && loadScriptFile(scriptFile, false) < 0) {
+          System.err.println("Error in scriptFile, exiting");
+          System.exit(-1);
+        }
+      } catch (IOException e) {
+       System.err.println("Error loading script file " + scriptFile);
+        e.printStackTrace();
+      }
+      if (durations[0] <= 0) {
+        System.err.println("A duration of zero or less is not allowed when running via MapReduce.");
+        System.exit(-1);
+      }
+    }
+  }
+
+  public static class ReducerThatCollectsLGdata extends MapReduceBase implements
+      Reducer<Text, IntWritable, Text, IntWritable> {
+    private IntWritable result = new IntWritable();
+    private JobConf jobConf;
+    
+    @Override
+    public void configure(JobConf job) {
+      this.jobConf = job;
+    }
+    
+    @Override
+    public void reduce(Text key, Iterator<IntWritable> values,
+        OutputCollector<Text, IntWritable> output, Reporter reporter)
+        throws IOException {
+      int sum = 0;
+      while (values.hasNext()) {
+        sum += values.next().get();
+      }
+      if (key.equals(OPEN_EXECTIME)){
+        executionTime[OPEN] = sum;
+      } else if (key.equals(NUMOPS_OPEN)){
+        numOfOps[OPEN] = sum;
+      } else if (key.equals(LIST_EXECTIME)){
+        executionTime[LIST] = sum;
+      } else if (key.equals(NUMOPS_LIST)){
+        numOfOps[LIST] = sum;
+      } else if (key.equals(DELETE_EXECTIME)){
+        executionTime[DELETE] = sum;
+      } else if (key.equals(NUMOPS_DELETE)){
+        numOfOps[DELETE] = sum;
+      } else if (key.equals(CREATE_EXECTIME)){
+        executionTime[CREATE] = sum;
+      } else if (key.equals(NUMOPS_CREATE)){
+        numOfOps[CREATE] = sum;
+      } else if (key.equals(WRITE_CLOSE_EXECTIME)){
+        System.out.println(WRITE_CLOSE_EXECTIME + " = " + sum);
+        executionTime[WRITE_CLOSE]= sum;
+      } else if (key.equals(NUMOPS_WRITE_CLOSE)){
+        numOfOps[WRITE_CLOSE] = sum;
+      } else if (key.equals(TOTALOPS)){
+        totalOps = sum;
+      } else if (key.equals(ELAPSED_TIME)){
+        totalTime = sum;
+      }
+      result.set(sum);
+      output.collect(key, result);
+      // System.out.println("Key = " + key + " Sum is =" + sum);
+      // printResults(System.out);
+    }
+
+    @Override
+    public void close() throws IOException {
+      // Output the result to a file Results in the output dir
+      FileContext fc;
+      try {
+        fc = FileContext.getFileContext(jobConf);
+      } catch (IOException ioe) {
+        System.err.println("Can not initialize the file system: " + 
+            ioe.getLocalizedMessage());
+        return;
+      }
+      FSDataOutputStream o = fc.create(FileOutputFormat.getTaskOutputPath(jobConf, "Results"),
+          EnumSet.of(CreateFlag.CREATE));
+         
+      PrintStream out = new PrintStream(o);
+      printResults(out);
+      out.close();
+      o.close();
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java
index 4dea0d7917512..265118a70f66d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java
@@ -81,7 +81,7 @@ public void reduce(K key, Iterator<Long> values,
   }
 
   private void cleanAndCreateInput(FileSystem fs) throws IOException {
-    fs.delete(INPUT_FILE, true);
+    fs.delete(INPUT_DIR, true);
     fs.delete(OUTPUT_DIR, true);
 
     OutputStream os = fs.create(INPUT_FILE);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 76a35988354d5..346953f90b3cf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -34,58 +34,6 @@
 
 public class TestMRTimelineEventHandling {
 
-  @Test
-  public void testTimelineServiceStartInMiniCluster() throws Exception {
-    Configuration conf = new YarnConfiguration();
-
-    /*
-     * Timeline service should not start if the config is set to false
-     * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
-     */
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
-    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
-    MiniMRYarnCluster cluster = null;
-    try {
-      cluster = new MiniMRYarnCluster(
-          TestJobHistoryEventHandler.class.getSimpleName(), 1);
-      cluster.init(conf);
-      cluster.start();
-
-      //verify that the timeline service is not started.
-      Assert.assertNull("Timeline Service should not have been started",
-          cluster.getApplicationHistoryServer());
-
-      //Run a MR job and verify it succeeds
-      Path inDir = new Path("input");
-      Path outDir = new Path("output");
-      RunningJob job =
-          UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
-      Assert.assertEquals(JobStatus.SUCCEEDED,
-          job.getJobStatus().getState().getValue());
-    }
-    finally {
-      if(cluster != null) {
-        cluster.stop();
-      }
-    }
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
-    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
-    cluster = null;
-    try {
-        cluster = new MiniMRYarnCluster(
-            TestJobHistoryEventHandler.class.getSimpleName(), 1);
-        cluster.init(conf);
-        cluster.start();
-        Assert.assertNull("Timeline Service should not have been started",
-            cluster.getApplicationHistoryServer());
-      }
-      finally {
-        if(cluster != null) {
-          cluster.stop();
-        }
-      }
-  }
-
   @Test
   public void testMRTimelineEventHandling() throws Exception {
     Configuration conf = new YarnConfiguration();
@@ -94,7 +42,7 @@ public void testMRTimelineEventHandling() throws Exception {
     MiniMRYarnCluster cluster = null;
     try {
       cluster = new MiniMRYarnCluster(
-          TestJobHistoryEventHandler.class.getSimpleName(), 1);
+              TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
       cluster.init(conf);
       cluster.start();
       TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -148,7 +96,7 @@ public void testMapreduceJobTimelineServiceEnabled()
     MiniMRYarnCluster cluster = null;
     try {
       cluster = new MiniMRYarnCluster(
-          TestJobHistoryEventHandler.class.getSimpleName(), 1);
+          TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
       cluster.init(conf);
       cluster.start();
       TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -185,7 +133,7 @@ public void testMapreduceJobTimelineServiceEnabled()
     cluster = null;
     try {
       cluster = new MiniMRYarnCluster(
-          TestJobHistoryEventHandler.class.getSimpleName(), 1);
+          TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
       cluster.init(conf);
       cluster.start();
       TimelineStore ts = cluster.getApplicationHistoryServer()
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 17b5fd28f1a8c..59aa00f4d2471 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -185,7 +185,7 @@ public void launchTest(JobConf conf,
     
     // Launch job with default option for temp dir. 
     // i.e. temp dir is ./tmp 
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     job.addFileToClassPath(APP_JAR);
     job.setJarByClass(TestMiniMRChildTask.class);
     job.setMaxMapAttempts(1); // speed up failures
@@ -391,36 +391,7 @@ public static void tearDown() {
       ioe.printStackTrace();           
     }
   }
-  
-  /**
-   * Tests task's temp directory.
-   * 
-   * In this test, we give different values to mapreduce.task.tmp.dir
-   * both relative and absolute. And check whether the temp directory 
-   * is created. We also check whether java.io.tmpdir value is same as 
-   * the directory specified. We create a temp file and check if is is 
-   * created in the directory specified.
-   */
-  @Test
-  public void testTaskTempDir(){
-    try {
-      JobConf conf = new JobConf(mr.getConfig());
-      
-      // intialize input, output directories
-      Path inDir = new Path("testing/wc/input");
-      Path outDir = new Path("testing/wc/output");
-      String input = "The input";
-      configure(conf, inDir, outDir, input, 
-          MapClass.class, IdentityReducer.class);
-      launchTest(conf, inDir, outDir, input);
-      
-    } catch(Exception e) {
-      e.printStackTrace();
-      fail("Exception in testing temp dir");
-      tearDown();
-    }
-  }
-
+ 
   /**
    * To test OS dependent setting of default execution path for a MapRed task.
    * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
@@ -566,7 +537,7 @@ void runTestTaskEnv(JobConf conf, Path inDir, Path outDir, boolean oldConfigs)
     conf.set(mapTaskJavaOptsKey, mapTaskJavaOpts);
     conf.set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);
 
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     job.addFileToClassPath(APP_JAR);
     job.setJarByClass(TestMiniMRChildTask.class);
     job.setMaxMapAttempts(1); // speed up failures
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
index d988c08a6f685..7630d18798eae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
@@ -187,7 +187,7 @@ private static void createFile(Path inFile, Configuration conf)
   }
 
   public static Job createJob() throws IOException {
-    final Job baseJob = new Job(mrCluster.getConfig());
+    final Job baseJob = Job.getInstance(mrCluster.getConfig());
     baseJob.setOutputKeyClass(Text.class);
     baseJob.setOutputValueClass(IntWritable.class);
     baseJob.setMapperClass(MyMapper.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/LargeSorter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/LargeSorter.java
index c5be44642e415..d6d0339d51c1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/LargeSorter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/LargeSorter.java
@@ -231,8 +231,7 @@ public int run(String[] args) throws Exception {
     conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMb);
     conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx" + (mapMb - 200) + "m");
 
-    @SuppressWarnings("deprecation")
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     job.setJarByClass(LargeSorter.class);
     job.setJobName("large-sorter");
     FileOutputFormat.setOutputPath(job, outDir);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
index eb3570445010b..40e101ab8fd59 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomTextWriter.java
@@ -195,7 +195,7 @@ public int run(String[] args) throws Exception {
     }
     conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
     
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     
     job.setJarByClass(RandomTextWriter.class);
     job.setJobName("random-text-writer");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomWriter.java
index def9e546ebbdc..a326c8ca2be2c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/RandomWriter.java
@@ -261,7 +261,7 @@ public int run(String[] args) throws Exception {
     }
     conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
 
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     
     job.setJarByClass(RandomWriter.class);
     job.setJobName("random-writer");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
index 83d689c1e9bb4..0215568c4786f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
@@ -17,8 +17,12 @@
  */
 package org.apache.hadoop.mapreduce;
 
+import java.io.IOException;
 import java.util.Random;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
 import org.junit.Test;
 import static org.junit.Assert.*;
 
@@ -70,7 +74,40 @@ public void testCounterValue() {
       testMaxGroups(new Counters());
     }
   }
-  
+
+  @Test public void testResetOnDeserialize() throws IOException {
+    // Allow only one counterGroup
+    Configuration conf = new Configuration();
+    conf.setInt(MRJobConfig.COUNTER_GROUPS_MAX_KEY, 1);
+    Limits.init(conf);
+
+    Counters countersWithOneGroup = new Counters();
+    countersWithOneGroup.findCounter("firstOf1Allowed", "First group");
+    boolean caughtExpectedException = false;
+    try {
+      countersWithOneGroup.findCounter("secondIsTooMany", "Second group");
+    }
+    catch (LimitExceededException _) {
+      caughtExpectedException = true;
+    }
+
+    assertTrue("Did not throw expected exception",
+        caughtExpectedException);
+
+    Counters countersWithZeroGroups = new Counters();
+    DataOutputBuffer out = new DataOutputBuffer();
+    countersWithZeroGroups.write(out);
+
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(out.getData(), out.getLength());
+
+    countersWithOneGroup.readFields(in);
+
+    // After reset one should be able to add a group
+    countersWithOneGroup.findCounter("firstGroupAfterReset", "After reset " +
+        "limit should be set back to zero");
+  }
+
   @Test
   public void testCountersIncrement() {
     Counters fCounters = new Counters();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLargeSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLargeSort.java
index ab99a2fe0cf73..eb48d6799be55 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLargeSort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLargeSort.java
@@ -55,6 +55,7 @@ public void testLargeSort() throws Exception {
     int[] ioSortMbs = {128, 256, 1536};
     for (int ioSortMb : ioSortMbs) {
       Configuration conf = new Configuration(cluster.getConfig());
+      conf.setInt(MRJobConfig.MAP_MEMORY_MB, 2048);
       conf.setInt(MRJobConfig.IO_SORT_MB, ioSortMb);
       conf.setInt(LargeSorter.NUM_MAP_TASKS, 1);
       conf.setInt(LargeSorter.MBS_PER_MAP, ioSortMb);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
index 48ad47af22000..912f0e352f743 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
@@ -465,11 +465,15 @@ private static void printSequenceFile(FileSystem fs, Path p,
   private static boolean isSequenceFile(FileSystem fs,
                                         Path f) throws IOException {
     DataInputStream in = fs.open(f);
-    byte[] seq = "SEQ".getBytes();
-    for(int i=0; i < seq.length; ++i) {
-      if (seq[i] != in.read()) {
-        return false;
+    try {
+      byte[] seq = "SEQ".getBytes();
+      for (int i = 0; i < seq.length; ++i) {
+        if (seq[i] != in.read()) {
+          return false;
+        }
       }
+    } finally {
+      in.close();
     }
     return true;
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 45b736e37d226..47b38a1365c58 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -72,7 +72,11 @@ public MiniMRYarnCluster(String testName) {
   }
 
   public MiniMRYarnCluster(String testName, int noOfNMs) {
-    super(testName, 1, noOfNMs, 4, 4);
+    this(testName, noOfNMs, false);
+  }
+
+  public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
+    super(testName, 1, noOfNMs, 4, 4, enableAHS);
     historyServerWrapper = new JobHistoryServerWrapper();
     addService(historyServerWrapper);
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 2b45049e08c07..60e56386a53df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -547,7 +547,7 @@ protected Job runFailingMapperJob()
     myConf.setInt(MRJobConfig.NUM_MAPS, 1);
     myConf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); //reduce the number of attempts
 
-    Job job = new Job(myConf);
+    Job job = Job.getInstance(myConf);
 
     job.setJarByClass(FailingMapper.class);
     job.setJobName("failmapper");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
index f2cd53cdfa228..b1dfe56dbf85d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
@@ -41,6 +41,10 @@
 import org.apache.hadoop.fs.DistributedFSCheck;
 import org.apache.hadoop.io.FileBench;
 import org.apache.hadoop.fs.JHLogAnalyzer;
+import org.apache.hadoop.fs.loadGenerator.DataGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGeneratorMR;
+import org.apache.hadoop.fs.loadGenerator.StructureGenerator;
 import org.apache.hadoop.fs.slive.SliveTest;
 
 /**
@@ -107,6 +111,14 @@ public MapredTestDriver(ProgramDriver pgd) {
       "Single process HDFS and MR cluster.");
       pgd.addClass("largesorter", LargeSorter.class,
           "Large-Sort tester");
+      pgd.addClass("NNloadGenerator", LoadGenerator.class,
+              "Generate load on Namenode using NN loadgenerator run WITHOUT MR");
+      pgd.addClass("NNloadGeneratorMR", LoadGeneratorMR.class,
+          "Generate load on Namenode using NN loadgenerator run as MR job");
+      pgd.addClass("NNstructureGenerator", StructureGenerator.class,
+          "Generate the structure to be used by NNdataGenerator");
+      pgd.addClass("NNdataGenerator", DataGenerator.class,
+          "Generate the data to be used by NNloadGenerator");
     } catch(Throwable e) {
       e.printStackTrace();
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
index 9ba3b776666d8..8ca2e162b4b91 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
@@ -77,7 +77,7 @@ public static void main(String [] args) throws Exception
   {
     Path outDir = new Path("output");
     Configuration conf = new Configuration();
-    Job job = new Job(conf, "user name check"); 
+    Job job = Job.getInstance(conf, "user name check"); 
 			
 			
     job.setJarByClass(UserNamePermission.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h
index 4c0c1a7297436..3bf5f767ed2b7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/primitives.h
@@ -97,11 +97,18 @@ inline void simple_memcpy(void * dest, const void * src, size_t len) {
  * little-endian to big-endian or vice versa
  */
 inline uint32_t bswap(uint32_t val) {
+#ifdef __aarch64__
+  __asm__("rev %w[dst], %w[src]" : [dst]"=r"(val) : [src]"r"(val));
+#else
   __asm__("bswap %0" : "=r" (val) : "0" (val));
+#endif
   return val;
 }
 
 inline uint64_t bswap64(uint64_t val) {
+#ifdef __aarch64__
+  __asm__("rev %[dst], %[src]" : [dst]"=r"(val) : [src]"r"(val));
+#else
 #ifdef __X64
   __asm__("bswapq %0" : "=r" (val) : "0" (val));
 #else
@@ -114,6 +121,7 @@ inline uint64_t bswap64(uint64_t val) {
 
   return (lower << 32) + higher;
 
+#endif
 #endif
   return val;
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
index 191e09389412d..be800c5685bcc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc
@@ -579,6 +579,11 @@ const uint32_t CRC32C_T8_7[256] = {0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB44876
     0xCF56CE31, 0x14124958, 0x5D2E347F, 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 0xC451B7CC,
     0x8D6DCAEB, 0x56294D82, 0x1F1530A5};
 
+#ifdef __aarch64__
+// Awaiting HW implementation
+#define SOFTWARE_CRC
+#endif
+
 #ifndef SOFTWARE_CRC
 #define USE_HARDWARE_CRC32C 1
 #endif
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 2d4515f6366f4..3b4c9c004ff47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -18,9 +18,9 @@
 package org.apache.hadoop.mapred.nativetask.kvtest;
 
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import java.util.zip.CRC32;
 
-import com.google.common.base.Stopwatch;
 import com.google.common.primitives.Longs;
 
 import org.apache.commons.logging.Log;
@@ -36,6 +36,7 @@
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.StopWatch;
 
 public class KVJob {
   public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
@@ -93,9 +94,10 @@ public KVJob(String jobname, Configuration conf,
       final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
           TestConstants.FILESIZE_KEY, "1000")),
           keyclass.getName(), valueclass.getName(), conf);
-      Stopwatch sw = new Stopwatch().start();
+      StopWatch sw = new StopWatch().start();
       testfile.createSequenceTestFile(inputpath);
-      LOG.info("Created test file " + inputpath + " in " + sw.elapsedMillis() + "ms");
+      LOG.info("Created test file " + inputpath + " in "
+          + sw.now(TimeUnit.MILLISECONDS) + "ms");
     }
     job.setInputFormatClass(SequenceFileInputFormat.class);
     FileInputFormat.addInputPath(job, new Path(inputpath));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 33220c6cbdb9c..6e069f1926398 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -493,9 +493,9 @@ private void startStore(Path recoveryRoot) throws IOException {
   @VisibleForTesting
   Version loadVersion() throws IOException {
     byte[] data = stateDb.get(bytes(STATE_DB_SCHEMA_VERSION_KEY));
-    // if version is not stored previously, treat it as 1.0.
+    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
     if (data == null || data.length == 0) {
-      return Version.newInstance(1, 0);
+      return getCurrentVersion();
     }
     Version version =
         new VersionPBImpl(VersionProto.parseFrom(data));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
index 36a953cb74050..7c023cff6b5f6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
@@ -315,7 +315,7 @@ public void close() {
   /** Create and setup a job */
   private static Job createJob(String name, Configuration conf
       ) throws IOException {
-    final Job job = new Job(conf, NAME + "_" + name);
+    final Job job = Job.getInstance(conf, NAME + "_" + name);
     final Configuration jobconf = job.getConfiguration();
     job.setJarByClass(BaileyBorweinPlouffe.class);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Grep.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Grep.java
index 421afcd7ace71..44a51e0a5ff60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Grep.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Grep.java
@@ -56,11 +56,12 @@ public int run(String[] args) throws Exception {
     if (args.length == 4)
       conf.set(RegexMapper.GROUP, args[3]);
 
-    Job grepJob = new Job(conf);
+    Job grepJob = Job.getInstance(conf);
     
     try {
       
       grepJob.setJobName("grep-search");
+      grepJob.setJarByClass(Grep.class);
 
       FileInputFormat.setInputPaths(grepJob, args[0]);
 
@@ -76,8 +77,9 @@ public int run(String[] args) throws Exception {
 
       grepJob.waitForCompletion(true);
 
-      Job sortJob = new Job(conf);
+      Job sortJob = Job.getInstance(conf);
       sortJob.setJobName("grep-sort");
+      sortJob.setJarByClass(Grep.class);
 
       FileInputFormat.setInputPaths(sortJob, tempDir);
       sortJob.setInputFormatClass(SequenceFileInputFormat.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
index 2063d04655a7f..d8318922cb5c1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
@@ -89,7 +89,7 @@ public int run(String[] args) throws Exception {
        num_reduces = cluster.getTaskTrackers() * 
                        Integer.parseInt(join_reduces);
     }
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     job.setJobName("join");
     job.setJarByClass(Sort.class);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/MultiFileWordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/MultiFileWordCount.java
index af5e370d1034d..d3df4b303d0fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/MultiFileWordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/MultiFileWordCount.java
@@ -229,7 +229,7 @@ public int run(String[] args) throws Exception {
       return 2;
     }
 
-    Job job = new Job(getConf());
+    Job job = Job.getInstance(getConf());
     job.setJobName("MultiFileWordCount");
     job.setJarByClass(MultiFileWordCount.class);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
index 8ddd64fd1f3a1..d565098e0a153 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
@@ -248,7 +248,7 @@ public void cleanup(Context context) throws IOException {
   public static BigDecimal estimatePi(int numMaps, long numPoints,
       Path tmpDir, Configuration conf
       ) throws IOException, ClassNotFoundException, InterruptedException {
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     //setup job conf
     job.setJobName(QuasiMonteCarlo.class.getSimpleName());
     job.setJarByClass(QuasiMonteCarlo.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
index c1f3f5eb15627..4d555c606d391 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomTextWriter.java
@@ -195,7 +195,7 @@ public int run(String[] args) throws Exception {
     }
     conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
     
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     
     job.setJarByClass(RandomTextWriter.class);
     job.setJobName("random-text-writer");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomWriter.java
index ea2e9efdd6b90..e1c13ecfa2735 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/RandomWriter.java
@@ -261,7 +261,7 @@ public int run(String[] args) throws Exception {
     }
     conf.setInt(MRJobConfig.NUM_MAPS, numMaps);
 
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     
     job.setJarByClass(RandomWriter.class);
     job.setJobName("random-writer");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/SecondarySort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/SecondarySort.java
index 6d07735ee007b..d536ec98c11cd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/SecondarySort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/SecondarySort.java
@@ -214,7 +214,7 @@ public static void main(String[] args) throws Exception {
       System.err.println("Usage: secondarysort <in> <out>");
       System.exit(2);
     }
-    Job job = new Job(conf, "secondary sort");
+    Job job = Job.getInstance(conf, "secondary sort");
     job.setJarByClass(SecondarySort.class);
     job.setMapperClass(MapClass.class);
     job.setReducerClass(Reduce.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java
index 5184bf0a58af3..a90c02b0e3a77 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java
@@ -132,7 +132,7 @@ public int run(String[] args) throws Exception {
       }
     }
     // Set user-supplied (possibly default) job configs
-    job = new Job(conf);
+    job = Job.getInstance(conf);
     job.setJobName("sorter");
     job.setJarByClass(Sort.class);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
index 86341307679ba..8a069630433a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
@@ -72,7 +72,7 @@ public static void main(String[] args) throws Exception {
       System.err.println("Usage: wordcount <in> [<in>...] <out>");
       System.exit(2);
     }
-    Job job = new Job(conf, "word count");
+    Job job = Job.getInstance(conf, "word count");
     job.setJarByClass(WordCount.class);
     job.setMapperClass(TokenizerMapper.class);
     job.setCombinerClass(IntSumReducer.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java
index fd7e9b692d86d..626bb18eaa952 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java
@@ -172,8 +172,7 @@ public int run(String[] args) throws Exception {
 
     Configuration conf = getConf();
 
-    @SuppressWarnings("deprecation")
-    Job job = new Job(conf, "word mean");
+    Job job = Job.getInstance(conf, "word mean");
     job.setJarByClass(WordMean.class);
     job.setMapperClass(WordMeanMapper.class);
     job.setCombinerClass(WordMeanReducer.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java
index 36a14719998b1..dd83122d9654f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java
@@ -181,8 +181,7 @@ public int run(String[] args) throws Exception {
     setConf(new Configuration());
     Configuration conf = getConf();
 
-    @SuppressWarnings("deprecation")
-    Job job = new Job(conf, "word median");
+    Job job = Job.getInstance(conf, "word median");
     job.setJarByClass(WordMedian.class);
     job.setMapperClass(WordMedianMapper.class);
     job.setCombinerClass(WordMedianReducer.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java
index 50a8f1edcc5d6..da727fab4f043 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java
@@ -189,8 +189,7 @@ public int run(String[] args) throws Exception {
 
     Configuration conf = getConf();
 
-    @SuppressWarnings("deprecation")
-    Job job = new Job(conf, "word stddev");
+    Job job = Job.getInstance(conf, "word stddev");
     job.setJarByClass(WordStandardDeviation.class);
     job.setMapperClass(WordStandardDeviationMapper.class);
     job.setCombinerClass(WordStandardDeviationReducer.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
index e97d9c3c424aa..29f1eb2c45e24 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
@@ -198,7 +198,7 @@ public int run(String[] args) throws Exception {
     Path input = new Path(output + "_input");
     FileSystem fileSys = FileSystem.get(conf);
     try {
-      Job job = new Job(conf);
+      Job job = Job.getInstance(conf);
       FileInputFormat.setInputPaths(job, input);
       FileOutputFormat.setOutputPath(job, output);
       job.setJarByClass(PentMap.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
index b365ba701d3fe..99f7c2485dff6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
@@ -432,7 +432,8 @@ private Machine chooseMachine(Configuration conf) throws IOException {
 
   /** Create a job */
   private Job createJob(String name, Summation sigma) throws IOException {
-    final Job job = new Job(getConf(), parameters.remoteDir + "/" + name);
+    final Job job = Job.getInstance(getConf(), parameters.remoteDir + "/" +
+                                    name);
     final Configuration jobconf = job.getConfiguration();
     job.setJarByClass(DistSum.class);
     jobconf.setInt(N_PARTS, parameters.nParts);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
index 187520a399181..a2db9d1bac69d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
@@ -151,11 +151,10 @@ Map<Parameter, List<TaskResult>> parse(String inputpath, String outputdir
   static <T extends Combinable<T>> Map<Parameter, T> combine(Map<Parameter, List<T>> m) {
     final Map<Parameter, T> combined = new TreeMap<Parameter, T>();
     for(Parameter p : Parameter.values()) {
+      //note: results would never be null due to the design of Util.combine
       final List<T> results = Util.combine(m.get(p));
       Util.out.format("%-6s => ", p); 
-      if (results == null)
-        Util.out.println("null");
-      else if (results.size() != 1) 
+      if (results.size() != 1)
         Util.out.println(results.toString().replace(", ", ",\n           "));
       else {
         final T r = results.get(0);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/Bellard.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/Bellard.java
index 90b608fd24cde..d909d92945151 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/Bellard.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/Bellard.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.NoSuchElementException;
 
 import org.apache.hadoop.examples.pi.Container;
 import org.apache.hadoop.examples.pi.Util;
@@ -255,7 +256,13 @@ public Iterator<Summation> iterator() {
         public boolean hasNext() {return i < parts.length;}
         /** {@inheritDoc} */
         @Override
-        public Summation next() {return parts[i++];}
+        public Summation next() throws NoSuchElementException {
+          if (hasNext()) {
+            return parts[i++];
+          } else {
+            throw new NoSuchElementException("Sum's iterator does not have next!");
+          }
+        }
         /** Unsupported */
         @Override
         public void remove() {throw new UnsupportedOperationException();}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
index 7e679343b4b6f..e8b65038e7971 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
@@ -289,10 +289,6 @@ public int run(String[] args)
     }
     setNumberOfRows(job, parseHumanLong(args[0]));
     Path outputDir = new Path(args[1]);
-    if (outputDir.getFileSystem(getConf()).exists(outputDir)) {
-      throw new IOException("Output directory " + outputDir + 
-                            " already exists.");
-    }
     FileOutputFormat.setOutputPath(job, outputDir);
     job.setJobName("TeraGen");
     job.setJarByClass(TeraGen.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
index 872e71917bec9..867f33ef7a393 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
@@ -20,10 +20,13 @@
 
 import java.io.IOException;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.InvalidJobConfException;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -87,9 +90,31 @@ public void checkOutputSpecs(JobContext job
       throw new InvalidJobConfException("Output directory not set in JobConf.");
     }
 
+    final Configuration jobConf = job.getConfiguration();
+
     // get delegation token for outDir's file system
     TokenCache.obtainTokensForNamenodes(job.getCredentials(),
-        new Path[] { outDir }, job.getConfiguration());
+        new Path[] { outDir }, jobConf);
+
+    final FileSystem fs = outDir.getFileSystem(jobConf);
+
+    if (fs.exists(outDir)) {
+      // existing output dir is considered empty iff its only content is the
+      // partition file.
+      //
+      final FileStatus[] outDirKids = fs.listStatus(outDir);
+      boolean empty = false;
+      if (outDirKids != null && outDirKids.length == 1) {
+        final FileStatus st = outDirKids[0];
+        final String fname = st.getPath().getName();
+        empty =
+          !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
+      }
+      if (TeraSort.getUseSimplePartitioner(job) || !empty) {
+        throw new FileAlreadyExistsException("Output directory " + outDir
+            + " already exists");
+      }
+    }
   }
 
   public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
index 4a11c9a331eb8..1956872b7ed57 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
@@ -20,17 +20,19 @@
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.util.ToolRunner;
-import org.junit.Ignore;
-@Ignore
 public class TestTeraSort extends HadoopTestCase {
+  private static Log LOG = LogFactory.getLog(TestTeraSort.class);
   
   public TestTeraSort()
       throws IOException {
-    super(CLUSTER_MR, DFS_FS, 1, 1);
+    super(LOCAL_MR, LOCAL_FS, 1, 1);
   }
 
   protected void tearDown() throws Exception {
@@ -45,42 +47,58 @@ protected void tearDown() throws Exception {
   private static final Path SORT_INPUT_PATH = new Path(TEST_DIR, "sortin");
   private static final Path SORT_OUTPUT_PATH = new Path(TEST_DIR, "sortout");
   private static final Path TERA_OUTPUT_PATH = new Path(TEST_DIR, "validate");
-  private static final String NUM_ROWS = "100"; 
+  private static final String NUM_ROWS = "100";
 
-  private void runTeraGen(Configuration conf, Path sortInput) 
+  private void runTeraGen(Configuration conf, Path sortInput)
       throws Exception {
     String[] genArgs = {NUM_ROWS, sortInput.toString()};
-    
+
     // Run TeraGen
     assertEquals(ToolRunner.run(conf, new TeraGen(), genArgs), 0);
   }
-  
+
   private void runTeraSort(Configuration conf,
       Path sortInput, Path sortOutput) throws Exception {
 
     // Setup command-line arguments to 'sort'
     String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
-    
+
     // Run Sort
     assertEquals(ToolRunner.run(conf, new TeraSort(), sortArgs), 0);
   }
-  
-  private void runTeraValidator(Configuration job, 
-                                       Path sortOutput, Path valOutput) 
+
+  private void runTeraValidator(Configuration job,
+                                       Path sortOutput, Path valOutput)
   throws Exception {
     String[] svArgs = {sortOutput.toString(), valOutput.toString()};
 
     // Run Tera-Validator
     assertEquals(ToolRunner.run(job, new TeraValidate(), svArgs), 0);
   }
-  
+
   public void testTeraSort() throws Exception {
     // Run TeraGen to generate input for 'terasort'
     runTeraGen(createJobConf(), SORT_INPUT_PATH);
 
+    // Run teragen again to check for FAE
+    try {
+      runTeraGen(createJobConf(), SORT_INPUT_PATH);
+      fail("Teragen output overwritten!");
+    } catch (FileAlreadyExistsException fae) {
+      LOG.info("Expected exception: ", fae);
+    }
+
     // Run terasort
     runTeraSort(createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
 
+    // Run terasort again to check for FAE
+    try {
+      runTeraSort(createJobConf(), SORT_INPUT_PATH, SORT_OUTPUT_PATH);
+      fail("Terasort output overwritten!");
+    } catch (FileAlreadyExistsException fae) {
+      LOG.info("Expected exception: ", fae);
+    }
+
     // Run tera-validator to check if sort worked correctly
     runTeraValidator(createJobConf(), SORT_OUTPUT_PATH,
       TERA_OUTPUT_PATH);
diff --git a/hadoop-mapreduce-project/shellprofile.d/mapreduce b/hadoop-mapreduce-project/shellprofile.d/mapreduce
new file mode 100644
index 0000000000000..0b3dab13a9782
--- /dev/null
+++ b/hadoop-mapreduce-project/shellprofile.d/mapreduce
@@ -0,0 +1,41 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile mapred
+
+function _mapred_hadoop_classpath
+{
+  #
+  # get all of the mapreduce jars+config in the path
+  #
+  # developers
+  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
+  fi
+
+  if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
+    hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
+  fi
+
+  hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
+  hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
+}
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index c271c91800a22..b48b9acee5216 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -14,7 +14,7 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
@@ -28,6 +28,7 @@
   <name>Apache Hadoop Maven Plugins</name>
   <properties>
     <maven.dependency.version>3.0</maven.dependency.version>
+    <maven.plugin-tools.version>3.4</maven.plugin-tools.version>
   </properties>
   <dependencies>
     <dependency>
@@ -43,30 +44,20 @@
     <dependency>
       <groupId>org.apache.maven.plugin-tools</groupId>
       <artifactId>maven-plugin-annotations</artifactId>
-      <version>${maven.dependency.version}</version>
+      <version>${maven.plugin-tools.version}</version>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
   <build>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-plugin-plugin</artifactId>
-        <version>${maven.dependency.version}</version>
-        <configuration>
-          <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
-        </configuration>
+        <version>${maven.plugin-tools.version}</version>
         <executions>
           <execution>
-            <id>mojo-descriptor</id>
-            <goals>
-              <goal>descriptor</goal>
-            </goals>
+            <id>default-descriptor</id>
+            <phase>process-classes</phase>
           </execution>
         </executions>
       </plugin>
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
index 86ba7bf5291e2..465b7136a9d53 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
@@ -31,7 +31,7 @@
 @Mojo(name="protoc", defaultPhase = LifecyclePhase.GENERATE_SOURCES)
 public class ProtocMojo extends AbstractMojo {
 
-  @Parameter(defaultValue="${project}")
+  @Parameter(defaultValue="${project}", readonly=true)
   private MavenProject project;
 
   @Parameter
@@ -43,7 +43,7 @@ public class ProtocMojo extends AbstractMojo {
   @Parameter(required=true)
   private FileSet source;
 
-  @Parameter
+  @Parameter(defaultValue="protoc")
   private String protocCommand;
 
   @Parameter(required=true)
@@ -51,9 +51,6 @@ public class ProtocMojo extends AbstractMojo {
 
   public void execute() throws MojoExecutionException {
     try {
-      if (protocCommand == null || protocCommand.trim().isEmpty()) {
-        protocCommand = "protoc";
-      }
       List<String> command = new ArrayList<String>();
       command.add(protocCommand);
       command.add("--version");
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
index 45b40c21266ee..141ab5afc3fb4 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
@@ -42,8 +42,8 @@ public Exec(Mojo mojo) {
    * Runs the specified command and saves each line of the command's output to
    * the given list.
    * 
-   * @param command List<String> containing command and all arguments
-   * @param output List<String> in/out parameter to receive command output
+   * @param command List containing command and all arguments
+   * @param output List in/out parameter to receive command output
    * @return int exit code of command
    */
   public int run(List<String> command, List<String> output) {
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java
index 6661f676dba74..2f2d2fbb60e3b 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/FileSetUtils.java
@@ -33,7 +33,7 @@ public class FileSetUtils {
    * @param list List of all elements
    * @return String containing every element, comma-separated
    */
-  private static String getCommaSeparatedList(List list) {
+  private static String getCommaSeparatedList(List<String> list) {
     StringBuilder buffer = new StringBuilder();
     String separator = "";
     for (Object e : list) {
@@ -47,7 +47,7 @@ private static String getCommaSeparatedList(List list) {
    * Converts a Maven FileSet to a list of File objects.
    * 
    * @param source FileSet to convert
-   * @return List<File> containing every element of the FileSet as a File
+   * @return List containing every element of the FileSet as a File
    * @throws IOException if an I/O error occurs while trying to find the files
    */
   @SuppressWarnings("unchecked")
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
index 563bd5948a59a..f342463370fc5 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
@@ -18,7 +18,6 @@
 import org.apache.maven.model.FileSet;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugins.annotations.LifecyclePhase;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.project.MavenProject;
@@ -47,7 +46,7 @@
 @Mojo(name="version-info")
 public class VersionInfoMojo extends AbstractMojo {
 
-  @Parameter(defaultValue="${project}")
+  @Parameter(defaultValue="${project}", readonly=true)
   private MavenProject project;
 
   @Parameter(required=true)
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index c9e54d3dba702..79cde7b55b138 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -13,7 +13,7 @@
  limitations under the License. See accompanying LICENSE file.
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d3c404e59f5bf..0c7cfc864d5a7 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -71,8 +71,42 @@
     <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
 
     <zookeeper.version>3.4.6</zookeeper.version>
+    <curator.version>2.7.1</curator.version>
+    <findbugs.version>3.0.0</findbugs.version>
 
     <tomcat.version>6.0.41</tomcat.version>
+
+    <!-- define the Java language version used by the compiler -->
+    <javac.version>1.7</javac.version>
+
+    <!-- The java version enforced by the maven enforcer -->
+    <!-- more complex patterns can be used here, such as
+       [${javac.version})
+    for an open-ended enforcement
+    -->
+    <enforced.java.version>[${javac.version},)</enforced.java.version>
+    <enforced.maven.version>[3.0.2,)</enforced.maven.version>
+
+    <!-- Plugin versions and config -->
+    <maven-surefire-plugin.argLine>-Xmx4096m -XX:MaxPermSize=768m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
+    <maven-surefire-plugin.version>2.17</maven-surefire-plugin.version>
+    <maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
+    <maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
+
+    <maven-clean-plugin.version>2.5</maven-clean-plugin.version>
+    <maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
+    <maven-install-plugin.version>2.5.1</maven-install-plugin.version>
+    <maven-resources-plugin.version>2.6</maven-resources-plugin.version>
+    <maven-jar-plugin.version>2.5</maven-jar-plugin.version>
+    <maven-war-plugin.version>2.4</maven-war-plugin.version>
+    <maven-source-plugin.version>2.3</maven-source-plugin.version>
+    <maven-checkstyle-plugin.version>2.12.1</maven-checkstyle-plugin.version>
+    <maven-project-info-reports-plugin.version>2.7</maven-project-info-reports-plugin.version>
+    <maven-pdf-plugin.version>1.2</maven-pdf-plugin.version>
+    <build-helper-maven-plugin.version>1.9</build-helper-maven-plugin.version>
+    <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
+    <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
+    <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
   </properties>
 
   <dependencyManagement>
@@ -657,17 +691,17 @@
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
-        <version>1.7.5</version>
+        <version>1.7.10</version>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
-        <version>1.7.5</version>
+        <version>1.7.10</version>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>jul-to-slf4j</artifactId>
-        <version>1.7.5</version>
+        <version>1.7.10</version>
       </dependency>
       <dependency>
         <groupId>org.eclipse.jdt</groupId>
@@ -750,9 +784,9 @@
         <version>0.1.42</version>
       </dependency>
       <dependency>
-        <groupId>org.htrace</groupId>
+        <groupId>org.apache.htrace</groupId>
         <artifactId>htrace-core</artifactId>
-        <version>3.0.4</version>
+        <version>3.1.0-incubating</version>
       </dependency>
       <dependency>
         <groupId>org.jdom</groupId>
@@ -826,7 +860,7 @@
       <dependency>
         <groupId>com.google.code.findbugs</groupId>
         <artifactId>jsr305</artifactId>
-        <version>1.3.9</version>
+        <version>${findbugs.version}</version>
       </dependency>
       <dependency>
         <groupId>javax.xml.bind</groupId>
@@ -883,22 +917,22 @@
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-recipes</artifactId>
-       <version>2.6.0</version>
+       <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-client</artifactId>
-       <version>2.6.0</version>
+       <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-framework</artifactId>
-       <version>2.6.0</version>
+       <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-test</artifactId>
-       <version>2.6.0</version>
+       <version>${curator.version}</version>
      </dependency>
      <dependency>
        <groupId>org.bouncycastle</groupId>
@@ -915,15 +949,15 @@
       <plugins>
         <plugin>
           <artifactId>maven-clean-plugin</artifactId>
-          <version>2.4.1</version>
+          <version>${maven-clean-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>2.5.1</version>
+          <version>${maven-compiler-plugin.version}</version>
           <configuration>
-            <source>1.6</source>
-            <target>1.6</target>
+            <source>${javac.version}</source>
+            <target>${javac.version}</target>
           </configuration>
         </plugin>
         <plugin>
@@ -934,22 +968,22 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>build-helper-maven-plugin</artifactId>
-          <version>1.5</version>
+          <version>${build-helper-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.16</version>
+          <version>${maven-surefire-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-install-plugin</artifactId>
-          <version>2.3.1</version>
+          <version>${maven-install-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-jar-plugin</artifactId>
-          <version>2.3.1</version>
+          <version>${maven-jar-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -959,32 +993,32 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-war-plugin</artifactId>
-          <version>2.1</version>
+          <version>${maven-war-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>findbugs-maven-plugin</artifactId>
-          <version>2.3.2</version>
+          <version>${findbugs.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-checkstyle-plugin</artifactId>
-          <version>2.6</version>
+          <version>${maven-checkstyle-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>native-maven-plugin</artifactId>
-          <version>1.0-alpha-7</version>
+          <version>${native-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>make-maven-plugin</artifactId>
-          <version>1.0-beta-1</version>
+          <version>${make-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-source-plugin</artifactId>
-          <version>2.1.2</version>
+          <version>${maven-source-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.avro</groupId>
@@ -994,22 +1028,22 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-project-info-reports-plugin</artifactId>
-          <version>2.4</version>
+          <version>${maven-project-info-reports-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-resources-plugin</artifactId>
-          <version>2.2</version>
+          <version>${maven-resources-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.2</version>
+          <version>${exec-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-pdf-plugin</artifactId>
-          <version>1.1</version>
+          <version>${maven-pdf-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.hadoop</groupId>
@@ -1050,7 +1084,7 @@
         <configuration>
           <reuseForks>false</reuseForks>
           <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
-          <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
+          <argLine>${maven-surefire-plugin.argLine}</argLine>
           <environmentVariables>
             <!-- HADOOP_HOME required for tests on Windows to find winutils -->
             <HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
@@ -1182,8 +1216,8 @@
             <artifactId>maven-compiler-plugin</artifactId>
             <configuration>
               <fork>true</fork>
-              <source>1.6</source>
-              <target>1.6</target>
+              <source>${javac.version}</source>
+              <target>${javac.version}</target>
               <compilerArguments>
                 <Xlint/>
                 <Xmaxwarns>9999</Xmaxwarns>
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 4a2c2f8d557fe..40777384a78df 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -38,7 +38,7 @@
   <body>
     <links position="left">
       <item name="Wiki" href="http://wiki.apache.org/hadoop" />
-      <item name="SVN" href="https://svn.apache.org/repos/asf/hadoop/" />
+      <item name="git" href="https://git-wip-us.apache.org/repos/asf/hadoop.git" />
     </links>
 
     <breadcrumbs position="left">
@@ -60,7 +60,7 @@
     <menu name="Common" inherit="top">
       <item name="CLI Mini Cluster" href="hadoop-project-dist/hadoop-common/CLIMiniCluster.html"/>
       <item name="Native Libraries" href="hadoop-project-dist/hadoop-common/NativeLibraries.html"/>
-      <item name="Superusers" href="hadoop-project-dist/hadoop-common/Superusers.html"/>
+      <item name="Proxy User" href="hadoop-project-dist/hadoop-common/Superusers.html"/>
       <item name="Secure Mode" href="hadoop-project-dist/hadoop-common/SecureMode.html"/>
       <item name="Service Level Authorization" href="hadoop-project-dist/hadoop-common/ServiceLevelAuth.html"/>
       <item name="HTTP Authentication" href="hadoop-project-dist/hadoop-common/HttpAuthentication.html"/>
@@ -102,9 +102,6 @@
       <item name="Encrypted Shuffle" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html"/>
       <item name="Pluggable Shuffle/Sort" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/PluggableShuffleAndPluggableSort.html"/>
       <item name="Distributed Cache Deploy" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistributedCacheDeploy.html"/>
-      <item name="Hadoop Streaming" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopStreaming.html"/>
-      <item name="Hadoop Archives" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html"/>
-      <item name="DistCp" href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html"/>
     </menu>
 
     <menu name="MapReduce REST APIs" inherit="top">
@@ -113,6 +110,7 @@
     </menu>
 
     <menu name="YARN" inherit="top">
+      <item name="Overview" href="hadoop-yarn/hadoop-yarn-site/index.html"/>
       <item name="YARN Architecture" href="hadoop-yarn/hadoop-yarn-site/YARN.html"/>
       <item name="Capacity Scheduler" href="hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html"/>
       <item name="Fair Scheduler" href="hadoop-yarn/hadoop-yarn-site/FairScheduler.html"/>
@@ -122,9 +120,11 @@
       <item name="YARN Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html"/>
       <item name="Writing YARN Applications" href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
       <item name="YARN Commands" href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
-      <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
       <item name="NodeManager Restart" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html"/>
       <item name="DockerContainerExecutor" href="hadoop-yarn/hadoop-yarn-site/DockerContainerExecutor.html"/>
+      <item name="Using CGroups" href="hadoop-yarn/hadoop-yarn-site/NodeManagerCGroups.html"/>
+      <item name="Secure Containers" href="hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
+      <item name="Registry" href="hadoop-yarn/hadoop-yarn-site/registry/index.html"/>
     </menu>
 
     <menu name="YARN REST APIs" inherit="top">
@@ -133,13 +133,28 @@
       <item name="Node Manager" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html"/>
     </menu>
     
+    <menu name="Hadoop Compatible File Systems" inherit="top">
+      <item name="Amazon S3" href="hadoop-aws/tools/hadoop-aws/index.html"/>
+      <item name="Azure Blob Storage" href="hadoop-azure/index.html"/>
+      <item name="OpenStack Swift" href="hadoop-openstack/index.html"/>
+    </menu>
+
     <menu name="Auth" inherit="top">
       <item name="Overview" href="hadoop-auth/index.html"/>
       <item name="Examples" href="hadoop-auth/Examples.html"/>
       <item name="Configuration" href="hadoop-auth/Configuration.html"/>
       <item name="Building" href="hadoop-auth/BuildingIt.html"/>
     </menu>
-    
+
+    <menu name="Tools" inherit="top">
+      <item name="Hadoop Streaming" href="hadoop-streaming/HadoopStreaming.html"/>
+      <item name="Hadoop Archives" href="hadoop-archives/HadoopArchives.html"/>
+      <item name="DistCp" href="hadoop-distcp/DistCp.html"/>
+      <item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
+      <item name="Rumen" href="hadoop-rumen/Rumen.html"/>
+      <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
+    </menu>
+
     <menu name="Reference" inherit="top">
       <item name="Release Notes" href="hadoop-project-dist/hadoop-common/releasenotes.html"/>
       <item name="API docs" href="api/index.html"/>
diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
index aa3027772d01a..18cd972af4e9e 100644
--- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
@@ -68,6 +68,7 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
 
 /**
  * a archive creation utility.
@@ -237,7 +238,6 @@ public InputSplit[] getSplits(JobConf jconf, int numSplits)
       ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
       LongWritable key = new LongWritable();
       final HarEntry value = new HarEntry();
-      SequenceFile.Reader reader = null;
       // the remaining bytes in the file split
       long remaining = fstatus.getLen();
       // the count of sizes calculated till now
@@ -249,8 +249,7 @@ public InputSplit[] getSplits(JobConf jconf, int numSplits)
       long targetSize = totalSize/numSplits;
       // create splits of size target size so that all the maps 
       // have equals sized data to read and write to.
-      try {
-        reader = new SequenceFile.Reader(fs, src, jconf);
+      try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, src, jconf)) {
         while(reader.next(key, value)) {
           if (currentCount + key.get() > targetSize && currentCount != 0){
             long size = lastPos - startPos;
@@ -267,9 +266,6 @@ public InputSplit[] getSplits(JobConf jconf, int numSplits)
           splits.add(new FileSplit(src, startPos, remaining, (String[])null));
         }
       }
-      finally { 
-        reader.close();
-      }
       return splits.toArray(new FileSplit[splits.size()]);
     }
 
@@ -741,7 +737,7 @@ public void configure(JobConf conf) {
         indexStream = fs.create(index);
         outStream = fs.create(masterIndex);
         String version = VERSION + " \n";
-        outStream.write(version.getBytes());
+        outStream.write(version.getBytes(Charsets.UTF_8));
         
       } catch(IOException e) {
         throw new RuntimeException(e);
@@ -760,7 +756,7 @@ public void reduce(IntWritable key, Iterator<Text> values,
       while(values.hasNext()) {
         Text value = values.next();
         String towrite = value.toString() + "\n";
-        indexStream.write(towrite.getBytes());
+        indexStream.write(towrite.getBytes(Charsets.UTF_8));
         written++;
         if (written > numIndexes -1) {
           // every 1000 indexes we report status
@@ -769,7 +765,7 @@ public void reduce(IntWritable key, Iterator<Text> values,
           endIndex = keyVal;
           String masterWrite = startIndex + " " + endIndex + " " + startPos 
                               +  " " + indexStream.getPos() + " \n" ;
-          outStream.write(masterWrite.getBytes());
+          outStream.write(masterWrite.getBytes(Charsets.UTF_8));
           startPos = indexStream.getPos();
           startIndex = endIndex;
           written = 0;
@@ -782,7 +778,7 @@ public void close() throws IOException {
       if (written > 0) {
         String masterWrite = startIndex + " " + keyVal + " " + startPos  +
                              " " + indexStream.getPos() + " \n";
-        outStream.write(masterWrite.getBytes());
+        outStream.write(masterWrite.getBytes(Charsets.UTF_8));
       }
       // close the streams
       outStream.close();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm b/hadoop-tools/hadoop-archives/src/site/markdown/HadoopArchives.md.vm
similarity index 100%
rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm
rename to hadoop-tools/hadoop-archives/src/site/markdown/HadoopArchives.md.vm
diff --git a/hadoop-tools/hadoop-archives/src/site/resources/css/site.css b/hadoop-tools/hadoop-archives/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-tools/hadoop-archives/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index e2e821e1bcae2..590778dbc2a0f 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -14,7 +14,7 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
index 241ec0f327706..5f46aea805d1d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
@@ -173,7 +173,7 @@ private InputStream get(String key, boolean checkMetadata)
       return object.getDataInputStream();
     } catch (S3ServiceException e) {
       if ("NoSuchKey".equals(e.getS3ErrorCode())) {
-        return null;
+        throw new IOException(key + " doesn't exist");
       }
       if (e.getCause() instanceof IOException) {
         throw (IOException) e.getCause();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index ee4bf684a7196..b6863bb162f9a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -28,7 +28,17 @@ public class Constants {
   // connect to s3 over ssl?
   public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled";
   public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
-  
+
+  //use a custom endpoint?
+  public static final String ENDPOINT = "fs.s3a.endpoint";
+  //connect to s3 through a proxy server?
+  public static final String PROXY_HOST = "fs.s3a.proxy.host";
+  public static final String PROXY_PORT = "fs.s3a.proxy.port";
+  public static final String PROXY_USERNAME = "fs.s3a.proxy.username";
+  public static final String PROXY_PASSWORD = "fs.s3a.proxy.password";
+  public static final String PROXY_DOMAIN = "fs.s3a.proxy.domain";
+  public static final String PROXY_WORKSTATION = "fs.s3a.proxy.workstation";
+
   // number of times we should retry errors
   public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
   public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
@@ -41,6 +51,23 @@ public class Constants {
   public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
   public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
 
+  // the maximum number of threads to allow in the pool used by TransferManager
+  public static final String MAX_THREADS = "fs.s3a.threads.max";
+  public static final int DEFAULT_MAX_THREADS = 256;
+
+  // the number of threads to keep in the pool used by TransferManager
+  public static final String CORE_THREADS = "fs.s3a.threads.core";
+  public static final int DEFAULT_CORE_THREADS = DEFAULT_MAXIMUM_CONNECTIONS;
+
+  // when the number of threads is greater than the core, this is the maximum time
+  // that excess idle threads will wait for new tasks before terminating.
+  public static final String KEEPALIVE_TIME = "fs.s3a.threads.keepalivetime";
+  public static final int DEFAULT_KEEPALIVE_TIME = 60;
+
+  // the maximum number of tasks that the LinkedBlockingQueue can hold
+  public static final String MAX_TOTAL_TASKS = "fs.s3a.max.total.tasks";
+  public static final int DEFAULT_MAX_TOTAL_TASKS = 1000;
+
   // size of each of or multipart pieces in bytes
   public static final String MULTIPART_SIZE = "fs.s3a.multipart.size";
   public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 6bdd233506574..22350bc40c266 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -26,6 +26,11 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.s3.S3Credentials;
 
@@ -52,6 +57,7 @@
 import com.amazonaws.event.ProgressListener;
 import com.amazonaws.event.ProgressEvent;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
@@ -77,6 +83,7 @@ public class S3AFileSystem extends FileSystem {
   private String bucket;
   private int maxKeys;
   private long partSize;
+  private TransferManager transfers;
   private int partSizeThreshold;
   public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
   private CannedAccessControlList cannedACL;
@@ -85,6 +92,55 @@ public class S3AFileSystem extends FileSystem {
   // The maximum number of entries that can be deleted in any call to s3
   private static final int MAX_ENTRIES_TO_DELETE = 1000;
 
+  private static final AtomicInteger poolNumber = new AtomicInteger(1);
+  /**
+   * Returns a {@link java.util.concurrent.ThreadFactory} that names each created thread uniquely,
+   * with a common prefix.
+   * @param prefix The prefix of every created Thread's name
+   * @return a {@link java.util.concurrent.ThreadFactory} that names threads
+   */
+  public static ThreadFactory getNamedThreadFactory(final String prefix) {
+    SecurityManager s = System.getSecurityManager();
+    final ThreadGroup threadGroup = (s != null) ? s.getThreadGroup() : Thread.currentThread()
+        .getThreadGroup();
+
+    return new ThreadFactory() {
+      final AtomicInteger threadNumber = new AtomicInteger(1);
+      private final int poolNum = poolNumber.getAndIncrement();
+      final ThreadGroup group = threadGroup;
+
+      @Override
+      public Thread newThread(Runnable r) {
+        final String name = prefix + "-pool" + poolNum + "-t" + threadNumber.getAndIncrement();
+        return new Thread(group, r, name);
+      }
+    };
+  }
+
+  /**
+   * Get a named {@link ThreadFactory} that just builds daemon threads.
+   * @param prefix name prefix for all threads created from the factory
+   * @return a thread factory that creates named, daemon threads with
+   *         the supplied exception handler and normal priority
+   */
+  private static ThreadFactory newDaemonThreadFactory(final String prefix) {
+    final ThreadFactory namedFactory = getNamedThreadFactory(prefix);
+    return new ThreadFactory() {
+      @Override
+      public Thread newThread(Runnable r) {
+        Thread t = namedFactory.newThread(r);
+        if (!t.isDaemon()) {
+          t.setDaemon(true);
+        }
+        if (t.getPriority() != Thread.NORM_PRIORITY) {
+          t.setPriority(Thread.NORM_PRIORITY);
+        }
+        return t;
+      }
+
+    };
+  }
+
   /** Called after a new FileSystem instance is constructed.
    * @param name a uri whose authority section names the host, port, etc.
    *   for this FileSystem
@@ -93,7 +149,6 @@ public class S3AFileSystem extends FileSystem {
   public void initialize(URI name, Configuration conf) throws IOException {
     super.initialize(name, conf);
 
-
     uri = URI.create(name.getScheme() + "://" + name.getAuthority());
     workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(this.uri,
         this.getWorkingDirectory());
@@ -114,14 +169,65 @@ public void initialize(URI name, Configuration conf) throws IOException {
     ClientConfiguration awsConf = new ClientConfiguration();
     awsConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS, 
       DEFAULT_MAXIMUM_CONNECTIONS));
-    awsConf.setProtocol(conf.getBoolean(SECURE_CONNECTIONS, 
-      DEFAULT_SECURE_CONNECTIONS) ?  Protocol.HTTPS : Protocol.HTTP);
+    boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
+        DEFAULT_SECURE_CONNECTIONS);
+    awsConf.setProtocol(secureConnections ?  Protocol.HTTPS : Protocol.HTTP);
     awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, 
       DEFAULT_MAX_ERROR_RETRIES));
     awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, 
       DEFAULT_SOCKET_TIMEOUT));
 
+    String proxyHost = conf.getTrimmed(PROXY_HOST,"");
+    int proxyPort = conf.getInt(PROXY_PORT, -1);
+    if (!proxyHost.isEmpty()) {
+      awsConf.setProxyHost(proxyHost);
+      if (proxyPort >= 0) {
+        awsConf.setProxyPort(proxyPort);
+      } else {
+        if (secureConnections) {
+          LOG.warn("Proxy host set without port. Using HTTPS default 443");
+          awsConf.setProxyPort(443);
+        } else {
+          LOG.warn("Proxy host set without port. Using HTTP default 80");
+          awsConf.setProxyPort(80);
+        }
+      }
+      String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
+      String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
+      if ((proxyUsername == null) != (proxyPassword == null)) {
+        String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+            PROXY_PASSWORD + " set without the other.";
+        LOG.error(msg);
+        throw new IllegalArgumentException(msg);
+      }
+      awsConf.setProxyUsername(proxyUsername);
+      awsConf.setProxyPassword(proxyPassword);
+      awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
+      awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
+                "domain {} as workstation {}", awsConf.getProxyHost(),
+            awsConf.getProxyPort(), String.valueOf(awsConf.getProxyUsername()),
+            awsConf.getProxyPassword(), awsConf.getProxyDomain(),
+            awsConf.getProxyWorkstation());
+      }
+    } else if (proxyPort >= 0) {
+      String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+      LOG.error(msg);
+      throw new IllegalArgumentException(msg);
+    }
+
     s3 = new AmazonS3Client(credentials, awsConf);
+    String endPoint = conf.getTrimmed(ENDPOINT,"");
+    if (!endPoint.isEmpty()) {
+      try {
+        s3.setEndpoint(endPoint);
+      } catch (IllegalArgumentException e) {
+        String msg = "Incorrect endpoint: "  + e.getMessage();
+        LOG.error(msg);
+        throw new IllegalArgumentException(msg, e);
+      }
+    }
 
     maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
     partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
@@ -138,6 +244,34 @@ public void initialize(URI name, Configuration conf) throws IOException {
       partSizeThreshold = 5 * 1024 * 1024;
     }
 
+    int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
+    int coreThreads = conf.getInt(CORE_THREADS, DEFAULT_CORE_THREADS);
+    if (maxThreads == 0) {
+      maxThreads = Runtime.getRuntime().availableProcessors() * 8;
+    }
+    if (coreThreads == 0) {
+      coreThreads = Runtime.getRuntime().availableProcessors() * 8;
+    }
+    long keepAliveTime = conf.getLong(KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
+    LinkedBlockingQueue<Runnable> workQueue =
+      new LinkedBlockingQueue<Runnable>(maxThreads *
+        conf.getInt(MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS));
+    ThreadPoolExecutor tpe = new ThreadPoolExecutor(
+        coreThreads,
+        maxThreads,
+        keepAliveTime,
+        TimeUnit.SECONDS,
+        workQueue,
+        newDaemonThreadFactory("s3a-transfer-shared-"));
+    tpe.allowCoreThreadTimeOut(true);
+
+    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
+    transferConfiguration.setMinimumUploadPartSize(partSize);
+    transferConfiguration.setMultipartUploadThreshold(partSizeThreshold);
+
+    transfers = new TransferManager(s3, tpe);
+    transfers.setConfiguration(transferConfiguration);
+
     String cannedACLName = conf.get(CANNED_ACL, DEFAULT_CANNED_ACL);
     if (!cannedACLName.isEmpty()) {
       cannedACL = CannedAccessControlList.valueOf(cannedACLName);
@@ -155,11 +289,9 @@ public void initialize(URI name, Configuration conf) throws IOException {
       DEFAULT_PURGE_EXISTING_MULTIPART_AGE);
 
     if (purgeExistingMultipart) {
-      TransferManager transferManager = new TransferManager(s3);
       Date purgeBefore = new Date(new Date().getTime() - purgeExistingMultipartAge*1000);
 
-      transferManager.abortMultipartUploads(bucket, purgeBefore);
-      transferManager.shutdownNow(false);
+      transfers.abortMultipartUploads(bucket, purgeBefore);
     }
 
     serverSideEncryptionAlgorithm = conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM);
@@ -181,6 +313,14 @@ public URI getUri() {
     return uri;
   }
 
+  /**
+   * Returns the S3 client used by this filesystem.
+   * @return AmazonS3Client
+   */
+  @VisibleForTesting
+  AmazonS3Client getAmazonS3Client() {
+    return s3;
+  }
 
   public S3AFileSystem() {
     super();
@@ -245,7 +385,7 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwr
     }
 
     // We pass null to FSDataOutputStream so it won't count writes that are being buffered to a file
-    return new FSDataOutputStream(new S3AOutputStream(getConf(), s3, this, 
+    return new FSDataOutputStream(new S3AOutputStream(getConf(), transfers, this,
       bucket, key, progress, cannedACL, statistics, 
       serverSideEncryptionAlgorithm), null);
   }
@@ -854,13 +994,6 @@ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
     LocalFileSystem local = getLocal(getConf());
     File srcfile = local.pathToFile(src);
 
-    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
-    transferConfiguration.setMinimumUploadPartSize(partSize);
-    transferConfiguration.setMultipartUploadThreshold(partSizeThreshold);
-
-    TransferManager transfers = new TransferManager(s3);
-    transfers.setConfiguration(transferConfiguration);
-
     final ObjectMetadata om = new ObjectMetadata();
     if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
       om.setServerSideEncryption(serverSideEncryptionAlgorithm);
@@ -875,6 +1008,8 @@ public void progressChanged(ProgressEvent progressEvent) {
           case ProgressEvent.PART_COMPLETED_EVENT_CODE:
             statistics.incrementWriteOps(1);
             break;
+          default:
+            break;
         }
       }
     };
@@ -886,8 +1021,6 @@ public void progressChanged(ProgressEvent progressEvent) {
       statistics.incrementWriteOps(1);
     } catch (InterruptedException e) {
       throw new IOException("Got interrupted, cancelling");
-    } finally {
-      transfers.shutdownNow(false);
     }
 
     // This will delete unnecessary fake parent directories
@@ -898,6 +1031,18 @@ public void progressChanged(ProgressEvent progressEvent) {
     }
   }
 
+  @Override
+  public void close() throws IOException {
+    try {
+      super.close();
+    } finally {
+      if (transfers != null) {
+        transfers.shutdownNow(true);
+        transfers = null;
+      }
+    }
+  }
+
   /**
   * Override getCononicalServiceName because we don't support token in S3A
   */
@@ -912,12 +1057,6 @@ private void copyFile(String srcKey, String dstKey) throws IOException {
       LOG.debug("copyFile " + srcKey + " -> " + dstKey);
     }
 
-    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
-    transferConfiguration.setMultipartCopyPartSize(partSize);
-
-    TransferManager transfers = new TransferManager(s3);
-    transfers.setConfiguration(transferConfiguration);
-
     ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
     final ObjectMetadata dstom = srcom.clone();
     if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
@@ -933,6 +1072,8 @@ public void progressChanged(ProgressEvent progressEvent) {
           case ProgressEvent.PART_COMPLETED_EVENT_CODE:
             statistics.incrementWriteOps(1);
             break;
+          default:
+            break;
         }
       }
     };
@@ -944,8 +1085,6 @@ public void progressChanged(ProgressEvent progressEvent) {
       statistics.incrementWriteOps(1);
     } catch (InterruptedException e) {
       throw new IOException("Got interrupted, cancelling");
-    } finally {
-      transfers.shutdownNow(false);
     }
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOutputStream.java
index 7783b998111c8..2b611b6fdf1f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOutputStream.java
@@ -49,7 +49,7 @@ public class S3AOutputStream extends OutputStream {
   private boolean closed;
   private String key;
   private String bucket;
-  private AmazonS3Client client;
+  private TransferManager transfers;
   private Progressable progress;
   private long partSize;
   private int partSizeThreshold;
@@ -61,14 +61,14 @@ public class S3AOutputStream extends OutputStream {
 
   public static final Logger LOG = S3AFileSystem.LOG;
 
-  public S3AOutputStream(Configuration conf, AmazonS3Client client, 
+  public S3AOutputStream(Configuration conf, TransferManager transfers,
     S3AFileSystem fs, String bucket, String key, Progressable progress, 
     CannedAccessControlList cannedACL, FileSystem.Statistics statistics, 
     String serverSideEncryptionAlgorithm)
       throws IOException {
     this.bucket = bucket;
     this.key = key;
-    this.client = client;
+    this.transfers = transfers;
     this.progress = progress;
     this.fs = fs;
     this.cannedACL = cannedACL;
@@ -114,13 +114,6 @@ public synchronized void close() throws IOException {
 
 
     try {
-      TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
-      transferConfiguration.setMinimumUploadPartSize(partSize);
-      transferConfiguration.setMultipartUploadThreshold(partSizeThreshold);
-
-      TransferManager transfers = new TransferManager(client);
-      transfers.setConfiguration(transferConfiguration);
-
       final ObjectMetadata om = new ObjectMetadata();
       if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
         om.setServerSideEncryption(serverSideEncryptionAlgorithm);
diff --git a/hadoop-tools/hadoop-aws/src/main/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
similarity index 99%
rename from hadoop-tools/hadoop-aws/src/main/site/markdown/tools/hadoop-aws/index.md
rename to hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 4a1956a62ad5f..375f82c8e6c9e 100644
--- a/hadoop-tools/hadoop-aws/src/main/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -48,7 +48,7 @@ the number of files, during which time partial updates may be visible. If
 the operations are interrupted, the filesystem is left in an intermediate state.
 
 For further discussion on these topics, please consult
-[/filesystem](The Hadoop FileSystem API Definition).
+[The Hadoop FileSystem API Definition](../../../hadoop-project-dist/hadoop-common/filesystem/index.html).
 
 ## Warning #2: your AWS credentials are valuable
 
@@ -172,7 +172,7 @@ If you do any of these: change your credentials immediately!
 
     <property>
       <name>fs.s3a.connection.timeout</name>
-      <value>5000</value>
+      <value>50000</value>
       <description>Socket connection timeout in seconds.</description>
     </property>
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 514647c37433a..d9726db7021ad 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -45,6 +45,9 @@ public static S3AFileSystem createTestFileSystem(Configuration conf) throws
           "No test filesystem in " + TestS3AFileSystemContract.TEST_FS_S3A_NAME);
     }
     S3AFileSystem fs1 = new S3AFileSystem();
+    //enable purging in tests
+    conf.setBoolean(Constants.PURGE_EXISTING_MULTIPART, true);
+    conf.setInt(Constants.PURGE_EXISTING_MULTIPART_AGE, 0);
     fs1.initialize(testURI, conf);
     return fs1;
   }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
new file mode 100644
index 0000000000000..25068f8d9aa53
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import org.apache.commons.lang.StringUtils;
+import com.amazonaws.AmazonClientException;
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TestS3AConfiguration {
+  private Configuration conf;
+  private S3AFileSystem fs;
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestS3AConfiguration.class);
+
+  private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
+
+  @Rule
+  public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+  /**
+   * Test if custom endpoint is picked up.
+   * <p/>
+   * The test expects TEST_ENDPOINT to be defined in the Configuration
+   * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
+   * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
+   * Evidently, the bucket has to be hosted in the region denoted by the
+   * endpoint for the test to succeed.
+   * <p/>
+   * More info and the list of endpoint identifiers:
+   * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+   *
+   * @throws Exception
+   */
+  @Test
+  public void TestEndpoint() throws Exception {
+    conf = new Configuration();
+    String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
+    if (endpoint.isEmpty()) {
+      LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
+          "setting was not detected");
+    } else {
+      conf.set(Constants.ENDPOINT, endpoint);
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      AmazonS3Client s3 = fs.getAmazonS3Client();
+      String endPointRegion = "";
+      // Differentiate handling of "s3-" and "s3." based endpoint identifiers
+      String[] endpointParts = StringUtils.split(endpoint, '.');
+      if (endpointParts.length == 3) {
+        endPointRegion = endpointParts[0].substring(3);
+      } else if (endpointParts.length == 4) {
+        endPointRegion = endpointParts[1];
+      } else {
+        fail("Unexpected endpoint");
+      }
+      assertEquals("Endpoint config setting and bucket location differ: ",
+          endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
+    }
+  }
+
+  @Test
+  public void TestProxyConnection() throws Exception {
+    conf = new Configuration();
+    conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+    conf.set(Constants.PROXY_HOST, "127.0.0.1");
+    conf.setInt(Constants.PROXY_PORT, 1);
+    String proxy =
+        conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a connection error for proxy server at " + proxy);
+    } catch (AmazonClientException e) {
+      if (!e.getMessage().contains(proxy + " refused")) {
+        throw e;
+      }
+    }
+  }
+
+  @Test
+  public void TestProxyPortWithoutHost() throws Exception {
+    conf = new Configuration();
+    conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+    conf.setInt(Constants.PROXY_PORT, 1);
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a proxy configuration error");
+    } catch (IllegalArgumentException e) {
+      String msg = e.toString();
+      if (!msg.contains(Constants.PROXY_HOST) &&
+          !msg.contains(Constants.PROXY_PORT)) {
+        throw e;
+      }
+    }
+  }
+
+  @Test
+  public void TestAutomaticProxyPortSelection() throws Exception {
+    conf = new Configuration();
+    conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+    conf.set(Constants.PROXY_HOST, "127.0.0.1");
+    conf.set(Constants.SECURE_CONNECTIONS, "true");
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a connection error for proxy server");
+    } catch (AmazonClientException e) {
+      if (!e.getMessage().contains("443")) {
+        throw e;
+      }
+    }
+    conf.set(Constants.SECURE_CONNECTIONS, "false");
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a connection error for proxy server");
+    } catch (AmazonClientException e) {
+      if (!e.getMessage().contains("80")) {
+        throw e;
+      }
+    }
+  }
+
+  @Test
+  public void TestUsernameInconsistentWithPassword() throws Exception {
+    conf = new Configuration();
+    conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+    conf.set(Constants.PROXY_HOST, "127.0.0.1");
+    conf.setInt(Constants.PROXY_PORT, 1);
+    conf.set(Constants.PROXY_USERNAME, "user");
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a connection error for proxy server");
+    } catch (IllegalArgumentException e) {
+      String msg = e.toString();
+      if (!msg.contains(Constants.PROXY_USERNAME) &&
+          !msg.contains(Constants.PROXY_PASSWORD)) {
+        throw e;
+      }
+    }
+    conf = new Configuration();
+    conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+    conf.set(Constants.PROXY_HOST, "127.0.0.1");
+    conf.setInt(Constants.PROXY_PORT, 1);
+    conf.set(Constants.PROXY_PASSWORD, "password");
+    try {
+      fs = S3ATestUtils.createTestFileSystem(conf);
+      fail("Expected a connection error for proxy server");
+    } catch (IllegalArgumentException e) {
+      String msg = e.toString();
+      if (!msg.contains(Constants.PROXY_USERNAME) &&
+          !msg.contains(Constants.PROXY_PASSWORD)) {
+        throw e;
+      }
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/README.txt b/hadoop-tools/hadoop-azure/README.txt
deleted file mode 100644
index a1d1a653ed8b9..0000000000000
--- a/hadoop-tools/hadoop-azure/README.txt
+++ /dev/null
@@ -1,166 +0,0 @@
-=============
-Building
-=============
-basic compilation:
-> mvn clean compile test-compile
-
-Compile, run tests and produce jar 
-> mvn clean package
-
-=============
-Unit tests
-=============
-Most of the tests will run without additional configuration.
-For complete testing, configuration in src/test/resources is required:
-  
-  src/test/resources/azure-test.xml -> Defines Azure storage dependencies, including account information 
-
-The other files in src/test/resources do not normally need alteration:
-  log4j.properties -> Test logging setup
-  hadoop-metrics2-azure-file-system.properties -> used to wire up instrumentation for testing
-  
-From command-line
-------------------
-Basic execution:
-> mvn test
-
-NOTES:
- - The mvn pom.xml includes src/test/resources in the runtime classpath
- - detailed output (such as log4j) appears in target\surefire-reports\TEST-{testName}.xml
-   including log4j messages.
-   
-Run the tests and generate report:
-> mvn site (at least once to setup some basics including images for the report)
-> mvn surefire-report:report  (run and produce report)
-> mvn mvn surefire-report:report-only  (produce report from last run)
-> mvn mvn surefire-report:report-only -DshowSuccess=false (produce report from last run, only show errors)
-> .\target\site\surefire-report.html (view the report)
-
-Via eclipse
--------------
-Manually add src\test\resources to the classpath for test run configuration:
-  - run menu|run configurations|{configuration}|classpath|User Entries|advanced|add folder
-
-Then run via junit test runner.
-NOTE:
- - if you change log4.properties, rebuild the project to refresh the eclipse cache.
-
-Run Tests against Mocked storage.
----------------------------------
-These run automatically and make use of an in-memory emulation of azure storage.
-
-
-Running tests against the Azure storage emulator  
----------------------------------------------------
-A selection of tests can run against the Azure Storage Emulator which is 
-a high-fidelity emulation of live Azure Storage.  The emulator is sufficient for high-confidence testing.
-The emulator is a Windows executable that runs on a local machine. 
-
-To use the emulator, install Azure SDK 2.3 and start the storage emulator
-See http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx
-
-Enable the Azure emulator tests by setting 
-  fs.azure.test.emulator -> true 
-in src\test\resources\azure-test.xml
-
-Known issues:
-  Symptom: When running tests for emulator, you see the following failure message
-           com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
-  Issue:   The emulator can get into a confused state.  
-  Fix:     Restart the Azure Emulator.  Ensure it is v3.2 or later.
- 
-Running tests against live Azure storage 
--------------------------------------------------------------------------
-In order to run WASB unit tests against a live Azure Storage account, add credentials to 
-src\test\resources\azure-test.xml. These settings augment the hadoop configuration object.
-
-For live tests, set the following in azure-test.xml:
- 1. "fs.azure.test.account.name -> {azureStorageAccountName} 
- 2. "fs.azure.account.key.{AccountName} -> {fullStorageKey}"
- 
-===================================
-Page Blob Support and Configuration
-===================================
-
-The Azure Blob Storage interface for Hadoop supports two kinds of blobs, block blobs
-and page blobs. Block blobs are the default kind of blob and are good for most 
-big-data use cases, like input data for Hive, Pig, analytical map-reduce jobs etc. 
-Page blob handling in hadoop-azure was introduced to support HBase log files. 
-Page blobs can be written any number of times, whereas block blobs can only be 
-appended to 50,000 times before you run out of blocks and your writes will fail.
-That won't work for HBase logs, so page blob support was introduced to overcome
-this limitation.
-
-Page blobs can be used for other purposes beyond just HBase log files though.
-They support the Hadoop FileSystem interface. Page blobs can be up to 1TB in
-size, larger than the maximum 200GB size for block blobs.
-
-In order to have the files you create be page blobs, you must set the configuration
-variable fs.azure.page.blob.dir to a comma-separated list of folder names.
-E.g. 
-
-    /hbase/WALs,/hbase/oldWALs,/data/mypageblobfiles
-    
-You can set this to simply / to make all files page blobs.
-
-The configuration option fs.azure.page.blob.size is the default initial 
-size for a page blob. It must be 128MB or greater, and no more than 1TB,
-specified as an integer number of bytes.
-
-====================
-Atomic Folder Rename
-====================
-
-Azure storage stores files as a flat key/value store without formal support
-for folders. The hadoop-azure file system layer simulates folders on top
-of Azure storage. By default, folder rename in the hadoop-azure file system
-layer is not atomic. That means that a failure during a folder rename 
-could, for example, leave some folders in the original directory and
-some in the new one.
-
-HBase depends on atomic folder rename. Hence, a configuration setting was
-introduced called fs.azure.atomic.rename.dir that allows you to specify a 
-comma-separated list of directories to receive special treatment so that 
-folder rename is made atomic. The default value of this setting is just /hbase.
-Redo will be applied to finish a folder rename that fails. A file  
-<folderName>-renamePending.json may appear temporarily and is the record of 
-the intention of the rename operation, to allow redo in event of a failure. 
-
-=============
-Findbugs
-=============
-Run findbugs and show interactive GUI for review of problems
-> mvn findbugs:gui 
-
-Run findbugs and fail build if errors are found:
-> mvn findbugs:check
-
-For help with findbugs plugin.
-> mvn findbugs:help
-
-=============
-Checkstyle
-=============
-Rules for checkstyle @ src\config\checkstyle.xml
- - these are based on a core set of standards, with exclusions for non-serious issues
- - as a general plan it would be good to turn on more rules over time.
- - Occasionally, run checkstyle with the default Sun rules by editing pom.xml.
-
-Command-line:
-> mvn checkstyle:check --> just test & fail build if violations found
-> mvn site checkstyle:checkstyle --> produce html report
-> . target\site\checkstyle.html  --> view report.
-
-Eclipse:
-- add the checkstyle plugin: Help|Install, site=http://eclipse-cs.sf.net/update
-- window|preferences|checkstyle. Add src/config/checkstyle.xml. Set as default.
-- project|properties|create configurations as required, eg src/main/java -> src/config/checkstyle.xml
-
-NOTE:
-- After any change to the checkstyle rules xml, use window|preferences|checkstyle|{refresh}|OK
-
-=============
-Javadoc
-============= 
-Command-line
-> mvn javadoc:javadoc
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index fbdc132e28856..d39dd769e1af0 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -14,7 +14,7 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
@@ -49,21 +49,6 @@
           <effort>Max</effort>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-site-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>org.apache.maven.doxia</groupId>
-            <artifactId>doxia-module-markdown</artifactId>
-            <version>1.3</version>
-          </dependency>
-        </dependencies>
-        <configuration>
-          <inputEncoding>UTF-8</inputEncoding>
-          <outputEncoding>UTF-8</outputEncoding>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-project-info-reports-plugin</artifactId>
@@ -84,6 +69,19 @@
           </configuration>
           
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
     </plugins>
 
     
diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle.xml
index 9df4f78bcc093..f68f6c8208983 100644
--- a/hadoop-tools/hadoop-azure/src/config/checkstyle.xml
+++ b/hadoop-tools/hadoop-azure/src/config/checkstyle.xml
@@ -147,7 +147,6 @@
         <!-- Checks for common coding problems               -->
         <!-- See http://checkstyle.sf.net/config_coding.html -->
         <!-- <module name="AvoidInlineConditionals"/> -->
-        <!-- <module name="DoubleCheckedLocking"/> -->
         <module name="EmptyStatement"/>
         <module name="EqualsHashCode"/>
         <!-- <module name="HiddenField"/> -->
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index c09176744c9c1..2412698490c65 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -789,8 +789,9 @@ private boolean isStorageEmulatorAccount(final String accountName) {
         STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME,
         DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME));
   }
-
-  static String getAccountKeyFromConfiguration(String accountName,
+  
+  @VisibleForTesting
+  public static String getAccountKeyFromConfiguration(String accountName,
       Configuration conf) throws KeyProviderException {
     String key = null;
     String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index ad2e2e6635c95..0248b85a283a5 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -25,6 +25,7 @@
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -153,7 +154,7 @@ public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs)
             "Error reading pending rename file contents -- "
                 + "maximum file size exceeded");
       }
-      String contents = new String(bytes, 0, l);
+      String contents = new String(bytes, 0, l, Charset.forName("UTF-8"));
 
       // parse the JSON
       ObjectMapper objMapper = new ObjectMapper();
@@ -253,7 +254,7 @@ public void writeFile(FileSystem fs) throws IOException {
       // Write file.
       try {
         output = fs.create(path);
-        output.write(contents.getBytes());
+        output.write(contents.getBytes(Charset.forName("UTF-8")));
       } catch (IOException e) {
         throw new IOException("Unable to write RenamePending file for folder rename from "
             + srcKey + " to " + dstKey, e);
@@ -2039,7 +2040,37 @@ private void updateParentFolderLastModifiedTime(String key)
               createPermissionStatus(FsPermission.getDefault()));
         }
 
-        store.updateFolderLastModifiedTime(parentKey, null);
+        if (store.isAtomicRenameKey(parentKey)) {
+          SelfRenewingLease lease = null;
+          try {
+            lease = leaseSourceFolder(parentKey);
+            store.updateFolderLastModifiedTime(parentKey, lease);
+          } catch (AzureException e) {
+            String errorCode = "";
+            try {
+              StorageException e2 = (StorageException) e.getCause();
+              errorCode = e2.getErrorCode();
+            } catch (Exception e3) {
+              // do nothing if cast fails
+            }
+            if (errorCode.equals("BlobNotFound")) {
+              throw new FileNotFoundException("Folder does not exist: " + parentKey);
+            }
+            LOG.warn("Got unexpected exception trying to get lease on "
+                + parentKey + ". " + e.getMessage());
+            throw e;
+          } finally {
+            try {
+              if (lease != null) {
+                lease.free();
+              }
+            } catch (Exception e) {
+              LOG.error("Unable to free lease on " + parentKey, e);
+            }
+          }
+        } else {
+          store.updateFolderLastModifiedTime(parentKey, null);
+        }
       }
     }
   }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
index 2d5c0c8ebde40..bda6006d60225 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.azure;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
@@ -27,6 +26,8 @@
 import com.microsoft.windowsazure.storage.StorageException;
 import com.microsoft.windowsazure.storage.blob.CloudBlob;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * An Azure blob lease that automatically renews itself indefinitely
  * using a background thread. Use it to synchronize distributed processes,
@@ -56,7 +57,7 @@ public class SelfRenewingLease {
   private static final Log LOG = LogFactory.getLog(SelfRenewingLease.class);
 
   // Used to allocate thread serial numbers in thread name
-  private static volatile int threadNumber = 0;
+  private static AtomicInteger threadNumber = new AtomicInteger(0);
 
 
   // Time to wait to retry getting the lease in milliseconds
@@ -99,7 +100,7 @@ public SelfRenewingLease(CloudBlobWrapper blobWrapper)
 
     // A Renewer running should not keep JVM from exiting, so make it a daemon.
     renewer.setDaemon(true);
-    renewer.setName("AzureLeaseRenewer-" + threadNumber++);
+    renewer.setName("AzureLeaseRenewer-" + threadNumber.getAndIncrement());
     renewer.start();
     LOG.debug("Acquired lease " + leaseID + " on " + blob.getUri()
         + " managed by thread " + renewer.getName());
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
new file mode 100644
index 0000000000000..0d69ccf737022
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -0,0 +1,243 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Hadoop Azure Support: Azure Blob Storage
+
+* [Introduction](#Introduction)
+* [Features](#Features)
+* [Limitations](#Limitations)
+* [Usage](#Usage)
+    * [Concepts](#Concepts)
+    * [Configuring Credentials](#Configuring_Credentials)
+    * [Page Blob Support and Configuration](#Page_Blob_Support_and_Configuration)
+    * [Atomic Folder Rename](#Atomic_Folder_Rename)
+    * [Accessing wasb URLs](#Accessing_wasb_URLs)
+* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
+
+## <a name="Introduction" />Introduction
+
+The hadoop-azure module provides support for integration with
+[Azure Blob Storage](http://azure.microsoft.com/en-us/documentation/services/storage/).
+The built jar file, named hadoop-azure.jar, also declares transitive dependencies
+on the additional artifacts it requires, notably the
+[Azure Storage SDK for Java](https://github.com/Azure/azure-storage-java).
+
+## <a name="Features" />Features
+
+* Read and write data stored in an Azure Blob Storage account.
+* Present a hierarchical file system view by implementing the standard Hadoop
+  [`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface.
+* Supports configuration of multiple Azure Blob Storage accounts.
+* Supports both page blobs (suitable for most use cases, such as MapReduce) and
+  block blobs (suitable for continuous write use cases, such as an HBase
+  write-ahead log).
+* Reference file system paths using URLs using the `wasb` scheme.
+* Also reference file system paths using URLs with the `wasbs` scheme for SSL
+  encrypted access.
+* Can act as a source of data in a MapReduce job, or a sink.
+* Tested on both Linux and Windows.
+* Tested at scale.
+
+## <a name="Limitations" />Limitations
+
+* The append operation is not implemented.
+* File owner and group are persisted, but the permissions model is not enforced.
+  Authorization occurs at the level of the entire Azure Blob Storage account.
+* File last access time is not tracked.
+
+## <a name="Usage" />Usage
+
+### <a name="Concepts" />Concepts
+
+The Azure Blob Storage data model presents 3 core concepts:
+
+* **Storage Account**: All access is done through a storage account.
+* **Container**: A container is a grouping of multiple blobs.  A storage account
+  may have multiple containers.  In Hadoop, an entire file system hierarchy is
+  stored in a single container.  It is also possible to configure multiple
+  containers, effectively presenting multiple file systems that can be referenced
+  using distinct URLs.
+* **Blob**: A file of any type and size.  In Hadoop, files are stored in blobs.
+  The internal implementation also uses blobs to persist the file system
+  hierarchy and other metadata.
+
+### <a name="Configuring_Credentials" />Configuring Credentials
+
+Usage of Azure Blob Storage requires configuration of credentials.  Typically
+this is set in core-site.xml.  The configuration property name is of the form
+`fs.azure.account.key.<account name>.blob.core.windows.net` and the value is the
+access key.  **The access key is a secret that protects access to your storage
+account.  Do not share the access key (or the core-site.xml file) with an
+untrusted party.**
+
+For example:
+
+    <property>
+      <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
+      <value>YOUR ACCESS KEY</value>
+    </property>
+
+In many Hadoop clusters, the core-site.xml file is world-readable.  If it's
+undesirable for the access key to be visible in core-site.xml, then it's also
+possible to configure it in encrypted form.  An additional configuration property
+specifies an external program to be invoked by Hadoop processes to decrypt the
+key.  The encrypted key value is passed to this external program as a command
+line argument:
+
+    <property>
+      <name>fs.azure.account.keyprovider.youraccount</name>
+      <value>org.apache.hadoop.fs.azure.ShellDecryptionKeyProvider</value>
+    </property>
+
+    <property>
+      <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
+      <value>YOUR ENCRYPTED ACCESS KEY</value>
+    </property>
+
+    <property>
+      <name>fs.azure.shellkeyprovider.script</name>
+      <value>PATH TO DECRYPTION PROGRAM</value>
+    </property>
+
+### <a name="Page_Blob_Support_and_Configuration" />Page Blob Support and Configuration
+
+The Azure Blob Storage interface for Hadoop supports two kinds of blobs,
+[block blobs and page blobs](http://msdn.microsoft.com/en-us/library/azure/ee691964.aspx).
+Block blobs are the default kind of blob and are good for most big-data use
+cases, like input data for Hive, Pig, analytical map-reduce jobs etc.  Page blob
+handling in hadoop-azure was introduced to support HBase log files.  Page blobs
+can be written any number of times, whereas block blobs can only be appended to
+50,000 times before you run out of blocks and your writes will fail.  That won't
+work for HBase logs, so page blob support was introduced to overcome this
+limitation.
+
+Page blobs can be used for other purposes beyond just HBase log files though.
+Page blobs can be up to 1TB in size, larger than the maximum 200GB size for block
+blobs.
+
+In order to have the files you create be page blobs, you must set the
+configuration variable `fs.azure.page.blob.dir` to a comma-separated list of
+folder names.
+
+For example:
+
+    <property>
+      <name>fs.azure.page.blob.dir</name>
+      <value>/hbase/WALs,/hbase/oldWALs,/data/mypageblobfiles</value>
+    </property>
+
+You can set this to simply / to make all files page blobs.
+
+The configuration option `fs.azure.page.blob.size` is the default initial
+size for a page blob. It must be 128MB or greater, and no more than 1TB,
+specified as an integer number of bytes.
+
+The configuration option `fs.azure.page.blob.extension.size` is the page blob
+extension size.  This defines the amount to extend a page blob if it starts to
+get full.  It must be 128MB or greater, specified as an integer number of bytes.
+
+### <a name="Atomic_Folder_Rename" />Atomic Folder Rename
+
+Azure storage stores files as a flat key/value store without formal support
+for folders.  The hadoop-azure file system layer simulates folders on top
+of Azure storage.  By default, folder rename in the hadoop-azure file system
+layer is not atomic.  That means that a failure during a folder rename
+could, for example, leave some folders in the original directory and
+some in the new one.
+
+HBase depends on atomic folder rename.  Hence, a configuration setting was
+introduced called `fs.azure.atomic.rename.dir` that allows you to specify a
+comma-separated list of directories to receive special treatment so that
+folder rename is made atomic.  The default value of this setting is just
+`/hbase`.  Redo will be applied to finish a folder rename that fails. A file
+`<folderName>-renamePending.json` may appear temporarily and is the record of
+the intention of the rename operation, to allow redo in event of a failure.
+
+For example:
+
+    <property>
+      <name>fs.azure.atomic.rename.dir</name>
+      <value>/hbase,/data</value>
+    </property>
+
+### <a name="Accessing_wasb_URLs" />Accessing wasb URLs
+
+After credentials are configured in core-site.xml, any Hadoop component may
+reference files in that Azure Blob Storage account by using URLs of the following
+format:
+
+    wasb[s]://<containername>@<accountname>.blob.core.windows.net/<path>
+
+The schemes `wasb` and `wasbs` identify a URL on a file system backed by Azure
+Blob Storage.  `wasb` utilizes unencrypted HTTP access for all interaction with
+the Azure Blob Storage API.  `wasbs` utilizes SSL encrypted HTTPS access.
+
+For example, the following
+[FileSystem Shell](../hadoop-project-dist/hadoop-common/FileSystemShell.html)
+commands demonstrate access to a storage account named `youraccount` and a
+container named `yourcontainer`.
+
+    > hadoop fs -mkdir wasb://yourcontainer@youraccount.blob.core.windows.net/testDir
+
+    > hadoop fs -put testFile wasb://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
+
+    > hadoop fs -cat wasbs://yourcontainer@youraccount.blob.core.windows.net/testDir/testFile
+    test file content
+
+It's also possible to configure `fs.defaultFS` to use a `wasb` or `wasbs` URL.
+This causes all bare paths, such as `/testDir/testFile` to resolve automatically
+to that file system.
+
+## <a name="Testing_the_hadoop-azure_Module" />Testing the hadoop-azure Module
+
+The hadoop-azure module includes a full suite of unit tests.  Most of the tests
+will run without additional configuration by running `mvn test`.  This includes
+tests against mocked storage, which is an in-memory emulation of Azure Storage.
+
+A selection of tests can run against the
+[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx)
+which is a high-fidelity emulation of live Azure Storage.  The emulator is
+sufficient for high-confidence testing.  The emulator is a Windows executable
+that runs on a local machine.
+
+To use the emulator, install Azure SDK 2.3 and start the storage emulator.  Then,
+edit `src/test/resources/azure-test.xml` and add the following property:
+
+    <property>
+      <name>fs.azure.test.emulator</name>
+      <value>true</value>
+    </property>
+
+There is a known issue when running tests with the emulator.  You may see the
+following failure message:
+
+    com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
+
+To resolve this, restart the Azure Emulator.  Ensure it v3.2 or later.
+
+It's also possible to run tests against a live Azure Storage account by adding
+credentials to `src/test/resources/azure-test.xml` and setting
+`fs.azure.test.account.name` to the name of the storage account.
+
+For example:
+
+    <property>
+      <name>fs.azure.account.key.youraccount.blob.core.windows.net</name>
+      <value>YOUR ACCESS KEY</value>
+    </property>
+
+    <property>
+      <name>fs.azure.test.account.name</name>
+      <value>youraccount</value>
+    </property>
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index a3232372acf76..b8ff912b4e156 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -482,7 +482,7 @@ private static Configuration createTestConfiguration(Configuration conf) {
     return conf;
   }
 
-  static CloudStorageAccount createTestAccount()
+  public static CloudStorageAccount createTestAccount()
       throws URISyntaxException, KeyProviderException
   {
     return createTestAccount(createTestConfiguration());
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
index 0fe93c2f1367b..2bc343e1727fd 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java
@@ -27,6 +27,7 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -74,7 +75,8 @@ private List<Path> fetchFileList(Path sourceListing) throws IOException {
     FileSystem fs = sourceListing.getFileSystem(getConf());
     BufferedReader input = null;
     try {
-      input = new BufferedReader(new InputStreamReader(fs.open(sourceListing)));
+      input = new BufferedReader(new InputStreamReader(fs.open(sourceListing),
+          Charset.forName("UTF-8")));
       String line = input.readLine();
       while (line != null) {
         result.add(new Path(line));
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
index f6fe11847a25b..d08a3012e27aa 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
@@ -115,7 +115,7 @@ public int read(long position, byte[] buffer, int offset, int length)
   }
 
   private void throttle() throws IOException {
-    if (getBytesPerSec() > maxBytesPerSec) {
+    while (getBytesPerSec() > maxBytesPerSec) {
       try {
         Thread.sleep(SLEEP_DURATION_MS);
         totalSleepTime += SLEEP_DURATION_MS;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
similarity index 100%
rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistCp.md.vm
rename to hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
diff --git a/hadoop-tools/hadoop-distcp/src/site/resources/css/site.css b/hadoop-tools/hadoop-distcp/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
index 8779e06555412..ed08139623582 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
@@ -238,11 +238,10 @@ public InputSplit[] getSplits(JobConf job, int numSplits
 
       Text key = new Text();
       FileOperation value = new FileOperation();
-      SequenceFile.Reader in = null;
       long prev = 0L;
       int count = 0; //count src
-      try {
-        for(in = new SequenceFile.Reader(fs, srcs, job); in.next(key, value); ) {
+      try (SequenceFile.Reader in = new SequenceFile.Reader(fs, srcs, job)) {
+        for ( ; in.next(key, value); ) {
           long curr = in.getPosition();
           long delta = curr - prev;
           if (++count > targetcount) {
@@ -252,9 +251,6 @@ public InputSplit[] getSplits(JobConf job, int numSplits
           }
         }
       }
-      finally {
-        in.close();
-      }
       long remaining = fs.getFileStatus(srcs).getLen() - prev;
       if (remaining != 0) {
         splits.add(new FileSplit(srcs, prev, remaining, (String[])null));
@@ -449,10 +445,8 @@ private boolean setup(List<FileOperation> ops, Path log)
     Path opList = new Path(jobdir, "_" + OP_LIST_LABEL);
     jobconf.set(OP_LIST_LABEL, opList.toString());
     int opCount = 0, synCount = 0;
-    SequenceFile.Writer opWriter = null;
-    try {
-      opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
-          FileOperation.class, SequenceFile.CompressionType.NONE);
+    try (SequenceFile.Writer opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
+            FileOperation.class, SequenceFile.CompressionType.NONE)) {
       for(FileOperation op : ops) {
         FileStatus srcstat = fs.getFileStatus(op.src); 
         if (srcstat.isDirectory() && op.isDifferent(srcstat)) {
@@ -479,8 +473,6 @@ private boolean setup(List<FileOperation> ops, Path log)
           }
         }
       }
-    } finally {
-      opWriter.close();
     }
 
     checkDuplication(fs, opList, new Path(jobdir, "_sorted"), jobconf);
@@ -496,9 +488,7 @@ private static void checkDuplication(FileSystem fs, Path file, Path sorted,
     SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
         new Text.Comparator(), Text.class, FileOperation.class, conf);
     sorter.sort(file, sorted);
-    SequenceFile.Reader in = null;
-    try {
-      in = new SequenceFile.Reader(fs, sorted, conf);
+    try (SequenceFile.Reader in = new SequenceFile.Reader(fs, sorted, conf)) {
       FileOperation curop = new FileOperation();
       Text prevsrc = null, cursrc = new Text(); 
       for(; in.next(cursrc, curop); ) {
@@ -512,9 +502,6 @@ private static void checkDuplication(FileSystem fs, Path file, Path sorted,
         curop = new FileOperation();
       }
     }
-    finally {
-      in.close();
-    }
   } 
 
   public static void main(String[] args) throws Exception {
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
index d1e65e2fe6f8b..f46c421607a7d 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
@@ -24,6 +24,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashSet;
@@ -50,9 +51,11 @@
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.SequenceFile.Writer;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.FileSplit;
@@ -72,6 +75,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
 
 /**
  * A Map-reduce program to recursively copy directories between
@@ -282,9 +286,8 @@ public InputSplit[] getSplits(JobConf job, int numSplits)
       long last = 0L;
       long acc = 0L;
       long cbrem = srcst.getLen();
-      SequenceFile.Reader sl = null;
-      try {
-        sl = new SequenceFile.Reader(fs, src, job);
+      try (SequenceFile.Reader sl =
+          new SequenceFile.Reader(job, Reader.file(src))) {
         for (; sl.next(key, value); last = sl.getPosition()) {
           // if adding this split would put this split past the target size,
           // cut the last split and put this next file in the next split.
@@ -298,9 +301,6 @@ public InputSplit[] getSplits(JobConf job, int numSplits)
           acc += key.get();
         }
       }
-      finally {
-        checkAndClose(sl);
-      }
       if (cbrem != 0) {
         splits.add(new FileSplit(src, pos, cbrem, (String[])null));
       }
@@ -437,32 +437,28 @@ private boolean skipCopyFile(FileStatus srcstat, Path absdst,
      */
     private long doCopyFile(FileStatus srcstat, Path tmpfile, Path absdst,
                             Reporter reporter) throws IOException {
-      FSDataInputStream in = null;
-      FSDataOutputStream out = null;
       long bytesCopied = 0L;
-      try {
-        Path srcPath = srcstat.getPath();
-        // open src file
-        in = srcPath.getFileSystem(job).open(srcPath);
+      Path srcPath = srcstat.getPath();
+      // open src file
+      try (FSDataInputStream in = srcPath.getFileSystem(job).open(srcPath)) {
         reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
         // open tmp file
-        out = create(tmpfile, reporter, srcstat);
-        LOG.info("Copying file " + srcPath + " of size " +
-                 srcstat.getLen() + " bytes...");
+        try (FSDataOutputStream out = create(tmpfile, reporter, srcstat)) {
+          LOG.info("Copying file " + srcPath + " of size " +
+                   srcstat.getLen() + " bytes...");
         
-        // copy file
-        for(int bytesRead; (bytesRead = in.read(buffer)) >= 0; ) {
-          out.write(buffer, 0, bytesRead);
-          bytesCopied += bytesRead;
-          reporter.setStatus(
-              String.format("%.2f ", bytesCopied*100.0/srcstat.getLen())
-              + absdst + " [ " +
-              StringUtils.humanReadableInt(bytesCopied) + " / " +
-              StringUtils.humanReadableInt(srcstat.getLen()) + " ]");
+          // copy file
+          for(int bytesRead; (bytesRead = in.read(buffer)) >= 0; ) {
+            out.write(buffer, 0, bytesRead);
+            bytesCopied += bytesRead;
+            reporter.setStatus(
+                String.format("%.2f ", bytesCopied*100.0/srcstat.getLen())
+                + absdst + " [ " +
+                TraditionalBinaryPrefix.long2String(bytesCopied, "", 1) + " / "
+                + TraditionalBinaryPrefix.long2String(srcstat.getLen(), "", 1)
+                + " ]");
+          }
         }
-      } finally {
-        checkAndClose(in);
-        checkAndClose(out);
       }
       return bytesCopied;
     }
@@ -470,7 +466,8 @@ private long doCopyFile(FileStatus srcstat, Path tmpfile, Path absdst,
     /**
      * Copy a file to a destination.
      * @param srcstat src path and metadata
-     * @param dstpath dst path
+     * @param relativedst relative dst path
+     * @param outc Log of skipped files
      * @param reporter
      * @throws IOException if copy fails(even if the validation of copy fails)
      */
@@ -569,7 +566,8 @@ private void updateDestStatus(FileStatus src, FileStatus dst
     }
 
     static String bytesString(long b) {
-      return b + " bytes (" + StringUtils.humanReadableInt(b) + ")";
+      return b + " bytes (" +
+          TraditionalBinaryPrefix.long2String(b, "", 1) + ")";
     }
 
     /**
@@ -697,16 +695,13 @@ private static List<Path> fetchFileList(Configuration conf, Path srcList)
       throws IOException {
     List<Path> result = new ArrayList<Path>();
     FileSystem fs = srcList.getFileSystem(conf);
-    BufferedReader input = null;
-    try {
-      input = new BufferedReader(new InputStreamReader(fs.open(srcList)));
+    try (BufferedReader input = new BufferedReader(new InputStreamReader(fs.open(srcList),
+            Charset.forName("UTF-8")))) {
       String line = input.readLine();
       while (line != null) {
         result.add(new Path(line));
         line = input.readLine();
       }
-    } finally {
-      checkAndClose(input);
     }
     return result;
   }
@@ -764,6 +759,7 @@ private static void checkSrcPath(JobConf jobConf, List<Path> srcPaths)
 
   /**
    * Driver to copy srcPath to destPath depending on required protocol.
+   * @param conf configuration
    * @param args arguments
    */
   static void copy(final Configuration conf, final Arguments args
@@ -840,10 +836,8 @@ static private void finalize(Configuration conf, JobConf jobconf,
 
     FileSystem dstfs = destPath.getFileSystem(conf);
     Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
-    SequenceFile.Reader in = null;
-    try {
-      in = new SequenceFile.Reader(dstdirlist.getFileSystem(jobconf),
-          dstdirlist, jobconf);
+    try (SequenceFile.Reader in =
+        new SequenceFile.Reader(jobconf, Reader.file(dstdirlist))) {
       Text dsttext = new Text();
       FilePair pair = new FilePair(); 
       for(; in.next(dsttext, pair); ) {
@@ -851,8 +845,6 @@ static private void finalize(Configuration conf, JobConf jobconf,
         updateDestStatus(pair.input, dstfs.getFileStatus(absdst),
             preseved, dstfs);
       }
-    } finally {
-      checkAndClose(in);
     }
   }
 
@@ -878,6 +870,8 @@ static class Arguments {
      * @param preservedAttributes Preserved attributes 
      * @param filelimit File limit
      * @param sizelimit Size limit
+     * @param mapredSslConf ssl configuration
+     * @param dryrun
      */
     Arguments(List<Path> srcs, Path basedir, Path dst, Path log,
         EnumSet<Options> flags, String preservedAttributes,
@@ -957,7 +951,7 @@ else if (opt[i] == Options.SIZE_LIMIT) {
             throw new IllegalArgumentException("num_maps not specified in -m");
           }
           try {
-            conf.setInt(MAX_MAPS_LABEL, Integer.valueOf(args[idx]));
+            conf.setInt(MAX_MAPS_LABEL, Integer.parseInt(args[idx]));
           } catch (NumberFormatException e) {
             throw new IllegalArgumentException("Invalid argument to -m: " +
                                                args[idx]);
@@ -1260,42 +1254,44 @@ static boolean setup(Configuration conf, JobConf jobConf,
     FileSystem jobfs = jobDirectory.getFileSystem(jobConf);
 
     Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
-    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
-    SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf,
-        srcfilelist, LongWritable.class, FilePair.class,
-        SequenceFile.CompressionType.NONE);
-
     Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
-    SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf,
-        dstfilelist, Text.class, Text.class,
-        SequenceFile.CompressionType.NONE);
-
     Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
+    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
     jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
-    SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf,
-        dstdirlist, Text.class, FilePair.class,
-        SequenceFile.CompressionType.NONE);
-
-    // handle the case where the destination directory doesn't exist
-    // and we've only a single src directory OR we're updating/overwriting
-    // the contents of the destination directory.
-    final boolean special =
-      (args.srcs.size() == 1 && !dstExists) || update || overwrite;
     int srcCount = 0, cnsyncf = 0, dirsyn = 0;
     long fileCount = 0L, dirCount = 0L, byteCount = 0L, cbsyncs = 0L,
          skipFileCount = 0L, skipByteCount = 0L;
-    
-    Path basedir = null;
-    HashSet<Path> parentDirsToCopy = new HashSet<Path>(); 
-    if (args.basedir != null) {
-      FileSystem basefs = args.basedir.getFileSystem(conf);
-      basedir = args.basedir.makeQualified(basefs);
-      if (!basefs.isDirectory(basedir)) {
-        throw new IOException("Basedir " + basedir + " is not a directory.");
+    try (
+        SequenceFile.Writer src_writer = SequenceFile.createWriter(jobConf,
+            Writer.file(srcfilelist), Writer.keyClass(LongWritable.class),
+            Writer.valueClass(FilePair.class), Writer.compression(
+            SequenceFile.CompressionType.NONE));
+        SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobConf,
+            Writer.file(dstfilelist), Writer.keyClass(Text.class),
+            Writer.valueClass(Text.class), Writer.compression(
+            SequenceFile.CompressionType.NONE));
+        SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobConf,
+            Writer.file(dstdirlist), Writer.keyClass(Text.class),
+            Writer.valueClass(FilePair.class), Writer.compression(
+            SequenceFile.CompressionType.NONE));
+    ) {
+      // handle the case where the destination directory doesn't exist
+      // and we've only a single src directory OR we're updating/overwriting
+      // the contents of the destination directory.
+      final boolean special =
+        (args.srcs.size() == 1 && !dstExists) || update || overwrite;
+
+      Path basedir = null;
+      HashSet<Path> parentDirsToCopy = new HashSet<Path>();
+      if (args.basedir != null) {
+        FileSystem basefs = args.basedir.getFileSystem(conf);
+        basedir = args.basedir.makeQualified(
+            basefs.getUri(), basefs.getWorkingDirectory());
+        if (!basefs.isDirectory(basedir)) {
+          throw new IOException("Basedir " + basedir + " is not a directory.");
+        }
       }
-    }
-    
-    try {
+
       for(Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
         final Path src = srcItr.next();
         FileSystem srcfs = src.getFileSystem(conf);
@@ -1311,7 +1307,8 @@ static boolean setup(Configuration conf, JobConf jobConf,
 
         if (basedir != null) {
           root = basedir;
-          Path parent = src.getParent().makeQualified(srcfs);
+          Path parent = src.getParent().makeQualified(
+              srcfs.getUri(), srcfs.getWorkingDirectory());
           while (parent != null && !parent.equals(basedir)) {
             if (!parentDirsToCopy.contains(parent)){
               parentDirsToCopy.add(parent);
@@ -1428,18 +1425,15 @@ static boolean setup(Configuration conf, JobConf jobConf,
           }
         }
       }
-    } finally {
-      checkAndClose(src_writer);
-      checkAndClose(dst_writer);
-      checkAndClose(dir_writer);
     }
     LOG.info("sourcePathsCount(files+directories)=" + srcCount);
     LOG.info("filesToCopyCount=" + fileCount);
-    LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
+    LOG.info("bytesToCopyCount=" +
+             TraditionalBinaryPrefix.long2String(byteCount, "", 1));
     if (update) {
       LOG.info("filesToSkipCopyCount=" + skipFileCount);
       LOG.info("bytesToSkipCopyCount=" +
-               StringUtils.humanReadableInt(skipByteCount));
+               TraditionalBinaryPrefix.long2String(skipByteCount, "", 1));
     }
     if (args.dryrun) {
       return false;
@@ -1483,7 +1477,8 @@ static boolean setup(Configuration conf, JobConf jobConf,
 
     LOG.info("sourcePathsCount=" + srcCount);
     LOG.info("filesToCopyCount=" + fileCount);
-    LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
+    LOG.info("bytesToCopyCount=" +
+             TraditionalBinaryPrefix.long2String(byteCount, "", 1));
     jobConf.setInt(SRC_COUNT_LABEL, srcCount);
     jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
     
@@ -1567,10 +1562,10 @@ static private long deleteNonexisting(
 
     //write dst lsr results
     final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
-    final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf,
-        dstlsr, Text.class, NullWritable.class,
-        SequenceFile.CompressionType.NONE);
-    try {
+    try (final SequenceFile.Writer writer = SequenceFile.createWriter(jobconf,
+        Writer.file(dstlsr), Writer.keyClass(Text.class),
+        Writer.valueClass(NullWritable.class), Writer.compression(
+        SequenceFile.CompressionType.NONE))) {
       //do lsr to get all file statuses in dstroot
       final Stack<FileStatus> lsrstack = new Stack<FileStatus>();
       for(lsrstack.push(dstroot); !lsrstack.isEmpty(); ) {
@@ -1583,8 +1578,6 @@ static private long deleteNonexisting(
           }
         }
       }
-    } finally {
-      checkAndClose(writer);
     }
 
     //sort lsr results
@@ -1594,13 +1587,11 @@ static private long deleteNonexisting(
     sorter.sort(dstlsr, sortedlsr);
 
     //compare lsr list and dst list  
-    SequenceFile.Reader lsrin = null;
-    SequenceFile.Reader dstin = null;
     long deletedPathsCount = 0;
-    try {
-      lsrin = new SequenceFile.Reader(jobfs, sortedlsr, jobconf);
-      dstin = new SequenceFile.Reader(jobfs, dstsorted, jobconf);
-
+    try (SequenceFile.Reader lsrin =
+             new SequenceFile.Reader(jobconf, Reader.file(sortedlsr));
+         SequenceFile.Reader  dstin =
+             new SequenceFile.Reader(jobconf, Reader.file(dstsorted))) {
       //compare sorted lsr list and sorted dst list
       final Text lsrpath = new Text();
       final Text dstpath = new Text();
@@ -1631,9 +1622,6 @@ static private long deleteNonexisting(
           }
         }
       }
-    } finally {
-      checkAndClose(lsrin);
-      checkAndClose(dstin);
     }
     return deletedPathsCount;
   }
@@ -1652,13 +1640,11 @@ static private boolean isAncestorPath(Path xp, Path yp) {
   /** Check whether the file list have duplication. */
   static private void checkDuplication(FileSystem fs, Path file, Path sorted,
     Configuration conf) throws IOException {
-    SequenceFile.Reader in = null;
-    try {
-      SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
-        new Text.Comparator(), Text.class, Text.class, conf);
-      sorter.sort(file, sorted);
-      in = new SequenceFile.Reader(fs, sorted, conf);
-
+    SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
+      new Text.Comparator(), Text.class, Text.class, conf);
+    sorter.sort(file, sorted);
+    try (SequenceFile.Reader in =
+         new SequenceFile.Reader(conf, Reader.file(sorted))) {
       Text prevdst = null, curdst = new Text();
       Text prevsrc = null, cursrc = new Text(); 
       for(; in.next(curdst, cursrc); ) {
@@ -1673,24 +1659,8 @@ static private void checkDuplication(FileSystem fs, Path file, Path sorted,
         cursrc = new Text();
       }
     }
-    finally {
-      checkAndClose(in);
-    }
   } 
 
-  static boolean checkAndClose(java.io.Closeable io) {
-    if (io != null) {
-      try {
-        io.close();
-      }
-      catch(IOException ioe) {
-        LOG.warn(StringUtils.stringifyException(ioe));
-        return false;
-      }
-    }
-    return true;
-  }
-
   /** An exception class for duplicated source files. */
   public static class DuplicationException extends IOException {
     private static final long serialVersionUID = 1L;
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistTool.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistTool.java
index 4e1a6aa77ca75..2c89cb084d53b 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistTool.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistTool.java
@@ -23,6 +23,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
@@ -96,14 +97,11 @@ protected static List<String> readFile(Configuration conf, Path inputfile
       ) throws IOException {
     List<String> result = new ArrayList<String>();
     FileSystem fs = inputfile.getFileSystem(conf);
-    BufferedReader input = null;
-    try {
-      input = new BufferedReader(new InputStreamReader(fs.open(inputfile)));
+    try (BufferedReader input = new BufferedReader(new InputStreamReader(fs.open(inputfile),
+            Charset.forName("UTF-8")))) {
       for(String line; (line = input.readLine()) != null;) {
         result.add(line);
       }
-    } finally {
-      input.close();
     }
     return result;
   }
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
index c3c8e90b2d222..050bfbe2a2011 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
@@ -21,6 +21,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -155,15 +156,15 @@ public int compare(byte[] b1, int s1, int l1,
         
         //Compare column-wise according to *sortSpec*
         for(int i=0; i < sortSpec.length; ++i) {
-          int column = (Integer.valueOf(sortSpec[i]).intValue());
+          int column = Integer.parseInt(sortSpec[i]);
           String c1 = logColumns1[column]; 
           String c2 = logColumns2[column];
           
           //Compare columns
           int comparision = super.compareBytes(
-                                               c1.getBytes(), 0, c1.length(),
-                                               c2.getBytes(), 0, c2.length()
-                                               );
+                                  c1.getBytes(Charset.forName("UTF-8")), 0, c1.length(),
+                                  c2.getBytes(Charset.forName("UTF-8")), 0, c2.length()
+                                  );
           
           //They differ!
           if (comparision != 0) {
diff --git a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
index 92458d4a2132d..cb043c4330eae 100644
--- a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
@@ -30,4 +30,11 @@
     <Bug pattern="REC_CATCH_EXCEPTION"/>
     <Bug code="REC"/>
   </Match>
+
+  <!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code -->
+  <Match>
+    <Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/>
+    <Field name="returnValue"/>
+    <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
+  </Match>
 </FindBugsFilter>
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
index 526878c91c360..bd7e878497b03 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -97,7 +98,9 @@ class CompressionEmulationUtil {
   
   private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE = 
     new CompressionRatioLookupTable();
-  
+
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   /**
    * This is a {@link Mapper} implementation for generating random text data.
    * It uses {@link RandomTextDataGenerator} for generating text data and the
@@ -133,7 +136,8 @@ public void map(NullWritable key, LongWritable value, Context context)
         String randomKey = rtg.getRandomWord();
         String randomValue = rtg.getRandomWord();
         context.write(new Text(randomKey), new Text(randomValue));
-        bytes -= (randomValue.getBytes().length + randomKey.getBytes().length);
+        bytes -= (randomValue.getBytes(charsetUTF8).length +
+            randomKey.getBytes(charsetUTF8).length);
       }
     }
   }
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
index ad3b1136c4b7c..d30a51d951998 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
@@ -41,6 +41,7 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -112,6 +113,8 @@ class DistributedCacheEmulator {
 
   Configuration conf; // gridmix configuration
 
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   // Pseudo local file system where local FS based distributed cache files are
   // created by gridmix.
   FileSystem pseudoLocalFs = null;
@@ -436,9 +439,10 @@ public int compare(Object dc1, Object dc2) {
     for (Iterator it = dcFiles.iterator(); it.hasNext();) {
       Map.Entry entry = (Map.Entry)it.next();
       LongWritable fileSize =
-          new LongWritable(Long.valueOf(entry.getValue().toString()));
+          new LongWritable(Long.parseLong(entry.getValue().toString()));
       BytesWritable filePath =
-          new BytesWritable(entry.getKey().toString().getBytes());
+          new BytesWritable(
+          entry.getKey().toString().getBytes(charsetUTF8));
 
       byteCount += fileSize.get();
       bytesSync += fileSize.get();
@@ -515,7 +519,7 @@ void configureDistCacheFiles(Configuration conf, JobConf jobConf)
             // local FS based distributed cache file.
             // Create this file on the pseudo local FS.
             String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString();
-            long fileSize = Long.valueOf(fileSizes[i]);
+            long fileSize = Long.parseLong(fileSizes[i]);
             Path mappedLocalFilePath =
                 PseudoLocalFs.generateFilePath(fileId, fileSize)
                     .makeQualified(pseudoLocalFs.getUri(),
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
index 0512746bf0306..4a75cdedf7388 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.mapred.gridmix;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
@@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob {
    */
   static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
 
+  private static final Charset charsetUTF8 = Charset.forName("UTF-8");
+
   public GenerateDistCacheData(Configuration conf) throws IOException {
     super(conf, 0L, JOB_NAME);
   }
@@ -152,7 +155,8 @@ protected void setup(Context context)
     public void map(LongWritable key, BytesWritable value, Context context)
         throws IOException, InterruptedException {
 
-      String fileName = new String(value.getBytes(), 0, value.getLength());
+      String fileName = new String(value.getBytes(), 0,
+          value.getLength(), charsetUTF8);
       Path path = new Path(fileName);
 
       FSDataOutputStream dos =
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 8ac590bfb6bf8..4386bc1bb2ff9 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -729,40 +729,40 @@ protected void printUsage(PrintStream out) {
     out.println("   -users <usersResourceURI> : URI that contains the users list.");
     out.println("Configuration parameters:");
     out.println("   General parameters:");
-    out.printf("       %-48s : Output directory\n", GRIDMIX_OUT_DIR);
-    out.printf("       %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
-    out.printf("       %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
-    out.printf("       %-48s : User resolution class\n", GRIDMIX_USR_RSV);
-    out.printf("       %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
+    out.printf("       %-48s : Output directory%n", GRIDMIX_OUT_DIR);
+    out.printf("       %-48s : Submitting threads%n", GRIDMIX_SUB_THR);
+    out.printf("       %-48s : Queued job desc%n", GRIDMIX_QUE_DEP);
+    out.printf("       %-48s : User resolution class%n", GRIDMIX_USR_RSV);
+    out.printf("       %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
     out.println("   Parameters related to job submission:");    
-    out.printf("       %-48s : Default queue\n",
+    out.printf("       %-48s : Default queue%n",
         GridmixJob.GRIDMIX_DEFAULT_QUEUE);
-    out.printf("       %-48s : Enable/disable using queues in trace\n",
+    out.printf("       %-48s : Enable/disable using queues in trace%n",
         GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
-    out.printf("       %-48s : Job submission policy (%s)\n",
+    out.printf("       %-48s : Job submission policy (%s)%n",
         GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
     out.println("   Parameters specific for LOADJOB:");
-    out.printf("       %-48s : Key fraction of rec\n",
+    out.printf("       %-48s : Key fraction of rec%n",
         AvgRecordFactory.GRIDMIX_KEY_FRC);
     out.println("   Parameters specific for SLEEPJOB:");
-    out.printf("       %-48s : Whether to ignore reduce tasks\n",
+    out.printf("       %-48s : Whether to ignore reduce tasks%n",
         SleepJob.SLEEPJOB_MAPTASK_ONLY);
-    out.printf("       %-48s : Number of fake locations for map tasks\n",
+    out.printf("       %-48s : Number of fake locations for map tasks%n",
         JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
-    out.printf("       %-48s : Maximum map task runtime in mili-sec\n",
+    out.printf("       %-48s : Maximum map task runtime in mili-sec%n",
         SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
-    out.printf("       %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
+    out.printf("       %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n",
         SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
     out.println("   Parameters specific for STRESS submission throttling policy:");
-    out.printf("       %-48s : jobs vs task-tracker ratio\n",
+    out.printf("       %-48s : jobs vs task-tracker ratio%n",
         StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
-    out.printf("       %-48s : maps vs map-slot ratio\n",
+    out.printf("       %-48s : maps vs map-slot ratio%n",
         StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
-    out.printf("       %-48s : reduces vs reduce-slot ratio\n",
+    out.printf("       %-48s : reduces vs reduce-slot ratio%n",
         StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
-    out.printf("       %-48s : map-slot share per job\n",
+    out.printf("       %-48s : map-slot share per job%n",
         StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
-    out.printf("       %-48s : reduce-slot share per job\n",
+    out.printf("       %-48s : reduce-slot share per job%n",
         StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
    }
 
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
index 1df625f03c78a..07d8878b88ec8 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
@@ -114,8 +114,8 @@ public Job run() throws IOException {
           String jobId = null == jobdesc.getJobID() 
                          ? "<unknown>" 
                          : jobdesc.getJobID().toString();
-          Job ret = new Job(conf,
-                            nameFormat.get().format("%06d", seq).toString());
+          Job ret = Job.getInstance(conf, nameFormat.get().format("%06d", seq)
+                                          .toString());
           ret.getConfiguration().setInt(GRIDMIX_JOB_SEQ, seq);
 
           ret.getConfiguration().set(Gridmix.ORIGINAL_JOB_ID, jobId);
@@ -343,7 +343,7 @@ protected GridmixJob(final Configuration conf, long submissionMillis,
     try {
       job = this.ugi.doAs(new PrivilegedExceptionAction<Job>() {
         public Job run() throws IOException {
-          Job ret = new Job(conf, name);
+          Job ret = Job.getInstance(conf, name);
           ret.getConfiguration().setInt(GRIDMIX_JOB_SEQ, seq);
           setJobQueue(ret, conf.get(GRIDMIX_DEFAULT_QUEUE));
           return ret;
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
index 497108a3a56d0..d7ef563c95aa2 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
@@ -124,7 +124,7 @@ long validateFileNameFormat(Path path) throws FileNotFoundException {
     } else {
       String[] parts = path.toUri().getPath().split("\\.");
       try {
-        fileSize = Long.valueOf(parts[parts.length - 1]);
+        fileSize = Long.parseLong(parts[parts.length - 1]);
         valid = (fileSize >= 0);
       } catch (NumberFormatException e) {
         valid = false;
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
index 322d7555a5140..915788bac1722 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
@@ -119,7 +119,23 @@ public static JobStats generateJobStats(Job job, JobStory jobdesc) {
     }
     return new JobStats(maps, reds, job);
   }
-  
+
+  private static void addToNumMapsSubmitted(int numMaps) {
+    numMapsSubmitted += numMaps;
+  }
+
+  private static void addToNumReducesSubmitted(int numReduces) {
+    numReducesSubmitted += numReduces;
+  }
+
+  private static void subtractFromNumMapsSubmitted(int numMaps) {
+    numMapsSubmitted -= numMaps;
+  }
+
+  private static void subtractFromNumReducesSubmitted(int numReduces) {
+    numReducesSubmitted -= numReduces;
+  }
+
   /**
    * Add a submitted job for monitoring.
    */
@@ -131,8 +147,8 @@ public void addJobStats(JobStats stats) {
       return;
     }
     submittedJobsMap.put(seq, stats);
-    numMapsSubmitted += stats.getNoOfMaps();
-    numReducesSubmitted += stats.getNoOfReds();
+    addToNumMapsSubmitted(stats.getNoOfMaps());
+    addToNumReducesSubmitted(stats.getNoOfReds());
   }
 
   /**
@@ -156,8 +172,8 @@ public void add(Statistics.JobStats job) {
     }
     
     // update the total number of submitted map/reduce task count
-    numMapsSubmitted -= stat.getNoOfMaps();
-    numReducesSubmitted -= stat.getNoOfReds();
+    subtractFromNumMapsSubmitted(stat.getNoOfMaps());
+    subtractFromNumReducesSubmitted(stat.getNoOfReds());
     
     completedJobsInCurrentInterval++;
     //check if we have reached the maximum level of job completions.
diff --git a/hadoop-tools/hadoop-gridmix/src/site/markdown/GridMix.md.vm b/hadoop-tools/hadoop-gridmix/src/site/markdown/GridMix.md.vm
index 53c88915b799f..5e4199b66407a 100644
--- a/hadoop-tools/hadoop-gridmix/src/site/markdown/GridMix.md.vm
+++ b/hadoop-tools/hadoop-gridmix/src/site/markdown/GridMix.md.vm
@@ -38,21 +38,14 @@ Overview
 
 GridMix is a benchmark for Hadoop clusters. It submits a mix of
 synthetic jobs, modeling a profile mined from production loads.
-
-There exist three versions of the GridMix tool. This document
-discusses the third (checked into `src/contrib` ), distinct
-from the two checked into the `src/benchmarks` sub-directory.
-While the first two versions of the tool included stripped-down versions
-of common jobs, both were principally saturation tools for stressing the
-framework at scale. In support of a broader range of deployments and
-finer-tuned job mixes, this version of the tool will attempt to model
+This version of the tool will attempt to model
 the resource profiles of production jobs to identify bottlenecks, guide
-development, and serve as a replacement for the existing GridMix
-benchmarks.
+development.
 
 To run GridMix, you need a MapReduce job trace describing the job mix
-for a given cluster. Such traces are typically generated by Rumen (see
-Rumen documentation). GridMix also requires input data from which the
+for a given cluster. Such traces are typically generated by
+[Rumen](../hadoop-rumen/Rumen.html).
+GridMix also requires input data from which the
 synthetic jobs will be reading bytes. The input data need not be in any
 particular format, as the synthetic jobs are currently binary readers.
 If you are running on a new cluster, an optional step generating input
@@ -62,10 +55,15 @@ on the same or another cluster, follow these steps:
 
 1.  Locate the job history files on the production cluster. This
     location is specified by the
-    `mapred.job.tracker.history.completed.location`
+    `mapreduce.jobhistory.done-dir` or 
+    `mapreduce.jobhistory.intermediate-done-dir`
     configuration property of the cluster.
-
-2.  Run Rumen to build a job trace in JSON format for all or select jobs.
+    ([MapReduce historyserver](../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#historyserver)
+    moves job history files from `mapreduce.jobhistory.done-dir`
+    to `mapreduce.jobhistory.intermediate-done-dir`.)
+    
+2.  Run [Rumen](../hadoop-rumen/Rumen.html)
+    to build a job trace in JSON format for all or select jobs.
 
 3.  Use GridMix with the job trace on the benchmark cluster.
 
@@ -79,13 +77,17 @@ Usage
 
 Basic command-line usage without configuration parameters:
 
-    org.apache.hadoop.mapred.gridmix.Gridmix [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
+java org.apache.hadoop.mapred.gridmix.Gridmix [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
 
 Basic command-line usage with configuration parameters:
 
-    org.apache.hadoop.mapred.gridmix.Gridmix \
-      -Dgridmix.client.submit.threads=10 -Dgridmix.output.directory=foo \
-      [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
+java org.apache.hadoop.mapred.gridmix.Gridmix \
+  -Dgridmix.client.submit.threads=10 -Dgridmix.output.directory=foo \
+  [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
 
 > Configuration parameters like
 > `-Dgridmix.client.submit.threads=10` and
@@ -102,6 +104,8 @@ The `-generate` option is used to generate input data and
 Distributed Cache files for the synthetic jobs. It accepts standard units
 of size suffixes, e.g. `100g` will generate
 100 * 2<sup>30</sup> bytes as input data.
+The minimum size of input data in compressed format (128MB by default)
+is defined by `gridmix.min.file.size`.
 `<iopath>/input` is the destination directory for
 generated input data and/or the directory from which input data will be
 read. HDFS-based Distributed Cache files are generated under the
@@ -121,16 +125,17 @@ uncompressed. Use "-" as the value of this parameter if you
 want to pass an  *uncompressed*  trace via the standard
 input-stream of GridMix.
 
-The class `org.apache.hadoop.mapred.gridmix.Gridmix` can
-be found in the JAR
-`contrib/gridmix/hadoop-gridmix-$VERSION.jar` inside your
-Hadoop installation, where `$VERSION` corresponds to the
-version of Hadoop installed. A simple way of ensuring that this class
-and all its dependencies are loaded correctly is to use the
-`hadoop` wrapper script in Hadoop:
+GridMix expects certain library *JARs* to be present in  the *CLASSPATH*.
+One simple way to run GridMix is to use `hadoop jar` command to run it.
+You also need to add the JAR of Rumen to classpath for both of client and tasks
+as example shown below.
 
-    hadoop jar <gridmix-jar> org.apache.hadoop.mapred.gridmix.Gridmix \
-      [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
+HADOOP_CLASSPATH=$HADOOP_HOME/share/hadoop/tools/lib/hadoop-rumen-2.5.1.jar \
+  $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/share/hadoop/tools/lib/hadoop-gridmix-2.5.1.jar \
+    -libjars $HADOOP_HOME/share/hadoop/tools/lib/hadoop-rumen-2.5.1.jar \
+    [-generate <size>] [-users <users-list>] <iopath> <trace>
+```
 
 The supported configuration parameters are explained in the
 following sections.
@@ -262,14 +267,14 @@ recorded in the trace. It constructs jobs of two types:
     </td>
     <td>A synthetic job where each task does  *nothing*  but sleep
     for a certain duration as observed in the production trace. The
-    scalability of the Job Tracker is often limited by how many
+    scalability of the ResourceManager is often limited by how many
     heartbeats it can handle every second. (Heartbeats are periodic
-    messages sent from Task Trackers to update their status and grab new
-    tasks from the Job Tracker.) Since a benchmark cluster is typically
+    messages sent from NodeManagers to update their status and grab new
+    tasks from the ResourceManager.) Since a benchmark cluster is typically
     a fraction in size of a production cluster, the heartbeat traffic
     generated by the slave nodes is well below the level of the
-    production cluster. One possible solution is to run multiple Task
-    Trackers on each slave node. This leads to the obvious problem that
+    production cluster. One possible solution is to run multiple
+    NodeManagers on each slave node. This leads to the obvious problem that
     the I/O workload generated by the synthetic jobs would thrash the
     slave nodes. Hence the need for such a job.</td>
   </tr>
@@ -334,7 +339,7 @@ Job Submission Policies
 
 GridMix controls the rate of job submission. This control can be
 based on the trace information or can be based on statistics it gathers
-from the Job Tracker. Based on the submission policies users define,
+from the ResourceManager. Based on the submission policies users define,
 GridMix uses the respective algorithm to control the job submission.
 There are currently three types of policies:
 
@@ -407,9 +412,9 @@ The following configuration parameters affect the job submission policy:
     <td>
       <code>gridmix.throttle.jobs-to-tracker-ratio</code>
     </td>
-    <td>In STRESS mode, the minimum ratio of running jobs to Task
-    Trackers in a cluster for the cluster to be considered
-     *overloaded* . This is the threshold TJ referred to earlier.
+    <td>In STRESS mode, the minimum ratio of running jobs to
+    NodeManagers in a cluster for the cluster to be considered
+    *overloaded* . This is the threshold TJ referred to earlier.
     The default is 1.0.</td>
   </tr>
   <tr>
@@ -688,20 +693,16 @@ correctly emulate compression.
 Emulating High-Ram jobs
 -----------------------
 
-MapReduce allows users to define a job as a High-Ram job. Tasks from a 
-High-Ram job can occupy multiple slots on the task-trackers. 
-Task-tracker assigns fixed virtual memory for each slot. Tasks from 
-High-Ram jobs can occupy multiple slots and thus can use up more 
-virtual memory as compared to a default task.
-
-Emulating this behavior is important because of the following reasons
+MapReduce allows users to define a job as a High-Ram job. Tasks from a
+High-Ram job can occupy larger fraction of memory in task processes.
+Emulating this behavior is important because of the following reasons.
 
 * Impact on scheduler:  Scheduling of tasks from High-Ram jobs 
-  impacts the scheduling behavior as it might result into slot 
-  reservation and slot/resource utilization.
+  impacts the scheduling behavior as it might result into
+  resource reservation and utilization.
 
-* Impact on the node : Since High-Ram tasks occupy multiple slots,
-  trackers do some bookkeeping for allocating extra resources for 
+* Impact on the node : Since High-Ram tasks occupy larger memory,
+  NodeManagers do some bookkeeping for allocating extra resources for 
   these tasks. Thus this becomes a precursor for memory emulation
   where tasks with high memory requirements needs to be considered
   as a High-Ram task.
@@ -808,11 +809,11 @@ job traces and cannot be accurately reproduced in GridMix:
 Appendix
 --------
 
+There exist older versions of the GridMix tool.
 Issues tracking the original implementations of
-<a href="https://issues.apache.org/jira/browse/HADOOP-2369">GridMix1</a>,
-<a href="https://issues.apache.org/jira/browse/HADOOP-3770">GridMix2</a>,
-and <a href="https://issues.apache.org/jira/browse/MAPREDUCE-776">GridMix3</a>
+[GridMix1](https://issues.apache.org/jira/browse/HADOOP-2369),
+[GridMix2](https://issues.apache.org/jira/browse/HADOOP-3770),
+and [GridMix3](https://issues.apache.org/jira/browse/MAPREDUCE-776)
 can be found on the Apache Hadoop MapReduce JIRA. Other issues tracking
 the current development of GridMix can be found by searching
-<a href="https://issues.apache.org/jira/browse/MAPREDUCE/component/12313086">
-the Apache Hadoop MapReduce JIRA</a>
+[the Apache Hadoop MapReduce JIRA](https://issues.apache.org/jira/browse/MAPREDUCE/component/12313086).
diff --git a/hadoop-tools/hadoop-gridmix/src/site/resources/css/site.css b/hadoop-tools/hadoop-gridmix/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-tools/hadoop-gridmix/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
index 883e87c6600d2..152a8ee8c1357 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
@@ -157,7 +157,7 @@ private static void runDataGenJob(Configuration conf, Path tempDir)
     // get the local job runner
     conf.setInt(MRJobConfig.NUM_MAPS, 1);
     
-    Job job = new Job(conf);
+    Job job = Job.getInstance(conf);
     
     CompressionEmulationUtil.configure(job);
     job.setInputFormatClass(CustomInputFormat.class);
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index bc43618a1ac9a..afdda992a16ad 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -14,7 +14,7 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
@@ -76,21 +76,6 @@
           <effort>Max</effort>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-site-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>org.apache.maven.doxia</groupId>
-            <artifactId>doxia-module-markdown</artifactId>
-            <version>1.3</version>
-          </dependency>
-        </dependencies>
-        <configuration>
-          <inputEncoding>UTF-8</inputEncoding>
-          <outputEncoding>UTF-8</outputEncoding>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-project-info-reports-plugin</artifactId>
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
index b3e6b9417950d..0138eae412d7d 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
@@ -45,6 +45,7 @@
 import java.io.InterruptedIOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -352,8 +353,8 @@ private List<FileStatus> listDirectory(SwiftObjectPath path,
     final CollectionType collectionType = JSONUtil.getJsonMapper().getTypeFactory().
             constructCollectionType(List.class, SwiftObjectFileStatus.class);
 
-    final List<SwiftObjectFileStatus> fileStatusList =
-            JSONUtil.toObject(new String(bytes), collectionType);
+    final List<SwiftObjectFileStatus> fileStatusList = JSONUtil.toObject(
+        new String(bytes, Charset.forName("UTF-8")), collectionType);
 
     //this can happen if user lists file /data/files/file
     //in this case swift will return empty array
@@ -447,7 +448,7 @@ public List<URI> getObjectLocation(Path path) throws IOException {
       //no object location, return an empty list
       return new LinkedList<URI>();
     }
-    return extractUris(new String(objectLocation), path);
+    return extractUris(new String(objectLocation, Charset.forName("UTF-8")), path);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
index 7e850e713de46..c9e26acf3d4e1 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
@@ -219,9 +219,9 @@ public static void compareByteArrays(byte[] src,
         byte actual = dest[i];
         byte expected = src[i];
         String letter = toChar(actual);
-        String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+        String line = String.format("[%04d] %2x %s%n", i, actual, letter);
         if (expected != actual) {
-          line = String.format("[%04d] %2x %s -expected %2x %s\n",
+          line = String.format("[%04d] %2x %s -expected %2x %s%n",
                                i,
                                actual,
                                letter,
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Hadoop20JHParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Hadoop20JHParser.java
index 9cfd85d5ca84f..08e825b4035e3 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Hadoop20JHParser.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Hadoop20JHParser.java
@@ -192,11 +192,6 @@ private String getFullLine() throws IOException {
 
     do {
       addedLine = getOneLine();
-
-      if (addedLine == null) {
-        return sb.toString();
-      }
-
       sb.append("\n");
       sb.append(addedLine);
     } while (addedLine.length() < endLineString.length()
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
index 653fff8550e1a..47fdb1ad55b54 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
@@ -559,7 +559,7 @@ private boolean setNextDirectoryInputStream() throws FileNotFoundException,
     input =
         maybeUncompressedPath(new Path(inputDirectoryPath, currentFileName));
 
-    return input != null;
+    return true;
   }
 
   private String readInputLine() throws IOException {
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
index f4de3ad679e86..6e73582d7c0ad 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/MapAttempt20LineHistoryEventEmitter.java
@@ -67,7 +67,7 @@ HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
         MapAttempt20LineHistoryEventEmitter that =
             (MapAttempt20LineHistoryEventEmitter) thatg;
 
-        if (finishTime != null && "success".equalsIgnoreCase(status)) {
+        if ("success".equalsIgnoreCase(status)) {
           return new MapAttemptFinishedEvent
             (taskAttemptID,
               that.originalTaskType, status,
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
index c99441e1e0a33..1d85872c08d7b 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
@@ -25,6 +25,8 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 
+import java.nio.charset.Charset;
+
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.ParserConfigurationException;
@@ -44,6 +46,7 @@ class ParsedConfigFile {
       Pattern.compile("_(job_[0-9]+_[0-9]+)_");
   private static final Pattern heapPattern =
       Pattern.compile("-Xmx([0-9]+)([mMgG])");
+  private static final Charset UTF_8 = Charset.forName("UTF-8");
 
   final int heapMegabytes;
 
@@ -100,7 +103,7 @@ private int maybeGetIntValue(String propName, String attr, String value,
     }
 
     try {
-      InputStream is = new ByteArrayInputStream(xmlString.getBytes());
+      InputStream is = new ByteArrayInputStream(xmlString.getBytes(UTF_8));
 
       DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
 
@@ -151,7 +154,7 @@ private int maybeGetIntValue(String propName, String attr, String value,
         
         properties.setProperty(attr, value);
 
-        if ("mapred.child.java.opts".equals(attr) && value != null) {
+        if ("mapred.child.java.opts".equals(attr)) {
           Matcher matcher = heapPattern.matcher(value);
           if (matcher.find()) {
             String heapSize = matcher.group(1);
@@ -164,11 +167,11 @@ private int maybeGetIntValue(String propName, String attr, String value,
           }
         }
 
-        if (MRJobConfig.QUEUE_NAME.equals(attr) && value != null) {
+        if (MRJobConfig.QUEUE_NAME.equals(attr)) {
           queue = value;
         }
 
-        if (MRJobConfig.JOB_NAME.equals(attr) && value != null) {
+        if (MRJobConfig.JOB_NAME.equals(attr)) {
           jobName = value;
         }
 
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/RandomSeedGenerator.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/RandomSeedGenerator.java
index 20ad66c540375..014fb6c33d26d 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/RandomSeedGenerator.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/RandomSeedGenerator.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.tools.rumen;
 
+import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
@@ -42,6 +43,7 @@
  */
 public class RandomSeedGenerator {
   private static Log LOG = LogFactory.getLog(RandomSeedGenerator.class);
+  private static final Charset UTF_8 = Charset.forName("UTF-8");
   
   /** MD5 algorithm instance, one for each thread. */
   private static final ThreadLocal<MessageDigest> md5Holder =
@@ -72,7 +74,7 @@ public static long getSeed(String streamId, long masterSeed) {
     // We could have fed the bytes of masterSeed one by one to md5.update()
     // instead
     String str = streamId + '/' + masterSeed;
-    byte[] digest = md5.digest(str.getBytes());
+    byte[] digest = md5.digest(str.getBytes(UTF_8));
     // Create a long from the first 8 bytes of the digest
     // This is fine as MD5 has the avalanche property.
     // Paranoids could have XOR folded the other 8 bytes in too. 
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
index 74bac99ece214..0261ea225f2cd 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ReduceAttempt20LineHistoryEventEmitter.java
@@ -66,7 +66,7 @@ HistoryEvent maybeEmitEvent(ParsedLine line, String taskAttemptIDName,
         String shuffleFinish = line.get("SHUFFLE_FINISHED");
         String sortFinish = line.get("SORT_FINISHED");
 
-        if (finishTime != null && shuffleFinish != null && sortFinish != null
+        if (shuffleFinish != null && sortFinish != null
             && "success".equalsIgnoreCase(status)) {
           ReduceAttempt20LineHistoryEventEmitter that =
               (ReduceAttempt20LineHistoryEventEmitter) thatg;
diff --git a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
index e25f3a794ae0f..bee976a0e9096 100644
--- a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
+++ b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
@@ -29,9 +29,7 @@ Rumen
     - [Components](#Components)
 - [How to use Rumen?](#How_to_use_Rumen)
     - [Trace Builder](#Trace_Builder)
-        - [Example](#Example)
     - [Folder](#Folder)
-        - [Examples](#Examples)
 - [Appendix](#Appendix)
     - [Resources](#Resources)
     - [Dependencies](#Dependencies)
@@ -128,18 +126,21 @@ can use the `Folder` utility to fold the current trace to the
 desired length. The remaining part of this section explains these 
 utilities in detail.
     
-> Examples in this section assumes that certain libraries are present 
-> in the java CLASSPATH. See <em>Section-3.2</em> for more details.
+Examples in this section assumes that certain libraries are present 
+in the java CLASSPATH. See [Dependencies](#Dependencies) for more details.
 
 
 $H3 Trace Builder
       
-`Command:`
+$H4 Command
 
-    java org.apache.hadoop.tools.rumen.TraceBuilder [options] <jobtrace-output> <topology-output> <inputs>
+```
+java org.apache.hadoop.tools.rumen.TraceBuilder [options] <jobtrace-output> <topology-output> <inputs>
+```
   
-This command invokes the `TraceBuilder` utility of
-*Rumen*. It converts the JobHistory files into a series of JSON
+This command invokes the `TraceBuilder` utility of *Rumen*.
+
+TraceBuilder converts the JobHistory files into a series of JSON
 objects and writes them into the `<jobtrace-output>`
 file. It also extracts the cluster layout (topology) and writes it in
 the`<topology-output>` file.
@@ -169,7 +170,7 @@ Cluster topology is used as follows :
 * To extrapolate splits information for tasks with missing splits
   details or synthetically generated tasks.
       
-`Options :`
+$H4 Options
 
 <table>
   <tr>
@@ -204,33 +205,45 @@ Cluster topology is used as follows :
 
 $H4 Example
 
-    java org.apache.hadoop.tools.rumen.TraceBuilder file:///home/user/job-trace.json file:///home/user/topology.output file:///home/user/logs/history/done
+*Rumen* expects certain library *JARs* to be present in  the *CLASSPATH*.
+One simple way to run Rumen is to use
+`$HADOOP_HOME/bin/hadoop jar` command to run it as example below.
 
-This will analyze all the jobs in 
+```
+java org.apache.hadoop.tools.rumen.TraceBuilder \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-topology.json \
+  hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
+```
 
-`/home/user/logs/history/done` stored on the 
-`local` FileSystem and output the jobtraces in 
-`/home/user/job-trace.json` along with topology 
-information in `/home/user/topology.output`.
+This will analyze all the jobs in 
+`/tmp/hadoop-yarn/staging/history/done_intermediate/testuser`
+stored on the  `HDFS` FileSystem
+and output the jobtraces in  `/tmp/job-trace.json`
+along with topology information in `/tmp/job-topology.json`
+stored on the `local` FileSystem.
 
 
 $H3 Folder
       
-`Command`:
+$H4 Command
 
-    java org.apache.hadoop.tools.rumen.Folder [options] [input] [output]
-      
-> Input and output to `Folder` is expected to be a fully 
-> qualified FileSystem path. So use file:// to specify 
-> files on the `local` FileSystem and hdfs:// to
-> specify files on HDFS.
+```
+java org.apache.hadoop.tools.rumen.Folder [options] [input] [output]
+```
       
 This command invokes the `Folder` utility of 
 *Rumen*. Folding essentially means that the output duration of 
 the resulting trace is fixed and job timelines are adjusted 
 to respect the final output duration. 
 
-`Options :`
+> Input and output to `Folder` is expected to be a fully 
+> qualified FileSystem path. So use `file://` to specify 
+> files on the `local` FileSystem and `hdfs://` to
+> specify files on HDFS.
+
+
+$H4 Options
 
 <table>
   <tr>
@@ -335,14 +348,28 @@ to respect the final output duration.
 $H4 Examples
 
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime
-               
-    java org.apache.hadoop.tools.rumen.Folder -output-duration 1h  -input-cycle 20m  file:///home/user/job-trace.json file:///home/user/job-trace-1hr.json
+
+```
+java org.apache.hadoop.tools.rumen.Folder \
+  -output-duration 1h \
+  -input-cycle 20m \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-trace-1hr.json
+```
 
 If the folded jobs are out of order then the command will bail out. 
 
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime and tolerate some skewness
 
-    java org.apache.hadoop.tools.rumen.Folder -output-duration 1h -input-cycle 20m  -allow-missorting -skew-buffer-length 100 file:///home/user/job-trace.json file:///home/user/job-trace-1hr.json
+```
+java org.apache.hadoop.tools.rumen.Folder \
+  -output-duration 1h \
+  -input-cycle 20m \
+  -allow-missorting \
+  -skew-buffer-length 100 \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-trace-1hr.json
+```
 
 If the folded jobs are out of order, then atmost
 100 jobs will be de-skewed. If the 101<sup>st</sup> job is 
@@ -350,23 +377,37 @@ If the folded jobs are out of order, then atmost
 
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime in debug mode
 
-    java org.apache.hadoop.tools.rumen.Folder -output-duration 1h -input-cycle 20m  -debug -temp-directory file:///tmp/debug file:///home/user/job-trace.json file:///home/user/job-trace-1hr.json
+```
+java org.apache.hadoop.tools.rumen.Folder \
+  -output-duration 1h \
+  -input-cycle 20m \
+  -debug -temp-directory file:///tmp/debug \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-trace-1hr.json
+```
 
 This will fold the 10hr job-trace file 
-`file:///home/user/job-trace.json` to finish within 1hr 
+`file:///tmp/job-trace.json` to finish within 1hr 
 and use `file:///tmp/debug` as the temporary directory. 
 The intermediate files in the temporary directory will not be cleaned
 up.
       
 $H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime with custom concentration.
 
-    java org.apache.hadoop.tools.rumen.Folder -output-duration 1h -input-cycle 20m  -concentration 2  file:///home/user/job-trace.json file:///home/user/job-trace-1hr.json</source>
+```
+java org.apache.hadoop.tools.rumen.Folder \
+  -output-duration 1h \
+  -input-cycle 20m \
+  -concentration 2 \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-trace-1hr.json
+```
 
 This will fold the 10hr job-trace file 
-`file:///home/user/job-trace.json` to finish within 1hr 
-with concentration of 2. `Example-2.3.2` will retain 10% 
-of the jobs. With *concentration* as 2, 20% of the total input 
-jobs will be retained.
+`file:///tmp/job-trace.json` to finish within 1hr 
+with concentration of 2.
+If the 10h job-trace is folded to 1h, it retains 10% of the jobs by default.
+With *concentration* as 2, 20% of the total input jobs will be retained.
 
 
 Appendix
@@ -377,21 +418,21 @@ $H3 Resources
 <a href="https://issues.apache.org/jira/browse/MAPREDUCE-751">MAPREDUCE-751</a>
 is the main JIRA that introduced *Rumen* to *MapReduce*. 
 Look at the MapReduce
-<a href="https://issues.apache.org/jira/browse/MAPREDUCE/component/12313617">
-rumen-component</a>for further details.
+<a href="https://issues.apache.org/jira/browse/MAPREDUCE/component/12313617">rumen-component</a>
+for further details.
 
 
 $H3 Dependencies
 
-*Rumen* expects certain library *JARs* to be present in 
-the *CLASSPATH*. The required libraries are
-
-* `Hadoop MapReduce Tools` (`hadoop-mapred-tools-{hadoop-version}.jar`)
-* `Hadoop Common` (`hadoop-common-{hadoop-version}.jar`)
-* `Apache Commons Logging` (`commons-logging-1.1.1.jar`)
-* `Apache Commons CLI` (`commons-cli-1.2.jar`)
-* `Jackson Mapper` (`jackson-mapper-asl-1.4.2.jar`)
-* `Jackson Core` (`jackson-core-asl-1.4.2.jar`)
-
-> One simple way to run Rumen is to use '$HADOOP_HOME/bin/hadoop jar' 
-> option  to run it.
+*Rumen* expects certain library *JARs* to be present in  the *CLASSPATH*.
+One simple way to run Rumen is to use
+`hadoop jar` command to run it as example below.
+
+```
+$HADOOP_HOME/bin/hadoop jar \
+  $HADOOP_HOME/share/hadoop/tools/lib/hadoop-rumen-2.5.1.jar \
+  org.apache.hadoop.tools.rumen.TraceBuilder \
+  file:///tmp/job-trace.json \
+  file:///tmp/job-topology.json \
+  hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
+```
diff --git a/hadoop-tools/hadoop-rumen/src/site/resources/css/site.css b/hadoop-tools/hadoop-rumen/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-tools/hadoop-rumen/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-tools/hadoop-sls/README b/hadoop-tools/hadoop-sls/README
index 86b554ebb91ad..3d7912e084b5f 100644
--- a/hadoop-tools/hadoop-sls/README
+++ b/hadoop-tools/hadoop-sls/README
@@ -8,5 +8,25 @@ SLS runs a regular RM without RPC endpoints and uses a NodeManager and
 Application Manager simulators to send and receive events simulating cluster
 and application load behavior.
 
-The size of the cluster and the application load is scripted in a configuration
-file.
+==== Quick Start ====
+
+Let $HADOOP_ROOT represent the Hadoop install directory. If you build Hadoop
+yourself, $HADOOP_ROOT is hadoop-dist/target/hadoop-$VERSION. The simulator 
+is located at $HADOOP_ROOT/share/hadoop/tools/sls. The folder sls contains 
+four directories: bin (running scripts), html (web portal to view progress),
+sample-conf (some example configurations), and sample-data (an example rumen
+trace).
+
+STEP 1: Copy all configuration files (under sample-conf) to $HADOOP_ROOT/etc/hadoop.
+STEP 2: Go to the $HADOOP_ROOT/share/hadoop/tools/sls directory, and run the simulator 
+using the sample rumen trace (under sample-data).
+
+bin/slsrun.sh —-input-rumen=sample-data/2jobs2min-rumen-jh.json —-output-dir=sample-output
+
+The simulator will start to run, and you can track the running progress 
+using its web portal (http://$HOST:10001/simulate, where $HOST is the place 
+where you run the simulator.). All collected scheduler metrics are stored 
+under the output-dir during running. This trace takes about 3 mins to finish.
+
+For more detailed setup, you can check out the document 
+(http://issues.apache.org/jira/secure/attachment/12604817/YARN-1021.pdf) 
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index e82390db0d188..0539a0885e091 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -14,7 +14,7 @@
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
index 080a71537582b..c8e38ac633c2d 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
@@ -13,94 +13,94 @@
 #  limitations under the License. See accompanying LICENSE file.
 #
 
-###############################################################################
-printUsage() {
+function hadoop_usage()
+{
   echo "Usage: rumen2sls.sh <OPTIONS>"
   echo "                 --rumen-file=<RUMEN_FILE>"
   echo "                 --output-dir=<SLS_OUTPUT_DIR>"
   echo "                 [--output-prefix=<PREFIX>] (default is sls)"
   echo
 }
-###############################################################################
-parseArgs() {
-  for i in $*
-  do
+
+function parse_args()
+{
+  for i in "$@"; do
     case $i in
-    --rumen-file=*)
-      rumenfile=${i#*=}
+      --rumen-file=*)
+        rumenfile=${i#*=}
       ;;
-    --output-dir=*)
-      outputdir=${i#*=}
+      --output-dir=*)
+        outputdir=${i#*=}
       ;;
-    --output-prefix=*)
-      outputprefix=${i#*=}
+      --output-prefix=*)
+        outputprefix=${i#*=}
       ;;
-    *)
-      echo "Invalid option"
-      echo
-      printUsage
-      exit 1
+      *)
+        hadoop_error "ERROR: Invalid option ${i}"
+        hadoop_exit_with_usage 1
       ;;
     esac
   done
-  if [[ "${rumenfile}" == "" || "${outputdir}" == "" ]] ; then
-    echo "Both --rumen-file ${rumenfile} and --output-dir \
-          ${outputfdir} must be specified"
-    echo
-    printUsage
-    exit 1
-  fi
-}
-###############################################################################
-calculateBasedir() {
-  # resolve links - $0 may be a softlink
-  PRG="${1}"
 
-  while [ -h "${PRG}" ]; do
-    ls=`ls -ld "${PRG}"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-      PRG="$link"
-    else
-      PRG=`dirname "${PRG}"`/"$link"
-    fi
-  done
+  if [[ -z "${rumenfile}" ]] ; then
+    hadoop_error "ERROR: --rumen-file must be specified."
+    hadoop_exit_with_usage 1
+  fi
 
-  BASEDIR=`dirname ${PRG}`
-  BASEDIR=`cd ${BASEDIR}/..;pwd`
+  if [[ -z "${outputdir}" ]] ; then
+    hadoop_error "ERROR: --output-dir must be specified."
+    hadoop_exit_with_usage 1
+  fi
 }
-###############################################################################
-calculateClasspath() {
-  HADOOP_BASE=`which hadoop`
-  HADOOP_BASE=`dirname $HADOOP_BASE`
-  DEFAULT_LIBEXEC_DIR=${HADOOP_BASE}/../libexec
-  HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-  . $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-  export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${TOOL_PATH}"
+
+function calculate_classpath()
+{
+  hadoop_debug "Injecting TOOL_PATH into CLASSPATH"
+  hadoop_add_classpath "${TOOL_PATH}"
 }
-###############################################################################
-runSLSGenerator() {
-  if [[ "${outputprefix}" == "" ]] ; then
+
+function run_sls_generator()
+{
+  if [[ -z "${outputprefix}" ]] ; then
     outputprefix="sls"
   fi
 
-  slsJobs=${outputdir}/${outputprefix}-jobs.json
-  slsNodes=${outputdir}/${outputprefix}-nodes.json
+  hadoop_add_param args -input "-input ${rumenfile}"
+  hadoop_add_param args -outputJobs "-outputJobs ${outputdir}/${outputprefix}-jobs.json"
+  hadoop_add_param args -outputNodes "-outputNodes ${outputdir}/${outputprefix}-nodes.json"
 
-  args="-input ${rumenfile} -outputJobs ${slsJobs}";
-  args="${args} -outputNodes ${slsNodes}";
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 
-  hadoop org.apache.hadoop.yarn.sls.RumenToSLSConverter ${args}
+  hadoop_finalize
+  # shellcheck disable=SC2086
+  hadoop_java_exec rumen2sls org.apache.hadoop.yarn.sls.RumenToSLSConverter ${args}
 }
-###############################################################################
 
-calculateBasedir $0
-calculateClasspath
-parseArgs "$@"
-runSLSGenerator
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
+  exit 1
+fi
+
+if [ $# = 0 ]; then
+  hadoop_exit_with_usage 1
+fi
 
-echo
-echo "SLS simulation files available at: ${outputdir}"
-echo
+parse_args "${@}"
+calculate_classpath
+run_sls_generator
 
-exit 0
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 1e402df2961be..463ae235759af 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -13,100 +13,120 @@
 #  limitations under the License. See accompanying LICENSE file.
 #
 
-###############################################################################
-printUsage() {
-  echo "Usage: slsrun.sh <OPTIONS>"
-  echo "                 --input-rumen|--input-sls=<FILE1,FILE2,...>"
+function hadoop_usage()
+{
+  echo "Usage: slsrun.sh <OPTIONS> "
+  echo "                 --input-rumen=<FILE1,FILE2,...>  | --input-sls=<FILE1,FILE2,...>"
   echo "                 --output-dir=<SLS_SIMULATION_OUTPUT_DIRECTORY>"
   echo "                 [--nodes=<SLS_NODES_FILE>]"
   echo "                 [--track-jobs=<JOBID1,JOBID2,...>]"
   echo "                 [--print-simulation]"
-  echo                  
 }
-###############################################################################
-parseArgs() {
-  for i in $*
-  do
+
+function parse_args()
+{
+  for i in "$@"; do
     case $i in
-    --input-rumen=*)
-      inputrumen=${i#*=}
+      --input-rumen=*)
+        inputrumen=${i#*=}
       ;;
-    --input-sls=*)
-      inputsls=${i#*=}
+      --input-sls=*)
+        inputsls=${i#*=}
       ;;
-    --output-dir=*)
-      outputdir=${i#*=}
+      --output-dir=*)
+        outputdir=${i#*=}
       ;;
-    --nodes=*)
-      nodes=${i#*=}
+      --nodes=*)
+        nodes=${i#*=}
       ;;
-    --track-jobs=*)
-      trackjobs=${i#*=}
+      --track-jobs=*)
+        trackjobs=${i#*=}
       ;;
-    --print-simulation)
-      printsimulation="true"
+      --print-simulation)
+        printsimulation="true"
       ;;
-    *)
-      echo "Invalid option"
-      echo
-      printUsage
-      exit 1
+      *)
+        hadoop_error "ERROR: Invalid option ${i}"
+        hadoop_exit_with_usage 1
       ;;
     esac
   done
 
-  if [[ "${inputrumen}" == "" && "${inputsls}" == "" ]] ; then
-    echo "Either --input-rumen or --input-sls must be specified"
-    echo
-    printUsage
-    exit 1
+  if [[ -z "${inputrumen}" && -z "${inputsls}" ]] ; then
+    hadoop_error "ERROR: Either --input-rumen or --input-sls must be specified."
+    hadoop_exit_with_usage 1
+  fi
+
+  if [[ -n "${inputrumen}" && -n "${inputsls}" ]] ; then
+    hadoop_error "ERROR: Only specify one of --input-rumen or --input-sls."
+    hadoop_exit_with_usage 1
   fi
 
-  if [[ "${outputdir}" == "" ]] ; then
-    echo "The output directory --output-dir must be specified"
-    echo
-    printUsage
-    exit 1
+  if [[ -z "${outputdir}" ]] ; then
+    hadoop_error "ERROR: The output directory --output-dir must be specified."
+    hadoop_exit_with_usage 1
   fi
 }
 
-###############################################################################
-calculateClasspath() {
-  HADOOP_BASE=`which hadoop`
-  HADOOP_BASE=`dirname $HADOOP_BASE`
-  DEFAULT_LIBEXEC_DIR=${HADOOP_BASE}/../libexec
-  HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-  . $HADOOP_LIBEXEC_DIR/hadoop-config.sh
-  export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${TOOL_PATH}:html"
+function calculate_classpath() {
+  hadoop_debug "Injecting TOOL_PATH into CLASSPATH"
+  hadoop_add_classpath "${TOOL_PATH}"
+  hadoop_debug "Injecting ${HADOOP_PREFIX}/share/hadoop/tools/sls/html into CLASSPATH"
+  hadoop_add_classpath "${HADOOP_PREFIX}/share/hadoop/tools/sls/html"
 }
-###############################################################################
-runSimulation() {
+
+function run_simulation() {
   if [[ "${inputsls}" == "" ]] ; then
-    args="-inputrumen ${inputrumen}"
+    hadoop_add_param args -inputrumen "-inputrumen ${inputrumen}"
   else
-    args="-inputsls ${inputsls}"
+    hadoop_add_param args -inputsls "-inputsls ${inputsls}"
   fi
 
-  args="${args} -output ${outputdir}"
+  hadoop_add_param args -output "-output ${outputdir}"
 
-  if [[ "${nodes}" != "" ]] ; then
-    args="${args} -nodes ${nodes}"
+  if [[ -n "${nodes}" ]] ; then
+    hadoop_add_param args -nodes "-nodes ${nodes}"
   fi
-  
-  if [[ "${trackjobs}" != "" ]] ; then
-    args="${args} -trackjobs ${trackjobs}"
+
+  if [[ -n "${trackjobs}" ]] ; then
+    hadoop_add_param args -trackjobs "-trackjobs ${trackjobs}"
   fi
-  
+
   if [[ "${printsimulation}" == "true" ]] ; then
-    args="${args} -printsimulation"
+    hadoop_add_param args -printsimulation "-printsimulation"
   fi
 
-  hadoop org.apache.hadoop.yarn.sls.SLSRunner ${args}
+  hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+  HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+
+  hadoop_finalize
+  # shellcheck disable=SC2086
+  hadoop_java_exec sls org.apache.hadoop.yarn.sls.SLSRunner ${args}
 }
-###############################################################################
 
-calculateClasspath
-parseArgs "$@"
-runSimulation
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+  DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+  this="${BASH_SOURCE-$0}"
+  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+  DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec"
+fi
+
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+else
+  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
+  exit 1
+fi
+
+if [[ $# = 0 ]]; then
+  hadoop_exit_with_usage 1
+fi
 
-exit 0
+parse_args "${@}"
+calculate_classpath
+run_simulation
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index 59ae8d755a51c..70ff8af515353 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -4657,7 +4657,6 @@
     "mapreduce.tasktracker.taskcontroller" : "org.apache.hadoop.mapred.DefaultTaskController",
     "yarn.scheduler.fair.preemption" : "true",
     "mapreduce.reduce.shuffle.parallelcopies" : "5",
-    "dfs.support.append" : "true",
     "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
     "mapreduce.jobtracker.heartbeats.in.second" : "100",
     "mapreduce.job.maxtaskfailures.per.tracker" : "3",
@@ -4674,7 +4673,6 @@
     "dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
     "ha.zookeeper.parent-znode" : "/hadoop-ha",
     "io.seqfile.lazydecompress" : "true",
-    "dfs.https.enable" : "false",
     "mapreduce.reduce.merge.inmem.threshold" : "1000",
     "mapreduce.input.fileinputformat.split.minsize" : "0",
     "dfs.replication" : "3",
@@ -4783,7 +4781,6 @@
     "io.map.index.skip" : "0",
     "net.topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
     "fs.s3.maxRetries" : "4",
-    "dfs.namenode.logging.level" : "info",
     "ha.failover-controller.new-active.rpc-timeout.ms" : "60000",
     "s3native.client-write-packet-size" : "65536",
     "yarn.resourcemanager.amliveliness-monitor.interval-ms" : "1000",
@@ -9770,7 +9767,6 @@
     "mapreduce.tasktracker.taskcontroller" : "org.apache.hadoop.mapred.DefaultTaskController",
     "yarn.scheduler.fair.preemption" : "true",
     "mapreduce.reduce.shuffle.parallelcopies" : "5",
-    "dfs.support.append" : "true",
     "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
     "mapreduce.jobtracker.heartbeats.in.second" : "100",
     "mapreduce.job.maxtaskfailures.per.tracker" : "3",
@@ -9787,7 +9783,6 @@
     "dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
     "ha.zookeeper.parent-znode" : "/hadoop-ha",
     "io.seqfile.lazydecompress" : "true",
-    "dfs.https.enable" : "false",
     "mapreduce.reduce.merge.inmem.threshold" : "1000",
     "mapreduce.input.fileinputformat.split.minsize" : "0",
     "dfs.replication" : "3",
@@ -9896,7 +9891,6 @@
     "io.map.index.skip" : "0",
     "net.topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
     "fs.s3.maxRetries" : "4",
-    "dfs.namenode.logging.level" : "info",
     "ha.failover-controller.new-active.rpc-timeout.ms" : "60000",
     "s3native.client-write-packet-size" : "65536",
     "yarn.resourcemanager.amliveliness-monitor.interval-ms" : "1000",
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index fdddcf4097c35..ee6eb7b55551c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode
         .UpdatedContainerInfo;
@@ -162,7 +163,7 @@ public String getNodeManagerVersion() {
 
     @Override
     public Set<String> getNodeLabels() {
-      return null;
+      return RMNodeLabelsManager.EMPTY_STRING_SET;
     }
   }
 
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 3b185ae928c16..b64be1b61a20e 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode
         .UpdatedContainerInfo;
@@ -150,6 +151,6 @@ public String getNodeManagerVersion() {
 
   @Override
   public Set<String> getNodeLabels() {
-    return null;
+    return RMNodeLabelsManager.EMPTY_STRING_SET;
   }
 }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index a65f7760d13f7..0a8029104c5dc 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -84,6 +84,7 @@
 import org.apache.hadoop.yarn.sls.SLSRunner;
 import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
 import org.apache.hadoop.yarn.sls.web.SLSWebApp;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.log4j.Logger;
 
@@ -866,6 +867,11 @@ public Resource getMaximumResourceCapability() {
     return scheduler.getMaximumResourceCapability();
   }
 
+  @Override
+  public ResourceCalculator getResourceCalculator() {
+    return scheduler.getResourceCalculator();
+  }
+
   @Override
   public int getNumClusterNodes() {
     return scheduler.getNumClusterNodes();
diff --git a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm b/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
index 399a1f5e8f880..a8b408c7e8255 100644
--- a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
+++ b/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
@@ -225,7 +225,8 @@ Yarn Scheduler Load Simulator (SLS)
   input traces. The script to start the simulator is <<<slsrun.sh>>>.
 
 +----+
-$ $HADOOP_ROOT/share/hadoop/tools/sls/bin/slsrun.sh
+$ cd $HADOOP_ROOT/share/hadoop/tools/sls
+$ bin/slsrun.sh
     --input-rumen|--input-sls=<TRACE_FILE1,TRACE_FILE2,...>
     --output-dir=<SLS_SIMULATION_OUTPUT_DIRECTORY> [--nodes=<SLS_NODES_FILE>]
     [--track-jobs=<JOBID1,JOBID2,...>] [--print-simulation]
@@ -258,7 +259,7 @@ $ $HADOOP_ROOT/share/hadoop/tools/sls/bin/slsrun.sh
   convert rumen traces to sls traces.
 
 +----+
-$ $HADOOP_ROOT/share/hadoop/tools/sls/bin/rumen2sls.sh
+$ bin/rumen2sls.sh
     --rumen-file=<RUMEN_FILE>
     --output-dir=<SLS_OUTPUT_DIRECTORY>
     [--output-prefix=<SLS_FILE_PREFIX>]
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/Environment.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/Environment.java
index bd76c31998256..98d8aa0306438 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/Environment.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/Environment.java
@@ -20,6 +20,7 @@
 
 import java.io.*;
 import java.net.InetAddress;
+import java.nio.charset.Charset;
 import java.util.*;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -62,7 +63,8 @@ public Environment() throws IOException {
     // Read the environment variables
 
     Process pid = Runtime.getRuntime().exec(command);
-    BufferedReader in = new BufferedReader(new InputStreamReader(pid.getInputStream()));
+    BufferedReader in = new BufferedReader(
+        new InputStreamReader(pid.getInputStream(), Charset.forName("UTF-8")));
     try {
       while (true) {
         String line = in.readLine();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/HadoopStreaming.apt.vm b/hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm
similarity index 100%
rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/HadoopStreaming.apt.vm
rename to hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm
diff --git a/hadoop-tools/hadoop-streaming/src/site/resources/css/site.css b/hadoop-tools/hadoop-streaming/src/site/resources/css/site.css
new file mode 100644
index 0000000000000..f830baafa8cc8
--- /dev/null
+++ b/hadoop-tools/hadoop-streaming/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+  height: 93px;
+  background: none;
+}
+
+#bannerLeft img {
+  margin-left: 30px;
+  margin-top: 10px;
+}
+
+#bannerRight img {
+  margin: 17px;
+}
+
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java
index 50f38bd20c24b..f2d9495efa892 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/mapreduce/TestStreamXmlRecordReader.java
@@ -109,7 +109,7 @@ public void createInput() throws IOException {
   @Test
   public void testStreamXmlRecordReader() throws Exception {
 
-    Job job = new Job();
+    Job job = Job.getInstance();
     Configuration conf = job.getConfiguration();
     job.setJarByClass(TestStreamXmlRecordReader.class);
     job.setMapperClass(Mapper.class);
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 43b19ec515330..7951eef27ad91 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -13,6 +13,8 @@ Trunk - Unreleased
     YARN-2472. yarn-daemons.sh should jsut call yarn directly (Masatake Iwasaki
     via aw)
 
+    YARN-2437. start-yarn.sh/stop-yarn should give info (Varun Saxena via aw)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -23,13 +25,15 @@ Trunk - Unreleased
     YARN-1471. The SLS simulator is not running the preemption policy 
     for CapacityScheduler (Carlo Curino via cdouglas)
 
-    YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk.
-    (Zhijie Shen via xgong)
-
     YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
 
     YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
 
+    YARN-3002. YARN documentation needs updating post-shell rewrite (aw)
+
+    YARN-2428. LCE default banned user list should have yarn (Varun
+    Saxena via aw)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -60,11 +64,28 @@ Release 2.7.0 - UNRELEASED
     YARN-2765. Added leveldb-based implementation for RMStateStore. (Jason Lowe
     via jianhe)
 
-    YARN-2880. Added a test to make sure node labels will be recovered
-    if RM restart is enabled. (Rohith Sharmaks via jianhe)
+    YARN-2203. [YARN-1492] Web UI for cache manager. (Chris Trezzo via kasha)
+
+    YARN-2738. [YARN-2574] Add FairReservationSystem for FairScheduler. 
+    (Anubhav Dhoot via kasha)
+
+    YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler. 
+    (Anubhav Dhoot via kasha)
+
+    YARN-2427. Added the API of moving apps between queues in RM web services.
+    (Varun Vasudev via zjshen)
+
+    YARN-2217. [YARN-1492] Shared cache client side changes. 
+    (Chris Trezzo via kasha)
 
   IMPROVEMENTS
 
+    YARN-3005. [JDK7] Use switch statement for String instead of if-else
+    statement in RegistrySecurity.java (Kengo Seki via aajisaka)
+
+    YARN-2950. Change message to mandate, not suggest JS requirement on UI.
+    (Dustin Cote via harsh)
+
     YARN-2891. Failed Container Executor does not provide a clear error
     message. (Dustin Cote via harsh)
 
@@ -128,10 +149,119 @@ Release 2.7.0 - UNRELEASED
 
     YARN-2056. Disable preemption at Queue level (Eric Payne via jlowe)
 
+    YARN-2762. Fixed RMAdminCLI to trim and check node-label related arguments
+    before sending to RM. (Rohith Sharmaks via jianhe)
+
+    YARN-2972. DelegationTokenRenewer thread pool never expands. (Jason Lowe
+    via junping_du)
+
+    YARN-2949. Add documentation for CGroups (Varun Vasudev via junping_du)
+
+    YARN-2970. NodeLabel operations in RMAdmin CLI get missing in help command.
+    (Varun Saxena via junping_du)
+
+    YARN-2837. Support TimeLine server to recover delegation token when
+    restarting. (Zhijie Shen via jianhe)
+
+    YARN-2993. Several fixes (missing acl check, error log msg ...) and some 
+    refinement in AdminService. (Yi Liu via junping_du)
+
+    YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
+
+    YARN-2998. Abstract out scheduler independent PlanFollower components. 
+    (Anubhav Dhoot via kasha)
+
+    YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the 
+    scheduler page. (Ashwin Shankar and Wei Yan via kasha)
+
+    YARN-2880. Added a test to make sure node labels will be recovered
+    if RM restart is enabled. (Rohith Sharmaks via jianhe)
+
+    YARN-2996. Improved synchronization and I/O operations of FS- and Mem-
+    RMStateStore. (Yi Liu via zjshen)
+
+    YARN-2956. Added missing links in YARN documentation. (Masatake Iwasaki via
+    jianhe)
+
+    YARN-2957. Create unit test to automatically compare YarnConfiguration
+    and yarn-default.xml. (rchiang via rkanter)
+
+    YARN-2643. Don't create a new DominantResourceCalculator on every
+    FairScheduler.allocate call. (kasha via rkanter)
+
+    YARN-3019. Make work-preserving-recovery the default mechanism for RM 
+    recovery. (Jian He via junping_du)
+
+    YARN-2807. Option "--forceactive" not works as described in usage of
+    "yarn rmadmin -transitionToActive". (Masatake Iwasaki via xgong)
+
+    YARN-2984. Metrics for container's actual memory usage. (kasha)
+
+    YARN-2800. Remove MemoryNodeLabelsStore and add a way to enable/disable
+    node labels feature. (Wangda Tan via ozawa)
+
+    YARN-3024. LocalizerRunner should give DIE action when all resources are
+    localized. (Chengbing Liu via xgong)
+
+    YARN-3092. Created a common ResourceUsage class to track labeled resource
+    usages in Capacity Scheduler. (Wangda Tan via jianhe)
+
+    YARN-3086. Make NodeManager memory configurable in MiniYARNCluster.
+    (Robert Metzger via ozawa)
+
+    YARN-2897. CrossOriginFilter needs more log statements (Mit Desai via
+    jeagles)
+
+    YARN-3028. Better syntax for replaceLabelsOnNode in RMAdmin CLI
+    (Rohith Sharmaks via wangda)
+
+    YARN-2932. Add entry for "preemptable" status (enabled/disabled) to 
+    scheduler web UI and queue initialize/refresh logging. 
+    (Eric Payne via wangda)
+
+    YARN-3108. ApplicationHistoryServer doesn't process -D arguments (Chang Li
+    via jeagles)
+
+    YARN-2808. Made YARN CLI list attempt’s finished containers of a running
+    application. (Naganarasimha G R via zjshen)
+
+    YARN-3085. Application summary should include the application type (Rohith
+    via jlowe)
+
+    YARN-3022. Expose Container resource information from NodeManager for
+    monitoring (adhoot via ranter)
+
+    YARN-3098. Created common QueueCapacities class in Capacity Scheduler to
+    track capacities-by-labels of queues. (Wangda Tan via jianhe)
+
+    YARN-3075. NodeLabelsManager implementation to retrieve label to node 
+    mapping (Varun Saxena via wangda)
+
+    YARN-1393. SLS: Add how-to-use instructions. (Wei Yan via kasha)
+
+    YARN-1723. AMRMClientAsync missing blacklist addition and removal
+    functionality. (Bartosz Ługowski via sseth)
+
+    YARN-3123. Made YARN CLI show a single completed container even if the app
+    is running. (Naganarasimha G R via zjshen)
+
+    YARN-1582. Capacity Scheduler: add a maximum-allocation-mb setting per
+    queue (Thomas Graves via jlowe)
+
+    YARN-1904. Ensure exceptions thrown in ClientRMService &
+    ApplicationHistoryClientService are uniform when application-attempt is
+    not found. (zjshen via acmurthy)
+
+    YARN-3144. Configuration for making delegation token failures to timeline
+    server not-fatal (Jonathan Eagles via jlowe)
+
   OPTIMIZATIONS
 
   BUG FIXES
 
+    YARN-3071. Remove invalid char from sample conf in doc of FairScheduler.
+    (Masatake Iwasaki via aajisaka)
+
     YARN-2254. TestRMWebServicesAppsModification should run against both 
     CS and FS. (Zhihai Xu via kasha)
 
@@ -182,9 +312,6 @@ Release 2.7.0 - UNRELEASED
     YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated
     log file is corrupted (Varun Saxena via jlowe)
 
-    YARN-2890. MiniYARNCluster should start the timeline server based on the
-    configuration. (Mit Desai via zjshen)
-
     YARN-2894. Fixed a bug regarding application view acl when RM fails over.
     (Rohith Sharmaks via jianhe)
 
@@ -200,6 +327,192 @@ Release 2.7.0 - UNRELEASED
     YARN-2927. [YARN-1492] InMemorySCMStore properties are inconsistent. 
     (Ray Chiang via kasha)
 
+    YARN-2931. PublicLocalizer may fail until directory is initialized by
+    LocalizeRunner. (Anubhav Dhoot via kasha)
+
+    YARN-2910. FSLeafQueue can throw ConcurrentModificationException. 
+    (Wilfred Spiegelenburg via kasha)
+
+    YARN-2930. Fixed TestRMRestart#testRMRestartRecoveringNodeLabelManager
+    intermittent failure. (Wangda Tan via jianhe)
+
+    YARN-2924. Fixed RMAdminCLI to not convert node labels to lower case.
+    (Wangda Tan via jianhe)
+
+    YARN-2917. Fixed potential deadlock when system.exit is called in AsyncDispatcher
+    (Rohith Sharmaks via jianhe)
+
+    YARN-2243. Order of arguments for Preconditions.checkNotNull() is wrong in 
+    SchedulerApplicationAttempt ctor. (devaraj)
+
+    YARN-2912 Jersey Tests failing with port in use. (varun saxena via stevel)
+
+    YARN-2356. yarn status command for non-existent application/application 
+    attempt/container is too verbose. (Sunil G via devaraj)
+
+    YARN-2914. [YARN-1492] Potential race condition in Singleton implementation of 
+    SharedCacheUploaderMetrics, CleanerMetrics, ClientSCMMetrics. (Varun Saxena via kasha)
+
+    YARN-2964. FSLeafQueue#assignContainer - document the reason for using both write and
+    read locks. (Tsuyoshi Ozawa via kasha)
+
+    YARN-2944. InMemorySCMStore can not be instantiated with ReflectionUtils#newInstance.
+    (Chris Trezzo via kasha)
+
+    YARN-2964. RM prematurely cancels tokens for jobs that submit jobs (oozie)
+    (Jian He via jlowe)
+
+    YARN-2675. containersKilled metrics is not updated when the container is killed 
+    during localization. (Zhihai Xu via kasha)
+
+    YARN-2952. Fixed incorrect version check in StateStore. (Rohith Sharmaks
+    via jianhe)
+
+    YARN-2975. FSLeafQueue app lists are accessed without required locks. (kasha)
+
+    YARN-2977. Fixed intermittent TestNMClient failure.
+    (Junping Du via ozawa)
+
+    YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via junping_du)
+
+    YARN-2920. Changed CapacityScheduler to kill containers on nodes where
+    node labels are changed. (Wangda Tan via jianhe)
+
+    YARN-2340. Fixed NPE when queue is stopped during RM restart.
+    (Rohith Sharmaks via jianhe)
+
+    YARN-2940. Fix new findbugs warnings in rest of the hadoop-yarn components. (Li Lu 
+    via junping_du)
+
+    YARN-2937. Fixed new findbugs warnings in hadoop-yarn-nodemanager. (Varun Saxena
+    via zjshen)
+
+    YARN-2946. Fixed potential deadlock in RMStateStore. (Rohith Sharmaks via
+    jianhe)
+
+    YARN-2988. Graph#save() may leak file descriptors. (Ted Yu via ozawa)
+
+    YARN-2992. ZKRMStateStore crashes due to session expiry. (Karthik Kambatla
+    via jianhe)
+
+    YARN-2938. Fixed new findbugs warnings in hadoop-yarn-resourcemanager and
+    hadoop-yarn-applicationhistoryservice. (Varun Saxena via zjshen)
+
+    YARN-2987. Fixed ClientRMService#getQueueInfo to check against queue and
+    app ACLs. (Varun Saxena via jianhe)
+
+    YARN-2991. Fixed DrainDispatcher to reuse the draining code path in
+    AsyncDispatcher. (Rohith Sharmaks via zjshen)
+
+    YARN-2922. ConcurrentModificationException in CapacityScheduler's LeafQueue.
+    (Rohith Sharmaks via ozawa)
+
+    YARN-2958. Made RMStateStore not update the last sequence number when updating the
+    delegation token. (Varun Saxena via zjshen)
+
+    YARN-2978. Fixed potential NPE while getting queue info. (Varun Saxena via
+    jianhe)
+
+    YARN-2230. Fixed few configs description in yarn-default.xml. (Vijay Bhat
+    via jianhe)
+
+    YARN-3010. Fixed findbugs warning in AbstractYarnScheduler. (Yi Liu via
+    jianhe)
+
+    YARN-2936. Changed YARNDelegationTokenIdentifier to set proto fields on
+    getProto method. (Varun Saxena via jianhe)
+
+    YARN-2997. Fixed NodeStatusUpdater to not send alreay-sent completed
+    container statuses on heartbeat. (Chengbing Liu via jianhe)
+
+    YARN-3014. Replaces labels on a host should update all NM's labels on that
+    host. (Wangda Tan via jianhe)
+
+    YARN-3027. Scheduler should use totalAvailable resource from node instead of
+    availableResource for maxAllocation. (adhoot via rkanter)
+
+    YARN-2637. Fixed max-am-resource-percent calculation in CapacityScheduler
+    when activating applications. (Craig Welch via jianhe)
+
+    YARN-2861. Fixed Timeline DT secret manager to not reuse RM's configs.
+    (Zhijie Shen via jianhe)
+
+    YARN-3064. TestRMRestart/TestContainerResourceUsage/TestNodeManagerResync 
+    failure with allocation timeout. (Jian He via junping_du)
+
+    YARN-2815. Excluded transitive dependency of JLine in hadoop-yarn-server-common.
+    (Ferdinand Xu via zjshen)
+
+    YARN-3070. TestRMAdminCLI#testHelp fails for transitionToActive command. 
+    (Contributed by Junping Du)
+
+    YARN-3015. yarn classpath command should support same options as hadoop
+    classpath. (Contributed by Varun Saxena)
+
+    YARN-2933. Capacity Scheduler preemption policy should only consider capacity 
+    without labels temporarily. (Mayank Bansal via wangda)
+
+    YARN-2731. Fixed RegisterApplicationMasterResponsePBImpl to properly invoke 
+    maybeInitBuilder. (Carlo Curino via wangda)
+
+    YARN-3078. LogCLIHelpers lacks of a blank space before string 'does not exist'.
+    (Sam Liu via ozawa)
+
+    YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse
+    processing. (Anubhav Dhoot via ozawa)
+
+    YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native
+    executor returns an error (Eric Payne via jlowe)
+
+    YARN-3011. Possible IllegalArgumentException in ResourceLocalizationService
+    might lead NM to crash. (Varun Saxena via jianhe)
+
+    YARN-3103. AMRMClientImpl does not update AMRM token properly. (Jason Lowe
+    via jianhe)
+
+    YARN-3079. Scheduler should also update maximumAllocation when updateNodeResource.
+    (Zhihai Xu via wangda)
+
+    YARN-3029. FSDownload.unpack() uses local locale for FS case conversion, may not
+    work everywhere. (Varun Saxena via ozawa)
+
+    YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage
+    to track used-resources-by-label.(Wangda Tan via jianhe)
+
+    YARN-3077. Fixed RM to create zk root path recursively. (Chun Chen via jianhe)
+
+    YARN-3113. Release audit warning for Sorting icons.psd. (stevel via kihwal)
+
+    YARN-3056. Add verification for containerLaunchDuration
+    in TestNodeManagerMetrics. (zhihai xu via xgong)
+
+    YARN-2543. Made resource usage be published to the timeline server too.
+    (Naganarasimha G R via zjshen)
+
+    YARN-3058. Fix error message of tokens' activation delay configuration.
+    (Yi Liu via ozawa)
+
+    YARN-3101. In Fair Scheduler, fix canceling of reservations for exceeding
+    max share (Anubhav Dhoot via Sandy Ryza)
+
+    YARN-3149. Fix typo in message for invalid application id.
+    (Bibin A Chundatt via xgong)
+
+    YARN-3145. Fixed ConcurrentModificationException on CapacityScheduler
+    ParentQueue#getQueueUserAclInfo. (Tsuyoshi OZAWA via jianhe)
+
+    YARN-1537. Fix race condition in
+    TestLocalResourcesTrackerImpl.testLocalResourceCache. (xgong via acmurthy)
+
+    YARN-2694. Ensure only single node label specified in ResourceRequest.
+    (Wangda Tan via jianhe)
+
+    YARN-3089. LinuxContainerExecutor does not handle file arguments to
+    deleteAsUser (Eric Payne via jlowe)
+
+    YARN-3143. RM Apps REST API can return NPE or entries missing id and other
+    fields (jlowe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES
@@ -440,9 +753,6 @@ Release 2.6.0 - 2014-11-18
     YARN-2406. Move RM recovery related proto to
     yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
 
-    YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the 
-    scheduler page. (Ashwin Shankar and Wei Yan via kasha)
-
     YARN-1506. Changed RMNode/SchedulerNode to update resource with event
     notification. (Junping Du via jianhe)
 
@@ -1450,6 +1760,9 @@ Release 2.5.0 - 2014-08-11
     YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler. 
     (Wei Yan via kasha)
 
+    YARN-2216. TestRMApplicationHistoryWriter sometimes fails in trunk.
+    (Zhijie Shen via xgong)
+
 Release 2.4.1 - 2014-06-23 
 
   INCOMPATIBLE CHANGES
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
index f44b199723254..3d0e772400b9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
@@ -42,8 +42,10 @@ else
 fi
 
 # start resourceManager
-"${HADOOP_YARN_HOME}/bin/yarn" --config "${YARN_CONF_DIR}" --daemon start resourcemanager
+echo "Starting resourcemanager" 
+"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon start resourcemanager
 # start nodeManager
-"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}"  start nodemanager
+echo "Starting nodemanagers" 
+"${bin}/yarn-daemons.sh" --config "${HADOOP_CONF_DIR}"  start nodemanager
 # start proxyserver
-#"${HADOOP_YARN_HOME}/bin/yarn" --config "${YARN_CONF_DIR}" --daemon start proxyserver
+#"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon start proxyserver
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
index c807f52097a74..6feb4a757cf85 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
@@ -43,9 +43,11 @@ else
   exit 1
 fi
 
-# start resourceManager
-"${HADOOP_YARN_HOME}/bin/yarn" --config "${YARN_CONF_DIR}" --daemon stop resourcemanager
-# start nodeManager
-"${bin}/yarn-daemons.sh" --config "${YARN_CONF_DIR}"  stop nodemanager
-# start proxyserver
-#"${HADOOP_YARN_HOME}/bin/yarn" --config "${YARN_CONF_DIR}" --daemon stop proxyserver
+# stop resourceManager
+echo "Stopping resourcemanager"
+"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon stop resourcemanager
+# stop nodeManager
+echo "Stopping nodemanagers"
+"${bin}/yarn-daemons.sh" --config "${HADOOP_CONF_DIR}"  stop nodemanager
+# stop proxyserver
+#"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon stop proxyserver
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index dfa27e4166660..b3fe1f3c2e7d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -17,9 +17,10 @@
 
 function hadoop_usage
 {
-  echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)]"
-  echo "           [--loglevel loglevel] COMMAND"
-  echo "where COMMAND is one of:"
+  echo "Usage: yarn [--config confdir] [COMMAND | CLASSNAME]"
+  echo "  CLASSNAME                             run the class named CLASSNAME"
+  echo " or"
+  echo "  where COMMAND is one of:"
   echo "  application                           prints application(s) report/kill application"
   echo "  applicationattempt                    prints applicationattempt(s) report"
   echo "  classpath                             prints the class path needed to get the"
@@ -35,12 +36,11 @@ function hadoop_usage
   echo "  resourcemanager                       run the ResourceManager"
   echo "  resourcemanager -format-state-store   deletes the RMStateStore"
   echo "  rmadmin                               admin tools"
-  echo "  sharedcachemanager                    run the SharedCacheManager daemon"
   echo "  scmadmin                              SharedCacheManager admin tools"
+  echo "  sharedcachemanager                    run the SharedCacheManager daemon"
   echo "  timelineserver                        run the timeline server"
   echo "  version                               print the version"
-  echo " or"
-  echo "  CLASSNAME                             run the class named CLASSNAME"
+  echo ""
   echo "Most commands print help when invoked w/o parameters."
 }
 
@@ -76,24 +76,22 @@ shift
 case "${COMMAND}" in
   application|applicationattempt|container)
     CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
     set -- "${COMMAND}" "$@"
   ;;
   classpath)
-    hadoop_finalize
-    echo "${CLASSPATH}"
-    exit
+    hadoop_do_classpath_subcommand "$@"
   ;;
   daemonlog)
     CLASS=org.apache.hadoop.log.LogLevel
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   jar)
     CLASS=org.apache.hadoop.util.RunJar
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   historyserver)
     supportdaemonization="true"
@@ -104,73 +102,79 @@ case "${COMMAND}" in
   ;;
   logs)
     CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   node)
     CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   nodemanager)
     supportdaemonization="true"
     CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
-    hadoop_debug "Append YARN_NODEMANAGER_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_NODEMANAGER_OPTS}"
+    hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
+    # Backwards compatibility
     if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
-      JAVA_HEAP_MAX="-Xmx${YARN_NODEMANAGER_HEAPSIZE}m"
+      HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
     fi
   ;;
   proxyserver)
     supportdaemonization="true"
     CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
-    hadoop_debug "Append YARN_PROXYSERVER_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_PROXYSERVER_OPTS}"
+    hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
+    # Backwards compatibility
     if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
-      JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m"
+      HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
     fi
   ;;
   queue)
     CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   resourcemanager)
     supportdaemonization="true"
     CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
-    YARN_OPTS="${YARN_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
-    hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto YARN_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
+    hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
+    # Backwards compatibility
     if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
-      JAVA_HEAP_MAX="-Xmx${YARN_RESOURCEMANAGER_HEAPSIZE}m"
+      HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
     fi
   ;;
   rmadmin)
     CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
-  timelineserver)
-    supportdaemonization="true"
-    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
-    hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_TIMELINESERVER_OPTS}"
-    if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
-      JAVA_HEAP_MAX="-Xmx${YARN_TIMELINESERVER_HEAPSIZE}m"
-    fi
+  scmadmin)
+    CLASS='org.apache.hadoop.yarn.client.SCMAdmin'
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   sharedcachemanager)
     supportdaemonization="true"
     CLASS='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
-    YARN_OPTS="$YARN_OPTS $YARN_SHAREDCACHEMANAGER_OPTS"
+    hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
   ;;
-  scmadmin)
-    CLASS='org.apache.hadoop.yarn.client.SCMAdmin'
-    YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
+  timelineserver)
+    supportdaemonization="true"
+    CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+    hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
+    # Backwards compatibility
+    if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
+      HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
+    fi
   ;;
   version)
     CLASS=org.apache.hadoop.util.VersionInfo
-    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
-    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+    HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
   ;;
   *)
     CLASS="${COMMAND}"
@@ -180,11 +184,7 @@ case "${COMMAND}" in
   ;;
 esac
 
-# set HADOOP_OPTS to YARN_OPTS so that we can use
-# finalize, etc, without doing anything funky
-hadoop_debug "Resetting HADOOP_OPTS=YARN_OPTS"
-# shellcheck disable=SC2034
-HADOOP_OPTS="${YARN_OPTS}"
+hadoop_verify_user "${COMMAND}"
 
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
@@ -192,23 +192,9 @@ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 if [[  "${HADOOP_DAEMON_MODE}" != "default" ]]; then
   # shellcheck disable=SC2034
   HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
-  YARN_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
   HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
 fi
 
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
-
-# Add YARN custom options to comamnd line in case someone actaully 
-# used these.
-#
-# Note that we are replacing ' ' with '\ ' so that when we exec
-# stuff it works
-#
-hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${HADOOP_LOG_DIR/ /\ }"
-hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE/ /\ }"
-hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${HADOOP_YARN_HOME/ /\ }"
-hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-
 hadoop_finalize
 
 if [[ -n "${supportdaemonization}" ]]; then
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
index 7fd7eada88788..7df9fa1019e01 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
@@ -33,49 +33,35 @@ function hadoop_subproject_init
     export HADOOP_YARN_ENV_PROCESSED=true
   fi
   
-  if [[ -n "${YARN_CONF_DIR}" ]]; then
-    HADOOP_CONF_DIR="${YARN_CONF_DIR}"
-  fi
-  
-  YARN_CONF_DIR="${HADOOP_CONF_DIR}"
-  
-  # YARN_CONF_DIR needs precedence over HADOOP_CONF_DIR
-  # and the various jar dirs
-  hadoop_add_classpath "${YARN_CONF_DIR}" before
-  
-  HADOOP_LOG_DIR="${YARN_LOG_DIR:-$HADOOP_LOG_DIR}"
-  YARN_LOG_DIR="${HADOOP_LOG_DIR}"
-  
-  HADOOP_LOGFILE="${YARN_LOGFILE:-$HADOOP_LOGFILE}"
-  YARN_LOGFILE="${HADOOP_LOGFILE}"
+  hadoop_deprecate_envvar YARN_CONF_DIR HADOOP_CONF_DIR
+
+  hadoop_deprecate_envvar YARN_LOG_DIR HADOOP_LOG_DIR
+
+  hadoop_deprecate_envvar YARN_LOGFILE HADOOP_LOGFILE
   
-  HADOOP_NICENESS="${YARN_NICENESS:-$HADOOP_NICENESS}"
-  YARN_NICENESS="${HADOOP_NICENESS}"
+  hadoop_deprecate_envvar YARN_NICENESS HADOOP_NICENESS
   
-  HADOOP_STOP_TIMEOUT="${YARN_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT}"
-  YARN_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}"
+  hadoop_deprecate_envvar YARN_STOP_TIMEOUT HADOOP_STOP_TIMEOUT
   
-  HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}"
-  YARN_PID_DIR="${HADOOP_PID_DIR}"
+  hadoop_deprecate_envvar YARN_PID_DIR HADOOP_PID_DIR
   
-  HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
-  YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
+  hadoop_deprecate_envvar YARN_ROOT_LOGGER HADOOP_ROOT_LOGGER
+
+  hadoop_deprecate_envvar YARN_IDENT_STRING HADOOP_IDENT_STRING
+
+  hadoop_deprecate_envvar YARN_OPTS HADOOP_OPTS
+
+  hadoop_deprecate_envvar YARN_SLAVES HADOOP_SLAVES
   
   HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}"
   
-  HADOOP_IDENT_STRING="${YARN_IDENT_STRING:-$HADOOP_IDENT_STRING}"
-  YARN_IDENT_STRING="${HADOOP_IDENT_STRING}"
-  
-  YARN_OPTS="${YARN_OPTS:-$HADOOP_OPTS}"
-  
   # YARN-1429 added the completely superfluous YARN_USER_CLASSPATH
   # env var.  We're going to override HADOOP_USER_CLASSPATH to keep
   # consistency with the rest of the duplicate/useless env vars
-  HADOOP_USER_CLASSPATH="${YARN_USER_CLASSPATH:-$HADOOP_USER_CLASSPATH}"
-  YARN_USER_CLASSPATH="${HADOOP_USER_CLASSPATH}"
-  
-  HADOOP_USER_CLASSPATH_FIRST="${YARN_USER_CLASSPATH_FIRST:-$HADOOP_USER_CLASSPATH_FIRST}"
-  YARN_USER_CLASSPATH_FIRST="${HADOOP_USER_CLASSPATH_FIRST}"
+
+  hadoop_deprecate_envvar YARN_USER_CLASSPATH HADOOP_USER_CLASSPATH
+
+  hadoop_deprecate_envvar YARN_USER_CLASSPATH_FIRST HADOOP_USER_CLASSPATH_FIRST
 }
 
 if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index b3286e23cef64..3f68b16a07083 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -142,13 +142,16 @@ if "%1" == "--loglevel" (
   set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
 
   if %yarn-command% == classpath (
-    @echo %CLASSPATH%
-    goto :eof
+    if not defined yarn-command-arguments (
+      @rem No need to bother starting up a JVM for this simple case.
+      @echo %CLASSPATH%
+      exit /b
+    )
   )
 
   set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
      application applicationattempt container node logs daemonlog historyserver ^
-     timelineserver
+     timelineserver classpath
   for %%i in ( %yarncommands% ) do (
     if %yarn-command% == %%i set yarncommand=true
   )
@@ -169,7 +172,7 @@ if "%1" == "--loglevel" (
 goto :eof
 
 :classpath
-  @echo %CLASSPATH%
+  set CLASS=org.apache.hadoop.util.Classpath
   goto :eof
 
 :rmadmin
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 3d3a036d73778..73aad83ca3d37 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -25,43 +25,20 @@
 ## YARN_xyz > HADOOP_xyz > hard-coded defaults
 ##
 
-###
-# YARN-specific overrides for generic settings
-###
-
-# By default, YARN will use HADOOP_LOG_DIR for YARN logging.  Specify a custom
-# log directory for YARN things here:
-# Java properties: hadoop.log.dir, yarn.log.dir
-# export YARN_LOG_DIR="${HADOOP_LOG_DIR}"
-
-# By default, YARN will use the value of HADOOP_LOGFILE as the 'fallback' log
-# file # when log4j settings are not defined.  Specify a custom YARN log file
-# here:
-# Java properties: hadoop.log.file, yarn.log.file
-# export YARN_LOGFILE=${HADOOP_LOGFILE}
-
-#Override the log4j settings for all YARN apps By default, YARN will use
-# HADOOP_ROOT_LOGGER.
-# Java properties: hadoop.root.logger, yarn.root.logger
-# export YARN_ROOT_LOGGER=${HADOOP_ROOT_LOGGER}
-
 ###
 # Resource Manager specific parameters
 ###
 
-# Specify the max heapsize for the ResourceManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
-# HADOOP_OPTS, and/or YARN_RESOURCEMANAGER_OPTS.
-# If not specified, the default value will be picked from either HADOOP_HEAPSIZE
-# or the built-in default.
-#
-#export YARN_RESOURCEMANAGER_HEAPSIZE=1000
+# Specify the max heapsize for the ResourceManager.  If no units are
+# given, it will be assumed to be in MB.
+# This value will be overridden by an Xmx setting specified in either
+# HADOOP_OPTS and/or YARN_RESOURCEMANAGER_OPTS.
+# Default is the same as HADOOP_HEAPSIZE_MAX
+#export YARN_RESOURCEMANAGER_HEAPSIZE=
 
 # Specify the JVM options to be used when starting the ResourceManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
 #
 # Examples for a Sun/Oracle JDK:
 # a) override the appsummary log file:
@@ -83,19 +60,16 @@
 # Node Manager specific parameters
 ###
 
-# Specify the max Heapsize for the NodeManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
-# HADOOP_OPTS, and/or YARN_NODEMANAGER_OPTS.
-# If not specified, the default value will be picked from either HADOOP_HEAPSIZE
-# or the built-in default.
-#
-#export YARN_NODEMANAGER_HEAPSIZE=1000
+# Specify the max heapsize for the NodeManager.  If no units are
+# given, it will be assumed to be in MB.
+# This value will be overridden by an Xmx setting specified in either
+# HADOOP_OPTS and/or YARN_NODEMANAGER_OPTS.
+# Default is the same as HADOOP_HEAPSIZE_MAX.
+#export YARN_NODEMANAGER_HEAPSIZE=
 
 # Specify the JVM options to be used when starting the NodeManager.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
 #
 # See ResourceManager for some examples
 #
@@ -105,19 +79,16 @@
 # TimeLineServer specifc parameters
 ###
 
-# Specify the max Heapsize for the timeline server using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
-# HADOOP_OPTS, and/or YARN_TIMELINESERVER_OPTS.
-# If not specified, the default value will be picked from either HADOOP_HEAPSIZE
-# or the built-in default.
-#
-#export YARN_TIMELINESERVER_HEAPSIZE=1000
+# Specify the max heapsize for the timelineserver.  If no units are
+# given, it will be assumed to be in MB.
+# This value will be overridden by an Xmx setting specified in either
+# HADOOP_OPTS and/or YARN_TIMELINESERVER_OPTS.
+# Default is the same as HADOOP_HEAPSIZE_MAX.
+#export YARN_TIMELINE_HEAPSIZE=
 
 # Specify the JVM options to be used when starting the TimeLineServer.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
 #
 # See ResourceManager for some examples
 #
@@ -127,22 +98,29 @@
 # Web App Proxy Server specifc parameters
 ###
 
-# Specify the max Heapsize for the proxy server using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either YARN_OPTS,
-# HADOOP_OPTS, and/or YARN_PROXYSERVER_OPTS.
-# If not specified, the default value will be picked from either HADOOP_HEAPSIZE
-# or the built-in default.
-#
-#export YARN_PROXYSERVER_HEAPSIZE=1000
+# Specify the max heapsize for the web app proxy server.  If no units are
+# given, it will be assumed to be in MB.
+# This value will be overridden by an Xmx setting specified in either
+# HADOOP_OPTS and/or YARN_PROXYSERVER_OPTS.
+# Default is the same as HADOOP_HEAPSIZE_MAX.
+#export YARN_PROXYSERVER_HEAPSIZE=
 
 # Specify the JVM options to be used when starting the proxy server.
-# These options will be appended to the options specified as YARN_OPTS
-# and therefore may override any similar flags set in YARN_OPTS
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
 #
 # See ResourceManager for some examples
 #
 #export YARN_PROXYSERVER_OPTS=
 
-
+###
+# Shared Cache Manager specific parameters
+###
+# Specify the JVM options to be used when starting the
+# shared cache manager server.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_SHAREDCACHEMANAGER_OPTS=
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 45d7294e87052..e0bbd7b125d67 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -173,6 +173,7 @@
       <Field name="userLimit" />
       <Field name="userLimitFactor" />
       <Field name="maxAMResourcePerQueuePercent" />
+      <Field name="lastClusterResource" />
     </Or>
     <Bug pattern="IS2_INCONSISTENT_SYNC" />
   </Match>
@@ -265,6 +266,11 @@
     <Class name="org.apache.hadoop.yarn.YarnUncaughtExceptionHandler"/>
     <Bug pattern="DM_EXIT"/>
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.event.AsyncDispatcher$2"/>
+    <Method name="run" />
+    <Bug pattern="DM_EXIT"/>
+  </Match>
 
   <!-- AsyncDispatcher will kill the process if there is an error dispatching -->
   <Match>
@@ -391,4 +397,57 @@
     <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE"/>
   </Match>
 
+  <!-- Ignore the false alarms on DM_DEFAULT_ENCODING (encoding already set) -->
+  <Match>
+    <Class name="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat$LogReader" />
+    <Bug pattern="DM_DEFAULT_ENCODING" />
+  </Match>
+
+  <!-- Ignore EI_EXPOSE_REP2 in timeline service -->
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils$KeyParser" />
+    <Bug pattern="EI_EXPOSE_REP2" />
+  </Match>
+
+  <!-- Ignore unrelated RV_RETURN_VALUE_IGNORED_BAD_PRACTICE warnings. No need to wait for result from executor service -->
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadService" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+  </Match>
+
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt" />
+    <Field name="queue" />
+    <Bug pattern="IS2_INCONSISTENT_SYNC" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" />
+    <Field name="allocConf" />
+    <Bug pattern="IS2_INCONSISTENT_SYNC" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+    <Field name="numContainers" />
+    <Bug pattern="VO_VOLATILE_INCREMENT" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue" />
+    <Field name="numContainers" />
+    <Bug pattern="VO_VOLATILE_INCREMENT" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue" />
+    <Field name="numApplications" />
+    <Bug pattern="VO_VOLATILE_INCREMENT" />
+  </Match>
+  <Match>
+    <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler" />
+    <Method name="recoverContainersOnNode" />
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE" />
+  </Match>
+
 </FindBugsFilter>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 7f86caed1ec2b..2f17ac96f5ad5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -253,24 +253,27 @@ public static boolean isAnyLocation(String hostName) {
   /**
    * Get node-label-expression for this Resource Request. If this is set, all
    * containers allocated to satisfy this resource-request will be only on those
-   * nodes that satisfy this node-label-expression
+   * nodes that satisfy this node-label-expression.
+   *  
+   * Please note that node label expression now can only take effect when the
+   * resource request has resourceName = ANY
    * 
    * @return node-label-expression
    */
   @Public
   @Evolving
-  public abstract String getNodeLabelExpression(); 
+  public abstract String getNodeLabelExpression();
   
   /**
-   * Set node label expression of this resource request. Now only
-   * support AND(&&), in the future will provide support for OR(||), NOT(!).
+   * Set node label expression of this resource request. Now only support
+   * specifying a single node label. In the future we will support more complex
+   * node label expression specification like AND(&&), OR(||), etc.
    * 
-   * Examples: 
-   * - GPU && LARGE_MEM, ask for node has label GPU and LARGE_MEM together
-   * - "" (empty) means ask for node doesn't have label on it, this is default
-   *   behavior
+   * Any please note that node label expression now can only take effect when
+   * the resource request has resourceName = ANY
    * 
-   * @param nodelabelExpression node-label-expression of this ResourceRequest
+   * @param nodelabelExpression
+   *          node-label-expression of this ResourceRequest
    */
   @Public
   @Evolving
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 55073c53e0fc7..d6f6dee1e8ce4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -356,18 +356,18 @@ private static void addDeprecatedKeys() {
   public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
       10;
 
-  //Delegation token related keys
-  public static final String  DELEGATION_KEY_UPDATE_INTERVAL_KEY = 
+  //RM delegation token related keys
+  public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
     RM_PREFIX + "delegation.key.update-interval";
-  public static final long    DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 
+  public static final long RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT =
     24*60*60*1000; // 1 day
-  public static final String  DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 
+  public static final String RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
     RM_PREFIX + "delegation.token.renew-interval";
-  public static final long    DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 
+  public static final long RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
     24*60*60*1000;  // 1 day
-  public static final String  DELEGATION_TOKEN_MAX_LIFETIME_KEY = 
+  public static final String RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY =
      RM_PREFIX + "delegation.token.max-lifetime";
-  public static final long    DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 
+  public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
     7*24*60*60*1000; // 7 days
   
   public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
@@ -378,7 +378,7 @@ private static void addDeprecatedKeys() {
       + "work-preserving-recovery.enabled";
   @Private
   public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED =
-      false;
+      true;
 
   public static final String RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS =
       RM_PREFIX + "work-preserving-recovery.scheduling-wait-ms";
@@ -822,6 +822,20 @@ private static void addDeprecatedKeys() {
       "container-monitor.procfs-tree.smaps-based-rss.enabled";
   public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED =
       false;
+
+  /** Enable/disable container metrics. */
+  @Private
+  public static final String NM_CONTAINER_METRICS_ENABLE =
+      NM_PREFIX + "container-metrics.enable";
+  @Private
+  public static final boolean DEFAULT_NM_CONTAINER_METRICS_ENABLE = true;
+
+  /** Container metrics flush period. -1 for flush on completion. */
+  @Private
+  public static final String NM_CONTAINER_METRICS_PERIOD_MS =
+      NM_PREFIX + "container-metrics.period-ms";
+  @Private
+  public static final int DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS = -1;
   
   /** Prefix for all node manager disk health checker configs. */
   private static final String NM_DISK_HEALTH_CHECK_PREFIX =
@@ -1137,7 +1151,7 @@ private static void addDeprecatedKeys() {
    * OS environment expansion syntax.
    * </p>
    * <p>
-   * Note: Use {@link DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+   * Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
    * cross-platform practice i.e. submit an application from a Windows client to
    * a Linux/Unix server or vice versa.
    * </p>
@@ -1191,6 +1205,11 @@ private static void addDeprecatedKeys() {
   public static final boolean
       DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
 
+  /** Allow changing the memory for the NodeManager in the MiniYARNCluster */
+  public static final String YARN_MINICLUSTER_NM_PMEM_MB =
+      YARN_MC_PREFIX + YarnConfiguration.NM_PMEM_MB;
+  public static final int DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB = 4 * 1024;
+
   /** The log directory for the containers */
   public static final String YARN_APP_CONTAINER_LOG_DIR =
       YARN_PREFIX + "app.container.log.dir";
@@ -1366,6 +1385,43 @@ private static void addDeprecatedKeys() {
   public static final long
       DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
 
+  /** Timeline client policy for whether connections are fatal */
+  public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT =
+      TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort";
+
+  public static final boolean
+      DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false;
+
+  /** Flag to enable recovery of timeline service */
+  public static final String TIMELINE_SERVICE_RECOVERY_ENABLED =
+      TIMELINE_SERVICE_PREFIX + "recovery.enabled";
+  public static final boolean DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED = false;
+
+  /** Timeline service state store class */
+  public static final String TIMELINE_SERVICE_STATE_STORE_CLASS =
+      TIMELINE_SERVICE_PREFIX + "state-store-class";
+
+  public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX =
+      TIMELINE_SERVICE_PREFIX + "leveldb-state-store.";
+
+  /** Timeline service state store leveldb path */
+  public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH =
+      TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX + "path";
+
+  // Timeline delegation token related keys
+  public static final String  TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL =
+      TIMELINE_SERVICE_PREFIX + "delegation.key.update-interval";
+  public static final long    DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL =
+      24*60*60*1000; // 1 day
+  public static final String  TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL =
+      TIMELINE_SERVICE_PREFIX + "delegation.token.renew-interval";
+  public static final long    DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL =
+      24*60*60*1000;  // 1 day
+  public static final String  TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
+      TIMELINE_SERVICE_PREFIX + "delegation.token.max-lifetime";
+  public static final long    DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
+      7*24*60*60*1000; // 7 days
+
   // ///////////////////////////////
   // Shared Cache Configs
   // ///////////////////////////////
@@ -1413,6 +1469,13 @@ private static void addDeprecatedKeys() {
       SHARED_CACHE_PREFIX + "admin.thread-count";
   public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1;
 
+  /** The address of the SCM web application. */
+  public static final String SCM_WEBAPP_ADDRESS =
+      SHARED_CACHE_PREFIX + "webapp.address";
+  public static final int DEFAULT_SCM_WEBAPP_PORT = 8788;
+  public static final String DEFAULT_SCM_WEBAPP_ADDRESS =
+      "0.0.0.0:" + DEFAULT_SCM_WEBAPP_PORT;
+
   // In-memory SCM store configuration
   
   public static final String IN_MEMORY_STORE_PREFIX =
@@ -1597,14 +1660,10 @@ private static void addDeprecatedKeys() {
   public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY
       .name();
   
-  public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
-
   /**
-   * Class for RMNodeLabelsManager Please note this value should be consistent
-   * in client nodes and RM node(s)
+   * Node-labels configurations
    */
-  public static final String RM_NODE_LABELS_MANAGER_CLASS = NODE_LABELS_PREFIX
-      + "manager-class";
+  public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
   
   /** URI for NodeLabelManager */
   public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
@@ -1613,6 +1672,14 @@ private static void addDeprecatedKeys() {
       NODE_LABELS_PREFIX + "fs-store.retry-policy-spec";
   public static final String DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC =
       "2000, 500";
+  
+  /**
+   * Flag to indicate if the node labels feature enabled, by default it's
+   * disabled
+   */
+  public static final String NODE_LABELS_ENABLED = NODE_LABELS_PREFIX
+      + "enabled";
+  public static final boolean DEFAULT_NODE_LABELS_ENABLED = false;
 
   public YarnConfiguration() {
     super();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index b3ea8659fc9d3..a9a7091d3bbbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -463,11 +463,11 @@ public boolean init(String[] args) throws ParseException, IOException {
       scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);
 
       if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
-        shellScriptPathTimestamp = Long.valueOf(envs
+        shellScriptPathTimestamp = Long.parseLong(envs
             .get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP));
       }
       if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) {
-        shellScriptPathLen = Long.valueOf(envs
+        shellScriptPathLen = Long.parseLong(envs
             .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
       }
       if (!scriptPath.isEmpty()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index a05b3b0d0aa18..46b5850ebb1ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -80,11 +80,12 @@ protected void setupInternal(int numNodeManager) throws Exception {
     conf.set("yarn.log.dir", "target");
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     
     if (yarnCluster == null) {
       yarnCluster =
           new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
-              numNodeManager, 1, 1);
+              numNodeManager, 1, 1, true);
       yarnCluster.init(conf);
       
       yarnCluster.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
index d41434e94dcae..b9c7e802b7c6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.InetAddress;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.Map;
@@ -232,11 +233,11 @@ public void launchAM(ApplicationAttemptId attemptId)
     Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM));
 
     final BufferedReader errReader = 
-        new BufferedReader(new InputStreamReader(amProc
-                                                 .getErrorStream()));
+        new BufferedReader(new InputStreamReader(
+            amProc.getErrorStream(), Charset.forName("UTF-8")));
     final BufferedReader inReader = 
-        new BufferedReader(new InputStreamReader(amProc
-                                                 .getInputStream()));
+        new BufferedReader(new InputStreamReader(
+            amProc.getInputStream(), Charset.forName("UTF-8")));
     
     // read error and input streams as this would free up the buffers
     // free the error stream buffer
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 6f8c65a9fa265..9923806367833 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -169,7 +169,8 @@ public ContainerRequest(Resource capability, String[] nodes,
      *          If true, containers for this request may be assigned on hosts
      *          and racks other than the ones explicitly requested.
      * @param nodeLabelsExpression
-     *          Set node labels to allocate resource
+     *          Set node labels to allocate resource, now we only support
+     *          asking for only a single node label
      */
     public ContainerRequest(Resource capability, String[] nodes,
         String[] racks, Priority priority, boolean relaxLocality,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
new file mode 100644
index 0000000000000..7cbe0e164b508
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api;
+
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * This is the client for YARN's shared cache.
+ */
+@Public
+@Unstable
+public abstract class SharedCacheClient extends AbstractService {
+
+  @Public
+  public static SharedCacheClient createSharedCacheClient() {
+    SharedCacheClient client = new SharedCacheClientImpl();
+    return client;
+  }
+
+  @Private
+  public SharedCacheClient(String name) {
+    super(name);
+  }
+
+  /**
+   * <p>
+   * The method to claim a resource with the <code>SharedCacheManager.</code>
+   * The client uses a checksum to identify the resource and an
+   * {@link ApplicationId} to identify which application will be using the
+   * resource.
+   * </p>
+   * 
+   * <p>
+   * The <code>SharedCacheManager</code> responds with whether or not the
+   * resource exists in the cache. If the resource exists, a <code>Path</code>
+   * to the resource in the shared cache is returned. If the resource does not
+   * exist, null is returned instead.
+   * </p>
+   * 
+   * @param applicationId ApplicationId of the application using the resource
+   * @param resourceKey the key (i.e. checksum) that identifies the resource
+   * @return Path to the resource, or null if it does not exist
+   */
+  @Public
+  @Unstable
+  public abstract Path use(ApplicationId applicationId, String resourceKey)
+      throws YarnException;
+
+  /**
+   * <p>
+   * The method to release a resource with the <code>SharedCacheManager.</code>
+   * This method is called once an application is no longer using a claimed
+   * resource in the shared cache. The client uses a checksum to identify the
+   * resource and an {@link ApplicationId} to identify which application is
+   * releasing the resource.
+   * </p>
+   * 
+   * <p>
+   * Note: This method is an optimization and the client is not required to call
+   * it for correctness.
+   * </p>
+   * 
+   * @param applicationId ApplicationId of the application releasing the
+   *          resource
+   * @param resourceKey the key (i.e. checksum) that identifies the resource
+   */
+  @Public
+  @Unstable
+  public abstract void release(ApplicationId applicationId, String resourceKey)
+      throws YarnException;
+
+  /**
+   * A convenience method to calculate the checksum of a specified file.
+   * 
+   * @param sourceFile A path to the input file
+   * @return A hex string containing the checksum digest
+   * @throws IOException
+   */
+  @Public
+  @Unstable
+  public abstract String getFileChecksum(Path sourceFile) throws IOException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index af26da1799a3a..f62e71b48f993 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -194,6 +194,17 @@ public abstract void unregisterApplicationMaster(
    */
   public abstract int getClusterNodeCount();
 
+  /**
+   * Update application's blacklist with addition or removal resources.
+   *
+   * @param blacklistAdditions list of resources which should be added to the
+   *        application blacklist
+   * @param blacklistRemovals list of resources which should be removed from the
+   *        application blacklist
+   */
+  public abstract void updateBlacklist(List<String> blacklistAdditions,
+                                       List<String> blacklistRemovals);
+
   /**
    * Wait for <code>check</code> to return true for each 1000 ms.
    * See also {@link #waitFor(com.google.common.base.Supplier, int)}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 82768bbf6101d..addc3b6daeccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -205,6 +205,19 @@ public Resource getAvailableResources() {
   public int getClusterNodeCount() {
     return client.getClusterNodeCount();
   }
+
+  /**
+   * Update application's blacklist with addition or removal resources.
+   *
+   * @param blacklistAdditions list of resources which should be added to the
+   *        application blacklist
+   * @param blacklistRemovals list of resources which should be removed from the
+   *        application blacklist
+   */
+  public void updateBlacklist(List<String> blacklistAdditions,
+                              List<String> blacklistRemovals) {
+    client.updateBlacklist(blacklistAdditions, blacklistRemovals);
+  }
   
   private class HeartbeatThread extends Thread {
     public HeartbeatThread() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 071c1eeaf1a5e..b1324c1d7b0a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -260,10 +260,9 @@ public AllocateResponse allocate(float progressIndicator)
         blacklistToAdd.addAll(blacklistAdditions);
         blacklistToRemove.addAll(blacklistRemovals);
         
-        ResourceBlacklistRequest blacklistRequest = 
-            (blacklistToAdd != null) || (blacklistToRemove != null) ? 
+        ResourceBlacklistRequest blacklistRequest =
             ResourceBlacklistRequest.newInstance(blacklistToAdd,
-                blacklistToRemove) : null;
+                blacklistToRemove);
         
         allocateRequest =
             AllocateRequest.newInstance(lastResponseId, progressIndicator,
@@ -422,6 +421,8 @@ public synchronized void addContainerRequest(T req) {
     checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true);
     checkLocalityRelaxationConflict(req.getPriority(), inferredRacks,
         req.getRelaxLocality());
+    // check if the node label expression specified is valid
+    checkNodeLabelExpression(req);
 
     if (req.getNodes() != null) {
       HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes());
@@ -587,6 +588,37 @@ private void checkLocalityRelaxationConflict(Priority priority,
       }
   }
   
+  /**
+   * Valid if a node label expression specified on container request is valid or
+   * not
+   * 
+   * @param containerRequest
+   */
+  private void checkNodeLabelExpression(T containerRequest) {
+    String exp = containerRequest.getNodeLabelExpression();
+    
+    if (null == exp || exp.isEmpty()) {
+      return;
+    }
+
+    // Don't support specifying >= 2 node labels in a node label expression now
+    if (exp.contains("&&") || exp.contains("||")) {
+      throw new InvalidContainerRequestException(
+          "Cannot specify more than two node labels"
+              + " in a single node label expression");
+    }
+    
+    // Don't allow specify node label against ANY request
+    if ((containerRequest.getRacks() != null && 
+        (!containerRequest.getRacks().isEmpty()))
+        || 
+        (containerRequest.getNodes() != null && 
+        (!containerRequest.getNodes().isEmpty()))) {
+      throw new InvalidContainerRequestException(
+          "Cannot specify node label with rack and node");
+    }
+  }
+  
   private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
     // This code looks weird but is needed because of the following scenario.
     // A ResourceRequest is removed from the remoteRequestTable. A 0 container 
@@ -641,7 +673,9 @@ private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
       resourceRequestInfo.containerRequests.add(req);
     }
     
-    resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+    if (ResourceRequest.ANY.equals(resourceName)) {
+      resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+    }
 
     // Note this down for next interaction with ResourceManager
     addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
@@ -756,11 +790,11 @@ private void updateAMRMToken(Token token) throws IOException {
         new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
           .getIdentifier().array(), token.getPassword().array(), new Text(
           token.getKind()), new Text(token.getService()));
-    amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
+    // Preserve the token service sent by the RM when adding the token
+    // to ensure we replace the previous token setup by the RM.
+    // Afterwards we can update the service address for the RPC layer.
     UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
-    if (UserGroupInformation.isSecurityEnabled()) {
-      currentUGI = UserGroupInformation.getLoginUser();
-    }
     currentUGI.addToken(amrmToken);
+    amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
new file mode 100644
index 0000000000000..0a61ee03c8bd8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.api.ClientSCMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.SharedCacheClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum;
+import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory;
+import org.apache.hadoop.yarn.util.Records;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * An implementation of the SharedCacheClient API.
+ */
+@Private
+@Unstable
+public class SharedCacheClientImpl extends SharedCacheClient {
+  private static final Log LOG = LogFactory
+      .getLog(SharedCacheClientImpl.class);
+
+  private ClientSCMProtocol scmClient;
+  private InetSocketAddress scmAddress;
+  private Configuration conf;
+  private SharedCacheChecksum checksum;
+
+  public SharedCacheClientImpl() {
+    super(SharedCacheClientImpl.class.getName());
+  }
+
+  private static InetSocketAddress getScmAddress(Configuration conf) {
+    return conf.getSocketAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS,
+        YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_ADDRESS,
+        YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_PORT);
+  }
+
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+    if (this.scmAddress == null) {
+      this.scmAddress = getScmAddress(conf);
+    }
+    this.conf = conf;
+    this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
+    super.serviceInit(conf);
+  }
+
+  @Override
+  protected void serviceStart() throws Exception {
+    this.scmClient = createClientProxy();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Connecting to Shared Cache Manager at " + this.scmAddress);
+    }
+    super.serviceStart();
+  }
+
+  @Override
+  protected void serviceStop() throws Exception {
+    stopClientProxy();
+    super.serviceStop();
+  }
+
+  @VisibleForTesting
+  protected ClientSCMProtocol createClientProxy() {
+    YarnRPC rpc = YarnRPC.create(getConfig());
+    return (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class,
+        this.scmAddress, getConfig());
+  }
+
+  @VisibleForTesting
+  protected void stopClientProxy() {
+    if (this.scmClient != null) {
+      RPC.stopProxy(this.scmClient);
+      this.scmClient = null;
+    }
+  }
+
+  @Override
+  public Path use(ApplicationId applicationId, String resourceKey)
+      throws YarnException {
+    Path resourcePath = null;
+    UseSharedCacheResourceRequest request = Records.newRecord(
+        UseSharedCacheResourceRequest.class);
+    request.setAppId(applicationId);
+    request.setResourceKey(resourceKey);
+    try {
+      UseSharedCacheResourceResponse response = this.scmClient.use(request);
+      if (response != null && response.getPath() != null) {
+        resourcePath = new Path(response.getPath());
+      }
+    } catch (Exception e) {
+      // Just catching IOException isn't enough.
+      // RPC call can throw ConnectionException.
+      // We don't handle different exceptions separately at this point.
+      throw new YarnException(e);
+    }
+    return resourcePath;
+  }
+
+  @Override
+  public void release(ApplicationId applicationId, String resourceKey)
+      throws YarnException {
+    ReleaseSharedCacheResourceRequest request = Records.newRecord(
+        ReleaseSharedCacheResourceRequest.class);
+    request.setAppId(applicationId);
+    request.setResourceKey(resourceKey);
+    try {
+      // We do not care about the response because it is empty.
+      this.scmClient.release(request);
+    } catch (Exception e) {
+      // Just catching IOException isn't enough.
+      // RPC call can throw ConnectionException.
+      throw new YarnException(e);
+    }
+  }
+
+  @Override
+  public String getFileChecksum(Path sourceFile)
+      throws IOException {
+    FileSystem fs = sourceFile.getFileSystem(this.conf);
+    FSDataInputStream in = null;
+    try {
+      in = fs.open(sourceFile);
+      return this.checksum.computeChecksum(in);
+    } finally {
+      if (in != null) {
+        in.close();
+      }
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index e4f31f20d848c..91fbd007193d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -22,6 +22,8 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -98,6 +100,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -118,7 +121,7 @@ public class YarnClientImpl extends YarnClient {
   protected long submitPollIntervalMillis;
   private long asyncApiPollIntervalMillis;
   private long asyncApiPollTimeoutMillis;
-  private AHSClient historyClient;
+  protected AHSClient historyClient;
   private boolean historyServiceEnabled;
   protected TimelineClient timelineClient;
   @VisibleForTesting
@@ -126,6 +129,7 @@ public class YarnClientImpl extends YarnClient {
   @VisibleForTesting
   String timelineDTRenewer;
   protected boolean timelineServiceEnabled;
+  protected boolean timelineServiceBestEffort;
 
   private static final String ROOT = "root";
 
@@ -160,14 +164,22 @@ protected void serviceInit(Configuration conf) throws Exception {
     if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
       timelineServiceEnabled = true;
-      timelineClient = TimelineClient.createTimelineClient();
+      timelineClient = createTimelineClient();
       timelineClient.init(conf);
       timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
       timelineService = TimelineUtils.buildTimelineTokenService(conf);
     }
+
+    timelineServiceBestEffort = conf.getBoolean(
+        YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
     super.serviceInit(conf);
   }
 
+  TimelineClient createTimelineClient() throws IOException, YarnException {
+    return TimelineClient.createTimelineClient();
+  }
+
   @Override
   protected void serviceStart() throws Exception {
     try {
@@ -322,7 +334,16 @@ private void addTimelineDelegationToken(
   @VisibleForTesting
   org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
       getTimelineDelegationToken() throws IOException, YarnException {
-    return timelineClient.getDelegationToken(timelineDTRenewer);
+        try {
+          return timelineClient.getDelegationToken(timelineDTRenewer);
+        } catch (Exception e ) {
+          if (timelineServiceBestEffort) {
+            LOG.warn("Failed to get delegation token from the timeline server: "
+                + e.getMessage());
+            return null;
+          }
+          throw e;
+        }
   }
 
   private static String getTimelineDelegationTokenRenewer(Configuration conf)
@@ -636,7 +657,8 @@ public ContainerReport getContainerReport(ContainerId containerId)
       }
       // Even if history-service is enabled, treat all exceptions still the same
       // except the following
-      if (e.getClass() != ApplicationNotFoundException.class) {
+      if (e.getClass() != ApplicationNotFoundException.class
+          && e.getClass() != ContainerNotFoundException.class) {
         throw e;
       }
       return historyClient.getContainerReport(containerId);
@@ -647,24 +669,79 @@ public ContainerReport getContainerReport(ContainerId containerId)
   public List<ContainerReport> getContainers(
       ApplicationAttemptId applicationAttemptId) throws YarnException,
       IOException {
+    List<ContainerReport> containersForAttempt =
+        new ArrayList<ContainerReport>();
+    boolean appNotFoundInRM = false;
     try {
-      GetContainersRequest request = Records
-          .newRecord(GetContainersRequest.class);
+      GetContainersRequest request =
+          Records.newRecord(GetContainersRequest.class);
       request.setApplicationAttemptId(applicationAttemptId);
       GetContainersResponse response = rmClient.getContainers(request);
-      return response.getContainerList();
+      containersForAttempt.addAll(response.getContainerList());
     } catch (YarnException e) {
-      if (!historyServiceEnabled) {
-        // Just throw it as usual if historyService is not enabled.
+      if (e.getClass() != ApplicationNotFoundException.class
+          || !historyServiceEnabled) {
+        // If Application is not in RM and history service is enabled then we
+        // need to check with history service else throw exception.
         throw e;
       }
-      // Even if history-service is enabled, treat all exceptions still the same
-      // except the following
-      if (e.getClass() != ApplicationNotFoundException.class) {
-        throw e;
+      appNotFoundInRM = true;
+    }
+
+    if (historyServiceEnabled) {
+      // Check with AHS even if found in RM because to capture info of finished
+      // containers also
+      List<ContainerReport> containersListFromAHS = null;
+      try {
+        containersListFromAHS =
+            historyClient.getContainers(applicationAttemptId);
+      } catch (IOException e) {
+        // History service access might be enabled but system metrics publisher
+        // is disabled hence app not found exception is possible
+        if (appNotFoundInRM) {
+          // app not found in bothM and RM then propagate the exception.
+          throw e;
+        }
+      }
+
+      if (null != containersListFromAHS && containersListFromAHS.size() > 0) {
+        // remove duplicates
+
+        Set<ContainerId> containerIdsToBeKeptFromAHS =
+            new HashSet<ContainerId>();
+        Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator();
+        while (tmpItr.hasNext()) {
+          containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId());
+        }
+
+        Iterator<ContainerReport> rmContainers =
+            containersForAttempt.iterator();
+        while (rmContainers.hasNext()) {
+          ContainerReport tmp = rmContainers.next();
+          containerIdsToBeKeptFromAHS.remove(tmp.getContainerId());
+          // Remove containers from AHS as container from RM will have latest
+          // information
+        }
+
+        if (containerIdsToBeKeptFromAHS.size() > 0
+            && containersListFromAHS.size() != containerIdsToBeKeptFromAHS
+                .size()) {
+          Iterator<ContainerReport> containersFromHS =
+              containersListFromAHS.iterator();
+          while (containersFromHS.hasNext()) {
+            ContainerReport containerReport = containersFromHS.next();
+            if (containerIdsToBeKeptFromAHS.contains(containerReport
+                .getContainerId())) {
+              containersForAttempt.add(containerReport);
+            }
+          }
+        } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS
+            .size()) {
+          containersForAttempt.addAll(containersListFromAHS);
+        }
       }
-      return historyClient.getContainers(applicationAttemptId);
     }
+    return containersForAttempt;
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 83d212dd9ae14..de8f740253425 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -19,7 +19,9 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.Charset;
 import java.text.DecimalFormat;
 import java.util.EnumSet;
 import java.util.HashSet;
@@ -41,7 +43,9 @@
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Times;
@@ -152,12 +156,14 @@ public int run(String[] args) throws Exception {
         return exitCode;
       }
       if (args[0].equalsIgnoreCase(APPLICATION)) {
-        printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
+        exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
       } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
-        printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
+        exitCode = printApplicationAttemptReport(cliParser
+            .getOptionValue(STATUS_CMD));
       } else if (args[0].equalsIgnoreCase(CONTAINER)) {
-        printContainerReport(cliParser.getOptionValue(STATUS_CMD));
+        exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
       }
+      return exitCode;
     } else if (cliParser.hasOption(LIST_CMD)) {
       if (args[0].equalsIgnoreCase(APPLICATION)) {
         allAppStates = false;
@@ -252,16 +258,28 @@ void printUsage(String title, Options opts) {
    * Prints the application attempt report for an application attempt id.
    * 
    * @param applicationAttemptId
+   * @return exitCode
    * @throws YarnException
    */
-  private void printApplicationAttemptReport(String applicationAttemptId)
+  private int printApplicationAttemptReport(String applicationAttemptId)
       throws YarnException, IOException {
-    ApplicationAttemptReport appAttemptReport = client
-        .getApplicationAttemptReport(ConverterUtils
-            .toApplicationAttemptId(applicationAttemptId));
+    ApplicationAttemptReport appAttemptReport = null;
+    try {
+      appAttemptReport = client.getApplicationAttemptReport(ConverterUtils
+          .toApplicationAttemptId(applicationAttemptId));
+    } catch (ApplicationNotFoundException e) {
+      sysout.println("Application for AppAttempt with id '"
+          + applicationAttemptId + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    } catch (ApplicationAttemptNotFoundException e) {
+      sysout.println("Application Attempt with id '" + applicationAttemptId
+          + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    }
     // Use PrintWriter.println, which uses correct platform line ending.
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintWriter appAttemptReportStr = new PrintWriter(baos);
+    PrintWriter appAttemptReportStr = new PrintWriter(
+        new OutputStreamWriter(baos, Charset.forName("UTF-8")));
     if (appAttemptReport != null) {
       appAttemptReportStr.println("Application Attempt Report : ");
       appAttemptReportStr.print("\tApplicationAttempt-Id : ");
@@ -282,25 +300,46 @@ private void printApplicationAttemptReport(String applicationAttemptId)
       appAttemptReportStr.print(appAttemptReport.getDiagnostics());
     } else {
       appAttemptReportStr.print("Application Attempt with id '"
-          + applicationAttemptId + "' doesn't exist in History Server.");
+          + applicationAttemptId + "' doesn't exist in Timeline Server.");
+      appAttemptReportStr.close();
+      sysout.println(baos.toString("UTF-8"));
+      return -1;
     }
     appAttemptReportStr.close();
     sysout.println(baos.toString("UTF-8"));
+    return 0;
   }
 
   /**
    * Prints the container report for an container id.
    * 
    * @param containerId
+   * @return exitCode
    * @throws YarnException
    */
-  private void printContainerReport(String containerId) throws YarnException,
+  private int printContainerReport(String containerId) throws YarnException,
       IOException {
-    ContainerReport containerReport = client.getContainerReport((ConverterUtils
-        .toContainerId(containerId)));
+    ContainerReport containerReport = null;
+    try {
+      containerReport = client.getContainerReport((ConverterUtils
+          .toContainerId(containerId)));
+    } catch (ApplicationNotFoundException e) {
+      sysout.println("Application for Container with id '" + containerId
+          + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    } catch (ApplicationAttemptNotFoundException e) {
+      sysout.println("Application Attempt for Container with id '"
+          + containerId + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    } catch (ContainerNotFoundException e) {
+      sysout.println("Container with id '" + containerId
+          + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    }
     // Use PrintWriter.println, which uses correct platform line ending.
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintWriter containerReportStr = new PrintWriter(baos);
+    PrintWriter containerReportStr = new PrintWriter(
+        new OutputStreamWriter(baos, Charset.forName("UTF-8")));
     if (containerReport != null) {
       containerReportStr.println("Container Report : ");
       containerReportStr.print("\tContainer-Id : ");
@@ -319,10 +358,14 @@ private void printContainerReport(String containerId) throws YarnException,
       containerReportStr.print(containerReport.getDiagnosticsInfo());
     } else {
       containerReportStr.print("Container with id '" + containerId
-          + "' doesn't exist in Hostory Server.");
+          + "' doesn't exist in Timeline Server.");
+      containerReportStr.close();
+      sysout.println(baos.toString("UTF-8"));
+      return -1;
     }
     containerReportStr.close();
     sysout.println(baos.toString("UTF-8"));
+    return 0;
   }
 
   /**
@@ -337,7 +380,8 @@ private void printContainerReport(String containerId) throws YarnException,
   private void listApplications(Set<String> appTypes,
       EnumSet<YarnApplicationState> appStates) throws YarnException,
       IOException {
-    PrintWriter writer = new PrintWriter(sysout);
+    PrintWriter writer = new PrintWriter(
+        new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
     if (allAppStates) {
       for (YarnApplicationState appState : YarnApplicationState.values()) {
         appStates.add(appState);
@@ -423,15 +467,24 @@ private void moveApplicationAcrossQueues(String applicationId, String queue)
    * Prints the application report for an application id.
    * 
    * @param applicationId
+   * @return exitCode
    * @throws YarnException
    */
-  private void printApplicationReport(String applicationId)
+  private int printApplicationReport(String applicationId)
       throws YarnException, IOException {
-    ApplicationReport appReport = client.getApplicationReport(ConverterUtils
-        .toApplicationId(applicationId));
+    ApplicationReport appReport = null;
+    try {
+      appReport = client.getApplicationReport(ConverterUtils
+          .toApplicationId(applicationId));
+    } catch (ApplicationNotFoundException e) {
+      sysout.println("Application with id '" + applicationId
+          + "' doesn't exist in RM or Timeline Server.");
+      return -1;
+    }
     // Use PrintWriter.println, which uses correct platform line ending.
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintWriter appReportStr = new PrintWriter(baos);
+    PrintWriter appReportStr = new PrintWriter(
+        new OutputStreamWriter(baos, Charset.forName("UTF-8")));
     if (appReport != null) {
       appReportStr.println("Application Report : ");
       appReportStr.print("\tApplication-Id : ");
@@ -478,9 +531,13 @@ private void printApplicationReport(String applicationId)
     } else {
       appReportStr.print("Application with id '" + applicationId
           + "' doesn't exist in RM.");
+      appReportStr.close();
+      sysout.println(baos.toString("UTF-8"));
+      return -1;
     }
     appReportStr.close();
     sysout.println(baos.toString("UTF-8"));
+    return 0;
   }
 
   private String getAllValidApplicationStates() {
@@ -503,7 +560,8 @@ private String getAllValidApplicationStates() {
    */
   private void listApplicationAttempts(String applicationId) throws YarnException,
       IOException {
-    PrintWriter writer = new PrintWriter(sysout);
+    PrintWriter writer = new PrintWriter(
+        new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
 
     List<ApplicationAttemptReport> appAttemptsReport = client
         .getApplicationAttempts(ConverterUtils.toApplicationId(applicationId));
@@ -529,7 +587,8 @@ private void listApplicationAttempts(String applicationId) throws YarnException,
    */
   private void listContainers(String appAttemptId) throws YarnException,
       IOException {
-    PrintWriter writer = new PrintWriter(sysout);
+    PrintWriter writer = new PrintWriter(
+        new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
 
     List<ContainerReport> appsReport = client
         .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index fa2779ebc65cd..22c240f7e38eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -19,7 +19,9 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -141,7 +143,8 @@ private void printUsage(Options opts) {
    */
   private void listClusterNodes(Set<NodeState> nodeStates) 
             throws YarnException, IOException {
-    PrintWriter writer = new PrintWriter(sysout);
+    PrintWriter writer = new PrintWriter(
+        new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
     List<NodeReport> nodesReport = client.getNodeReports(
                                        nodeStates.toArray(new NodeState[0]));
     writer.println("Total Nodes:" + nodesReport.size());
@@ -167,7 +170,8 @@ private void printNodeStatus(String nodeIdStr) throws YarnException,
     List<NodeReport> nodesReport = client.getNodeReports();
     // Use PrintWriter.println, which uses correct platform line ending.
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintWriter nodeReportStr = new PrintWriter(baos);
+    PrintWriter nodeReportStr = new PrintWriter(
+        new OutputStreamWriter(baos, Charset.forName("UTF-8")));
     NodeReport nodeReport = null;
     for (NodeReport report : nodesReport) {
       if (!report.getNodeId().equals(nodeId)) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
index 4d50e7f13ac35..8a5521dc4b6db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
@@ -18,7 +18,9 @@
 package org.apache.hadoop.yarn.client.cli;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.Charset;
 import java.text.DecimalFormat;
 import java.util.Set;
 
@@ -102,7 +104,8 @@ void printUsage(Options opts) {
    */
   private int listQueue(String queueName) throws YarnException, IOException {
     int rc;
-    PrintWriter writer = new PrintWriter(sysout);
+    PrintWriter writer = new PrintWriter(
+        new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
 
     QueueInfo queueInfo = client.getQueueInfo(queueName);
     if (queueInfo != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 89d87cfc4e2a0..af83102c96dd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -59,6 +59,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 
 @Private
@@ -69,6 +70,10 @@ public class RMAdminCLI extends HAAdmin {
     RecordFactoryProvider.getRecordFactory(null);
   private boolean directlyAccessNodeLabelStore = false;
   static CommonNodeLabelsManager localNodeLabelsManager = null;
+  private static final String NO_LABEL_ERR_MSG =
+      "No cluster node-labels are specified";
+  private static final String NO_MAPPING_ERR_MSG =
+      "No node-to-labels mappings are specified";
 
   protected final static Map<String, UsageInfo> ADMIN_USAGE =
       ImmutableMap.<String, UsageInfo>builder()
@@ -89,9 +94,6 @@ public class RMAdminCLI extends HAAdmin {
                   "ResoureceManager will reload the authorization policy file."))
           .put("-getGroups", new UsageInfo("[username]",
               "Get the groups which given user belongs to."))
-          .put("-help", new UsageInfo("[cmd]",
-              "Displays help for the given command or all commands if none " +
-                  "is specified."))
           .put("-addToClusterNodeLabels",
               new UsageInfo("[label1,label2,label3] (label splitted by \",\")",
                   "add to cluster node labels "))
@@ -99,8 +101,11 @@ public class RMAdminCLI extends HAAdmin {
               new UsageInfo("[label1,label2,label3] (label splitted by \",\")",
                   "remove from cluster node labels"))
           .put("-replaceLabelsOnNode",
-              new UsageInfo("[node1:port,label1,label2 node2:port,label1,label2]",
-                  "replace labels on nodes"))
+              new UsageInfo(
+                  "[node1[:port]=label1,label2 node2[:port]=label1,label2]",
+                  "replace labels on nodes"
+                      + " (please note that we do not support specifying multiple"
+                      + " labels on a single host for now.)"))
           .put("-directlyAccessNodeLabelStore",
               new UsageInfo("", "Directly access node label store, "
                   + "with this option, all node label related operations"
@@ -180,6 +185,7 @@ private static void buildUsageMsg(StringBuilder builder,
         }
       }
     }
+    builder.append("   -help" + " [cmd]\n");
   }
 
   private static void printHelp(String cmd, boolean isHAEnabled) {
@@ -195,10 +201,14 @@ private static void printHelp(String cmd, boolean isHAEnabled) {
       " [-refreshAdminAcls]" +
       " [-refreshServiceAcl]" +
       " [-getGroup [username]]" +
-      " [-help [cmd]]");
+      " [[-addToClusterNodeLabels [label1,label2,label3]]" +
+      " [-removeFromClusterNodeLabels [label1,label2,label3]]" +
+      " [-replaceLabelsOnNode [node1[:port]=label1,label2 node2[:port]=label1]" +
+      " [-directlyAccessNodeLabelStore]]");
     if (isHAEnabled) {
       appendHAUsage(summary);
     }
+    summary.append(" [-help [cmd]]");
     summary.append("\n");
 
     StringBuilder helpBuilder = new StringBuilder();
@@ -215,6 +225,8 @@ private static void printHelp(String cmd, boolean isHAEnabled) {
         }
       }
     }
+    helpBuilder.append("   -help [cmd]: Displays help for the given command or all commands" +
+        " if none is specified.");
     System.out.println(helpBuilder);
     System.out.println();
     ToolRunner.printGenericCommandUsage(System.out);
@@ -332,18 +344,24 @@ private int getGroups(String[] usernames) throws IOException {
     return localNodeLabelsManager;
   }
   
-  private int addToClusterNodeLabels(String args) throws IOException,
-      YarnException {
+  private Set<String> buildNodeLabelsSetFromStr(String args) {
     Set<String> labels = new HashSet<String>();
     for (String p : args.split(",")) {
-      labels.add(p);
+      if (!p.trim().isEmpty()) {
+        labels.add(p.trim());
+      }
     }
 
-    return addToClusterNodeLabels(labels);
+    if (labels.isEmpty()) {
+      throw new IllegalArgumentException(NO_LABEL_ERR_MSG);
+    }
+    return labels;
   }
 
-  private int addToClusterNodeLabels(Set<String> labels) throws IOException,
+  private int addToClusterNodeLabels(String args) throws IOException,
       YarnException {
+    Set<String> labels = buildNodeLabelsSetFromStr(args);
+
     if (directlyAccessNodeLabelStore) {
       getNodeLabelManagerInstance(getConf()).addToCluserNodeLabels(labels);
     } else {
@@ -358,10 +376,7 @@ private int addToClusterNodeLabels(Set<String> labels) throws IOException,
 
   private int removeFromClusterNodeLabels(String args) throws IOException,
       YarnException {
-    Set<String> labels = new HashSet<String>();
-    for (String p : args.split(",")) {
-      labels.add(p);
-    }
+    Set<String> labels = buildNodeLabelsSetFromStr(args);
 
     if (directlyAccessNodeLabelStore) {
       getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels(
@@ -377,8 +392,7 @@ private int removeFromClusterNodeLabels(String args) throws IOException,
     return 0;
   }
   
-  private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args)
-      throws IOException {
+  private Map<NodeId, Set<String>> buildNodeLabelsMapFromStr(String args) {
     Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
 
     for (String nodeToLabels : args.split("[ \n]")) {
@@ -387,29 +401,46 @@ private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args)
         continue;
       }
 
-      String[] splits = nodeToLabels.split(",");
-      String nodeIdStr = splits[0];
+      // "," also supported for compatibility
+      String[] splits = nodeToLabels.split("=");
+      int index = 0;
+      if (splits.length != 2) {
+        splits = nodeToLabels.split(",");
+        index = 1;
+      }
 
-      if (nodeIdStr.trim().isEmpty()) {
-        throw new IOException("node name cannot be empty");
+      String nodeIdStr = splits[0];
+      if (index == 0) {
+        splits = splits[1].split(",");
       }
+      
+      Preconditions.checkArgument(!nodeIdStr.trim().isEmpty(),
+          "node name cannot be empty");
 
       NodeId nodeId = ConverterUtils.toNodeIdWithDefaultPort(nodeIdStr);
       map.put(nodeId, new HashSet<String>());
 
-      for (int i = 1; i < splits.length; i++) {
+      for (int i = index; i < splits.length; i++) {
         if (!splits[i].trim().isEmpty()) {
-          map.get(nodeId).add(splits[i].trim().toLowerCase());
+          map.get(nodeId).add(splits[i].trim());
         }
       }
+      
+      int nLabels = map.get(nodeId).size();
+      Preconditions.checkArgument(nLabels <= 1, "%d labels specified on host=%s"
+          + ", please note that we do not support specifying multiple"
+          + " labels on a single host for now.", nLabels, nodeIdStr);
     }
 
+    if (map.isEmpty()) {
+      throw new IllegalArgumentException(NO_MAPPING_ERR_MSG);
+    }
     return map;
   }
 
   private int replaceLabelsOnNodes(String args) throws IOException,
       YarnException {
-    Map<NodeId, Set<String>> map = buildNodeLabelsFromStr(args);
+    Map<NodeId, Set<String>> map = buildNodeLabelsMapFromStr(args);
     return replaceLabelsOnNodes(map);
   }
 
@@ -507,21 +538,21 @@ public int run(String[] args) throws Exception {
         exitCode = getGroups(usernames);
       } else if ("-addToClusterNodeLabels".equals(cmd)) {
         if (i >= args.length) {
-          System.err.println("No cluster node-labels are specified");
+          System.err.println(NO_LABEL_ERR_MSG);
           exitCode = -1;
         } else {
           exitCode = addToClusterNodeLabels(args[i]);
         }
       } else if ("-removeFromClusterNodeLabels".equals(cmd)) {
         if (i >= args.length) {
-          System.err.println("No cluster node-labels are specified");
+          System.err.println(NO_LABEL_ERR_MSG);
           exitCode = -1;
         } else {
           exitCode = removeFromClusterNodeLabels(args[i]);
         }
       } else if ("-replaceLabelsOnNode".equals(cmd)) {
         if (i >= args.length) {
-          System.err.println("No cluster node-labels are specified");
+          System.err.println(NO_MAPPING_ERR_MSG);
           exitCode = -1;
         } else {
           exitCode = replaceLabelsOnNodes(args[i]);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 4b2b4bcc43084..da7d50529ac9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -295,7 +295,7 @@ protected void startHACluster(int numOfNMs, boolean overrideClientRMService,
     conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     cluster =
         new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
-            numOfNMs, 1, 1, overrideClientRMService, overrideRTS,
+            numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
             overrideApplicationMasterService);
     cluster.resetStartFailoverFlag(false);
     cluster.init(conf);
@@ -326,10 +326,10 @@ public class MiniYARNClusterForHATesting extends MiniYARNCluster {
 
     public MiniYARNClusterForHATesting(String testName,
         int numResourceManagers, int numNodeManagers, int numLocalDirs,
-        int numLogDirs, boolean overrideClientRMService,
+        int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
         boolean overrideRTS, boolean overrideApplicationMasterService) {
       super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
-          numLogDirs);
+          numLogDirs, enableAHS);
       this.overrideClientRMService = overrideClientRMService;
       this.overrideRTS = overrideRTS;
       this.overrideApplicationMasterService = overrideApplicationMasterService;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index e24b5f6491c3b..7d29d0529272a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import com.google.common.base.Supplier;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -40,7 +38,6 @@
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.junit.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
@@ -75,6 +72,7 @@
 import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
 import org.apache.hadoop.yarn.client.api.NMTokenCache;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -89,6 +87,7 @@
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -96,6 +95,8 @@
 import org.mockito.stubbing.Answer;
 import org.mortbay.log.Log;
 
+import com.google.common.base.Supplier;
+
 public class TestAMRMClient {
   static Configuration conf = null;
   static MiniYARNCluster yarnCluster = null;
@@ -148,7 +149,6 @@ public static void setup() throws Exception {
     racks = new String[]{ rack };
   }
   
-  @SuppressWarnings("deprecation")
   @Before
   public void startApp() throws Exception {
     // submit new app
@@ -200,8 +200,11 @@ Collections.<String, LocalResource> emptyMap(),
     // of testing.
     UserGroupInformation.setLoginUser(UserGroupInformation
       .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
-    appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
+
+    // emulate RM setup of AMRM token in credentials by adding the token
+    // *before* setting the token service
     UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+    appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
   }
   
   @After
@@ -675,21 +678,57 @@ public void testAskWithNodeLabels() {
     AMRMClientImpl<ContainerRequest> client =
         new AMRMClientImpl<ContainerRequest>();
 
-    // add x, y to ANY
+    // add exp=x to ANY
     client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
-        1), null, null, Priority.UNDEFINED, true, "x && y"));
+        1), null, null, Priority.UNDEFINED, true, "x"));
     Assert.assertEquals(1, client.ask.size());
-    Assert.assertEquals("x && y", client.ask.iterator().next()
+    Assert.assertEquals("x", client.ask.iterator().next()
         .getNodeLabelExpression());
 
-    // add x, y and a, b to ANY, only a, b should be kept
+    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
     client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
-        1), null, null, Priority.UNDEFINED, true, "x && y"));
+        1), null, null, Priority.UNDEFINED, true, "x"));
     client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
-        1), null, null, Priority.UNDEFINED, true, "a && b"));
+        1), null, null, Priority.UNDEFINED, true, "a"));
     Assert.assertEquals(1, client.ask.size());
-    Assert.assertEquals("a && b", client.ask.iterator().next()
+    Assert.assertEquals("a", client.ask.iterator().next()
         .getNodeLabelExpression());
+    
+    // add exp=x to ANY, rack and node, only resource request has ANY resource
+    // name will be assigned the label expression
+    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
+    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
+        1), null, null, Priority.UNDEFINED, true,
+        "y"));
+    Assert.assertEquals(1, client.ask.size());
+    for (ResourceRequest req : client.ask) {
+      if (ResourceRequest.ANY.equals(req.getResourceName())) {
+        Assert.assertEquals("y", req.getNodeLabelExpression());
+      } else {
+        Assert.assertNull(req.getNodeLabelExpression());
+      }
+    }
+  }
+  
+  private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
+      ContainerRequest request) {
+    try {
+      client.addContainerRequest(request);
+    } catch (InvalidContainerRequestException e) {
+      return;
+    }
+    Assert.fail();
+  }
+  
+  @Test(timeout=30000)
+  public void testAskWithInvalidNodeLabels() {
+    AMRMClientImpl<ContainerRequest> client =
+        new AMRMClientImpl<ContainerRequest>();
+
+    // specified exp with more than one node labels
+    verifyAddRequestFailed(client,
+        new ContainerRequest(Resource.newInstance(1024, 1), null, null,
+            Priority.UNDEFINED, true, "x && y"));
   }
     
   private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient)  
@@ -1026,13 +1065,18 @@ public ApplicationMasterProtocol run() {
         UserGroupInformation.getCurrentUser().getCredentials();
     Iterator<org.apache.hadoop.security.token.Token<?>> iter =
         credentials.getAllTokens().iterator();
+    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> result = null;
     while (iter.hasNext()) {
       org.apache.hadoop.security.token.Token<?> token = iter.next();
       if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
-        return (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>)
+        if (result != null) {
+          Assert.fail("credentials has more than one AMRM token."
+              + " token1: " + result + " token2: " + token);
+        }
+        result = (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>)
             token;
       }
     }
-    return null;
+    return result;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index 88dbf81d2afed..0d4a2714f8279 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -344,10 +344,11 @@ private void testContainerManagement(NMClientImpl nmClient,
         // getContainerStatus can be called after stopContainer
         try {
           // O is possible if CLEANUP_CONTAINER is executed too late
-          // 137 is possible if the container is not terminated but killed
+          // -105 is possible if the container is not terminated but killed
           testGetContainerStatus(container, i, ContainerState.COMPLETE,
               "Container killed by the ApplicationMaster.", Arrays.asList(
-                  new Integer[] {ContainerExitStatus.KILLED_BY_APPMASTER}));
+                  new Integer[] {ContainerExitStatus.KILLED_BY_APPMASTER,
+                  ContainerExitStatus.SUCCESS}));
         } catch (YarnException e) {
           // The exception is possible because, after the container is stopped,
           // it may be removed from NM's context.
@@ -383,7 +384,10 @@ private void testGetContainerStatus(Container container, int index,
           assertEquals(container.getId(), status.getContainerId());
           assertTrue("" + index + ": " + status.getDiagnostics(),
               status.getDiagnostics().contains(diagnostics));
-          assertTrue(exitStatuses.contains(status.getExitStatus()));
+          
+          assertTrue("Exit Statuses are supposed to be in: " + exitStatuses +
+              ", but the actual exit status code is: " + status.getExitStatus(),
+              exitStatuses.contains(status.getExitStatus()));
           break;
         }
         Thread.sleep(100);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
new file mode 100644
index 0000000000000..3985e5444767f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.DataOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ClientSCMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestSharedCacheClientImpl {
+
+  private static final Log LOG = LogFactory
+      .getLog(TestSharedCacheClientImpl.class);
+
+  public static SharedCacheClientImpl client;
+  public static ClientSCMProtocol cProtocol;
+  private static Path TEST_ROOT_DIR;
+  private static FileSystem localFs;
+  private static String input = "This is a test file.";
+  private static String inputChecksumSHA256 =
+      "f29bc64a9d3732b4b9035125fdb3285f5b6455778edca72414671e0ca3b2e0de";
+
+  @BeforeClass
+  public static void beforeClass() throws IOException {
+    localFs = FileSystem.getLocal(new Configuration());
+    TEST_ROOT_DIR =
+        new Path("target", TestSharedCacheClientImpl.class.getName()
+            + "-tmpDir").makeQualified(localFs.getUri(),
+            localFs.getWorkingDirectory());
+  }
+
+  @AfterClass
+  public static void afterClass() {
+    try {
+      if (localFs != null) {
+        localFs.close();
+      }
+    } catch (IOException ioe) {
+      LOG.info("IO exception in closing file system)");
+      ioe.printStackTrace();
+    }
+  }
+
+  @Before
+  public void setup() {
+    cProtocol = mock(ClientSCMProtocol.class);
+    client = new SharedCacheClientImpl() {
+      @Override
+      protected ClientSCMProtocol createClientProxy() {
+        return cProtocol;
+      }
+
+      @Override
+      protected void stopClientProxy() {
+        // do nothing because it is mocked
+      }
+    };
+    client.init(new Configuration());
+    client.start();
+  }
+
+  @After
+  public void cleanup() {
+    if (client != null) {
+      client.stop();
+      client = null;
+    }
+  }
+
+  @Test
+  public void testUse() throws Exception {
+    Path file = new Path("viewfs://test/path");
+    UseSharedCacheResourceResponse response =
+        new UseSharedCacheResourceResponsePBImpl();
+    response.setPath(file.toString());
+    when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
+        response);
+    Path newPath = client.use(mock(ApplicationId.class), "key");
+    assertEquals(file, newPath);
+  }
+
+  @Test(expected = YarnException.class)
+  public void testUseError() throws Exception {
+    String message = "Mock IOExcepiton!";
+    when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
+        new IOException(message));
+    client.use(mock(ApplicationId.class), "key");
+  }
+
+  @Test
+  public void testRelease() throws Exception {
+    // Release does not care about the return value because it is empty
+    when(cProtocol.release(isA(ReleaseSharedCacheResourceRequest.class)))
+        .thenReturn(null);
+    client.release(mock(ApplicationId.class), "key");
+  }
+
+  @Test(expected = YarnException.class)
+  public void testReleaseError() throws Exception {
+    String message = "Mock IOExcepiton!";
+    when(cProtocol.release(isA(ReleaseSharedCacheResourceRequest.class)))
+        .thenThrow(new IOException(message));
+    client.release(mock(ApplicationId.class), "key");
+  }
+
+  @Test
+  public void testChecksum() throws Exception {
+    String filename = "test1.txt";
+    Path file = makeFile(filename);
+    assertEquals(inputChecksumSHA256, client.getFileChecksum(file));
+  }
+
+  @Test(expected = FileNotFoundException.class)
+  public void testNonexistantFileChecksum() throws Exception {
+    Path file = new Path(TEST_ROOT_DIR, "non-existant-file");
+    client.getFileChecksum(file);
+  }
+
+  private Path makeFile(String filename) throws Exception {
+    Path file = new Path(TEST_ROOT_DIR, filename);
+    DataOutputStream out = null;
+    try {
+      out = localFs.create(file);
+      out.write(input.getBytes("UTF-8"));
+    } finally {
+      if(out != null) {
+        out.close();
+      }
+    }
+    return file;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 02f2882155541..73e06afd9b08b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -35,6 +35,7 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
@@ -91,11 +92,14 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AHSClient;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -338,6 +342,9 @@ public void testGetApplicationAttempt() throws YarnException, IOException {
   @Test(timeout = 10000)
   public void testGetContainers() throws YarnException, IOException {
     Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+        true);
+    
     final YarnClient client = new MockYarnClient();
     client.init(conf);
     client.start();
@@ -351,12 +358,25 @@ public void testGetContainers() throws YarnException, IOException {
         (ContainerId.newContainerId(appAttemptId, 1)));
     Assert.assertEquals(reports.get(1).getContainerId(),
         (ContainerId.newContainerId(appAttemptId, 2)));
+    Assert.assertEquals(reports.get(2).getContainerId(),
+        (ContainerId.newContainerId(appAttemptId, 3)));
+    
+    //First2 containers should come from RM with updated state information and 
+    // 3rd container is not there in RM and should
+    Assert.assertEquals(ContainerState.RUNNING,
+        (reports.get(0).getContainerState()));
+    Assert.assertEquals(ContainerState.RUNNING,
+        (reports.get(1).getContainerState()));
+    Assert.assertEquals(ContainerState.COMPLETE,
+        (reports.get(2).getContainerState()));
     client.stop();
   }
 
   @Test(timeout = 10000)
   public void testGetContainerReport() throws YarnException, IOException {
     Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+        true);
     final YarnClient client = new MockYarnClient();
     client.init(conf);
     client.start();
@@ -373,6 +393,12 @@ public void testGetContainerReport() throws YarnException, IOException {
     Assert.assertEquals(report.getContainerId().toString(),
         (ContainerId.newContainerId(expectedReports.get(0)
             .getCurrentApplicationAttemptId(), 1)).toString());
+    containerId = ContainerId.newContainerId(appAttemptId, 3);
+    report = client.getContainerReport(containerId);
+    Assert.assertNotNull(report);
+    Assert.assertEquals(report.getContainerId().toString(),
+        (ContainerId.newContainerId(expectedReports.get(0)
+            .getCurrentApplicationAttemptId(), 3)).toString());
     client.stop();
   }
 
@@ -383,6 +409,9 @@ private static class MockYarnClient extends YarnClientImpl {
       new HashMap<ApplicationId, List<ApplicationAttemptReport>>();
     private HashMap<ApplicationAttemptId, List<ContainerReport>> containers = 
       new HashMap<ApplicationAttemptId, List<ContainerReport>>();
+    private HashMap<ApplicationAttemptId, List<ContainerReport>> containersFromAHS =
+      new HashMap<ApplicationAttemptId, List<ContainerReport>>();
+
     GetApplicationsResponse mockAppResponse =
       mock(GetApplicationsResponse.class);
     GetApplicationAttemptsResponse mockAppAttemptsResponse = 
@@ -428,6 +457,9 @@ public void start() {
 
         when(rmClient.getContainerReport(any(GetContainerReportRequest.class)))
             .thenReturn(mockContainerResponse);
+        
+        historyClient = mock(AHSClient.class);
+        
       } catch (YarnException e) {
         Assert.fail("Exception is not expected.");
       } catch (IOException e) {
@@ -501,15 +533,37 @@ private List<ApplicationReport> createAppReports() {
       ContainerReport container = ContainerReport.newInstance(
           ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
           NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
-          "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+          "diagnosticInfo", "logURL", 0, ContainerState.RUNNING);
       containerReports.add(container);
 
       ContainerReport container1 = ContainerReport.newInstance(
           ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
           NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
-          "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+          "diagnosticInfo", "logURL", 0, ContainerState.RUNNING);
       containerReports.add(container1);
       containers.put(attempt.getApplicationAttemptId(), containerReports);
+      
+      //add containers to be sent from AHS
+      List<ContainerReport> containerReportsForAHS =
+          new ArrayList<ContainerReport>();
+      
+      container = ContainerReport.newInstance(
+          ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
+          NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+          "diagnosticInfo", "logURL", 0, null);
+      containerReportsForAHS.add(container);
+
+      container1 = ContainerReport.newInstance(
+          ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
+          NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+          "diagnosticInfo", "HSlogURL", 0, null);
+      containerReportsForAHS.add(container1);
+      ContainerReport container2 = ContainerReport.newInstance(
+          ContainerId.newContainerId(attempt.getApplicationAttemptId(),3), null,
+          NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+          "diagnosticInfo", "HSlogURL", 0, ContainerState.COMPLETE);
+      containerReportsForAHS.add(container2);
+      containersFromAHS.put(attempt.getApplicationAttemptId(), containerReportsForAHS);
 
       ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
       ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
@@ -586,14 +640,36 @@ public ApplicationAttemptReport getApplicationAttemptReport(
             IOException {
       when(mockContainersResponse.getContainerList()).thenReturn(
         getContainersReport(appAttemptId));
+      when(historyClient.getContainers(any(ApplicationAttemptId.class)))
+      .thenReturn(getContainersFromAHS(appAttemptId));
       return super.getContainers(appAttemptId);
     }
 
+    private List<ContainerReport> getContainersFromAHS(
+        ApplicationAttemptId appAttemptId) {
+      return containersFromAHS.get(appAttemptId);
+    }
+
     @Override
     public ContainerReport getContainerReport(ContainerId containerId)
         throws YarnException, IOException {
-      when(mockContainerResponse.getContainerReport()).thenReturn(
-        getContainer(containerId));
+      try {
+        ContainerReport container = getContainer(containerId, containers);
+        when(mockContainerResponse.getContainerReport()).thenReturn(container);
+      } catch (YarnException e) {
+        when(rmClient.getContainerReport(any(GetContainerReportRequest.class)))
+        .thenThrow(e).thenReturn(mockContainerResponse);
+      }
+      try {
+        ContainerReport container =
+            getContainer(containerId, containersFromAHS);
+        when(historyClient.getContainerReport(any(ContainerId.class)))
+            .thenReturn(container);
+      } catch (YarnException e) {
+        when(historyClient.getContainerReport(any(ContainerId.class)))
+            .thenThrow(e);
+      }
+
       return super.getContainerReport(containerId);
     }
     
@@ -611,8 +687,25 @@ public List<ContainerReport> getContainersReport(
       return containers.get(appAttemptId);
     }
 
-    public ContainerReport getContainer(ContainerId containerId) {
-      return containers.get(containerId.getApplicationAttemptId()).get(0);
+    private ContainerReport getContainer(
+        ContainerId containerId,
+        HashMap<ApplicationAttemptId, List<ContainerReport>> containersToAppAttemptMapping)
+        throws YarnException, IOException {
+      List<ContainerReport> containersForAppAttempt =
+          containersToAppAttemptMapping.get(containerId
+              .getApplicationAttemptId());
+      if (containersForAppAttempt == null) {
+        throw new ApplicationNotFoundException(containerId
+            .getApplicationAttemptId().getApplicationId() + " is not found ");
+      }
+      Iterator<ContainerReport> iterator = containersForAppAttempt.iterator();
+      while (iterator.hasNext()) {
+        ContainerReport next = iterator.next();
+        if (next.getContainerId().equals(containerId)) {
+          return next;
+        }
+      }
+      throw new ContainerNotFoundException(containerId + " is not found ");
     }
   }
 
@@ -760,6 +853,42 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
     }
   }
 
+  @Test
+  public void testBestEffortTimelineDelegationToken()
+      throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+
+    YarnClientImpl client = spy(new YarnClientImpl() {
+
+      @Override
+      TimelineClient createTimelineClient() throws IOException, YarnException {
+        timelineClient = mock(TimelineClient.class);
+        when(timelineClient.getDelegationToken(any(String.class)))
+          .thenThrow(new IOException("Best effort test exception"));
+        return timelineClient;
+      }
+    });
+
+    client.init(conf);
+    try {
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, true);
+      client.serviceInit(conf);
+      client.getTimelineDelegationToken();
+    } catch (Exception e) {
+      Assert.fail("Should not have thrown an exception");
+    }
+    try {
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false);
+      client.serviceInit(conf);
+      client.getTimelineDelegationToken();
+      Assert.fail("Get delegation token should have thrown an exception");
+    } catch (Exception e) {
+      // Success
+    }
+  }
+
   @Test
   public void testAutomaticTimelineDelegationTokenLoading()
       throws Exception {
@@ -771,22 +900,18 @@ public void testAutomaticTimelineDelegationTokenLoading()
     final Token<TimelineDelegationTokenIdentifier> dToken =
         new Token<TimelineDelegationTokenIdentifier>(
             timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
-    // crate a mock client
+    // create a mock client
     YarnClientImpl client = spy(new YarnClientImpl() {
+
       @Override
-      protected void serviceInit(Configuration conf) throws Exception {
-        if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
-            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-          timelineServiceEnabled = true;
-          timelineClient = mock(TimelineClient.class);
-          when(timelineClient.getDelegationToken(any(String.class)))
-              .thenReturn(dToken);
-          timelineClient.init(getConfig());
-          timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
-        }
-        this.setConfig(conf);
+      TimelineClient createTimelineClient() throws IOException, YarnException {
+        timelineClient = mock(TimelineClient.class);
+        when(timelineClient.getDelegationToken(any(String.class)))
+          .thenReturn(dToken);
+        return timelineClient;
       }
 
+
       @Override
       protected void serviceStart() throws Exception {
         rmClient = mock(ApplicationClientProtocol.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 6176a3e39c3cc..d4480906d27cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -73,7 +73,6 @@ public class TestRMAdminCLI {
   @Before
   public void configure() throws IOException, YarnException {
     remoteAdminServiceAccessed = false;
-    dummyNodeLabelsManager = new DummyCommonNodeLabelsManager();
     admin = mock(ResourceManagerAdministrationProtocol.class);
     when(admin.addToClusterNodeLabels(any(AddToClusterNodeLabelsRequest.class)))
         .thenAnswer(new Answer<AddToClusterNodeLabelsResponse>() {
@@ -105,6 +104,7 @@ protected HAServiceTarget resolveTarget(String rmId) {
         return haServiceTarget;
       }
     };
+    initDummyNodeLabelsManager();
     rmAdminCLI.localNodeLabelsManager = dummyNodeLabelsManager;
 
     YarnConfiguration conf = new YarnConfiguration();
@@ -124,6 +124,13 @@ protected HAServiceTarget resolveTarget(String rmId) {
     };
   }
   
+  private void initDummyNodeLabelsManager() {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    dummyNodeLabelsManager = new DummyCommonNodeLabelsManager();
+    dummyNodeLabelsManager.init(conf);
+  }
+  
   @Test(timeout=500)
   public void testRefreshQueues() throws Exception {
     String[] args = { "-refreshQueues" };
@@ -279,7 +286,10 @@ public void testHelp() throws Exception {
               "yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshSuper" +
               "UserGroupsConfiguration] [-refreshUserToGroupsMappings] " +
               "[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup" +
-              " [username]] [-help [cmd]]"));
+              " [username]] [[-addToClusterNodeLabels [label1,label2,label3]]" +
+              " [-removeFromClusterNodeLabels [label1,label2,label3]] [-replaceLabelsOnNode " +
+              "[node1[:port]=label1,label2 node2[:port]=label1] [-directlyAccessNodeLabelStore]] " +
+              "[-help [cmd]]"));
       assertTrue(dataOut
           .toString()
           .contains(
@@ -330,8 +340,8 @@ public void testHelp() throws Exception {
       testError(new String[] { "-help", "-getGroups" },
           "Usage: yarn rmadmin [-getGroups [username]]", dataErr, 0);
       testError(new String[] { "-help", "-transitionToActive" },
-          "Usage: yarn rmadmin [-transitionToActive <serviceId>" +
-          " [--forceactive]]", dataErr, 0);
+          "Usage: yarn rmadmin [-transitionToActive [--forceactive]" +
+          " <serviceId>]", dataErr, 0);
       testError(new String[] { "-help", "-transitionToStandby" },
           "Usage: yarn rmadmin [-transitionToStandby <serviceId>]", dataErr, 0);
       testError(new String[] { "-help", "-getServiceState" },
@@ -352,16 +362,21 @@ public void testHelp() throws Exception {
       // Test -help when RM HA is enabled
       assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
       oldOutPrintStream.println(dataOut);
-      assertTrue(dataOut
-          .toString()
-          .contains(
-              "yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshSuper" +
+      String expectedHelpMsg = 
+          "yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshSuper" +
               "UserGroupsConfiguration] [-refreshUserToGroupsMappings] " +
               "[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup" +
-              " [username]] [-help [cmd]] [-transitionToActive <serviceId>" + 
-              " [--forceactive]] [-transitionToStandby <serviceId>] [-failover" +
+              " [username]] [[-addToClusterNodeLabels [label1,label2,label3]]" +
+              " [-removeFromClusterNodeLabels [label1,label2,label3]] [-replaceLabelsOnNode " +
+              "[node1[:port]=label1,label2 node2[:port]=label1] [-directlyAccessNodeLabelStore]] " +
+              "[-transitionToActive [--forceactive] <serviceId>] " + 
+              "[-transitionToStandby <serviceId>] [-failover" +
               " [--forcefence] [--forceactive] <serviceId> <serviceId>] " +
-              "[-getServiceState <serviceId>] [-checkHealth <serviceId>]"));
+              "[-getServiceState <serviceId>] [-checkHealth <serviceId>] [-help [cmd]]";
+      String actualHelpMsg = dataOut.toString();
+      assertTrue(String.format("Help messages: %n " + actualHelpMsg + " %n doesn't include expected " +
+          "messages: %n" + expectedHelpMsg), actualHelpMsg.contains(expectedHelpMsg
+              ));
     } finally {
       System.setOut(oldOutPrintStream);
       System.setErr(oldErrPrintStream);
@@ -443,14 +458,31 @@ public void testAddToClusterNodeLabels() throws Exception {
         new String[] { "-addToClusterNodeLabels",
             "-directlyAccessNodeLabelStore" };
     assertTrue(0 != rmAdminCLI.run(args));
+
+    // no labels, should fail at client validation
+    args = new String[] { "-addToClusterNodeLabels", " " };
+    assertTrue(0 != rmAdminCLI.run(args));
+
+    // no labels, should fail at client validation
+    args = new String[] { "-addToClusterNodeLabels", " , " };
+    assertTrue(0 != rmAdminCLI.run(args));
+
+    // successfully add labels
+    args =
+        new String[] { "-addToClusterNodeLabels", ",x,,",
+            "-directlyAccessNodeLabelStore" };
+    assertEquals(0, rmAdminCLI.run(args));
+    assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll(
+        ImmutableSet.of("x")));
   }
   
   @Test
   public void testRemoveFromClusterNodeLabels() throws Exception {
     // Successfully remove labels
-    dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x"));
+    dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
     String[] args =
-        { "-removeFromClusterNodeLabels", "x", "-directlyAccessNodeLabelStore" };
+        { "-removeFromClusterNodeLabels", "x,,y",
+            "-directlyAccessNodeLabelStore" };
     assertEquals(0, rmAdminCLI.run(args));
     assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().isEmpty());
     
@@ -463,50 +495,70 @@ public void testRemoveFromClusterNodeLabels() throws Exception {
         new String[] { "-removeFromClusterNodeLabels",
             "-directlyAccessNodeLabelStore" };
     assertTrue(0 != rmAdminCLI.run(args));
+
+    // no labels, should fail at client validation
+    args = new String[] { "-removeFromClusterNodeLabels", " " };
+    assertTrue(0 != rmAdminCLI.run(args));
+
+    // no labels, should fail at client validation
+    args = new String[] { "-removeFromClusterNodeLabels", ", " };
+    assertTrue(0 != rmAdminCLI.run(args));
   }
   
   @Test
   public void testReplaceLabelsOnNode() throws Exception {
     // Successfully replace labels
-    dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
+    dummyNodeLabelsManager
+        .addToCluserNodeLabels(ImmutableSet.of("x", "y", "Y"));
     String[] args =
-        { "-replaceLabelsOnNode", "node1,x,y node2,y",
+        { "-replaceLabelsOnNode",
+            "node1:8000,x node2:8000=y node3,x node4=Y",
             "-directlyAccessNodeLabelStore" };
     assertEquals(0, rmAdminCLI.run(args));
     assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
-        NodeId.newInstance("node1", 0)));
+        NodeId.newInstance("node1", 8000)));
     assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
-        NodeId.newInstance("node2", 0)));
-    
+        NodeId.newInstance("node2", 8000)));
+    assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
+        NodeId.newInstance("node3", 0)));
+    assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
+        NodeId.newInstance("node4", 0)));
+
     // no labels, should fail
     args = new String[] { "-replaceLabelsOnNode" };
     assertTrue(0 != rmAdminCLI.run(args));
-    
+
     // no labels, should fail
     args =
-        new String[] { "-replaceLabelsOnNode",
-            "-directlyAccessNodeLabelStore" };
+        new String[] { "-replaceLabelsOnNode", "-directlyAccessNodeLabelStore" };
+    assertTrue(0 != rmAdminCLI.run(args));
+
+    // no labels, should fail
+    args = new String[] { "-replaceLabelsOnNode", " " };
     assertTrue(0 != rmAdminCLI.run(args));
-  }
 
+    args = new String[] { "-replaceLabelsOnNode", ", " };
+    assertTrue(0 != rmAdminCLI.run(args));
+  }
+  
   @Test
-  public void testReplaceLabelsOnNodeWithPort() throws Exception {
+  public void testReplaceMultipleLabelsOnSingleNode() throws Exception {
     // Successfully replace labels
     dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
     String[] args =
-      { "-replaceLabelsOnNode", "node1:8000,x,y node2:8000,y",
-      "-directlyAccessNodeLabelStore" };
-    assertEquals(0, rmAdminCLI.run(args));
-    assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
-        NodeId.newInstance("node1", 8000)));
-    assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
-        NodeId.newInstance("node2", 8000)));
+        { "-replaceLabelsOnNode", "node1,x,y",
+            "-directlyAccessNodeLabelStore" };
+    assertTrue(0 != rmAdminCLI.run(args));
   }
 
   private void testError(String[] args, String template,
       ByteArrayOutputStream data, int resultCode) throws Exception {
-    assertEquals(resultCode, rmAdminCLI.run(args));
-    assertTrue(data.toString().contains(template));
+    int actualResultCode = rmAdminCLI.run(args);
+    assertEquals("Expected result code: " + resultCode + 
+        ", actual result code is: " + actualResultCode, resultCode, actualResultCode);
+    assertTrue(String.format("Expected error message: %n" + template + 
+        " is not included in messages: %n" + data.toString()), 
+        data.toString().contains(template));
     data.reset();
   }
   
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 194d7d19ec57e..fa81f14eb6407 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -62,7 +62,9 @@
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.Assert;
 import org.junit.Before;
@@ -319,14 +321,12 @@ public void testGetApplicationReportException() throws Exception {
     when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
         new ApplicationNotFoundException("History file for application"
             + applicationId + " is not found"));
-    try {
-      cli.run(new String[] { "application", "-status", applicationId.toString() });
-      Assert.fail();
-    } catch (Exception ex) {
-      Assert.assertTrue(ex instanceof ApplicationNotFoundException);
-      Assert.assertEquals("History file for application"
-          + applicationId + " is not found", ex.getMessage());
-    }
+    int exitCode = cli.run(new String[] { "application", "-status",
+        applicationId.toString() });
+    verify(sysOut).println(
+        "Application with id '" + applicationId
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
   }
 
   @Test
@@ -1318,6 +1318,80 @@ public void testGetQueueInfoWithNonExistedQueue() throws Exception {
     Assert.assertEquals(queueInfoStr, sysOutStream.toString());
   }
 
+  @Test
+  public void testGetApplicationAttemptReportException() throws Exception {
+    ApplicationCLI cli = createAndGetAppCLI();
+    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+    ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(
+        applicationId, 1);
+    when(client.getApplicationAttemptReport(attemptId1)).thenThrow(
+        new ApplicationNotFoundException("History file for application"
+            + applicationId + " is not found"));
+
+    int exitCode = cli.run(new String[] { "applicationattempt", "-status",
+        attemptId1.toString() });
+    verify(sysOut).println(
+        "Application for AppAttempt with id '" + attemptId1
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
+
+    ApplicationAttemptId attemptId2 = ApplicationAttemptId.newInstance(
+        applicationId, 2);
+    when(client.getApplicationAttemptReport(attemptId2)).thenThrow(
+        new ApplicationAttemptNotFoundException(
+            "History file for application attempt" + attemptId2
+                + " is not found"));
+
+    exitCode = cli.run(new String[] { "applicationattempt", "-status",
+        attemptId2.toString() });
+    verify(sysOut).println(
+        "Application Attempt with id '" + attemptId2
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
+  }
+
+  @Test
+  public void testGetContainerReportException() throws Exception {
+    ApplicationCLI cli = createAndGetAppCLI();
+    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+        applicationId, 1);
+    long cntId = 1;
+    ContainerId containerId1 = ContainerId.newContainerId(attemptId, cntId++);
+    when(client.getContainerReport(containerId1)).thenThrow(
+        new ApplicationNotFoundException("History file for application"
+            + applicationId + " is not found"));
+
+    int exitCode = cli.run(new String[] { "container", "-status",
+        containerId1.toString() });
+    verify(sysOut).println(
+        "Application for Container with id '" + containerId1
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
+    ContainerId containerId2 = ContainerId.newContainerId(attemptId, cntId++);
+    when(client.getContainerReport(containerId2)).thenThrow(
+        new ApplicationAttemptNotFoundException(
+            "History file for application attempt" + attemptId
+                + " is not found"));
+
+    exitCode = cli.run(new String[] { "container", "-status",
+        containerId2.toString() });
+    verify(sysOut).println(
+        "Application Attempt for Container with id '" + containerId2
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
+
+    ContainerId containerId3 = ContainerId.newContainerId(attemptId, cntId++);
+    when(client.getContainerReport(containerId3)).thenThrow(
+        new ContainerNotFoundException("History file for container"
+            + containerId3 + " is not found"));
+    exitCode = cli.run(new String[] { "container", "-status",
+        containerId3.toString() });
+    verify(sysOut).println(
+        "Container with id '" + containerId3
+            + "' doesn't exist in RM or Timeline Server.");
+    Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
+  }
 
   private void verifyUsageInfo(YarnCLI cli) throws Exception {
     cli.setSysErrPrintStream(sysErr);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 3adfe8b788174..6bc5b71d4af30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -142,6 +142,11 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.sun.jersey.jersey-test-framework</groupId>
+      <artifactId>jersey-test-framework-grizzly2</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
@@ -208,6 +213,7 @@
             <exclude>src/main/resources/webapps/node/.keep</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.9.4/css/jui-dt.css</exclude>
             <exclude>src/main/resources/webapps/static/dt-1.9.4/css/demo_table.css</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.9.4/images/Sorting icons.psd</exclude>
             <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
           </excludes>
         </configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index 32dc85d6f38fd..a95aadfbef834 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -229,11 +229,11 @@ public void setApplicationACLs(
   
   @Override
   public void setClientToAMTokenMasterKey(ByteBuffer key) {
+    maybeInitBuilder();
     if (key == null) {
       builder.clearClientToAmTokenMasterKey();
       return;
     }
-    maybeInitBuilder();
     builder.setClientToAmTokenMasterKey(ByteString.copyFrom(key));
   }
   
@@ -316,6 +316,7 @@ public List<NMToken> getNMTokensFromPreviousAttempts() {
   
   @Override
   public void setNMTokensFromPreviousAttempts(final List<NMToken> nmTokens) {
+    maybeInitBuilder();
     if (nmTokens == null || nmTokens.isEmpty()) {
       if (this.nmTokens != null) {
         this.nmTokens.clear();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
index c8abdbe587e66..b4d92afeebea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
@@ -68,7 +68,7 @@ public void setPriority(int priority) {
   
   @Override
   public String toString() {
-    return Integer.valueOf(getPriority()).toString();
+    return Integer.toString(getPriority());
   }
 
 }  
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 605d60b3b8e9c..de9d8da766dd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -136,9 +136,13 @@ static class TimelineClientConnectionRetry {
 
     // Indicates if retries happened last time. Only tests should read it.
     // In unit tests, retryOn() calls should _not_ be concurrent.
+    private boolean retried = false;
+
     @Private
     @VisibleForTesting
-    public boolean retried = false;
+    boolean getRetired() {
+      return retried;
+    }
 
     // Constructor with default retry settings
     public TimelineClientConnectionRetry(Configuration conf) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 370b0f7324f15..d36d841772a96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -34,6 +34,8 @@
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Dispatches {@link Event}s in a separate thread. Currently only single thread
  * does that. Potentially there could be multiple channels for each event type
@@ -181,8 +183,9 @@ protected void dispatch(Event event) {
       if (exitOnDispatchException
           && (ShutdownHookManager.get().isShutdownInProgress()) == false
           && stopped == false) {
-        LOG.info("Exiting, bbye..");
-        System.exit(-1);
+        Thread shutDownThread = new Thread(createShutDownThread());
+        shutDownThread.setName("AsyncDispatcher ShutDown handler");
+        shutDownThread.start();
       }
     }
   }
@@ -271,4 +274,19 @@ void addHandler(EventHandler<Event> handler) {
     }
 
   }
+
+  Runnable createShutDownThread() {
+    return new Runnable() {
+      @Override
+      public void run() {
+        LOG.info("Exiting, bbye..");
+        System.exit(-1);
+      }
+    };
+  }
+
+  @VisibleForTesting
+  protected boolean isDrained() {
+    return this.drained;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 7eebcb334ffd7..b6693326eaef6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -30,6 +30,7 @@
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.Writer;
+import java.nio.charset.Charset;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -263,7 +264,7 @@ public void write(DataOutputStream out, Set<File> pendingUploadFiles)
           this.uploadedFiles.add(logFile);
         } catch (IOException e) {
           String message = logErrorMessage(logFile, e);
-          out.write(message.getBytes());
+          out.write(message.getBytes(Charset.forName("UTF-8")));
         } finally {
           IOUtils.cleanup(LOG, in);
         }
@@ -651,7 +652,7 @@ public static void readAcontainerLogs(DataInputStream valueStream,
       OutputStream os = null;
       PrintStream ps = null;
       try {
-        os = new WriterOutputStream(writer);
+        os = new WriterOutputStream(writer, Charset.forName("UTF-8"));
         ps = new PrintStream(os);
         while (true) {
           try {
@@ -781,7 +782,8 @@ public String nextLog() throws IOException {
         currentLogData =
             new BoundedInputStream(valueStream, currentLogLength);
         currentLogData.setPropagateClose(false);
-        currentLogISR = new InputStreamReader(currentLogData);
+        currentLogISR = new InputStreamReader(currentLogData,
+            Charset.forName("UTF-8"));
         currentLogType = logType;
       } catch (EOFException e) {
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 1546ece9b99ee..df9bd3293d445 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -212,11 +212,11 @@ private static void containerLogNotFound(String containerId) {
   }
 
   private static void logDirNotExist(String remoteAppLogDir) {
-    System.out.println(remoteAppLogDir + "does not exist.");
+    System.out.println(remoteAppLogDir + " does not exist.");
     System.out.println("Log aggregation has not completed or is not enabled.");
   }
 
   private static void emptyLogDir(String remoteAppLogDir) {
-    System.out.println(remoteAppLogDir + "does not have any log files.");
+    System.out.println(remoteAppLogDir + " does not have any log files.");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index daefe8d33f08d..25ff417320090 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -40,17 +41,20 @@
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
 import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEvent;
 import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType;
 import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels;
+import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
 import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 
 public class CommonNodeLabelsManager extends AbstractService {
@@ -63,6 +67,14 @@ public class CommonNodeLabelsManager extends AbstractService {
   private static final Pattern LABEL_PATTERN = Pattern
       .compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
   public static final int WILDCARD_PORT = 0;
+  
+  /**
+   * Error messages
+   */
+  @VisibleForTesting
+  public static final String NODE_LABELS_NOT_ENABLED_ERR =
+      "Node-label-based scheduling is disabled. Please check "
+          + YarnConfiguration.NODE_LABELS_ENABLED;
 
   /**
    * If a user doesn't specify label of a queue or node, it belongs
@@ -72,8 +84,8 @@ public class CommonNodeLabelsManager extends AbstractService {
 
   protected Dispatcher dispatcher;
 
-  protected ConcurrentMap<String, Label> labelCollections =
-      new ConcurrentHashMap<String, Label>();
+  protected ConcurrentMap<String, NodeLabel> labelCollections =
+      new ConcurrentHashMap<String, NodeLabel>();
   protected ConcurrentMap<String, Host> nodeCollections =
       new ConcurrentHashMap<String, Host>();
 
@@ -81,14 +93,7 @@ public class CommonNodeLabelsManager extends AbstractService {
   protected final WriteLock writeLock;
 
   protected NodeLabelsStore store;
-
-  protected static class Label {
-    public Resource resource;
-
-    protected Label() {
-      this.resource = Resource.newInstance(0, 0);
-    }
-  }
+  private boolean nodeLabelsEnabled = false;
 
   /**
    * A <code>Host</code> can have multiple <code>Node</code>s 
@@ -117,15 +122,17 @@ protected static class Node {
     public Set<String> labels;
     public Resource resource;
     public boolean running;
+    public NodeId nodeId;
     
-    protected Node() {
+    protected Node(NodeId nodeid) {
       labels = null;
       resource = Resource.newInstance(0, 0);
       running = false;
+      nodeId = nodeid;
     }
     
     public Node copy() {
-      Node c = new Node();
+      Node c = new Node(nodeId);
       if (labels != null) {
         c.labels =
             Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
@@ -138,6 +145,12 @@ public Node copy() {
       return c;
     }
   }
+  
+  private enum NodeLabelUpdateOperation {
+    ADD,
+    REMOVE,
+    REPLACE
+  }
 
   private final class ForwardingEventHandler implements
       EventHandler<NodeLabelsStoreEvent> {
@@ -194,9 +207,15 @@ protected void initDispatcher(Configuration conf) {
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
-    initNodeLabelStore(conf);
+    // set if node labels enabled
+    nodeLabelsEnabled =
+        conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED,
+            YarnConfiguration.DEFAULT_NODE_LABELS_ENABLED);
+    if (nodeLabelsEnabled) {
+      initNodeLabelStore(conf);
+    }
     
-    labelCollections.put(NO_LABEL, new Label());
+    labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
   }
 
   protected void initNodeLabelStore(Configuration conf) throws Exception {
@@ -251,6 +270,10 @@ protected void serviceStop() throws Exception {
    */
   @SuppressWarnings("unchecked")
   public void addToCluserNodeLabels(Set<String> labels) throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
     if (null == labels || labels.isEmpty()) {
       return;
     }
@@ -266,7 +289,7 @@ public void addToCluserNodeLabels(Set<String> labels) throws IOException {
     for (String label : labels) {
       // shouldn't overwrite it to avoid changing the Label.resource
       if (this.labelCollections.get(label) == null) {
-        this.labelCollections.put(label, new Label());
+        this.labelCollections.put(label, new NodeLabel(label));
         newLabels.add(label);
       }
     }
@@ -287,54 +310,36 @@ protected void checkAddLabelsToNode(
     // check all labels being added existed
     Set<String> knownLabels = labelCollections.keySet();
     for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) {
-      if (!knownLabels.containsAll(entry.getValue())) {
+      NodeId nodeId = entry.getKey();
+      Set<String> labels = entry.getValue();
+      
+      if (!knownLabels.containsAll(labels)) {
         String msg =
             "Not all labels being added contained by known "
                 + "label collections, please check" + ", added labels=["
-                + StringUtils.join(entry.getValue(), ",") + "]";
+                + StringUtils.join(labels, ",") + "]";
         LOG.error(msg);
         throw new IOException(msg);
       }
-    }
-  }
-  
-  @SuppressWarnings("unchecked")
-  protected void internalAddLabelsToNode(
-      Map<NodeId, Set<String>> addedLabelsToNode) throws IOException {
-    // do add labels to nodes
-    Map<NodeId, Set<String>> newNMToLabels =
-        new HashMap<NodeId, Set<String>>();
-    for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) {
-      NodeId nodeId = entry.getKey();
-      Set<String> labels = entry.getValue();
- 
-      createHostIfNonExisted(nodeId.getHost());
-      if (nodeId.getPort() == WILDCARD_PORT) {
-        Host host = nodeCollections.get(nodeId.getHost());
-        host.labels.addAll(labels);
-        newNMToLabels.put(nodeId, host.labels);
-      } else {
-        createNodeIfNonExisted(nodeId);
-        Node nm = getNMInNodeSet(nodeId);
-        if (nm.labels == null) {
-          nm.labels = new HashSet<String>();
+      
+      // In YARN-2694, we temporarily disable user add more than 1 labels on a
+      // same host
+      if (!labels.isEmpty()) {
+        Set<String> newLabels = new HashSet<String>(getLabelsByNode(nodeId));
+        newLabels.addAll(labels);
+        // we don't allow number of labels on a node > 1 after added labels
+        if (newLabels.size() > 1) {
+          String msg =
+              String.format(
+                      "%d labels specified on host=%s after add labels to node"
+                          + ", please note that we do not support specifying multiple"
+                          + " labels on a single host for now.",
+                      newLabels.size(), nodeId.getHost());
+          LOG.error(msg);
+          throw new IOException(msg);
         }
-        nm.labels.addAll(labels);
-        newNMToLabels.put(nodeId, nm.labels);
       }
     }
-
-    if (null != dispatcher) {
-      dispatcher.getEventHandler().handle(
-          new UpdateNodeToLabelsMappingsEvent(newNMToLabels));
-    }
-
-    // shows node->labels we added
-    LOG.info("addLabelsToNode:");
-    for (Entry<NodeId, Set<String>> entry : newNMToLabels.entrySet()) {
-      LOG.info("  NM=" + entry.getKey() + ", labels=["
-          + StringUtils.join(entry.getValue().iterator(), ",") + "]");
-    }
   }
   
   /**
@@ -344,9 +349,13 @@ protected void internalAddLabelsToNode(
    */
   public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
     addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
     checkAddLabelsToNode(addedLabelsToNode);
-    internalAddLabelsToNode(addedLabelsToNode);
+    internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD);
   }
   
   protected void checkRemoveFromClusterNodeLabels(
@@ -410,6 +419,11 @@ protected void internalRemoveFromClusterNodeLabels(Collection<String> labelsToRe
    */
   public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     labelsToRemove = normalizeLabels(labelsToRemove);
     
     checkRemoveFromClusterNodeLabels(labelsToRemove);
@@ -475,26 +489,111 @@ protected void checkRemoveLabelsFromNode(
       }
     }
   }
-  
+
+  private void addNodeToLabels(NodeId node, Set<String> labels) {
+    for(String l : labels) {
+      labelCollections.get(l).addNodeId(node);
+    }
+  }
+
+  private void removeNodeFromLabels(NodeId node, Set<String> labels) {
+    for(String l : labels) {
+      labelCollections.get(l).removeNodeId(node);
+    }
+  }
+
+  private void replaceNodeForLabels(NodeId node, Set<String> oldLabels,
+      Set<String> newLabels) {
+    if(oldLabels != null) {
+      removeNodeFromLabels(node, oldLabels);
+    }
+    addNodeToLabels(node, newLabels);
+  }
+
   @SuppressWarnings("unchecked")
-  protected void internalRemoveLabelsFromNode(
-      Map<NodeId, Set<String>> removeLabelsFromNode) {
-    // do remove labels from nodes
+  protected void internalUpdateLabelsOnNodes(
+      Map<NodeId, Set<String>> nodeToLabels, NodeLabelUpdateOperation op)
+      throws IOException {
+    // do update labels from nodes
     Map<NodeId, Set<String>> newNMToLabels =
         new HashMap<NodeId, Set<String>>();
-    for (Entry<NodeId, Set<String>> entry : removeLabelsFromNode.entrySet()) {
+    Set<String> oldLabels;
+    for (Entry<NodeId, Set<String>> entry : nodeToLabels.entrySet()) {
       NodeId nodeId = entry.getKey();
       Set<String> labels = entry.getValue();
       
+      createHostIfNonExisted(nodeId.getHost());
       if (nodeId.getPort() == WILDCARD_PORT) {
         Host host = nodeCollections.get(nodeId.getHost());
-        host.labels.removeAll(labels);
+        switch (op) {
+        case REMOVE:
+          removeNodeFromLabels(nodeId, labels);
+          host.labels.removeAll(labels);
+          for (Node node : host.nms.values()) {
+            if (node.labels != null) {
+              node.labels.removeAll(labels);
+            }
+            removeNodeFromLabels(node.nodeId, labels);
+          }
+          break;
+        case ADD:
+          addNodeToLabels(nodeId, labels);
+          host.labels.addAll(labels);
+          for (Node node : host.nms.values()) {
+            if (node.labels != null) {
+              node.labels.addAll(labels);
+            }
+            addNodeToLabels(node.nodeId, labels);
+          }
+          break;
+        case REPLACE:
+          replaceNodeForLabels(nodeId, host.labels, labels);
+          host.labels.clear();
+          host.labels.addAll(labels);
+          for (Node node : host.nms.values()) {
+            replaceNodeForLabels(node.nodeId, node.labels, labels);
+            node.labels = null;
+          }
+          break;
+        default:
+          break;
+        }
         newNMToLabels.put(nodeId, host.labels);
       } else {
-        Node nm = getNMInNodeSet(nodeId);
-        if (nm.labels != null) {
-          nm.labels.removeAll(labels);
+        if (EnumSet.of(NodeLabelUpdateOperation.ADD,
+            NodeLabelUpdateOperation.REPLACE).contains(op)) {
+          // Add and replace
+          createNodeIfNonExisted(nodeId);
+          Node nm = getNMInNodeSet(nodeId);
+          switch (op) {
+          case ADD:
+            addNodeToLabels(nodeId, labels);
+            if (nm.labels == null) { 
+              nm.labels = new HashSet<String>();
+            }
+            nm.labels.addAll(labels);
+            break;
+          case REPLACE:
+            oldLabels = getLabelsByNode(nodeId);
+            replaceNodeForLabels(nodeId, oldLabels, labels);
+            if (nm.labels == null) { 
+              nm.labels = new HashSet<String>();
+            }
+            nm.labels.clear();
+            nm.labels.addAll(labels);
+            break;
+          default:
+            break;
+          }
           newNMToLabels.put(nodeId, nm.labels);
+        } else {
+          // remove
+          removeNodeFromLabels(nodeId, labels);
+          Node nm = getNMInNodeSet(nodeId);
+          if (nm.labels != null) {
+            nm.labels.removeAll(labels);
+            newNMToLabels.put(nodeId, nm.labels);
+          }
         }
       }
     }
@@ -505,7 +604,7 @@ protected void internalRemoveLabelsFromNode(
     }
 
     // shows node->labels we added
-    LOG.info("removeLabelsFromNode:");
+    LOG.info(op.name() + " labels on nodes:");
     for (Entry<NodeId, Set<String>> entry : newNMToLabels.entrySet()) {
       LOG.info("  NM=" + entry.getKey() + ", labels=["
           + StringUtils.join(entry.getValue().iterator(), ",") + "]");
@@ -521,11 +620,17 @@ protected void internalRemoveLabelsFromNode(
   public void
       removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
           throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
     
     checkRemoveLabelsFromNode(removeLabelsFromNode);
 
-    internalRemoveLabelsFromNode(removeLabelsFromNode);
+    internalUpdateLabelsOnNodes(removeLabelsFromNode,
+        NodeLabelUpdateOperation.REMOVE);
   }
   
   protected void checkReplaceLabelsOnNode(
@@ -537,57 +642,30 @@ protected void checkReplaceLabelsOnNode(
     // check all labels being added existed
     Set<String> knownLabels = labelCollections.keySet();
     for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) {
-      if (!knownLabels.containsAll(entry.getValue())) {
+      NodeId nodeId = entry.getKey();
+      Set<String> labels = entry.getValue();
+      
+      // As in YARN-2694, we disable user add more than 1 labels on a same host
+      if (labels.size() > 1) {
+        String msg = String.format("%d labels specified on host=%s"
+            + ", please note that we do not support specifying multiple"
+            + " labels on a single host for now.", labels.size(),
+            nodeId.getHost());
+        LOG.error(msg);
+        throw new IOException(msg);
+      }
+      
+      if (!knownLabels.containsAll(labels)) {
         String msg =
             "Not all labels being replaced contained by known "
                 + "label collections, please check" + ", new labels=["
-                + StringUtils.join(entry.getValue(), ",") + "]";
+                + StringUtils.join(labels, ",") + "]";
         LOG.error(msg);
         throw new IOException(msg);
       }
     }
   }
-  
-  @SuppressWarnings("unchecked")
-  protected void internalReplaceLabelsOnNode(
-      Map<NodeId, Set<String>> replaceLabelsToNode) throws IOException {
-    // do replace labels to nodes
-    Map<NodeId, Set<String>> newNMToLabels = new HashMap<NodeId, Set<String>>();
-    for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) {
-      NodeId nodeId = entry.getKey();
-      Set<String> labels = entry.getValue();
-
-      createHostIfNonExisted(nodeId.getHost());      
-      if (nodeId.getPort() == WILDCARD_PORT) {
-        Host host = nodeCollections.get(nodeId.getHost());
-        host.labels.clear();
-        host.labels.addAll(labels);
-        newNMToLabels.put(nodeId, host.labels);
-      } else {
-        createNodeIfNonExisted(nodeId);
-        Node nm = getNMInNodeSet(nodeId);
-        if (nm.labels == null) {
-          nm.labels = new HashSet<String>();
-        }
-        nm.labels.clear();
-        nm.labels.addAll(labels);
-        newNMToLabels.put(nodeId, nm.labels);
-      }
-    }
-
-    if (null != dispatcher) {
-      dispatcher.getEventHandler().handle(
-          new UpdateNodeToLabelsMappingsEvent(newNMToLabels));
-    }
 
-    // shows node->labels we added
-    LOG.info("setLabelsToNode:");
-    for (Entry<NodeId, Set<String>> entry : newNMToLabels.entrySet()) {
-      LOG.info("  NM=" + entry.getKey() + ", labels=["
-          + StringUtils.join(entry.getValue().iterator(), ",") + "]");
-    }
-  }
-  
   /**
    * replace labels to nodes
    * 
@@ -595,11 +673,17 @@ protected void internalReplaceLabelsOnNode(
    */
   public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
       throws IOException {
+    if (!nodeLabelsEnabled) {
+      LOG.error(NODE_LABELS_NOT_ENABLED_ERR);
+      throw new IOException(NODE_LABELS_NOT_ENABLED_ERR);
+    }
+    
     replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
     
     checkReplaceLabelsOnNode(replaceLabelsToNode);
 
-    internalReplaceLabelsOnNode(replaceLabelsToNode);
+    internalUpdateLabelsOnNodes(replaceLabelsToNode,
+        NodeLabelUpdateOperation.REPLACE);
   }
 
   /**
@@ -633,6 +717,52 @@ public Map<NodeId, Set<String>> getNodeLabels() {
     }
   }
 
+  /**
+   * Get mapping of labels to nodes for all the labels.
+   *
+   * @return labels to nodes map
+   */
+  public Map<String, Set<NodeId>> getLabelsToNodes() {
+    try {
+      readLock.lock();
+      return getLabelsToNodes(labelCollections.keySet());
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  /**
+   * Get mapping of labels to nodes for specified set of labels.
+   *
+   * @param labels set of labels for which labels to nodes mapping will be
+   *        returned.
+   * @return labels to nodes map
+   */
+  public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels) {
+    try {
+      readLock.lock();
+      Map<String, Set<NodeId>> labelsToNodes =
+          new HashMap<String, Set<NodeId>>();
+      for (String label : labels) {
+        if(label.equals(NO_LABEL)) {
+          continue;
+        }
+        NodeLabel nodeLabelInfo = labelCollections.get(label);
+        if(nodeLabelInfo != null) {
+          Set<NodeId> nodeIds = nodeLabelInfo.getAssociatedNodeIds();
+          if (!nodeIds.isEmpty()) {
+            labelsToNodes.put(label, nodeIds);
+          }
+        } else {
+          LOG.warn("getLabelsToNodes : Label [" + label + "] cannot be found");
+        }
+      }      
+      return Collections.unmodifiableMap(labelsToNodes);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
   /**
    * Get existing valid labels in repository
    * 
@@ -728,7 +858,7 @@ protected void createNodeIfNonExisted(NodeId nodeId) throws IOException {
     }
     Node nm = host.nms.get(nodeId);
     if (null == nm) {
-      host.nms.put(nodeId, new Node());
+      host.nms.put(nodeId, new Node(nodeId));
     }
   }
   
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java
new file mode 100644
index 0000000000000..1765a65b26c30
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+public class NodeLabel implements Comparable<NodeLabel> {
+  private Resource resource;
+  private int numActiveNMs;
+  private String labelName;
+  private Set<NodeId> nodeIds;
+
+  public NodeLabel(String labelName) {
+    this(labelName, Resource.newInstance(0, 0), 0);
+  }
+  
+  protected NodeLabel(String labelName, Resource res, int activeNMs) {
+    this.labelName = labelName;
+    this.resource = res;
+    this.numActiveNMs = activeNMs;
+    this.nodeIds = new HashSet<NodeId>();
+  }
+
+  public void addNodeId(NodeId node) {
+    nodeIds.add(node);
+  }
+
+  public void removeNodeId(NodeId node) {
+    nodeIds.remove(node);
+  }
+  
+  public Set<NodeId> getAssociatedNodeIds() {
+    return new HashSet<NodeId>(nodeIds);
+  }
+
+  public void addNode(Resource nodeRes) {
+    Resources.addTo(resource, nodeRes);
+    numActiveNMs++;
+  }
+  
+  public void removeNode(Resource nodeRes) {
+    Resources.subtractFrom(resource, nodeRes);
+    numActiveNMs--;
+  }
+
+  public Resource getResource() {
+    return this.resource;
+  }
+
+  public int getNumActiveNMs() {
+    return numActiveNMs;
+  }
+  
+  public String getLabelName() {
+    return labelName;
+  }
+  
+  public NodeLabel getCopy() {
+    return new NodeLabel(labelName, resource, numActiveNMs);
+  }
+  
+  @Override
+  public int compareTo(NodeLabel o) {
+    // We should always put empty label entry first after sorting
+    if (labelName.isEmpty() != o.getLabelName().isEmpty()) {
+      if (labelName.isEmpty()) {
+        return -1;
+      }
+      return 1;
+    }
+    
+    return labelName.compareTo(o.getLabelName());
+  }
+  
+  @Override
+  public boolean equals(Object obj) {
+    if (obj instanceof NodeLabel) {
+      NodeLabel other = (NodeLabel) obj;
+      return Resources.equals(resource, other.getResource())
+          && StringUtils.equals(labelName, other.getLabelName())
+          && (other.getNumActiveNMs() == numActiveNMs); 
+    }
+    return false;
+  }
+  
+  @Override
+  public int hashCode() {
+    final int prime = 502357;
+    return (int) ((((long) labelName.hashCode() << 8)
+        + (resource.hashCode() << 4) + numActiveNMs) % prime);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
index 033c0343ce0b1..857d81b115b85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
@@ -29,8 +29,7 @@
 
 public abstract class NodeLabelsStore implements Closeable {
   protected final CommonNodeLabelsManager mgr;
-  protected Configuration conf;
-  
+
   public NodeLabelsStore(CommonNodeLabelsManager mgr) {
     this.mgr = mgr;
   }
@@ -59,9 +58,7 @@ public abstract void removeClusterNodeLabels(Collection<String> labels)
    */
   public abstract void recover() throws IOException;
   
-  public void init(Configuration conf) throws Exception {
-    this.conf = conf;
-  }
+  public void init(Configuration conf) throws Exception {}
   
   public CommonNodeLabelsManager getNodeLabelsManager() {
     return mgr;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
index 7ccb9238a81ec..40ea858addd41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
@@ -64,19 +64,41 @@ public synchronized void readFields(DataInput in) throws IOException {
     setMasterKeyId(builder.getMasterKeyId());
   }
 
+  private void setBuilderFields() {
+    if (builder.getOwner() != null &&
+        !builder.getOwner().equals(getOwner().toString())) {
+      builder.setOwner(getOwner().toString());
+    }
+    if (builder.getRenewer() != null &&
+        !builder.getRenewer().equals(getRenewer().toString())) { 
+      builder.setRenewer(getRenewer().toString());
+    }
+    if (builder.getRealUser() != null &&
+        !builder.getRealUser().equals(getRealUser().toString())) {
+      builder.setRealUser(getRealUser().toString());
+    }
+    if (builder.getIssueDate() != getIssueDate()) {
+      builder.setIssueDate(getIssueDate());
+    }
+    if (builder.getMaxDate() != getMaxDate()) {
+      builder.setMaxDate(getMaxDate());
+    }
+    if (builder.getSequenceNumber() != getSequenceNumber()) {
+      builder.setSequenceNumber(getSequenceNumber());
+    }
+    if (builder.getMasterKeyId() != getMasterKeyId()) {
+      builder.setMasterKeyId(getMasterKeyId());
+    }
+  }
+
   @Override
   public synchronized void write(DataOutput out) throws IOException {
-    builder.setOwner(getOwner().toString());
-    builder.setRenewer(getRenewer().toString());
-    builder.setRealUser(getRealUser().toString());
-    builder.setIssueDate(getIssueDate());
-    builder.setMaxDate(getMaxDate());
-    builder.setSequenceNumber(getSequenceNumber());
-    builder.setMasterKeyId(getMasterKeyId());
+    setBuilderFields();
     builder.build().writeTo((DataOutputStream) out);
   }
 
   public YARNDelegationTokenIdentifierProto getProto() {
+    setBuilderFields();
     return builder.build();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
index 235d6739426c2..390deb21bf577 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/Graph.java
@@ -17,8 +17,10 @@
  */
 package org.apache.hadoop.yarn.state;
 
-import java.io.FileWriter;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -149,7 +151,7 @@ public String generateGraphViz(String indent) {
     StringBuilder sb = new StringBuilder();
     if (this.parent == null) {
       sb.append("digraph " + name + " {\n");
-      sb.append(String.format("graph [ label=%s, fontsize=24, fontname=Helvetica];\n",
+      sb.append(String.format("graph [ label=%s, fontsize=24, fontname=Helvetica];%n",
           wrapSafeString(name)));
       sb.append("node [fontsize=12, fontname=Helvetica];\n");
       sb.append("edge [fontsize=9, fontcolor=blue, fontname=Arial];\n");
@@ -163,14 +165,14 @@ public String generateGraphViz(String indent) {
     }
     for (Node n : nodes) {
       sb.append(String.format(
-          "%s%s [ label = %s ];\n",
+          "%s%s [ label = %s ];%n",
           indent,
           wrapSafeString(n.getUniqueId()),
           n.id));
       List<Edge> combinedOuts = combineEdges(n.outs);
       for (Edge e : combinedOuts) {
         sb.append(String.format(
-            "%s%s -> %s [ label = %s ];\n",
+            "%s%s -> %s [ label = %s ];%n",
             indent,
             wrapSafeString(e.from.getUniqueId()),
             wrapSafeString(e.to.getUniqueId()),
@@ -186,9 +188,10 @@ public String generateGraphViz() {
   }
 
   public void save(String filepath) throws IOException {
-    FileWriter fout = new FileWriter(filepath);
-    fout.write(generateGraphViz());
-    fout.close();
+    try (OutputStreamWriter fout = new OutputStreamWriter(
+        new FileOutputStream(filepath), Charset.forName("UTF-8"));) {
+      fout.write(generateGraphViz());
+    }
   }
 
   public static List<Edge> combineEdges(List<Edge> edges) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
index 2482fe34e1e82..26e93c693e6c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
@@ -56,7 +56,7 @@ public static Graph getGraphFromClasses(String graphName, List<String> classes)
 
   public static void main(String [] args) throws Exception {
     if (args.length < 3) {
-      System.err.printf("Usage: %s <GraphName> <class[,class[,...]]> <OutputFile>\n",
+      System.err.printf("Usage: %s <GraphName> <class[,class[,...]]> <OutputFile>%n",
           VisualizeStateMachine.class.getName());
       System.exit(1);
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index 73ec90607e221..0aa68a2268131 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -204,7 +204,7 @@ public static ApplicationId toApplicationId(
     try {
       return toApplicationId(it);
     } catch (NumberFormatException n) {
-      throw new IllegalArgumentException("Invalid AppAttemptId: "
+      throw new IllegalArgumentException("Invalid ApplicationId: "
           + appIdStr, n);
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 436cb312938c9..870aa9521221a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -23,6 +23,7 @@
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.security.PrivilegedExceptionAction;
+import java.util.Locale;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -271,7 +272,7 @@ private Path copy(Path sCopy, Path dstdir) throws IOException {
   private long unpack(File localrsrc, File dst) throws IOException {
     switch (resource.getType()) {
     case ARCHIVE: {
-      String lowerDst = dst.getName().toLowerCase();
+      String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH);
       if (lowerDst.endsWith(".jar")) {
         RunJar.unJar(localrsrc, dst);
       } else if (lowerDst.endsWith(".zip")) {
@@ -290,7 +291,7 @@ private long unpack(File localrsrc, File dst) throws IOException {
     }
     break;
     case PATTERN: {
-      String lowerDst = dst.getName().toLowerCase();
+      String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH);
       if (lowerDst.endsWith(".jar")) {
         String p = resource.getPattern();
         RunJar.unJar(localrsrc, dst,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
index 56a81cde67390..2347f4041cc92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
@@ -19,9 +19,11 @@
 package org.apache.hadoop.yarn.util;
 
 import java.io.BufferedReader;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileReader;
+import java.io.InputStreamReader;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -147,9 +149,10 @@ private void readProcMemInfoFile(boolean readAgain) {
 
     // Read "/proc/memInfo" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsMemFile);
+      fReader = new InputStreamReader(
+          new FileInputStream(procfsMemFile), Charset.forName("UTF-8"));
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....
@@ -206,9 +209,10 @@ private void readProcCpuInfoFile() {
     }
     // Read "/proc/cpuinfo" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsCpuFile);
+      fReader = new InputStreamReader(
+          new FileInputStream(procfsCpuFile), Charset.forName("UTF-8"));
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....
@@ -253,9 +257,10 @@ private void readProcCpuInfoFile() {
   private void readProcStatFile() {
     // Read "/proc/stat" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsStatFile);
+      fReader = new InputStreamReader(
+          new FileInputStream(procfsStatFile), Charset.forName("UTF-8"));
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index eeb559c827aae..69aa96dafa705 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -20,10 +20,12 @@
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileReader;
+import java.io.InputStreamReader;
 import java.io.IOException;
 import java.math.BigInteger;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -297,7 +299,7 @@ public static boolean checkPidPgrpidForMatch(String _pid, String procfs) {
   }
 
   private static final String PROCESSTREE_DUMP_FORMAT =
-      "\t|- %s %s %d %d %s %d %d %d %d %s\n";
+      "\t|- %s %s %d %d %s %d %d %d %d %s%n";
 
   public List<String> getCurrentProcessIDs() {
     List<String> currentPIDs = new ArrayList<String>();
@@ -317,7 +319,7 @@ public String getProcessTreeDump() {
     // The header.
     ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME "
         + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
-        + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
+        + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n"));
     for (ProcessInfo p : processTree.values()) {
       if (p != null) {
         ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p
@@ -489,10 +491,12 @@ private static ProcessInfo constructProcessInfo(ProcessInfo pinfo,
     ProcessInfo ret = null;
     // Read "procfsDir/<pid>/stat" file - typically /proc/<pid>/stat
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
       File pidDir = new File(procfsDir, pinfo.getPid());
-      fReader = new FileReader(new File(pidDir, PROCFS_STAT_FILE));
+      fReader = new InputStreamReader(
+          new FileInputStream(
+              new File(pidDir, PROCFS_STAT_FILE)), Charset.forName("UTF-8"));
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // The process vanished in the interim!
@@ -671,11 +675,12 @@ public String getCmdLine(String procfsDir) {
         return ret;
       }
       BufferedReader in = null;
-      FileReader fReader = null;
+      InputStreamReader fReader = null;
       try {
-        fReader =
-            new FileReader(new File(new File(procfsDir, pid.toString()),
-                PROCFS_CMDLINE_FILE));
+        fReader = new InputStreamReader(
+            new FileInputStream(
+                new File(new File(procfsDir, pid.toString()), PROCFS_CMDLINE_FILE)),
+                Charset.forName("UTF-8"));
       } catch (FileNotFoundException f) {
         // The process vanished in the interim!
         return ret;
@@ -725,14 +730,15 @@ public String getCmdLine(String procfsDir) {
   private static void constructProcessSMAPInfo(ProcessTreeSmapMemInfo pInfo,
       String procfsDir) {
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
       File pidDir = new File(procfsDir, pInfo.getPid());
       File file = new File(pidDir, SMAPS);
       if (!file.exists()) {
         return;
       }
-      fReader = new FileReader(file);
+      fReader = new InputStreamReader(
+          new FileInputStream(file), Charset.forName("UTF-8"));
       in = new BufferedReader(fReader);
       ProcessSmapMemoryInfo memoryMappingInfo = null;
       List<String> lines = IOUtils.readLines(in);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index a2346a88a2840..a70074bae13d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -32,18 +32,6 @@
 @InterfaceAudience.LimitedPrivate({"YARN", "MAPREDUCE"})
 @InterfaceStability.Unstable
 public abstract class ResourceCalculatorPlugin extends Configured {
-  
-  protected String processPid = null;
-
-  /**
-   * set the pid of the process for which <code>getProcResourceValues</code>
-   * will be invoked
-   * 
-   * @param pid
-   */
-  public void setProcessPid(String pid) {
-    processPid = pid;
-  }
 
   /**
    * Obtain the total size of the virtual memory present in the system.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
index 3759026d1f664..143d236f01ac6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
@@ -162,10 +162,10 @@ public String getProcessTreeDump() {
     StringBuilder ret = new StringBuilder();
     // The header.
     ret.append(String.format("\t|- PID " + "CPU_TIME(MILLIS) "
-        + "VMEM(BYTES) WORKING_SET(BYTES)\n"));
+        + "VMEM(BYTES) WORKING_SET(BYTES)%n"));
     for (ProcessInfo p : processTree.values()) {
       if (p != null) {
-        ret.append(String.format("\t|- %s %d %d %d\n", p.pid,
+        ret.append(String.format("\t|- %s %d %d %d%n", p.pid,
             p.cpuTimeMs, p.vmem, p.workingSet));
       }
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 91d2a2019ab8f..62c3c7a0ef3d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -32,4 +32,5 @@ public interface YarnWebParams {
   String APP_STATE = "app.state";
   String QUEUE_NAME = "queue.name";
   String NODE_STATE = "node.state";
+  String NODE_LABEL = "node.label";
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
index 7c311bcebd71c..6a64d1cd332c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
@@ -91,7 +91,8 @@ protected void render(Block html) {
   public static void jsnotice(HTML html) {
     html.
       div("#jsnotice.ui-state-error").
-          _("This page works best with javascript enabled.")._();
+          _("This page will not function without javascript enabled."
+            + " Please enable javascript on your browser.")._();
     html.
       script().$type("text/javascript").
         _("$('#jsnotice').hide();")._();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/NavBlock.java
index febc818711397..cdc13eb59cb6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/NavBlock.java
@@ -36,6 +36,6 @@ public class NavBlock extends HtmlBlock {
           li().a("/conf", "Configuration")._().
           li().a("/stacks", "Thread dump")._().
           li().a("/logs", "Logs")._().
-          li().a("/metrics", "Metrics")._()._()._();
+          li().a("/jmx?qry=Hadoop:*", "Metrics")._()._()._();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 73a6b5d55e057..971ce0b6fc164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -229,32 +229,32 @@
 
   <property>
     <description>The minimum allocation for every container request at the RM,
-    in MBs. Memory requests lower than this won't take effect,
-    and the specified value will get allocated at minimum.</description>
+    in MBs. Memory requests lower than this will throw a
+    InvalidResourceRequestException.</description>
     <name>yarn.scheduler.minimum-allocation-mb</name>
     <value>1024</value>
   </property>
 
   <property>
     <description>The maximum allocation for every container request at the RM,
-    in MBs. Memory requests higher than this won't take effect,
-    and will get capped to this value.</description>
+    in MBs. Memory requests higher than this will throw a
+    InvalidResourceRequestException.</description>
     <name>yarn.scheduler.maximum-allocation-mb</name>
     <value>8192</value>
   </property>
 
   <property>
     <description>The minimum allocation for every container request at the RM,
-    in terms of virtual CPU cores. Requests lower than this won't take effect,
-    and the specified value will get allocated the minimum.</description>
+    in terms of virtual CPU cores. Requests lower than this will throw a
+    InvalidResourceRequestException.</description>
     <name>yarn.scheduler.minimum-allocation-vcores</name>
     <value>1</value>
   </property>
 
   <property>
     <description>The maximum allocation for every container request at the RM,
-    in terms of virtual CPU cores. Requests higher than this won't take effect,
-    and will get capped to this value.</description>
+    in terms of virtual CPU cores. Requests higher than this will throw a
+    InvalidResourceRequestException.</description>
     <name>yarn.scheduler.maximum-allocation-vcores</name>
     <value>32</value>
   </property>
@@ -271,7 +271,7 @@
     to YARN for experimenting the feature.
     </description>
     <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
   <property>
@@ -1370,6 +1370,12 @@
     <value>30</value>
   </property>
 
+  <property>
+    <description>Client policy for whether timeline operations are non-fatal</description>
+    <name>yarn.timeline-service.client.best-effort</name>
+    <value>false</value>
+  </property>
+
   <property>
     <description>
     Default retry time interval for timeline servive client.
@@ -1378,6 +1384,26 @@
     <value>1000</value>
   </property>
 
+  <property>
+    <description>Enable timeline server to recover state after starting. If
+    true, then yarn.timeline-service.state-store-class must be specified.
+    </description>
+    <name>yarn.timeline-service.recovery.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>Store class name for timeline state store.</description>
+    <name>yarn.timeline-service.state-store-class</name>
+    <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
+  </property>
+
+  <property>
+    <description>Store file name for leveldb state store.</description>
+    <name>yarn.timeline-service.leveldb-state-store.path</name>
+    <value>${hadoop.tmp.dir}/yarn/timeline</value>
+  </property>
+
   <!--  Shared Cache Configuration -->
   <property>
     <description>Whether the shared cache is enabled</description>
@@ -1444,6 +1470,12 @@
     <value>1</value>
   </property>
 
+  <property>
+    <description>The address of the web application in the SCM (shared cache manager)</description>
+    <name>yarn.sharedcache.webapp.address</name>
+    <value>0.0.0.0:8788</value>
+  </property>
+
   <property>
     <description>The frequency at which a cleaner task runs.
     Specified in minutes.</description>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 7da3bbf4645a7..c8027a202151e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -197,7 +197,7 @@ public void testCheckRetryCount() throws Exception {
         ce.getMessage().contains("Connection retries limit exceeded"));
       // we would expect this exception here, check if the client has retried
       Assert.assertTrue("Retry filter didn't perform any retries! ", client
-        .connectionRetry.retried);
+        .connectionRetry.getRetired());
     }
   }
 
@@ -272,7 +272,7 @@ private void assertException(TimelineClientImpl client, RuntimeException ce) {
             .getMessage().contains("Connection retries limit exceeded"));
     // we would expect this exception here, check if the client has retried
     Assert.assertTrue("Retry filter didn't perform any retries! ",
-        client.connectionRetry.retried);
+        client.connectionRetry.getRetired());
   }
 
   private static ClientResponse mockEntityClientResponse(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
new file mode 100644
index 0000000000000..9075d9f6dd3e5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.conf;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Unit test class to compare
+ * {@link org.apache.hadoop.yarn.conf.YarnConfiguration} and
+ * yarn-default.xml for missing properties.  Currently only throws an error
+ * if the class is missing a property.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
+
+  @Override
+  public void initializeMemberVariables() {
+    xmlFilename = new String("yarn-default.xml");
+    configurationClasses = new Class[] { YarnConfiguration.class };
+        
+
+    // Allocate for usage
+    configurationPropsToSkipCompare = new HashSet<String>();
+
+    // Set error modes
+    errorIfMissingConfigProps = true;
+    errorIfMissingXmlProps = false;
+
+    // Specific properties to skip
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_IPC_CLIENT_FACTORY_CLASS);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_IPC_SERVER_FACTORY_CLASS);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_IPC_RPC_IMPL);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration.DEFAULT_RM_SCHEDULER);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL);
+    configurationPropsToSkipCompare
+        .add(YarnConfiguration
+	    .YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
+
+    // Allocate for usage
+    xmlPropsToSkipCompare = new HashSet<String>();
+    xmlPrefixToSkipCompare = new HashSet<String>();
+
+    // Should probably be moved from yarn-default.xml to mapred-default.xml
+    xmlPropsToSkipCompare.add("mapreduce.job.hdfs-servers");
+    xmlPropsToSkipCompare.add("mapreduce.job.jar");
+
+    // Possibly obsolete, but unable to verify 100%
+    xmlPropsToSkipCompare.add("yarn.nodemanager.aux-services.mapreduce_shuffle.class");
+    xmlPropsToSkipCompare.add("yarn.resourcemanager.container.liveness-monitor.interval-ms");
+
+    // Used in the XML file as a variable reference internal to the XML file
+    xmlPropsToSkipCompare.add("yarn.nodemanager.hostname");
+    xmlPropsToSkipCompare.add("yarn.timeline-service.hostname");
+
+    // Currently defined in TimelineAuthenticationFilterInitializer
+    xmlPrefixToSkipCompare.add("yarn.timeline-service.http-authentication");
+
+    // Currently defined in RegistryConstants
+    xmlPrefixToSkipCompare.add("hadoop.registry");
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index 803b2bb2b3be0..da5ae443ae081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -23,68 +23,20 @@
 @SuppressWarnings("rawtypes")
 public class DrainDispatcher extends AsyncDispatcher {
 
-// flagrant initialize abuse throughout, but safe per
-// http://java.sun.com/docs/books/jls/third_edition/html/typesValues.html#96595
-// and similar grotesqueries
-  private volatile boolean drained = false;
-  private final BlockingQueue<Event> queue;
-  final Object mutex;
-
   public DrainDispatcher() {
     this(new LinkedBlockingQueue<Event>());
   }
 
   private DrainDispatcher(BlockingQueue<Event> eventQueue) {
     super(eventQueue);
-    this.queue = eventQueue;
-    this.mutex = this;
   }
 
   /**
    * Busy loop waiting for all queued events to drain.
    */
   public void await() {
-    while (!drained) {
+    while (!isDrained()) {
       Thread.yield();
     }
   }
-
-  @Override
-  Runnable createThread() {
-    return new Runnable() {
-      @Override
-      public void run() {
-        while (!Thread.currentThread().isInterrupted()) {
-          synchronized (mutex) {
-            // !drained if dispatch queued new events on this dispatcher
-            drained = queue.isEmpty();
-          }
-          Event event;
-          try {
-            event = queue.take();
-          } catch(InterruptedException ie) {
-            return;
-          }
-          if (event != null) {
-            dispatch(event);
-          }
-        }
-      }
-    };
-  }
-
-  @Override
-  public EventHandler getEventHandler() {
-    final EventHandler actual = super.getEventHandler();
-    return new EventHandler() {
-      @Override
-      public void handle(Event event) {
-        synchronized (mutex) {
-          actual.handle(event);
-          drained = false;
-        }
-      }
-    };
-  }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
index ff0e1019b74b8..3b9825d1e03cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
@@ -19,9 +19,11 @@
 package org.apache.hadoop.yarn.nodelabels;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.Map.Entry;
 
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.junit.Assert;
@@ -39,6 +41,37 @@ public static void assertMapEquals(Map<NodeId, Set<String>> m1,
     }
   }
 
+  public static void assertLabelsToNodesEquals(Map<String, Set<NodeId>> m1,
+      ImmutableMap<String, Set<NodeId>> m2) {
+    Assert.assertEquals(m1.size(), m2.size());
+    for (String k : m1.keySet()) {
+      Assert.assertTrue(m2.containsKey(k));
+      Set<NodeId> s1 = new HashSet<NodeId>(m1.get(k));
+      Set<NodeId> s2 = new HashSet<NodeId>(m2.get(k));
+      Assert.assertEquals(s1, s2);
+      Assert.assertTrue(s1.containsAll(s2));
+    }
+  }
+
+  public static ImmutableMap<String, Set<NodeId>> transposeNodeToLabels(
+      Map<NodeId, Set<String>> mapNodeToLabels) {
+    Map<String, Set<NodeId>> mapLabelsToNodes =
+        new HashMap<String, Set<NodeId>>();
+    for(Entry<NodeId, Set<String>> entry : mapNodeToLabels.entrySet()) {
+      NodeId node = entry.getKey();
+      Set<String> setLabels = entry.getValue();
+      for(String label : setLabels) {
+        Set<NodeId> setNode = mapLabelsToNodes.get(label);
+        if (setNode == null) {
+          setNode = new HashSet<NodeId>();
+        }
+        setNode.add(NodeId.newInstance(node.getHost(), node.getPort()));
+        mapLabelsToNodes.put(label, setNode);
+      }
+    }
+    return ImmutableMap.copyOf(mapLabelsToNodes);
+  }
+
   public static void assertMapContains(Map<NodeId, Set<String>> m1,
       ImmutableMap<NodeId, Set<String>> m2) {
     for (NodeId k : m2.keySet()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
index a56a5955770e6..d05c75ce63bd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.nodelabels;
 
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -26,6 +28,8 @@
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -41,7 +45,9 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
   @Before
   public void before() {
     mgr = new DummyCommonNodeLabelsManager();
-    mgr.init(new Configuration());
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    mgr.init(conf);
     mgr.start();
   }
 
@@ -221,19 +227,17 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception {
         ImmutableMap.of(toNodeId("n1"), CommonNodeLabelsManager.EMPTY_STRING_SET));
 
     // add label on node
-    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
-        toNodeId("n2"), toSet("p2")));
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     assertMapEquals(
         mgr.getNodeLabels(),
         ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
-            toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+            toSet("p3"), toNodeId("n3"), toSet("p3")));
     assertMapEquals(mgr.lastNodeToLabels,
-        ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
-            toSet("p2", "p3")));
+        ImmutableMap.of(toNodeId("n1"), toSet("p1")));
 
     // remove labels on node
     mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
-        toNodeId("n2"), toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+        toNodeId("n2"), toSet("p3"), toNodeId("n3"), toSet("p3")));
     Assert.assertEquals(0, mgr.getNodeLabels().size());
     assertMapEquals(mgr.lastNodeToLabels, ImmutableMap.of(toNodeId("n1"),
         CommonNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n2"),
@@ -270,10 +274,10 @@ public void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException {
   @Test(timeout = 5000) 
   public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
     mgr.addToCluserNodeLabels(toSet(" p1", "p2"));
-    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ", "p2")));
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ")));
     assertMapEquals(
         mgr.getNodeLabels(),
-        ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+        ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2")));
     assertMapEquals(
         mgr.getNodeLabels(),
@@ -281,4 +285,255 @@ public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
     mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("  p2 ")));
     Assert.assertTrue(mgr.getNodeLabels().isEmpty());
   }
-}
\ No newline at end of file
+  
+  @Test(timeout = 5000)
+  public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo()
+      throws IOException {
+    mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    assertMapEquals(
+        mgr.getNodeLabels(),
+        ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    
+    // Replace labels on n1:1 to P2
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2"),
+        toNodeId("n1:2"), toSet("p2")));
+    assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"),
+        toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
+        toSet("p2")));
+    
+    // Replace labels on n1 to P1, both n1:1/n1 will be P1 now
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"),
+        toSet("p1"), toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"),
+        toSet("p1")));
+    
+    // Set labels on n1:1 to P2 again to verify if add/remove works
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2")));
+  }
+
+  private void assertNodeLabelsDisabledErrorMessage(IOException e) {
+    Assert.assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR,
+        e.getMessage());
+  }
+  
+  @Test(timeout = 5000)
+  public void testNodeLabelsDisabled() throws IOException {
+    DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager();
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
+    mgr.init(conf);
+    mgr.start();
+    boolean caught = false;
+    
+    // add labels
+    try {
+      mgr.addToCluserNodeLabels(ImmutableSet.of("x"));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+      caught = true;
+    }
+    // check exception caught
+    Assert.assertTrue(caught);
+    caught = false;
+    
+    // remove labels
+    try {
+      mgr.removeFromClusterNodeLabels(ImmutableSet.of("x"));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+      caught = true;
+    }
+    // check exception caught
+    Assert.assertTrue(caught);
+    caught = false;
+    
+    // add labels to node
+    try {
+      mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+      caught = true;
+    }
+    // check exception caught
+    Assert.assertTrue(caught);
+    caught = false;
+    
+    // remove labels from node
+    try {
+      mgr.removeLabelsFromNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+      caught = true;
+    }
+    // check exception caught
+    Assert.assertTrue(caught);
+    caught = false;
+    
+    // replace labels on node
+    try {
+      mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("host", 0),
+          CommonNodeLabelsManager.EMPTY_STRING_SET));
+    } catch (IOException e) {
+      assertNodeLabelsDisabledErrorMessage(e);
+      caught = true;
+    }
+    // check exception caught
+    Assert.assertTrue(caught);
+    caught = false;
+    
+    mgr.close();
+  }  
+
+  @Test(timeout = 5000)
+  public void testLabelsToNodes()
+      throws IOException {
+    mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    Map<String, Set<NodeId>> labelsToNodes = mgr.getLabelsToNodes();
+    assertLabelsToNodesEquals(
+        labelsToNodes,
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n1"))));
+    assertLabelsToNodesEquals(
+        labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Replace labels on n1:1 to P2
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2"),
+        toNodeId("n1:2"), toSet("p2")));
+    labelsToNodes = mgr.getLabelsToNodes();
+    assertLabelsToNodesEquals(
+        labelsToNodes,
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n1")),
+        "p2", toSet(toNodeId("n1:1"),toNodeId("n1:2"))));
+    assertLabelsToNodesEquals(
+        labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Replace labels on n1 to P1, both n1:1/n1 will be P1 now
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    labelsToNodes = mgr.getLabelsToNodes();
+    assertLabelsToNodesEquals(
+        labelsToNodes,
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n1"),toNodeId("n1:1"),toNodeId("n1:2"))));
+    assertLabelsToNodesEquals(
+        labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Set labels on n1:1 to P2 again to verify if add/remove works
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2")));
+    // Add p3 to n1, should makes n1:1 to be p2/p3, and n1:2 to be p1/p3
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n2"), toSet("p3")));
+    labelsToNodes = mgr.getLabelsToNodes();
+    assertLabelsToNodesEquals(
+        labelsToNodes,
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n1"),toNodeId("n1:2")),
+        "p2", toSet(toNodeId("n1:1")),
+        "p3", toSet(toNodeId("n2"))));
+    assertLabelsToNodesEquals(
+        labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Remove P3 from n1, should makes n1:1 to be p2, and n1:2 to be p1
+    mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n2"), toSet("p3")));
+    labelsToNodes = mgr.getLabelsToNodes();
+    assertLabelsToNodesEquals(
+        labelsToNodes,
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n1"),toNodeId("n1:2")),
+        "p2", toSet(toNodeId("n1:1"))));
+    assertLabelsToNodesEquals(
+        labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
+  }
+
+  @Test(timeout = 5000)
+  public void testLabelsToNodesForSelectedLabels()
+      throws IOException {
+    mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+    mgr.addLabelsToNode(
+        ImmutableMap.of(
+        toNodeId("n1:1"), toSet("p1"),
+        toNodeId("n1:2"), toSet("p2")));
+    Set<String> setlabels =
+        new HashSet<String>(Arrays.asList(new String[]{"p1"}));
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of("p1", toSet(toNodeId("n1:1"))));
+
+    // Replace labels on n1:1 to P3
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p3")));
+    assertTrue(mgr.getLabelsToNodes(setlabels).isEmpty());
+    setlabels = new HashSet<String>(Arrays.asList(new String[]{"p2", "p3"}));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of(
+        "p3", toSet(toNodeId("n1"), toNodeId("n1:1"),toNodeId("n1:2"))));
+
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n2"), toSet("p2")));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of(
+        "p2", toSet(toNodeId("n2")),
+        "p3", toSet(toNodeId("n1"), toNodeId("n1:1"),toNodeId("n1:2"))));
+
+    mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p3")));
+    setlabels =
+        new HashSet<String>(Arrays.asList(new String[]{"p1", "p2", "p3"}));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of(
+        "p2", toSet(toNodeId("n2"))));
+
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n3"), toSet("p1")));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n3")),
+        "p2", toSet(toNodeId("n2"))));
+
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2:2"), toSet("p3")));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of(
+        "p1", toSet(toNodeId("n3")),
+        "p2", toSet(toNodeId("n2")),
+        "p3", toSet(toNodeId("n2:2"))));
+    setlabels = new HashSet<String>(Arrays.asList(new String[]{"p1"}));
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(setlabels),
+        ImmutableMap.of("p1", toSet(toNodeId("n3"))));
+  }
+
+  @Test(timeout = 5000)
+  public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
+    boolean failed = false;
+    // As in YARN-2694, we temporarily disable no more than one label existed in
+    // one host
+    mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+    try {
+      mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+    } catch (IOException e) {
+      failed = true;
+    }
+    Assert.assertTrue("Should failed when set > 1 labels on a host", failed);
+
+    try {
+      mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+    } catch (IOException e) {
+      failed = true;
+    }
+    Assert.assertTrue("Should failed when add > 1 labels on a host", failed);
+
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    // add a same label to a node, #labels in this node is still 1, shouldn't
+    // fail
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+    try {
+      mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
+    } catch (IOException e) {
+      failed = true;
+    }
+    Assert.assertTrue("Should failed when #labels > 1 on a host after add",
+        failed);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index 45a2d8d32f214..5cc026a74ccc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -63,6 +63,7 @@ private FileSystemNodeLabelsStore getStore() {
   public void before() throws IOException {
     mgr = new MockNodeLabelManager();
     conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     File tempDir = File.createTempFile("nlb", ".tmp");
     tempDir.delete();
     tempDir.mkdirs();
@@ -115,6 +116,11 @@ public void testRecoverWithMirror() throws Exception {
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
+            ImmutableMap.of(
+            "p6", toSet(toNodeId("n6"), toNodeId("n7")),
+            "p4", toSet(toNodeId("n4")),
+            "p2", toSet(toNodeId("n2"))));
 
     // stutdown mgr and start a new mgr
     mgr.stop();
@@ -129,6 +135,11 @@ public void testRecoverWithMirror() throws Exception {
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
+            ImmutableMap.of(
+            "p6", toSet(toNodeId("n6"), toNodeId("n7")),
+            "p4", toSet(toNodeId("n4")),
+            "p2", toSet(toNodeId("n2"))));
     mgr.stop();
   }
 
@@ -168,6 +179,11 @@ public void testEditlogRecover() throws Exception {
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
+            ImmutableMap.of(
+            "p6", toSet(toNodeId("n6"), toNodeId("n7")),
+            "p4", toSet(toNodeId("n4")),
+            "p2", toSet(toNodeId("n2"))));
     mgr.stop();
   }
   
@@ -217,7 +233,12 @@ public void testSerilizationAfterRecovery() throws Exception {
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
-    
+    assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
+        ImmutableMap.of(
+        "p6", toSet(toNodeId("n6"), toNodeId("n7")),
+        "p4", toSet(toNodeId("n4")),
+        "p2", toSet(toNodeId("n2"))));
+
     /*
      * Add label p7,p8 then shutdown
      */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
index 834dcf131c498..5fe75bc12c9e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.yarn.security;
 
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
@@ -31,6 +33,7 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
@@ -224,7 +227,7 @@ public void testRMDelegationTokenIdentifier() throws IOException {
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
-        
+    dib.close();
     // verify the whole record equals with original record
     Assert.assertEquals("Token is not the same after serialization " +
         "and deserialization.", token, anotherToken);
@@ -249,6 +252,32 @@ public void testRMDelegationTokenIdentifier() throws IOException {
     
     Assert.assertEquals("masterKeyId from proto is not the same with original token",
         anotherToken.getMasterKeyId(), masterKeyId);
+    
+    // Test getProto    
+    RMDelegationTokenIdentifier token1 = 
+        new RMDelegationTokenIdentifier(owner, renewer, realUser);
+    token1.setIssueDate(issueDate);
+    token1.setMaxDate(maxDate);
+    token1.setSequenceNumber(sequenceNumber);
+    token1.setMasterKeyId(masterKeyId);
+    YARNDelegationTokenIdentifierProto tokenProto = token1.getProto();
+    // Write token proto to stream
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream out = new DataOutputStream(baos);
+    tokenProto.writeTo(out);
+
+    // Read token
+    byte[] tokenData = baos.toByteArray();
+    RMDelegationTokenIdentifier readToken = new RMDelegationTokenIdentifier();
+    DataInputBuffer db = new DataInputBuffer();
+    db.reset(tokenData, tokenData.length);
+    readToken.readFields(db);
+
+    // Verify if read token equals with original token
+    Assert.assertEquals("Token from getProto is not the same after " +
+        "serialization and deserialization.", token1, readToken);
+    db.close();
+    out.close();
   }
   
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 07dd63006fb60..3597b31471521 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -33,6 +33,7 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.Callable;
@@ -67,7 +68,6 @@
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -232,7 +232,7 @@ static LocalResource createZipFile(FileContext files, Path p, int len,
     byte[] bytes = new byte[len];
     r.nextBytes(bytes);
 
-    File archiveFile = new File(p.toUri().getPath() + ".zip");
+    File archiveFile = new File(p.toUri().getPath() + ".ZIP");
     archiveFile.createNewFile();
     ZipOutputStream out = new ZipOutputStream(
         new FileOutputStream(archiveFile));
@@ -243,11 +243,11 @@ static LocalResource createZipFile(FileContext files, Path p, int len,
 
     LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
     ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
-        + ".zip")));
+        + ".ZIP")));
     ret.setSize(len);
     ret.setType(LocalResourceType.ARCHIVE);
     ret.setVisibility(vis);
-    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
+    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".ZIP"))
         .getModificationTime());
     return ret;
   }
@@ -474,8 +474,8 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
 
     int size = rand.nextInt(512) + 512;
     LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
-
     Path p = new Path(basedir, "" + 1);
+    String strFileName = "";
     LocalResource rsrc = null;
     switch (fileType) {
     case TAR:
@@ -487,6 +487,7 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
       break;
     case ZIP:
       rsrc = createZipFile(files, p, size, rand, vis);
+      strFileName = p.getName() + ".ZIP";
       break;
     case TGZ:
       rsrc = createTgzFile(files, p, size, rand, vis);
@@ -509,6 +510,13 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
           FileStatus[] childFiles = files.getDefaultFileSystem().listStatus(
               filestatus.getPath());
           for (FileStatus childfile : childFiles) {
+            if(strFileName.endsWith(".ZIP") &&
+               childfile.getPath().getName().equals(strFileName) &&
+               !childfile.isDirectory()) {
+               Assert.fail("Failure...After unzip, there should have been a" +
+                 " directory formed with zip file name but found a file. "
+                 + childfile.getPath());
+            }
             if (childfile.getPath().getName().startsWith("tmp")) {
               Assert.fail("Tmp File should not have been there "
                   + childfile.getPath());
@@ -539,6 +547,21 @@ public void testDownloadArchiveZip() throws IOException, URISyntaxException,
     downloadWithFileType(TEST_FILE_TYPE.ZIP);
   }
 
+  /*
+   * To test fix for YARN-3029
+   */
+  @Test (timeout=10000)
+  public void testDownloadArchiveZipWithTurkishLocale() throws IOException,
+      URISyntaxException, InterruptedException {
+    Locale defaultLocale = Locale.getDefault();
+    // Set to Turkish
+    Locale turkishLocale = new Locale("tr", "TR");
+    Locale.setDefault(turkishLocale);
+    downloadWithFileType(TEST_FILE_TYPE.ZIP);
+    // Set the locale back to original default locale
+    Locale.setDefault(defaultLocale);
+  }
+
   @Test (timeout=10000)
   public void testDownloadArchiveTgz() throws IOException, URISyntaxException,
       InterruptedException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java
new file mode 100644
index 0000000000000..0b177f91d665f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp;
+
+import org.junit.Before;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+public abstract class JerseyTestBase extends JerseyTest {
+  public JerseyTestBase(WebAppDescriptor appDescriptor) {
+    super(appDescriptor);
+  }
+
+  @Before
+  public void initializeJerseyPort() {
+    int jerseyPort = 9998;
+    String port = System.getProperty("jersey.test.port");
+    if(null != port) {
+      jerseyPort = Integer.parseInt(port) + 10;
+      if(jerseyPort > 65535) {
+        jerseyPort = 9998;
+      }
+    }
+    System.setProperty("jersey.test.port", Integer.toString(jerseyPort));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index 0b68b0a41b0d7..7f35c3fe62843 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -491,7 +491,10 @@ public List<ACL> zkGetACLS(String path) throws IOException {
   public boolean zkPathExists(String path) throws IOException {
     checkServiceLive();
     try {
-      return zkStat(path) != null;
+      // if zkStat(path) returns without throwing an exception, the return value
+      // is guaranteed to be not null
+      zkStat(path);
+      return true;
     } catch (PathNotFoundException e) {
       return false;
     } catch (IOException e) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 5370880be2885..1c75e43987e8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -192,14 +192,17 @@ protected void serviceInit(Configuration conf) throws Exception {
     String auth = conf.getTrimmed(KEY_REGISTRY_CLIENT_AUTH,
         REGISTRY_CLIENT_AUTH_ANONYMOUS);
 
-    // TODO JDK7 SWITCH
-    if (REGISTRY_CLIENT_AUTH_KERBEROS.equals(auth)) {
+    switch (auth) {
+    case REGISTRY_CLIENT_AUTH_KERBEROS:
       access = AccessPolicy.sasl;
-    } else if (REGISTRY_CLIENT_AUTH_DIGEST.equals(auth)) {
+      break;
+    case REGISTRY_CLIENT_AUTH_DIGEST:
       access = AccessPolicy.digest;
-    } else if (REGISTRY_CLIENT_AUTH_ANONYMOUS.equals(auth)) {
+      break;
+    case REGISTRY_CLIENT_AUTH_ANONYMOUS:
       access = AccessPolicy.anon;
-    } else {
+      break;
+    default:
       throw new ServiceStateException(E_UNKNOWN_AUTHENTICATION_MECHANISM
                                       + "\"" + auth + "\"");
     }
@@ -592,17 +595,17 @@ public static String getKerberosAuthModuleForJVM() {
    * Note the semicolon on the last entry
    */
   private static final String JAAS_ENTRY =
-      "%s { \n"
-      + " %s required\n"
+      "%s { %n"
+      + " %s required%n"
       // kerberos module
-      + " keyTab=\"%s\"\n"
-      + " debug=true\n"
-      + " principal=\"%s\"\n"
-      + " useKeyTab=true\n"
-      + " useTicketCache=false\n"
-      + " doNotPrompt=true\n"
-      + " storeKey=true;\n"
-      + "}; \n"
+      + " keyTab=\"%s\"%n"
+      + " debug=true%n"
+      + " principal=\"%s\"%n"
+      + " useKeyTab=true%n"
+      + " useTicketCache=false%n"
+      + " doNotPrompt=true%n"
+      + " storeKey=true;%n"
+      + "}; %n"
       ;
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java
index 693bb0b911fa9..513d7ac0aaf5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/RegistryAdminService.java
@@ -217,9 +217,9 @@ protected void serviceStart() throws Exception {
 
       String message = String.format(Locale.ENGLISH,
           "Failed to create root paths {%s};" +
-          "\ndiagnostics={%s}" +
-          "\ncurrent registry is:" +
-          "\n{%s}",
+          "%ndiagnostics={%s}" +
+          "%ncurrent registry is:" +
+          "%n{%s}",
           e,
           bindingDiagnosticDetails(),
           dumpRegistryRobustly(true));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 968f21fb512ff..287a45ace7dc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -181,4 +181,40 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-maven-plugins</artifactId>
+        <executions>
+          <execution>
+            <id>compile-protoc</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>protoc</goal>
+            </goals>
+            <configuration>
+              <protocVersion>${protobuf.version}</protocVersion>
+              <protocCommand>${protoc.path}</protocCommand>
+              <imports>
+                <param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
+                <param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
+                <param>${basedir}/../../hadoop-yarn-common/src/main/proto/server/</param>
+                <param>${basedir}/../hadoop-yarn-server-common/src/main/proto</param>
+                <param>${basedir}/src/main/proto</param>
+              </imports>
+              <source>
+                <directory>${basedir}/src/main/proto</directory>
+                <includes>
+                  <include>yarn_server_timelineserver_recovery.proto</include>
+                </includes>
+              </source>
+              <output>${project.build.directory}/generated-sources/java</output>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 4c04c5c84f9c5..8da1ea1aea6fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -49,9 +49,11 @@
 import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
@@ -153,13 +155,17 @@ public CancelDelegationTokenResponse cancelDelegationToken(
     public GetApplicationAttemptReportResponse getApplicationAttemptReport(
         GetApplicationAttemptReportRequest request) throws YarnException,
         IOException {
+      ApplicationAttemptId appAttemptId = request.getApplicationAttemptId();
       try {
         GetApplicationAttemptReportResponse response =
             GetApplicationAttemptReportResponse.newInstance(history
-              .getApplicationAttempt(request.getApplicationAttemptId()));
+              .getApplicationAttempt(appAttemptId));
         return response;
       } catch (IOException e) {
-        throw new ApplicationAttemptNotFoundException(e.getMessage());
+        String msg = "ApplicationAttempt with id '" + appAttemptId +
+            "' doesn't exist in the history store.";
+        LOG.error(msg, e);
+        throw new ApplicationAttemptNotFoundException(msg);
       }
     }
 
@@ -177,14 +183,17 @@ public GetApplicationAttemptsResponse getApplicationAttempts(
     @Override
     public GetApplicationReportResponse getApplicationReport(
         GetApplicationReportRequest request) throws YarnException, IOException {
+      ApplicationId applicationId = request.getApplicationId();
       try {
-        ApplicationId applicationId = request.getApplicationId();
         GetApplicationReportResponse response =
             GetApplicationReportResponse.newInstance(history
               .getApplication(applicationId));
         return response;
       } catch (IOException e) {
-        throw new ApplicationNotFoundException(e.getMessage());
+        String msg = "Application with id '" + applicationId +
+            "' doesn't exist in the history store.";
+        LOG.error(msg, e);
+        throw new ApplicationNotFoundException(msg);
       }
     }
 
@@ -200,13 +209,17 @@ public GetApplicationsResponse getApplications(
     @Override
     public GetContainerReportResponse getContainerReport(
         GetContainerReportRequest request) throws YarnException, IOException {
+      ContainerId containerId = request.getContainerId();
       try {
         GetContainerReportResponse response =
-            GetContainerReportResponse.newInstance(history.getContainer(request
-              .getContainerId()));
+            GetContainerReportResponse.newInstance(
+                history.getContainer(containerId));
         return response;
       } catch (IOException e) {
-        throw new ContainerNotFoundException(e.getMessage());
+        String msg = "Container with id '" + containerId +
+            "' doesn't exist in the history store.";
+        LOG.error(msg, e);
+        throw new ContainerNotFoundException(msg);
       }
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index c88ccccf727b8..22418a8535c06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
@@ -222,6 +223,7 @@ private static ApplicationReportExt convertToApplicationReport(
     String diagnosticsInfo = null;
     FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
     YarnApplicationState state = null;
+    ApplicationResourceUsageReport appResources = null;
     Map<ApplicationAccessType, String> appViewACLs =
         new HashMap<ApplicationAccessType, String>();
     Map<String, Object> entityInfo = entity.getOtherInfo();
@@ -260,6 +262,14 @@ private static ApplicationReportExt convertToApplicationReport(
             entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
                 .toString();
       }
+      if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
+        long vcoreSeconds=Long.parseLong(entityInfo.get(
+                ApplicationMetricsConstants.APP_CPU_METRICS).toString());
+        long memorySeconds=Long.parseLong(entityInfo.get(
+                ApplicationMetricsConstants.APP_MEM_METRICS).toString());
+        appResources=ApplicationResourceUsageReport
+            .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
+      }
     }
     List<TimelineEvent> events = entity.getEvents();
     if (events != null) {
@@ -310,7 +320,7 @@ private static ApplicationReportExt convertToApplicationReport(
     return new ApplicationReportExt(ApplicationReport.newInstance(
         ConverterUtils.toApplicationId(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
-        diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+        diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
         null, 1.0F, type, null), appViewACLs);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index c7e305cb2144e..0bafd3695eedd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
@@ -106,9 +107,8 @@ protected void serviceStart() throws Exception {
     } catch(IOException ie) {
       throw new YarnRuntimeException("Failed to login", ie);
     }
-
-    startWebApp();
     super.serviceStart();
+    startWebApp();
   }
 
   @Override
@@ -154,6 +154,7 @@ static ApplicationHistoryServer launchAppHistoryServer(String[] args) {
         new CompositeServiceShutdownHook(appHistoryServer),
         SHUTDOWN_HOOK_PRIORITY);
       YarnConfiguration conf = new YarnConfiguration();
+      new GenericOptionsParser(conf, args);
       appHistoryServer.init(conf);
       appHistoryServer.start();
     } catch (Throwable t) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index ba75c141c26bc..5f153bdea6724 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -18,28 +18,8 @@
 
 package org.apache.hadoop.yarn.server.timeline;
 
-import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
-import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
-import static org.fusesource.leveldbjni.JniDBFactory.bytes;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -53,30 +33,31 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.*;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyParser;
 import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
 import org.fusesource.leveldbjni.JniDBFactory;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Options;
-import org.iq80.leveldb.ReadOptions;
-import org.iq80.leveldb.WriteBatch;
-import org.iq80.leveldb.WriteOptions;
+import org.iq80.leveldb.*;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
+import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
+import static org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.prefixMatches;
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
 
 /**
  * <p>An implementation of an application timeline store backed by leveldb.</p>
@@ -140,25 +121,25 @@ public class LeveldbTimelineStore extends AbstractService
   @VisibleForTesting
   static final String FILENAME = "leveldb-timeline-store.ldb";
 
-  private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes();
-  private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes();
-  private static final byte[] INDEXED_ENTRY_PREFIX = "i".getBytes();
+  private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] INDEXED_ENTRY_PREFIX = "i".getBytes(Charset.forName("UTF-8"));
 
-  private static final byte[] EVENTS_COLUMN = "e".getBytes();
-  private static final byte[] PRIMARY_FILTERS_COLUMN = "f".getBytes();
-  private static final byte[] OTHER_INFO_COLUMN = "i".getBytes();
-  private static final byte[] RELATED_ENTITIES_COLUMN = "r".getBytes();
+  private static final byte[] EVENTS_COLUMN = "e".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] PRIMARY_FILTERS_COLUMN = "f".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] OTHER_INFO_COLUMN = "i".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] RELATED_ENTITIES_COLUMN = "r".getBytes(Charset.forName("UTF-8"));
   private static final byte[] INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN =
-      "z".getBytes();
-  private static final byte[] DOMAIN_ID_COLUMN = "d".getBytes();
+      "z".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] DOMAIN_ID_COLUMN = "d".getBytes(Charset.forName("UTF-8"));
 
-  private static final byte[] DOMAIN_ENTRY_PREFIX = "d".getBytes();
-  private static final byte[] OWNER_LOOKUP_PREFIX = "o".getBytes();
-  private static final byte[] DESCRIPTION_COLUMN = "d".getBytes();
-  private static final byte[] OWNER_COLUMN = "o".getBytes();
-  private static final byte[] READER_COLUMN = "r".getBytes();
-  private static final byte[] WRITER_COLUMN = "w".getBytes();
-  private static final byte[] TIMESTAMP_COLUMN = "t".getBytes();
+  private static final byte[] DOMAIN_ENTRY_PREFIX = "d".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] OWNER_LOOKUP_PREFIX = "o".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] DESCRIPTION_COLUMN = "d".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] OWNER_COLUMN = "o".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] READER_COLUMN = "r".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] WRITER_COLUMN = "w".getBytes(Charset.forName("UTF-8"));
+  private static final byte[] TIMESTAMP_COLUMN = "t".getBytes(Charset.forName("UTF-8"));
 
   private static final byte[] EMPTY_BYTES = new byte[0];
   
@@ -357,102 +338,6 @@ synchronized void returnLock(CountingReentrantLock<K> lock) {
     }
   }
 
-  private static class KeyBuilder {
-    private static final int MAX_NUMBER_OF_KEY_ELEMENTS = 10;
-    private byte[][] b;
-    private boolean[] useSeparator;
-    private int index;
-    private int length;
-
-    public KeyBuilder(int size) {
-      b = new byte[size][];
-      useSeparator = new boolean[size];
-      index = 0;
-      length = 0;
-    }
-
-    public static KeyBuilder newInstance() {
-      return new KeyBuilder(MAX_NUMBER_OF_KEY_ELEMENTS);
-    }
-
-    public KeyBuilder add(String s) {
-      return add(s.getBytes(), true);
-    }
-
-    public KeyBuilder add(byte[] t) {
-      return add(t, false);
-    }
-
-    public KeyBuilder add(byte[] t, boolean sep) {
-      b[index] = t;
-      useSeparator[index] = sep;
-      length += t.length;
-      if (sep) {
-        length++;
-      }
-      index++;
-      return this;
-    }
-
-    public byte[] getBytes() throws IOException {
-      ByteArrayOutputStream baos = new ByteArrayOutputStream(length);
-      for (int i = 0; i < index; i++) {
-        baos.write(b[i]);
-        if (i < index-1 && useSeparator[i]) {
-          baos.write(0x0);
-        }
-      }
-      return baos.toByteArray();
-    }
-
-    public byte[] getBytesForLookup() throws IOException {
-      ByteArrayOutputStream baos = new ByteArrayOutputStream(length);
-      for (int i = 0; i < index; i++) {
-        baos.write(b[i]);
-        if (useSeparator[i]) {
-          baos.write(0x0);
-        }
-      }
-      return baos.toByteArray();
-    }
-  }
-
-  private static class KeyParser {
-    private final byte[] b;
-    private int offset;
-
-    public KeyParser(byte[] b, int offset) {
-      this.b = b;
-      this.offset = offset;
-    }
-
-    public String getNextString() throws IOException {
-      if (offset >= b.length) {
-        throw new IOException(
-            "tried to read nonexistent string from byte array");
-      }
-      int i = 0;
-      while (offset+i < b.length && b[offset+i] != 0x0) {
-        i++;
-      }
-      String s = new String(b, offset, i);
-      offset = offset + i + 1;
-      return s;
-    }
-
-    public long getNextLong() throws IOException {
-      if (offset+8 >= b.length) {
-        throw new IOException("byte array ran out when trying to read long");
-      }
-      long l = readReverseOrderedLong(b, offset);
-      offset += 8;
-      return l;
-    }
-
-    public int getOffset() {
-      return offset;
-    }
-  }
 
   @Override
   public TimelineEntity getEntity(String entityId, String entityType,
@@ -556,7 +441,7 @@ private static TimelineEntity getEntity(String entityId, String entityType,
         }
       } else if (key[prefixlen] == DOMAIN_ID_COLUMN[0]) {
         byte[] v = iterator.peekNext().getValue();
-        String domainId = new String(v);
+        String domainId = new String(v, Charset.forName("UTF-8"));
         entity.setDomainId(domainId);
       } else {
         if (key[prefixlen] !=
@@ -660,18 +545,6 @@ public int compare(byte[] o1, byte[] o2) {
     return events;
   }
 
-  /**
-   * Returns true if the byte array begins with the specified prefix.
-   */
-  private static boolean prefixMatches(byte[] prefix, int prefixlen,
-      byte[] b) {
-    if (b.length < prefixlen) {
-      return false;
-    }
-    return WritableComparator.compareBytes(prefix, 0, prefixlen, b, 0,
-        prefixlen) == 0;
-  }
-
   @Override
   public TimelineEntities getEntities(String entityType,
       Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs,
@@ -918,7 +791,7 @@ private void put(TimelineEntity entity, TimelinePutResponse response,
               if (domainIdBytes == null) {
                 domainId = TimelineDataManager.DEFAULT_DOMAIN_ID;
               } else {
-                domainId = new String(domainIdBytes);
+                domainId = new String(domainIdBytes, Charset.forName("UTF-8"));
               }
               if (!domainId.equals(entity.getDomainId())) {
                 // in this case the entity will be put, but the relation will be
@@ -973,9 +846,9 @@ private void put(TimelineEntity entity, TimelinePutResponse response,
           return;
         }
       } else {
-        writeBatch.put(key, entity.getDomainId().getBytes());
+        writeBatch.put(key, entity.getDomainId().getBytes(Charset.forName("UTF-8")));
         writePrimaryFilterEntries(writeBatch, primaryFilters, key,
-            entity.getDomainId().getBytes());
+            entity.getDomainId().getBytes(Charset.forName("UTF-8")));
       }
       db.write(writeBatch);
     } catch (DBException de) {
@@ -1007,7 +880,7 @@ private void put(TimelineEntity entity, TimelinePutResponse response,
           // This is the new entity, the domain should be the same
         byte[] key = createDomainIdKey(relatedEntity.getId(),
             relatedEntity.getType(), relatedEntityStartTime);
-        db.put(key, entity.getDomainId().getBytes());
+        db.put(key, entity.getDomainId().getBytes(Charset.forName("UTF-8")));
         db.put(createRelatedEntityKey(relatedEntity.getId(),
             relatedEntity.getType(), relatedEntityStartTime,
             entity.getEntityId(), entity.getEntityType()), EMPTY_BYTES);
@@ -1334,7 +1207,7 @@ private static byte[] createOtherInfoKey(String entityId, String entityType,
    * to the end of the array (for parsing other info keys).
    */
   private static String parseRemainingKey(byte[] b, int offset) {
-    return new String(b, offset, b.length - offset);
+    return new String(b, offset, b.length - offset, Charset.forName("UTF-8"));
   }
 
   /**
@@ -1637,9 +1510,9 @@ LeveldbIterator getDbIterator(boolean fillCache) {
   Version loadVersion() throws IOException {
     try {
       byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY));
-      // if version is not stored previously, treat it as 1.0.
+      // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
       if (data == null || data.length == 0) {
-        return Version.newInstance(1, 0);
+        return getCurrentVersion();
       }
       Version version =
           new VersionPBImpl(VersionProto.parseFrom(data));
@@ -1717,8 +1590,10 @@ public void put(TimelineDomain domain) throws IOException {
       byte[] ownerLookupEntryKey = createOwnerLookupKey(
           domain.getOwner(), domain.getId(), DESCRIPTION_COLUMN);
       if (domain.getDescription() != null) {
-        writeBatch.put(domainEntryKey, domain.getDescription().getBytes());
-        writeBatch.put(ownerLookupEntryKey, domain.getDescription().getBytes());
+        writeBatch.put(domainEntryKey, domain.getDescription().
+                       getBytes(Charset.forName("UTF-8")));
+        writeBatch.put(ownerLookupEntryKey, domain.getDescription().
+                       getBytes(Charset.forName("UTF-8")));
       } else {
         writeBatch.put(domainEntryKey, EMPTY_BYTES);
         writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
@@ -1729,16 +1604,17 @@ public void put(TimelineDomain domain) throws IOException {
       ownerLookupEntryKey = createOwnerLookupKey(
           domain.getOwner(), domain.getId(), OWNER_COLUMN);
       // Null check for owner is done before
-      writeBatch.put(domainEntryKey, domain.getOwner().getBytes());
-      writeBatch.put(ownerLookupEntryKey, domain.getOwner().getBytes());
+      writeBatch.put(domainEntryKey, domain.getOwner().getBytes(Charset.forName("UTF-8")));
+      writeBatch.put(ownerLookupEntryKey, domain.getOwner().getBytes(Charset.forName("UTF-8")));
 
       // Write readers
       domainEntryKey = createDomainEntryKey(domain.getId(), READER_COLUMN);
       ownerLookupEntryKey = createOwnerLookupKey(
           domain.getOwner(), domain.getId(), READER_COLUMN);
       if (domain.getReaders() != null && domain.getReaders().length() > 0) {
-        writeBatch.put(domainEntryKey, domain.getReaders().getBytes());
-        writeBatch.put(ownerLookupEntryKey, domain.getReaders().getBytes());
+        writeBatch.put(domainEntryKey, domain.getReaders().getBytes(Charset.forName("UTF-8")));
+        writeBatch.put(ownerLookupEntryKey, domain.getReaders().
+                       getBytes(Charset.forName("UTF-8")));
       } else {
         writeBatch.put(domainEntryKey, EMPTY_BYTES);
         writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
@@ -1749,8 +1625,9 @@ public void put(TimelineDomain domain) throws IOException {
       ownerLookupEntryKey = createOwnerLookupKey(
           domain.getOwner(), domain.getId(), WRITER_COLUMN);
       if (domain.getWriters() != null && domain.getWriters().length() > 0) {
-        writeBatch.put(domainEntryKey, domain.getWriters().getBytes());
-        writeBatch.put(ownerLookupEntryKey, domain.getWriters().getBytes());
+        writeBatch.put(domainEntryKey, domain.getWriters().getBytes(Charset.forName("UTF-8")));
+        writeBatch.put(ownerLookupEntryKey, domain.getWriters().
+                       getBytes(Charset.forName("UTF-8")));
       } else {
         writeBatch.put(domainEntryKey, EMPTY_BYTES);
         writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
@@ -1887,13 +1764,13 @@ private static TimelineDomain getTimelineDomain(
       byte[] value = iterator.peekNext().getValue();
       if (value != null && value.length > 0) {
         if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
-          domain.setDescription(new String(value));
+          domain.setDescription(new String(value, Charset.forName("UTF-8")));
         } else if (key[prefix.length] == OWNER_COLUMN[0]) {
-          domain.setOwner(new String(value));
+          domain.setOwner(new String(value, Charset.forName("UTF-8")));
         } else if (key[prefix.length] == READER_COLUMN[0]) {
-          domain.setReaders(new String(value));
+          domain.setReaders(new String(value, Charset.forName("UTF-8")));
         } else if (key[prefix.length] == WRITER_COLUMN[0]) {
-          domain.setWriters(new String(value));
+          domain.setWriters(new String(value, Charset.forName("UTF-8")));
         } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
           domain.setCreatedTime(readReverseOrderedLong(value, 0));
           domain.setModifiedTime(readReverseOrderedLong(value, 8));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
index af714b17b800f..9c5419e44c1cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
@@ -398,7 +398,7 @@ public synchronized TimelinePutResponse put(TimelineEntities data) {
   public void put(TimelineDomain domain) throws IOException {
     TimelineDomain domainToReplace =
         domainsById.get(domain.getId());
-    long currentTimestamp = System.currentTimeMillis();
+    Long currentTimestamp = System.currentTimeMillis();
     TimelineDomain domainToStore = createTimelineDomain(
         domain.getId(), domain.getDescription(), domain.getOwner(),
         domain.getReaders(), domain.getWriters(),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/LeveldbTimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/LeveldbTimelineStateStore.java
new file mode 100644
index 0000000000000..b62a54111af35
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/LeveldbTimelineStateStore.java
@@ -0,0 +1,420 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.recovery;
+
+import static org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.prefixMatches;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.timeline.recovery.records.TimelineDelegationTokenIdentifierData;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
+import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.Options;
+import org.iq80.leveldb.WriteBatch;
+
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+
+/**
+ * A timeline service state storage implementation that supports any persistent
+ * storage that adheres to the LevelDB interface.
+ */
+public class LeveldbTimelineStateStore extends
+    TimelineStateStore {
+
+  public static final Log LOG =
+      LogFactory.getLog(LeveldbTimelineStateStore.class);
+
+  private static final String DB_NAME = "timeline-state-store.ldb";
+  private static final FsPermission LEVELDB_DIR_UMASK = FsPermission
+      .createImmutable((short) 0700);
+
+  private static final byte[] TOKEN_ENTRY_PREFIX = bytes("t");
+  private static final byte[] TOKEN_MASTER_KEY_ENTRY_PREFIX = bytes("k");
+  private static final byte[] LATEST_SEQUENCE_NUMBER_KEY = bytes("s");
+
+  private static final Version CURRENT_VERSION_INFO = Version.newInstance(1, 0);
+  private static final byte[] TIMELINE_STATE_STORE_VERSION_KEY = bytes("v");
+
+  private DB db;
+
+  public LeveldbTimelineStateStore() {
+    super(LeveldbTimelineStateStore.class.getName());
+  }
+
+  @Override
+  protected void initStorage(Configuration conf) throws IOException {
+  }
+
+  @Override
+  protected void startStorage() throws IOException {
+    Options options = new Options();
+    Path dbPath =
+        new Path(
+            getConfig().get(
+                YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH),
+            DB_NAME);
+    FileSystem localFS = null;
+    try {
+      localFS = FileSystem.getLocal(getConfig());
+      if (!localFS.exists(dbPath)) {
+        if (!localFS.mkdirs(dbPath)) {
+          throw new IOException("Couldn't create directory for leveldb " +
+              "timeline store " + dbPath);
+        }
+        localFS.setPermission(dbPath, LEVELDB_DIR_UMASK);
+      }
+    } finally {
+      IOUtils.cleanup(LOG, localFS);
+    }
+    JniDBFactory factory = new JniDBFactory();
+    try {
+      options.createIfMissing(false);
+      db = factory.open(new File(dbPath.toString()), options);
+      LOG.info("Loading the existing database at th path: " + dbPath.toString());
+      checkVersion();
+    } catch (NativeDB.DBException e) {
+      if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+        try {
+          options.createIfMissing(true);
+          db = factory.open(new File(dbPath.toString()), options);
+          LOG.info("Creating a new database at th path: " + dbPath.toString());
+          storeVersion(CURRENT_VERSION_INFO);
+        } catch (DBException ex) {
+          throw new IOException(ex);
+        }
+      } else {
+        throw new IOException(e);
+      }
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  protected void closeStorage() throws IOException {
+    IOUtils.cleanup(LOG, db);
+  }
+
+  @Override
+  public TimelineServiceState loadState() throws IOException {
+    LOG.info("Loading timeline service state from leveldb");
+    TimelineServiceState state = new TimelineServiceState();
+    int numKeys = loadTokenMasterKeys(state);
+    int numTokens = loadTokens(state);
+    loadLatestSequenceNumber(state);
+    LOG.info("Loaded " + numKeys + " master keys and " + numTokens
+        + " tokens from leveldb, and latest sequence number is "
+        + state.getLatestSequenceNumber());
+    return state;
+  }
+
+  @Override
+  public void storeToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException {
+    DataOutputStream ds = null;
+    WriteBatch batch = null;
+    try {
+      byte[] k = createTokenEntryKey(tokenId.getSequenceNumber());
+      if (db.get(k) != null) {
+        throw new IOException(tokenId + " already exists");
+      }
+      byte[] v = buildTokenData(tokenId, renewDate);
+      ByteArrayOutputStream bs = new ByteArrayOutputStream();
+      ds = new DataOutputStream(bs);
+      ds.writeInt(tokenId.getSequenceNumber());
+      batch = db.createWriteBatch();
+      batch.put(k, v);
+      batch.put(LATEST_SEQUENCE_NUMBER_KEY, bs.toByteArray());
+      db.write(batch);
+    } catch (DBException e) {
+      throw new IOException(e);
+    } finally {
+      IOUtils.cleanup(LOG, ds);
+      IOUtils.cleanup(LOG, batch);
+    }
+  }
+
+  @Override
+  public void updateToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException {
+    try {
+      byte[] k = createTokenEntryKey(tokenId.getSequenceNumber());
+      if (db.get(k) == null) {
+        throw new IOException(tokenId + " doesn't exist");
+      }
+      byte[] v = buildTokenData(tokenId, renewDate);
+      db.put(k, v);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void removeToken(TimelineDelegationTokenIdentifier tokenId)
+      throws IOException {
+    try {
+      byte[] key = createTokenEntryKey(tokenId.getSequenceNumber());
+      db.delete(key);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void storeTokenMasterKey(DelegationKey key) throws IOException {
+    try {
+      byte[] k = createTokenMasterKeyEntryKey(key.getKeyId());
+      if (db.get(k) != null) {
+        throw new IOException(key + " already exists");
+      }
+      byte[] v = buildTokenMasterKeyData(key);
+      db.put(k, v);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void removeTokenMasterKey(DelegationKey key) throws IOException {
+    try {
+      byte[] k = createTokenMasterKeyEntryKey(key.getKeyId());
+      db.delete(k);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private static byte[] buildTokenData(
+      TimelineDelegationTokenIdentifier tokenId, Long renewDate)
+      throws IOException {
+    TimelineDelegationTokenIdentifierData data =
+        new TimelineDelegationTokenIdentifierData(tokenId, renewDate);
+    return data.toByteArray();
+  }
+
+  private static byte[] buildTokenMasterKeyData(DelegationKey key)
+      throws IOException {
+    ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+    DataOutputStream dataStream = new DataOutputStream(memStream);
+    try {
+      key.write(dataStream);
+      dataStream.close();
+    } finally {
+      IOUtils.cleanup(LOG, dataStream);
+    }
+    return memStream.toByteArray();
+  }
+
+  private static void loadTokenMasterKeyData(TimelineServiceState state,
+      byte[] keyData)
+      throws IOException {
+    DelegationKey key = new DelegationKey();
+    DataInputStream in =
+        new DataInputStream(new ByteArrayInputStream(keyData));
+    try {
+      key.readFields(in);
+    } finally {
+      IOUtils.cleanup(LOG, in);
+    }
+    state.tokenMasterKeyState.add(key);
+  }
+
+  private static void loadTokenData(TimelineServiceState state, byte[] tokenData)
+      throws IOException {
+    TimelineDelegationTokenIdentifierData data =
+        new TimelineDelegationTokenIdentifierData();
+    DataInputStream in =
+        new DataInputStream(new ByteArrayInputStream(tokenData));
+    try {
+      data.readFields(in);
+    } finally {
+      IOUtils.cleanup(LOG, in);
+    }
+    state.tokenState.put(data.getTokenIdentifier(), data.getRenewDate());
+  }
+
+  private int loadTokenMasterKeys(TimelineServiceState state)
+      throws IOException {
+    byte[] base = KeyBuilder.newInstance().add(TOKEN_MASTER_KEY_ENTRY_PREFIX)
+        .getBytesForLookup();
+    int numKeys = 0;
+    LeveldbIterator iterator = null;
+    try {
+      for (iterator = new LeveldbIterator(db), iterator.seek(base);
+          iterator.hasNext(); iterator.next()) {
+        byte[] k = iterator.peekNext().getKey();
+        if (!prefixMatches(base, base.length, k)) {
+          break;
+        }
+        byte[] v = iterator.peekNext().getValue();
+        loadTokenMasterKeyData(state, v);
+        ++numKeys;
+      }
+    } finally {
+      IOUtils.cleanup(LOG, iterator);
+    }
+    return numKeys;
+  }
+
+  private int loadTokens(TimelineServiceState state) throws IOException {
+    byte[] base = KeyBuilder.newInstance().add(TOKEN_ENTRY_PREFIX)
+        .getBytesForLookup();
+    int numTokens = 0;
+    LeveldbIterator iterator = null;
+    try {
+      for (iterator = new LeveldbIterator(db), iterator.seek(base);
+          iterator.hasNext(); iterator.next()) {
+        byte[] k = iterator.peekNext().getKey();
+        if (!prefixMatches(base, base.length, k)) {
+          break;
+        }
+        byte[] v = iterator.peekNext().getValue();
+        loadTokenData(state, v);
+        ++numTokens;
+      }
+    } catch (DBException e) {
+      throw new IOException(e);
+    } finally {
+      IOUtils.cleanup(LOG, iterator);
+    }
+    return numTokens;
+  }
+
+  private void loadLatestSequenceNumber(TimelineServiceState state)
+      throws IOException {
+    byte[] data = null;
+    try {
+      data = db.get(LATEST_SEQUENCE_NUMBER_KEY);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+    if (data != null) {
+      DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+      try {
+        state.latestSequenceNumber = in.readInt();
+      } finally {
+        IOUtils.cleanup(LOG, in);
+      }
+    }
+  }
+  /**
+   * Creates a domain entity key with column name suffix, of the form
+   * TOKEN_ENTRY_PREFIX + sequence number.
+   */
+  private static byte[] createTokenEntryKey(int seqNum) throws IOException {
+    return KeyBuilder.newInstance().add(TOKEN_ENTRY_PREFIX)
+        .add(Integer.toString(seqNum)).getBytes();
+  }
+
+  /**
+   * Creates a domain entity key with column name suffix, of the form
+   * TOKEN_MASTER_KEY_ENTRY_PREFIX + sequence number.
+   */
+  private static byte[] createTokenMasterKeyEntryKey(int keyId)
+      throws IOException {
+    return KeyBuilder.newInstance().add(TOKEN_MASTER_KEY_ENTRY_PREFIX)
+        .add(Integer.toString(keyId)).getBytes();
+  }
+
+  @VisibleForTesting
+  Version loadVersion() throws IOException {
+    try {
+      byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY);
+      // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
+      if (data == null || data.length == 0) {
+        return getCurrentVersion();
+      }
+      Version version =
+          new VersionPBImpl(
+              YarnServerCommonProtos.VersionProto.parseFrom(data));
+      return version;
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @VisibleForTesting
+  void storeVersion(Version state) throws IOException {
+    byte[] data =
+        ((VersionPBImpl) state).getProto().toByteArray();
+    try {
+      db.put(TIMELINE_STATE_STORE_VERSION_KEY, data);
+    } catch (DBException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @VisibleForTesting
+  Version getCurrentVersion() {
+    return CURRENT_VERSION_INFO;
+  }
+
+  /**
+   * 1) Versioning timeline state store:
+   * major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
+   * 2) Any incompatible change of TS-store is a major upgrade, and any
+   * compatible change of TS-store is a minor upgrade.
+   * 3) Within a minor upgrade, say 1.1 to 1.2:
+   * overwrite the version info and proceed as normal.
+   * 4) Within a major upgrade, say 1.2 to 2.0:
+   * throw exception and indicate user to use a separate upgrade tool to
+   * upgrade timeline store or remove incompatible old state.
+   */
+  private void checkVersion() throws IOException {
+    Version loadedVersion = loadVersion();
+    LOG.info("Loaded timeline state store version info " + loadedVersion);
+    if (loadedVersion.equals(getCurrentVersion())) {
+      return;
+    }
+    if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+      LOG.info("Storing timeline state store version info " + getCurrentVersion());
+      storeVersion(CURRENT_VERSION_INFO);
+    } else {
+      String incompatibleMessage =
+          "Incompatible version for timeline state store: expecting version "
+              + getCurrentVersion() + ", but loading version " + loadedVersion;
+      LOG.fatal(incompatibleMessage);
+      throw new IOException(incompatibleMessage);
+    }
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/MemoryTimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/MemoryTimelineStateStore.java
new file mode 100644
index 0000000000000..f50cd5dc3931b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/MemoryTimelineStateStore.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timeline.recovery;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+/**
+ * A state store backed by memory for unit tests
+ */
+public class MemoryTimelineStateStore
+    extends TimelineStateStore {
+
+  private TimelineServiceState state;
+
+  @Override
+  protected void initStorage(Configuration conf) throws IOException {
+  }
+
+  @Override
+  protected void startStorage() throws IOException {
+    state = new TimelineServiceState();
+  }
+
+  @Override
+  protected void closeStorage() throws IOException {
+    state = null;
+  }
+
+  @Override
+  public TimelineServiceState loadState() throws IOException {
+    TimelineServiceState result = new TimelineServiceState();
+    result.tokenState.putAll(state.tokenState);
+    result.tokenMasterKeyState.addAll(state.tokenMasterKeyState);
+    result.latestSequenceNumber = state.latestSequenceNumber;
+    return result;
+  }
+
+  @Override
+  public void storeToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException {
+    if (state.tokenState.containsKey(tokenId)) {
+      throw new IOException("token " + tokenId + " was stored twice");
+    }
+    state.tokenState.put(tokenId, renewDate);
+    state.latestSequenceNumber = tokenId.getSequenceNumber();
+  }
+
+  @Override
+  public void updateToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException {
+    if (!state.tokenState.containsKey(tokenId)) {
+      throw new IOException("token " + tokenId + " not in store");
+    }
+    state.tokenState.put(tokenId, renewDate);
+  }
+
+  @Override
+  public void removeToken(TimelineDelegationTokenIdentifier tokenId)
+      throws IOException {
+    state.tokenState.remove(tokenId);
+  }
+
+  @Override
+  public void storeTokenMasterKey(DelegationKey key)
+      throws IOException {
+    if (state.tokenMasterKeyState.contains(key)) {
+      throw new IOException("token master key " + key + " was stored twice");
+    }
+    state.tokenMasterKeyState.add(key);
+  }
+
+  @Override
+  public void removeTokenMasterKey(DelegationKey key)
+      throws IOException {
+    state.tokenMasterKeyState.remove(key);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/TimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/TimelineStateStore.java
new file mode 100644
index 0000000000000..7bfda55d13b28
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/TimelineStateStore.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.recovery;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+@Private
+@Unstable
+/**
+ * Base class for timeline service state storage.
+ * Storage implementations need to implement blocking store and load methods
+ * to actually store and load the state.
+ */
+public abstract class TimelineStateStore extends AbstractService {
+
+  public static class TimelineServiceState {
+    int latestSequenceNumber = 0;
+    Map<TimelineDelegationTokenIdentifier, Long> tokenState =
+        new HashMap<TimelineDelegationTokenIdentifier, Long>();
+    Set<DelegationKey> tokenMasterKeyState = new HashSet<DelegationKey>();
+
+    public int getLatestSequenceNumber() {
+      return latestSequenceNumber;
+    }
+
+    public Map<TimelineDelegationTokenIdentifier, Long> getTokenState() {
+      return tokenState;
+    }
+
+    public Set<DelegationKey> getTokenMasterKeyState() {
+      return tokenMasterKeyState;
+    }
+  }
+
+  public TimelineStateStore() {
+    super(TimelineStateStore.class.getName());
+  }
+
+  public TimelineStateStore(String name) {
+    super(name);
+  }
+
+  /**
+   * Initialize the state storage
+   *
+   * @param conf the configuration
+   * @throws IOException
+   */
+  @Override
+  public void serviceInit(Configuration conf) throws IOException {
+    initStorage(conf);
+  }
+
+  /**
+   * Start the state storage for use
+   *
+   * @throws IOException
+   */
+  @Override
+  public void serviceStart() throws IOException {
+    startStorage();
+  }
+
+  /**
+   * Shutdown the state storage.
+   *
+   * @throws IOException
+   */
+  @Override
+  public void serviceStop() throws IOException {
+    closeStorage();
+  }
+
+  /**
+   * Implementation-specific initialization.
+   *
+   * @param conf the configuration
+   * @throws IOException
+   */
+  protected abstract void initStorage(Configuration conf) throws IOException;
+
+  /**
+   * Implementation-specific startup.
+   *
+   * @throws IOException
+   */
+  protected abstract void startStorage() throws IOException;
+
+  /**
+   * Implementation-specific shutdown.
+   *
+   * @throws IOException
+   */
+  protected abstract void closeStorage() throws IOException;
+
+  /**
+   * Load the timeline service state from the state storage.
+   *
+   * @throws IOException
+   */
+  public abstract TimelineServiceState loadState() throws IOException;
+
+  /**
+   * Blocking method to store a delegation token along with the current token
+   * sequence number to the state storage.
+   *
+   * Implementations must not return from this method until the token has been
+   * committed to the state store.
+   *
+   * @param tokenId the token to store
+   * @param renewDate the token renewal deadline
+   * @throws IOException
+   */
+  public abstract void storeToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException;
+
+  /**
+   * Blocking method to update the expiration of a delegation token
+   * in the state storage.
+   *
+   * Implementations must not return from this method until the expiration
+   * date of the token has been updated in the state store.
+   *
+   * @param tokenId the token to update
+   * @param renewDate the new token renewal deadline
+   * @throws IOException
+   */
+  public abstract void updateToken(TimelineDelegationTokenIdentifier tokenId,
+      Long renewDate) throws IOException;
+
+  /**
+   * Blocking method to remove a delegation token from the state storage.
+   *
+   * Implementations must not return from this method until the token has been
+   * removed from the state store.
+   *
+   * @param tokenId the token to remove
+   * @throws IOException
+   */
+  public abstract void removeToken(TimelineDelegationTokenIdentifier tokenId)
+      throws IOException;
+
+  /**
+   * Blocking method to store a delegation token master key.
+   *
+   * Implementations must not return from this method until the key has been
+   * committed to the state store.
+   *
+   * @param key the master key to store
+   * @throws IOException
+   */
+  public abstract void storeTokenMasterKey(
+      DelegationKey key) throws IOException;
+
+  /**
+   * Blocking method to remove a delegation token master key.
+   *
+   * Implementations must not return from this method until the key has been
+   * removed from the state store.
+   *
+   * @param key the master key to remove
+   * @throws IOException
+   */
+  public abstract void removeTokenMasterKey(DelegationKey key)
+      throws IOException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/records/TimelineDelegationTokenIdentifierData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/records/TimelineDelegationTokenIdentifierData.java
new file mode 100644
index 0000000000000..f60170d435635
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/recovery/records/TimelineDelegationTokenIdentifierData.java
@@ -0,0 +1,63 @@
+package org.apache.hadoop.yarn.server.timeline.recovery.records;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.yarn.proto.YarnServerTimelineServerRecoveryProtos.TimelineDelegationTokenIdentifierDataProto;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+public class TimelineDelegationTokenIdentifierData {
+  TimelineDelegationTokenIdentifierDataProto.Builder builder =
+      TimelineDelegationTokenIdentifierDataProto.newBuilder();
+
+  public TimelineDelegationTokenIdentifierData() {
+  }
+
+  public TimelineDelegationTokenIdentifierData(
+      TimelineDelegationTokenIdentifier identifier, long renewdate) {
+    builder.setTokenIdentifier(identifier.getProto());
+    builder.setRenewDate(renewdate);
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    builder.mergeFrom((DataInputStream) in);
+  }
+
+  public byte[] toByteArray() throws IOException {
+    return builder.build().toByteArray();
+  }
+
+  public TimelineDelegationTokenIdentifier getTokenIdentifier()
+      throws IOException {
+    ByteArrayInputStream in =
+        new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray());
+    TimelineDelegationTokenIdentifier identifer =
+        new TimelineDelegationTokenIdentifier();
+    identifer.readFields(new DataInputStream(in));
+    return identifer;
+  }
+
+  public long getRenewDate() {
+    return builder.getRenewDate();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
index d041d5d3c1ddd..39c10fb1fdf5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
@@ -18,9 +18,12 @@
 
 package org.apache.hadoop.yarn.server.timeline.security;
 
-import java.io.FileReader;
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -113,7 +116,9 @@ public void initFilter(FilterContainer container, Configuration conf) {
       Reader reader = null;
       try {
         StringBuilder secret = new StringBuilder();
-        reader = new FileReader(signatureSecretFile);
+        reader = new InputStreamReader(new FileInputStream(new File(signatureSecretFile)),
+                                      Charset.forName("UTF-8"));
+
         int c = reader.read();
         while (c > -1) {
           secret.append((char) c);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
index 11a64e6645f2a..c940eea703704 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java
@@ -18,33 +18,34 @@
 
 package org.apache.hadoop.yarn.server.timeline.security;
 
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
 import java.io.IOException;
-import java.net.InetSocketAddress;
+import java.util.Map.Entry;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
 
 /**
  * The service wrapper of {@link TimelineDelegationTokenSecretManager}
  */
 @Private
 @Unstable
-public class TimelineDelegationTokenSecretManagerService extends AbstractService {
+public class TimelineDelegationTokenSecretManagerService extends
+    AbstractService {
 
   private TimelineDelegationTokenSecretManager secretManager = null;
-  private InetSocketAddress serviceAddr = null;
+  private TimelineStateStore stateStore = null;
 
   public TimelineDelegationTokenSecretManagerService() {
     super(TimelineDelegationTokenSecretManagerService.class.getName());
@@ -52,60 +53,93 @@ public TimelineDelegationTokenSecretManagerService() {
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
+    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED)) {
+      stateStore = createStateStore(conf);
+      stateStore.init(conf);
+    }
+
     long secretKeyInterval =
-        conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY,
-            YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+        conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL,
+            YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL);
     long tokenMaxLifetime =
-        conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
-            YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+        conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME,
+            YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME);
     long tokenRenewInterval =
-        conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
-            YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+        conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL,
+            YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL);
     secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval,
-        tokenMaxLifetime, tokenRenewInterval,
-        3600000);
-    secretManager.startThreads();
-
-    serviceAddr = TimelineUtils.getTimelineTokenServiceAddress(getConfig());
+        tokenMaxLifetime, tokenRenewInterval, 3600000, stateStore);
     super.init(conf);
   }
 
+  @Override
+  protected void serviceStart() throws Exception {
+    if (stateStore != null) {
+      stateStore.start();
+      TimelineServiceState state = stateStore.loadState();
+      secretManager.recover(state);
+    }
+
+    secretManager.startThreads();
+    super.serviceStart();
+  }
+
   @Override
   protected void serviceStop() throws Exception {
+    if (stateStore != null) {
+      stateStore.stop();
+    }
+
     secretManager.stopThreads();
     super.stop();
   }
 
+  protected TimelineStateStore createStateStore(
+      Configuration conf) {
+    return ReflectionUtils.newInstance(
+        conf.getClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+            LeveldbTimelineStateStore.class,
+            TimelineStateStore.class), conf);
+  }
+
   /**
    * Ge the instance of {link #TimelineDelegationTokenSecretManager}
+   *
    * @return the instance of {link #TimelineDelegationTokenSecretManager}
    */
-  public TimelineDelegationTokenSecretManager getTimelineDelegationTokenSecretManager() {
+  public TimelineDelegationTokenSecretManager
+  getTimelineDelegationTokenSecretManager() {
     return secretManager;
   }
 
-  /**
-   * Create a timeline secret manager
-   * 
-   * @param delegationKeyUpdateInterval
-   *          the number of seconds for rolling new secret keys.
-   * @param delegationTokenMaxLifetime
-   *          the maximum lifetime of the delegation tokens
-   * @param delegationTokenRenewInterval
-   *          how often the tokens must be renewed
-   * @param delegationTokenRemoverScanInterval
-   *          how often the tokens are scanned for expired tokens
-   */
   @Private
   @Unstable
   public static class TimelineDelegationTokenSecretManager extends
       AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> {
 
-    public TimelineDelegationTokenSecretManager(long delegationKeyUpdateInterval,
-        long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
-        long delegationTokenRemoverScanInterval) {
+    public static final Log LOG =
+        LogFactory.getLog(TimelineDelegationTokenSecretManager.class);
+
+    private TimelineStateStore stateStore;
+
+    /**
+     * Create a timeline secret manager
+     *
+     * @param delegationKeyUpdateInterval the number of seconds for rolling new secret keys.
+     * @param delegationTokenMaxLifetime the maximum lifetime of the delegation tokens
+     * @param delegationTokenRenewInterval how often the tokens must be renewed
+     * @param delegationTokenRemoverScanInterval how often the tokens are scanned for expired tokens
+     */
+    public TimelineDelegationTokenSecretManager(
+        long delegationKeyUpdateInterval,
+        long delegationTokenMaxLifetime,
+        long delegationTokenRenewInterval,
+        long delegationTokenRemoverScanInterval,
+        TimelineStateStore stateStore) {
       super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
           delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+      this.stateStore = stateStore;
     }
 
     @Override
@@ -113,6 +147,90 @@ public TimelineDelegationTokenIdentifier createIdentifier() {
       return new TimelineDelegationTokenIdentifier();
     }
 
+    @Override
+    protected void storeNewMasterKey(DelegationKey key) throws IOException {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Storing master key " + key.getKeyId());
+      }
+      try {
+        if (stateStore != null) {
+          stateStore.storeTokenMasterKey(key);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to store master key " + key.getKeyId(), e);
+      }
+    }
+
+    @Override
+    protected void removeStoredMasterKey(DelegationKey key) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Removing master key " + key.getKeyId());
+      }
+      try {
+        if (stateStore != null) {
+          stateStore.removeTokenMasterKey(key);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to remove master key " + key.getKeyId(), e);
+      }
+    }
+
+    @Override
+    protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId,
+        long renewDate) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Storing token " + tokenId.getSequenceNumber());
+      }
+      try {
+        if (stateStore != null) {
+          stateStore.storeToken(tokenId, renewDate);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to store token " + tokenId.getSequenceNumber(), e);
+      }
+    }
+
+    @Override
+    protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId)
+        throws IOException {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Storing token " + tokenId.getSequenceNumber());
+      }
+      try {
+        if (stateStore != null) {
+          stateStore.removeToken(tokenId);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to remove token " + tokenId.getSequenceNumber(), e);
+      }
+    }
+
+    @Override
+    protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId,
+        long renewDate) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Updating token " + tokenId.getSequenceNumber());
+      }
+      try {
+        if (stateStore != null) {
+          stateStore.updateToken(tokenId, renewDate);
+        }
+      } catch (IOException e) {
+        LOG.error("Unable to update token " + tokenId.getSequenceNumber(), e);
+      }
+    }
+
+    public void recover(TimelineServiceState state) throws IOException {
+      LOG.info("Recovering " + getClass().getSimpleName());
+      for (DelegationKey key : state.getTokenMasterKeyState()) {
+        addKey(key);
+      }
+      this.delegationTokenSequenceNumber = state.getLatestSequenceNumber();
+      for (Entry<TimelineDelegationTokenIdentifier, Long> entry :
+          state.getTokenState().entrySet()) {
+        addPersistedDelegationToken(entry.getKey(), entry.getValue());
+      }
+    }
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java
new file mode 100644
index 0000000000000..fc6cc7d9db811
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.util;
+
+
+import org.apache.hadoop.io.WritableComparator;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
+
+public class LeveldbUtils {
+
+  public static class KeyBuilder {
+    private static final int MAX_NUMBER_OF_KEY_ELEMENTS = 10;
+    private byte[][] b;
+    private boolean[] useSeparator;
+    private int index;
+    private int length;
+
+    public KeyBuilder(int size) {
+      b = new byte[size][];
+      useSeparator = new boolean[size];
+      index = 0;
+      length = 0;
+    }
+
+    public static KeyBuilder newInstance() {
+      return new KeyBuilder(MAX_NUMBER_OF_KEY_ELEMENTS);
+    }
+
+    public KeyBuilder add(String s) {
+      return add(s.getBytes(Charset.forName("UTF-8")), true);
+    }
+
+    public KeyBuilder add(byte[] t) {
+      return add(t, false);
+    }
+
+    public KeyBuilder add(byte[] t, boolean sep) {
+      b[index] = t;
+      useSeparator[index] = sep;
+      length += t.length;
+      if (sep) {
+        length++;
+      }
+      index++;
+      return this;
+    }
+
+    public byte[] getBytes() throws IOException {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream(length);
+      for (int i = 0; i < index; i++) {
+        baos.write(b[i]);
+        if (i < index - 1 && useSeparator[i]) {
+          baos.write(0x0);
+        }
+      }
+      return baos.toByteArray();
+    }
+
+    public byte[] getBytesForLookup() throws IOException {
+      ByteArrayOutputStream baos = new ByteArrayOutputStream(length);
+      for (int i = 0; i < index; i++) {
+        baos.write(b[i]);
+        if (useSeparator[i]) {
+          baos.write(0x0);
+        }
+      }
+      return baos.toByteArray();
+    }
+  }
+
+  public static class KeyParser {
+    private final byte[] b;
+    private int offset;
+
+    public KeyParser(byte[] b, int offset) {
+      this.b = b;
+      this.offset = offset;
+    }
+
+    public String getNextString() throws IOException {
+      if (offset >= b.length) {
+        throw new IOException(
+            "tried to read nonexistent string from byte array");
+      }
+      int i = 0;
+      while (offset + i < b.length && b[offset + i] != 0x0) {
+        i++;
+      }
+      String s = new String(b, offset, i, Charset.forName("UTF-8"));
+      offset = offset + i + 1;
+      return s;
+    }
+
+    public long getNextLong() throws IOException {
+      if (offset + 8 >= b.length) {
+        throw new IOException("byte array ran out when trying to read long");
+      }
+      long l = readReverseOrderedLong(b, offset);
+      offset += 8;
+      return l;
+    }
+
+    public int getOffset() {
+      return offset;
+    }
+  }
+
+  /**
+   * Returns true if the byte array begins with the specified prefix.
+   */
+  public static boolean prefixMatches(byte[] prefix, int prefixlen,
+      byte[] b) {
+    if (b.length < prefixlen) {
+      return false;
+    }
+    return WritableComparator.compareBytes(prefix, 0, prefixlen, b, 0,
+        prefixlen) == 0;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d5fab7ac64c18..9edaefbdfe3db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -106,25 +106,43 @@ private void doCrossFilter(HttpServletRequest req, HttpServletResponse res) {
 
     String originsList = encodeHeader(req.getHeader(ORIGIN));
     if (!isCrossOrigin(originsList)) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Header origin is null. Returning");
+      }
       return;
     }
 
     if (!areOriginsAllowed(originsList)) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Header origins '" + originsList + "' not allowed. Returning");
+      }
       return;
     }
 
     String accessControlRequestMethod =
         req.getHeader(ACCESS_CONTROL_REQUEST_METHOD);
     if (!isMethodAllowed(accessControlRequestMethod)) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Access control method '" + accessControlRequestMethod +
+            "' not allowed. Returning");
+      }
       return;
     }
 
     String accessControlRequestHeaders =
         req.getHeader(ACCESS_CONTROL_REQUEST_HEADERS);
     if (!areHeadersAllowed(accessControlRequestHeaders)) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Access control headers '" + accessControlRequestHeaders +
+            "' not allowed. Returning");
+      }
       return;
     }
 
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("Completed cross origin filter checks. Populating " +
+          "HttpServletResponse");
+    }
     res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, originsList);
     res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString());
     res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, getAllowedMethodsHeader());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/proto/yarn_server_timelineserver_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/proto/yarn_server_timelineserver_recovery.proto
new file mode 100644
index 0000000000000..fc141c2a7300a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/proto/yarn_server_timelineserver_recovery.proto
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "YarnServerTimelineServerRecoveryProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_security_token.proto";
+
+message TimelineDelegationTokenIdentifierDataProto {
+  optional YARNDelegationTokenIdentifierProto token_identifier = 1;
+  optional int64 renewDate = 2;
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 2a38d0a4bd771..32d011eaa68a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -80,6 +80,10 @@ public void testApplicationReport() throws IOException, YarnException {
         clientService.getClientHandler().getApplicationReport(request);
     ApplicationReport appReport = response.getApplicationReport();
     Assert.assertNotNull(appReport);
+    Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+        .getMemorySeconds());
+    Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+        .getVcoreSeconds());
     Assert.assertEquals("application_0_0001", appReport.getApplicationId()
       .toString());
     Assert.assertEquals("test app type",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index a093f19f9eaf1..50a15f1bfd357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -201,6 +202,12 @@ public ApplicationReport run() throws Exception {
             app.getOriginalTrackingUrl());
         Assert.assertEquals("test diagnostics info", app.getDiagnostics());
       }
+      ApplicationResourceUsageReport applicationResourceUsageReport =
+          app.getApplicationResourceUsageReport();
+      Assert.assertEquals(123,
+          applicationResourceUsageReport.getMemorySeconds());
+      Assert
+          .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
       Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
           app.getFinalApplicationStatus());
       Assert.assertEquals(YarnApplicationState.FINISHED,
@@ -416,6 +423,8 @@ private static TimelineEntity createApplicationTimelineEntity(
     entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
     entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
         Integer.MAX_VALUE + 1L);
+    entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);
+    entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345);
     if (emptyACLs) {
       entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
     } else {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index 930122120568b..a7e7daa1c58ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -31,6 +31,8 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
 import org.junit.After;
 import org.junit.Assert;
@@ -48,6 +50,8 @@ public void testStartStopServer() throws Exception {
     Configuration config = new YarnConfiguration();
     config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
         MemoryTimelineStore.class, TimelineStore.class);
+    config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+        MemoryTimelineStateStore.class, TimelineStateStore.class);
     config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
     try {
       try {
@@ -102,6 +106,32 @@ public void testLaunch() throws Exception {
     }
   }
 
+ //test launch method with -D arguments
+ @Test(timeout = 60000)
+ public void testLaunchWithArguments() throws Exception {
+   ExitUtil.disableSystemExit();
+   ApplicationHistoryServer historyServer = null;
+   try {
+     // Not able to modify the config of this test case,
+     // but others have been customized to avoid conflicts
+     String[] args = new String[2];
+     args[0]="-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000";
+     args[1]="-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200";
+     historyServer =
+         ApplicationHistoryServer.launchAppHistoryServer(args);
+     Configuration conf = historyServer.getConfig();
+     assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
+     assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
+   } catch (ExitUtil.ExitException e) {
+     assertEquals(0, e.status);
+     ExitUtil.resetFirstExitException();
+     fail();
+   } finally {
+     if (historyServer != null) {
+       historyServer.stop();
+     }
+   }
+ }
   @Test(timeout = 240000)
   public void testFilterOverrides() throws Exception {
 
@@ -128,6 +158,8 @@ public void testFilterOverrides() throws Exception {
       Configuration config = new YarnConfiguration();
       config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
           MemoryTimelineStore.class, TimelineStore.class);
+      config.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+          MemoryTimelineStateStore.class, TimelineStateStore.class);
       config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "localhost:0");
       try {
         config.set("hadoop.http.filter.initializers", filterInitializer);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 76bf8c3c75594..41dda91479901 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.codehaus.jettison.json.JSONArray;
@@ -73,11 +74,10 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
 @RunWith(Parameterized.class)
-public class TestAHSWebServices extends JerseyTest {
+public class TestAHSWebServices extends JerseyTestBase {
 
   private static ApplicationHistoryManagerOnTimelineStore historyManager;
   private static final String[] USERS = new String[] { "foo" , "bar" };
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
new file mode 100644
index 0000000000000..a35477dc70b99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timeline.recovery;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestLeveldbTimelineStateStore {
+
+  private FileContext fsContext;
+  private File fsPath;
+  private Configuration conf;
+  private TimelineStateStore store;
+
+  @Before
+  public void setup() throws Exception {
+    fsPath = new File("target", getClass().getSimpleName() +
+        "-tmpDir").getAbsoluteFile();
+    fsContext = FileContext.getLocalFSFileContext();
+    fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
+    conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, true);
+    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+        LeveldbTimelineStateStore.class,
+        TimelineStateStore.class);
+    conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH,
+        fsPath.getAbsolutePath());
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (store != null) {
+      store.stop();
+    }
+    if (fsContext != null) {
+      fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
+    }
+  }
+
+  private LeveldbTimelineStateStore initAndStartTimelineServiceStateStoreService() {
+    store = new LeveldbTimelineStateStore();
+    store.init(conf);
+    store.start();
+    return (LeveldbTimelineStateStore) store;
+  }
+
+  @Test
+  public void testTokenStore() throws Exception {
+    initAndStartTimelineServiceStateStoreService();
+    TimelineServiceState state = store.loadState();
+    assertTrue("token state not empty", state.tokenState.isEmpty());
+    assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty());
+
+    final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
+    final TimelineDelegationTokenIdentifier token1 =
+        new TimelineDelegationTokenIdentifier(new Text("tokenOwner1"),
+            new Text("tokenRenewer1"), new Text("tokenUser1"));
+    token1.setSequenceNumber(1);
+    token1.getBytes();
+    final Long tokenDate1 = 1L;
+    final TimelineDelegationTokenIdentifier token2 =
+        new TimelineDelegationTokenIdentifier(new Text("tokenOwner2"),
+            new Text("tokenRenewer2"), new Text("tokenUser2"));
+    token2.setSequenceNumber(12345678);
+    token2.getBytes();
+    final Long tokenDate2 = 87654321L;
+
+    store.storeTokenMasterKey(key1);
+    try {
+      store.storeTokenMasterKey(key1);
+      fail("redundant store of key undetected");
+    } catch (IOException e) {
+      // expected
+    }
+    store.storeToken(token1, tokenDate1);
+    store.storeToken(token2, tokenDate2);
+    try {
+      store.storeToken(token1, tokenDate1);
+      fail("redundant store of token undetected");
+    } catch (IOException e) {
+      // expected
+    }
+    store.close();
+
+    initAndStartTimelineServiceStateStoreService();
+    state = store.loadState();
+    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
+    assertTrue("missing token 1", state.tokenState.containsKey(token1));
+    assertEquals("incorrect token 1 date", tokenDate1,
+        state.tokenState.get(token1));
+    assertTrue("missing token 2", state.tokenState.containsKey(token2));
+    assertEquals("incorrect token 2 date", tokenDate2,
+        state.tokenState.get(token2));
+    assertEquals("incorrect master key count", 1,
+        state.tokenMasterKeyState.size());
+    assertTrue("missing master key 1",
+        state.tokenMasterKeyState.contains(key1));
+    assertEquals("incorrect latest sequence number", 12345678,
+        state.getLatestSequenceNumber());
+
+    final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
+    final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
+    final TimelineDelegationTokenIdentifier token3 =
+        new TimelineDelegationTokenIdentifier(new Text("tokenOwner3"),
+            new Text("tokenRenewer3"), new Text("tokenUser3"));
+    token3.setSequenceNumber(12345679);
+    token3.getBytes();
+    final Long tokenDate3 = 87654321L;
+
+    store.removeToken(token1);
+    store.storeTokenMasterKey(key2);
+    final Long newTokenDate2 = 975318642L;
+    store.updateToken(token2, newTokenDate2);
+    store.removeTokenMasterKey(key1);
+    store.storeTokenMasterKey(key3);
+    store.storeToken(token3, tokenDate3);
+    store.close();
+
+    initAndStartTimelineServiceStateStoreService();
+    state = store.loadState();
+    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
+    assertFalse("token 1 not removed", state.tokenState.containsKey(token1));
+    assertTrue("missing token 2", state.tokenState.containsKey(token2));
+    assertEquals("incorrect token 2 date", newTokenDate2,
+        state.tokenState.get(token2));
+    assertTrue("missing token 3", state.tokenState.containsKey(token3));
+    assertEquals("incorrect token 3 date", tokenDate3,
+        state.tokenState.get(token3));
+    assertEquals("incorrect master key count", 2,
+        state.tokenMasterKeyState.size());
+    assertFalse("master key 1 not removed",
+        state.tokenMasterKeyState.contains(key1));
+    assertTrue("missing master key 2",
+        state.tokenMasterKeyState.contains(key2));
+    assertTrue("missing master key 3",
+        state.tokenMasterKeyState.contains(key3));
+    assertEquals("incorrect latest sequence number", 12345679,
+        state.getLatestSequenceNumber());
+    store.close();
+  }
+
+  @Test
+  public void testCheckVersion() throws IOException {
+    LeveldbTimelineStateStore store =
+        initAndStartTimelineServiceStateStoreService();
+    // default version
+    Version defaultVersion = store.getCurrentVersion();
+    Assert.assertEquals(defaultVersion, store.loadVersion());
+
+    // compatible version
+    Version compatibleVersion =
+        Version.newInstance(defaultVersion.getMajorVersion(),
+            defaultVersion.getMinorVersion() + 2);
+    store.storeVersion(compatibleVersion);
+    Assert.assertEquals(compatibleVersion, store.loadVersion());
+    store.stop();
+
+    // overwrite the compatible version
+    store = initAndStartTimelineServiceStateStoreService();
+    Assert.assertEquals(defaultVersion, store.loadVersion());
+
+    // incompatible version
+    Version incompatibleVersion =
+        Version.newInstance(defaultVersion.getMajorVersion() + 1,
+            defaultVersion.getMinorVersion());
+    store.storeVersion(incompatibleVersion);
+    store.stop();
+
+    try {
+      initAndStartTimelineServiceStateStoreService();
+      Assert.fail("Incompatible version, should expect fail here.");
+    } catch (ServiceStateException e) {
+      Assert.assertTrue("Exception message mismatch",
+          e.getMessage().contains("Incompatible version for timeline state store"));
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
index fe2ed5c8b9772..7e96d2a36465a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
@@ -60,6 +60,7 @@
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.junit.Assert;
 import org.junit.Test;
@@ -72,10 +73,9 @@
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestTimelineWebServices extends JerseyTest {
+public class TestTimelineWebServices extends JerseyTestBase {
 
   private static TimelineStore store;
   private static TimelineACLsManager timelineACLsManager;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 35eacc537312c..1a4dab8c2bfe8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -84,6 +84,12 @@
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>jline</groupId>
+          <artifactId>jline</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.fusesource.leveldbjni</groupId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/lib/ZKClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/lib/ZKClient.java
index 84e41cecccfc7..e67530874f34d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/lib/ZKClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/lib/ZKClient.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.lib;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.List;
 
 import org.apache.zookeeper.CreateMode;
@@ -55,8 +56,8 @@ public ZKClient(String string) throws IOException {
   public void registerService(String path, String data) throws
     IOException, InterruptedException {
     try {
-      zkClient.create(path, data.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, 
-          CreateMode.EPHEMERAL);
+      zkClient.create(path, data.getBytes(Charset.forName("UTF-8")),
+          ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
     } catch(KeeperException ke) {
       throw new IOException(ke);
     }
@@ -109,7 +110,7 @@ public String getServiceData(String path) throws IOException,
     try {
       Stat stat = new Stat();
       byte[] byteData = zkClient.getData(path, false, stat);
-      data = new String(byteData);
+      data = new String(byteData, Charset.forName("UTF-8"));
     } catch(KeeperException ke) {
       throw new IOException(ke);
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 1e91514384223..630a5bf58891d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -104,7 +104,8 @@ private void addSystemCredentialsToProto() {
     for (Map.Entry<ApplicationId, ByteBuffer> entry : systemCredentials.entrySet()) {
       builder.addSystemCredentialsForApps(SystemCredentialsForAppsProto.newBuilder()
         .setAppId(convertToProtoFormat(entry.getKey()))
-        .setCredentialsForApp(ProtoUtils.convertToProtoFormat(entry.getValue())));
+        .setCredentialsForApp(ProtoUtils.convertToProtoFormat(
+            entry.getValue().duplicate())));
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index ee34c495af2f6..df8eecb40b20e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -63,6 +63,12 @@ public class ApplicationMetricsConstants {
 
   public static final String STATE_EVENT_INFO =
       "YARN_APPLICATION_STATE";
+  
+  public static final String APP_CPU_METRICS =
+      "YARN_APPLICATION_CPU_METRIC";
+  
+  public static final String APP_MEM_METRICS =
+      "YARN_APPLICATION_MEM_METRIC";
 
   public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
       "YARN_APPLICATION_LATEST_APP_ATTEMPT";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 1b326716af864..68d4ef9fe77aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -118,7 +118,7 @@ public static LocalResource newLocalResource(URI uri,
   public static ApplicationId newApplicationId(RecordFactory recordFactory,
       long clustertimestamp, CharSequence id) {
     return ApplicationId.newInstance(clustertimestamp,
-        Integer.valueOf(id.toString()));
+        Integer.parseInt(id.toString()));
   }
 
   public static ApplicationId newApplicationId(RecordFactory recordFactory,
@@ -137,7 +137,7 @@ public static ApplicationAttemptId newApplicationAttemptId(
 
   public static ApplicationId convert(long clustertimestamp, CharSequence id) {
     return ApplicationId.newInstance(clustertimestamp,
-        Integer.valueOf(id.toString()));
+        Integer.parseInt(id.toString()));
   }
 
   public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 327f8824a4b08..77193df328096 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -226,7 +226,7 @@ public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Ma
 
     PrintStream pout = null;
     try {
-      pout = new PrintStream(out);
+      pout = new PrintStream(out, false, "UTF-8");
       sb.write(pout);
     } finally {
       if (out != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index cc2de999bc838..f3d21210b8552 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -32,12 +32,11 @@
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Random;
 import java.util.Map;
 
+import org.apache.commons.lang.math.RandomUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -292,7 +291,7 @@ public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOExcep
 
       try {
         out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out);
+        pout = new PrintStream(out, false, "UTF-8");
         writeLocalWrapperScript(launchDst, pidFile, pout);
       } finally {
         IOUtils.cleanup(LOG, pout, out);
@@ -345,7 +344,7 @@ private void writeSessionScript(Path launchDst, Path pidFile)
       PrintStream pout = null;
       try {
         out = lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out);
+        pout = new PrintStream(out, false, "UTF-8");
         // We need to do a move as writing to a file is not atomic
         // Process reading a file being written to may get garbled data
         // hence write pid to tmp file first followed by a mv
@@ -539,8 +538,7 @@ protected Path getWorkingDir(List<String> localDirs, String user,
 
     // make probability to pick a directory proportional to
     // the available space on the directory.
-    Random r = new Random();
-    long randomPosition = Math.abs(r.nextLong()) % totalAvailable;
+    long randomPosition = RandomUtils.nextLong() % totalAvailable;
     int dir = 0;
     // skip zero available space directory,
     // because totalAvailable is greater than 0 and randomPosition
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index d8dd8907117d2..c854173fad36e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -22,6 +22,8 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+
+import org.apache.commons.lang.math.RandomUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -59,7 +61,6 @@
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.net.InetSocketAddress;
-
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
 import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
 
@@ -310,9 +311,9 @@ public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Ma
     PrintStream ps = null;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     try {
-      pout = new PrintStream(out);
+      pout = new PrintStream(out, false, "UTF-8");
       if (LOG.isDebugEnabled()) {
-        ps = new PrintStream(baos);
+        ps = new PrintStream(baos, false, "UTF-8");
         sb.write(ps);
       }
       sb.write(pout);
@@ -326,7 +327,7 @@ public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Ma
       }
     }
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Script: " + baos.toString());
+      LOG.debug("Script: " + baos.toString("UTF-8"));
     }
   }
 
@@ -440,7 +441,7 @@ public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOExcep
 
       try {
         out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out);
+        pout = new PrintStream(out, false, "UTF-8");
         writeLocalWrapperScript(launchDst, pidFile, pout);
       } finally {
         IOUtils.cleanup(LOG, pout, out);
@@ -498,7 +499,7 @@ private void writeSessionScript(Path launchDst, Path pidFile)
       PrintStream pout = null;
       try {
         out = lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out);
+        pout = new PrintStream(out, false, "UTF-8");
         // We need to do a move as writing to a file is not atomic
         // Process reading a file being written to may get garbled data
         // hence write pid to tmp file first followed by a mv
@@ -736,8 +737,7 @@ protected Path getWorkingDir(List<String> localDirs, String user,
 
     // make probability to pick a directory proportional to
     // the available space on the directory.
-    Random r = new Random();
-    long randomPosition = Math.abs(r.nextLong()) % totalAvailable;
+    long randomPosition = RandomUtils.nextLong() % totalAvailable;
     int dir = 0;
     // skip zero available space directory,
     // because totalAvailable is greater than 0 and randomPosition
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 4db4ef262798a..d6e6894974d6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -301,9 +301,6 @@ public int launchContainer(Container container,
         return ExitCode.TERMINATED.getExitCode();
       }
     } catch (ExitCodeException e) {
-      if (null == shExec) {
-        return -1;
-      }
       int exitCode = shExec.getExitCode();
       LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
       // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
@@ -395,18 +392,23 @@ public void deleteAsUser(String user, Path dir, Path... baseDirs) {
     verifyUsernamePattern(user);
     String runAsUser = getRunAsUser(user);
 
+    String dirString = dir == null ? "" : dir.toUri().getPath();
+
     List<String> command = new ArrayList<String>(
         Arrays.asList(containerExecutorExe,
                     runAsUser,
                     user,
                     Integer.toString(Commands.DELETE_AS_USER.getValue()),
-                    dir == null ? "" : dir.toUri().getPath()));
+                    dirString));
+    List<String> pathsToDelete = new ArrayList<String>();
     if (baseDirs == null || baseDirs.length == 0) {
       LOG.info("Deleting absolute path : " + dir);
+      pathsToDelete.add(dirString);
     } else {
       for (Path baseDir : baseDirs) {
         Path del = dir == null ? baseDir : new Path(baseDir, dir);
         LOG.info("Deleting path : " + del);
+        pathsToDelete.add(del.toString());
         command.add(baseDir.toUri().getPath());
       }
     }
@@ -422,7 +424,7 @@ public void deleteAsUser(String user, Path dir, Path... baseDirs) {
       }
     } catch (IOException e) {
       int exitCode = shExec.getExitCode();
-      LOG.error("DeleteAsUser for " + dir.toUri().getPath()
+      LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete)
           + " returned with exit code: " + exitCode, e);
       LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
       logOutput(shExec.getOutput());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index f561dbb4760e2..6ddd7e4af5d37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -106,6 +106,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
   // the AM finishes it informs the RM to stop the may-be-already-completed
   // containers.
   private final Map<ContainerId, Long> recentlyStoppedContainers;
+  // Save the reported completed containers in case of lost heartbeat responses.
+  // These completed containers will be sent again till a successful response.
+  private final Map<ContainerId, ContainerStatus> pendingCompletedContainers;
   // Duration for which to track recently stopped container.
   private long durationToTrackStoppedContainers;
 
@@ -126,6 +129,8 @@ public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
     this.metrics = metrics;
     this.recentlyStoppedContainers =
         new LinkedHashMap<ContainerId, Long>();
+    this.pendingCompletedContainers =
+        new HashMap<ContainerId, ContainerStatus>();
   }
 
   @Override
@@ -358,11 +363,10 @@ protected List<ContainerStatus> getContainerStatuses() throws IOException {
     List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
     for (Container container : this.context.getContainers().values()) {
       ContainerId containerId = container.getContainerId();
-      ApplicationId applicationId = container.getContainerId()
-          .getApplicationAttemptId().getApplicationId();
+      ApplicationId applicationId = containerId.getApplicationAttemptId()
+          .getApplicationId();
       org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
           container.cloneAndGetContainerStatus();
-      containerStatuses.add(containerStatus);
       if (containerStatus.getState() == ContainerState.COMPLETE) {
         if (isApplicationStopped(applicationId)) {
           if (LOG.isDebugEnabled()) {
@@ -370,14 +374,21 @@ protected List<ContainerStatus> getContainerStatuses() throws IOException {
                 + containerId + " from NM context.");
           }
           context.getContainers().remove(containerId);
+          pendingCompletedContainers.put(containerId, containerStatus);
         } else {
-          // Adding to finished containers cache. Cache will keep it around at
-          // least for #durationToTrackStoppedContainers duration. In the
-          // subsequent call to stop container it will get removed from cache.
-          addCompletedContainer(container.getContainerId());
+          if (!isContainerRecentlyStopped(containerId)) {
+            pendingCompletedContainers.put(containerId, containerStatus);
+            // Adding to finished containers cache. Cache will keep it around at
+            // least for #durationToTrackStoppedContainers duration. In the
+            // subsequent call to stop container it will get removed from cache.
+            addCompletedContainer(containerId);
+          }
         }
+      } else {
+        containerStatuses.add(containerStatus);
       }
     }
+    containerStatuses.addAll(pendingCompletedContainers.values());
     if (LOG.isDebugEnabled()) {
       LOG.debug("Sending out " + containerStatuses.size()
           + " container statuses: " + containerStatuses);
@@ -397,8 +408,8 @@ private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
         new ArrayList<NMContainerStatus>();
     for (Container container : this.context.getContainers().values()) {
       ContainerId containerId = container.getContainerId();
-      ApplicationId applicationId = container.getContainerId()
-          .getApplicationAttemptId().getApplicationId();
+      ApplicationId applicationId = containerId.getApplicationAttemptId()
+          .getApplicationId();
       if (!this.context.getApplications().containsKey(applicationId)) {
         context.getContainers().remove(containerId);
         continue;
@@ -410,7 +421,7 @@ private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
         // Adding to finished containers cache. Cache will keep it around at
         // least for #durationToTrackStoppedContainers duration. In the
         // subsequent call to stop container it will get removed from cache.
-        addCompletedContainer(container.getContainerId());
+        addCompletedContainer(containerId);
       }
     }
     LOG.info("Sending out " + containerStatuses.size()
@@ -457,7 +468,9 @@ public void removeOrTrackCompletedContainersFromContext(
       ContainerId containerId = iter.next();
       // remove the container only if the container is at DONE state
       Container nmContainer = context.getContainers().get(containerId);
-      if (nmContainer != null && nmContainer.getContainerState().equals(
+      if (nmContainer == null) {
+        iter.remove();
+      } else if (nmContainer.getContainerState().equals(
         org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE)) {
         context.getContainers().remove(containerId);
         removedContainers.add(containerId);
@@ -469,6 +482,7 @@ public void removeOrTrackCompletedContainersFromContext(
       LOG.info("Removed completed containers from NM context: "
           + removedContainers);
     }
+    pendingCompletedContainers.clear();
   }
 
   private void trackAppsForKeepAlive(List<ApplicationId> appIds) {
@@ -507,7 +521,7 @@ public void clearFinishedContainersFromCache() {
       recentlyStoppedContainers.clear();
     }
   }
-  
+
   @Private
   @VisibleForTesting
   public void removeVeryOldStoppedContainersFromCache() {
@@ -605,6 +619,7 @@ public void run() {
                   ResourceManagerConstants.RM_INVALID_IDENTIFIER;
               dispatcher.getEventHandler().handle(
                   new NodeManagerEvent(NodeManagerEventType.RESYNC));
+              pendingCompletedContainers.clear();
               break;
             }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index ec9c65e01115c..cd3e71a8d623c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -29,6 +29,7 @@
 import java.io.PrintStream;
 import java.net.InetSocketAddress;
 import java.net.URISyntaxException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.nativeio.NativeIO.Windows;
 import org.apache.hadoop.io.nativeio.NativeIOException;
 import org.apache.hadoop.util.NativeCodeLoader;
@@ -313,21 +315,23 @@ private static class ElevatedFileSystem extends DelegateToFileSystem {
     private static class ElevatedRawLocalFilesystem extends RawLocalFileSystem {
       
       @Override
-      protected boolean mkOneDir(File p2f) throws IOException {
-        Path path = new Path(p2f.getAbsolutePath());
+      protected boolean mkOneDirWithMode(Path path, File p2f,
+          FsPermission permission) throws IOException {
         if (LOG.isDebugEnabled()) {
-          LOG.debug(String.format("EFS:mkOneDir: %s", path));
+          LOG.debug(String.format("EFS:mkOneDirWithMode: %s %s", path,
+              permission));
         }
         boolean ret = false;
 
         // File.mkdir returns false, does not throw. Must mimic it.
         try {
           Native.Elevated.mkdir(path);
+          setPermission(path, permission);
           ret = true;
         }
         catch(Throwable e) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug(String.format("EFS:mkOneDir: %s", 
+            LOG.debug(String.format("EFS:mkOneDirWithMode: %s",
                 org.apache.hadoop.util.StringUtils.stringifyException(e)));
           }
         }
@@ -354,12 +358,23 @@ public void setOwner(Path p, String username, String groupname)
       }
       
       @Override
-      protected OutputStream createOutputStream(Path f, boolean append) 
-          throws IOException {
+      protected OutputStream createOutputStreamWithMode(Path f, boolean append,
+          FsPermission permission) throws IOException {
         if (LOG.isDebugEnabled()) {
-          LOG.debug(String.format("EFS:create: %s %b", f, append));
+          LOG.debug(String.format("EFS:createOutputStreamWithMode: %s %b %s", f,
+              append, permission));
+        }
+        boolean success = false;
+        OutputStream os = Native.Elevated.create(f, append);
+        try {
+          setPermission(f, permission);
+          success = true;
+          return os;
+        } finally {
+          if (!success) {
+            IOUtils.cleanup(LOG, os);
+          }
         }
-        return Native.Elevated.create(f, append); 
       }
       
       @Override
@@ -487,7 +502,7 @@ public void run() {
           try
           {
             BufferedReader lines = new BufferedReader(
-                new InputStreamReader(stream));
+                new InputStreamReader(stream, Charset.forName("UTF-8")));
             char[] buf = new char[512];
             int nRead;
             while ((nRead = lines.read(buf, 0, buf.length)) > 0) {
@@ -704,7 +719,7 @@ public void startLocalizer(Path nmPrivateContainerTokens,
        }
        catch(Throwable e) {
          LOG.warn(String.format(
-             "An exception occured during the cleanup of localizer job %s:\n%s", 
+             "An exception occured during the cleanup of localizer job %s:%n%s", 
              localizerPid, 
              org.apache.hadoop.util.StringUtils.stringifyException(e)));
        }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 58b1e2299e70a..57b0bdac2b71c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -159,9 +158,6 @@ public ContainerImpl(Configuration conf, Dispatcher dispatcher,
     this.diagnostics.append(diagnostics);
   }
 
-  private static final ContainerDoneTransition CONTAINER_DONE_TRANSITION =
-    new ContainerDoneTransition();
-
   private static final ContainerDiagnosticsUpdateTransition UPDATE_DIAGNOSTICS_TRANSITION =
       new ContainerDiagnosticsUpdateTransition();
 
@@ -202,7 +198,7 @@ ContainerEventType.RESOURCE_LOCALIZED, new LocalizedTransition())
     .addTransition(ContainerState.LOCALIZATION_FAILED,
         ContainerState.DONE,
         ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
-        CONTAINER_DONE_TRANSITION)
+        new LocalizationFailedToDoneTransition())
     .addTransition(ContainerState.LOCALIZATION_FAILED,
         ContainerState.LOCALIZATION_FAILED,
         ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
@@ -254,7 +250,7 @@ ContainerEventType.KILL_CONTAINER, new KillTransition())
     // From CONTAINER_EXITED_WITH_SUCCESS State
     .addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
         ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
-        CONTAINER_DONE_TRANSITION)
+        new ExitedWithSuccessToDoneTransition())
     .addTransition(ContainerState.EXITED_WITH_SUCCESS,
         ContainerState.EXITED_WITH_SUCCESS,
         ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
@@ -266,7 +262,7 @@ ContainerEventType.KILL_CONTAINER, new KillTransition())
     // From EXITED_WITH_FAILURE State
     .addTransition(ContainerState.EXITED_WITH_FAILURE, ContainerState.DONE,
             ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
-            CONTAINER_DONE_TRANSITION)
+            new ExitedWithFailureToDoneTransition())
     .addTransition(ContainerState.EXITED_WITH_FAILURE,
         ContainerState.EXITED_WITH_FAILURE,
         ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
@@ -301,7 +297,7 @@ ContainerEventType.KILL_CONTAINER, new KillTransition())
     .addTransition(ContainerState.KILLING,
             ContainerState.DONE,
             ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
-            CONTAINER_DONE_TRANSITION)
+            new KillingToDoneTransition())
     // Handle a launched container during killing stage is a no-op
     // as cleanup container is always handled after launch container event
     // in the container launcher
@@ -313,7 +309,7 @@ ContainerEventType.KILL_CONTAINER, new KillTransition())
     .addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
             ContainerState.DONE,
             ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP,
-            CONTAINER_DONE_TRANSITION)
+            new ContainerCleanedupAfterKillToDoneTransition())
     .addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
         ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
         ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
@@ -463,47 +459,6 @@ public ContainerTokenIdentifier getContainerTokenIdentifier() {
     }
   }
 
-  @SuppressWarnings("fallthrough")
-  private void finished() {
-    ApplicationId applicationId =
-        containerId.getApplicationAttemptId().getApplicationId();
-    switch (getContainerState()) {
-      case EXITED_WITH_SUCCESS:
-        metrics.endRunningContainer();
-        metrics.completedContainer();
-        NMAuditLogger.logSuccess(user,
-            AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
-            applicationId, containerId);
-        break;
-      case EXITED_WITH_FAILURE:
-        if (wasLaunched) {
-          metrics.endRunningContainer();
-        }
-        // fall through
-      case LOCALIZATION_FAILED:
-        metrics.failedContainer();
-        NMAuditLogger.logFailure(user,
-            AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
-            "Container failed with state: " + getContainerState(),
-            applicationId, containerId);
-        break;
-      case CONTAINER_CLEANEDUP_AFTER_KILL:
-        if (wasLaunched) {
-          metrics.endRunningContainer();
-        }
-        // fall through
-      case NEW:
-        metrics.killedContainer();
-        NMAuditLogger.logSuccess(user,
-            AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
-            applicationId,
-            containerId);
-    }
-
-    metrics.releaseContainer(this.resource);
-    sendFinishedEvents();
-  }
-
   @SuppressWarnings("unchecked")
   private void sendFinishedEvents() {
     // Inform the application
@@ -539,10 +494,11 @@ private void sendContainerMonitorStartEvent() {
           YarnConfiguration.NM_VMEM_PMEM_RATIO,
           YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
       long vmemBytes = (long) (pmemRatio * pmemBytes);
+      int cpuVcores = getResource().getVirtualCores();
 
       dispatcher.getEventHandler().handle(
           new ContainerStartMonitoringEvent(containerId,
-              vmemBytes, pmemBytes));
+              vmemBytes, pmemBytes, cpuVcores));
   }
 
   private void addDiagnostics(String... diags) {
@@ -611,7 +567,13 @@ public ContainerState transition(ContainerImpl container,
       } else if (container.recoveredAsKilled &&
           container.recoveredStatus == RecoveredContainerStatus.REQUESTED) {
         // container was killed but never launched
-        container.finished();
+        container.metrics.killedContainer();
+        NMAuditLogger.logSuccess(container.user,
+            AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
+            container.containerId.getApplicationAttemptId().getApplicationId(),
+            container.containerId);
+        container.metrics.releaseContainer(container.resource);
+        container.sendFinishedEvents();
         return ContainerState.DONE;
       }
 
@@ -994,7 +956,8 @@ static class ContainerDoneTransition implements
     @Override
     @SuppressWarnings("unchecked")
     public void transition(ContainerImpl container, ContainerEvent event) {
-      container.finished();
+      container.metrics.releaseContainer(container.resource);
+      container.sendFinishedEvents();
       //if the current state is NEW it means the CONTAINER_INIT was never 
       // sent for the event, thus no need to send the CONTAINER_STOP
       if (container.getCurrentState() 
@@ -1016,6 +979,105 @@ public void transition(ContainerImpl container, ContainerEvent event) {
       container.exitCode = killEvent.getContainerExitStatus();
       container.addDiagnostics(killEvent.getDiagnostic(), "\n");
       container.addDiagnostics("Container is killed before being launched.\n");
+      container.metrics.killedContainer();
+      NMAuditLogger.logSuccess(container.user,
+          AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
+      super.transition(container, event);
+    }
+  }
+
+  /**
+   * Handle the following transition:
+   * - LOCALIZATION_FAILED -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
+   */
+  static class LocalizationFailedToDoneTransition extends
+      ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      container.metrics.failedContainer();
+      NMAuditLogger.logFailure(container.user,
+          AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
+          "Container failed with state: " + container.getContainerState(),
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
+      super.transition(container, event);
+    }
+  }
+
+  /**
+   * Handle the following transition:
+   * - EXITED_WITH_SUCCESS -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
+   */
+  static class ExitedWithSuccessToDoneTransition extends
+      ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      container.metrics.endRunningContainer();
+      container.metrics.completedContainer();
+      NMAuditLogger.logSuccess(container.user,
+          AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
+      super.transition(container, event);
+    }
+  }
+
+  /**
+   * Handle the following transition:
+   * - EXITED_WITH_FAILURE -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
+   */
+  static class ExitedWithFailureToDoneTransition extends
+      ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      if (container.wasLaunched) {
+        container.metrics.endRunningContainer();
+      }
+      container.metrics.failedContainer();
+      NMAuditLogger.logFailure(container.user,
+          AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
+          "Container failed with state: " + container.getContainerState(),
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
+      super.transition(container, event);
+    }
+  }
+
+  /**
+   * Handle the following transition:
+   * - KILLING -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
+   */
+  static class KillingToDoneTransition extends
+      ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      container.metrics.killedContainer();
+      NMAuditLogger.logSuccess(container.user,
+          AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
+      super.transition(container, event);
+    }
+  }
+
+  /**
+   * Handle the following transition:
+   * CONTAINER_CLEANEDUP_AFTER_KILL -> DONE upon CONTAINER_RESOURCES_CLEANEDUP
+   */
+  static class ContainerCleanedupAfterKillToDoneTransition extends
+      ContainerDoneTransition {
+    @Override
+    public void transition(ContainerImpl container, ContainerEvent event) {
+      if (container.wasLaunched) {
+        container.metrics.endRunningContainer();
+      }
+      container.metrics.killedContainer();
+      NMAuditLogger.logSuccess(container.user,
+          AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
+          container.containerId.getApplicationAttemptId().getApplicationId(),
+          container.containerId);
       super.transition(container, event);
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index f4b6221a7bf4e..8c84132ab0220 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -763,7 +763,7 @@ public void addResource(LocalizerResourceRequestEvent request) {
        */
 
       if (rsrc.tryAcquire()) {
-        if (rsrc.getState().equals(ResourceState.DOWNLOADING)) {
+        if (rsrc.getState() == ResourceState.DOWNLOADING) {
           LocalResource resource = request.getResource().getRequest();
           try {
             Path publicRootPath =
@@ -775,6 +775,12 @@ public void addResource(LocalizerResourceRequestEvent request) {
             if (!publicDirDestPath.getParent().equals(publicRootPath)) {
               DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath()));
             }
+
+            // In case this is not a newly initialized nm state, ensure
+            // initialized local/log dirs similar to LocalizerRunner
+            getInitializedLocalDirs();
+            getInitializedLogDirs();
+
             // explicitly synchronize pending here to avoid future task
             // completing and being dequeued before pending updated
             synchronized (pending) {
@@ -788,6 +794,13 @@ public void addResource(LocalizerResourceRequestEvent request) {
               .getResource().getRequest(), e.getMessage()));
             LOG.error("Local path for public localization is not found. "
                 + " May be disks failed.", e);
+          } catch (IllegalArgumentException ie) {
+            rsrc.unlock();
+            publicRsrc.handle(new ResourceFailedLocalizationEvent(request
+                .getResource().getRequest(), ie.getMessage()));
+            LOG.error("Local path for public localization is not found. "
+                + " Incorrect path. " + request.getResource().getRequest()
+                .getPath(), ie);
           } catch (RejectedExecutionException re) {
             rsrc.unlock();
             publicRsrc.handle(new ResourceFailedLocalizationEvent(request
@@ -889,7 +902,7 @@ private LocalResource findNextResource() {
          LocalizedResource nRsrc = evt.getResource();
          // Resource download should take place ONLY if resource is in
          // Downloading state
-         if (!ResourceState.DOWNLOADING.equals(nRsrc.getState())) {
+         if (nRsrc.getState() != ResourceState.DOWNLOADING) {
            i.remove();
            continue;
          }
@@ -900,7 +913,7 @@ private LocalResource findNextResource() {
           * 2) Resource is still in DOWNLOADING state
           */
          if (nRsrc.tryAcquire()) {
-           if (nRsrc.getState().equals(ResourceState.DOWNLOADING)) {
+           if (nRsrc.getState() == ResourceState.DOWNLOADING) {
              LocalResourceRequest nextRsrc = nRsrc.getRequest();
              LocalResource next =
                  recordFactory.newRecordInstance(LocalResource.class);
@@ -930,41 +943,9 @@ LocalizerHeartbeatResponse update(
       String user = context.getUser();
       ApplicationId applicationId =
           context.getContainerId().getApplicationAttemptId().getApplicationId();
-      // The localizer has just spawned. Start giving it resources for
-      // remote-fetching.
-      if (remoteResourceStatuses.isEmpty()) {
-        LocalResource next = findNextResource();
-        if (next != null) {
-          response.setLocalizerAction(LocalizerAction.LIVE);
-          try {
-            ArrayList<ResourceLocalizationSpec> rsrcs =
-                new ArrayList<ResourceLocalizationSpec>();
-            ResourceLocalizationSpec rsrc =
-                NodeManagerBuilderUtils.newResourceLocalizationSpec(next,
-                  getPathForLocalization(next));
-            rsrcs.add(rsrc);
-            response.setResourceSpecs(rsrcs);
-          } catch (IOException e) {
-            LOG.error("local path for PRIVATE localization could not be found."
-                + "Disks might have failed.", e);
-          } catch (URISyntaxException e) {
-            // TODO fail? Already translated several times...
-          }
-        } else if (pending.isEmpty()) {
-          // TODO: Synchronization
-          response.setLocalizerAction(LocalizerAction.DIE);
-        } else {
-          response.setLocalizerAction(LocalizerAction.LIVE);
-        }
-        return response;
-      }
-      ArrayList<ResourceLocalizationSpec> rsrcs =
-          new ArrayList<ResourceLocalizationSpec>();
-       /*
-        * TODO : It doesn't support multiple downloads per ContainerLocalizer
-        * at the same time. We need to think whether we should support this.
-        */
 
+      LocalizerAction action = LocalizerAction.LIVE;
+      // Update resource statuses.
       for (LocalResourceStatus stat : remoteResourceStatuses) {
         LocalResource rsrc = stat.getResource();
         LocalResourceRequest req = null;
@@ -993,30 +974,8 @@ LocalizerHeartbeatResponse update(
             // list
             assoc.getResource().unlock();
             scheduled.remove(req);
-            
-            if (pending.isEmpty()) {
-              // TODO: Synchronization
-              response.setLocalizerAction(LocalizerAction.DIE);
-              break;
-            }
-            response.setLocalizerAction(LocalizerAction.LIVE);
-            LocalResource next = findNextResource();
-            if (next != null) {
-              try {
-                ResourceLocalizationSpec resource =
-                    NodeManagerBuilderUtils.newResourceLocalizationSpec(next,
-                      getPathForLocalization(next));
-                rsrcs.add(resource);
-              } catch (IOException e) {
-                LOG.error("local path for PRIVATE localization could not be " +
-                  "found. Disks might have failed.", e);
-              } catch (URISyntaxException e) {
-                  //TODO fail? Already translated several times...
-              }
-            }
             break;
           case FETCH_PENDING:
-            response.setLocalizerAction(LocalizerAction.LIVE);
             break;
           case FETCH_FAILURE:
             final String diagnostics = stat.getException().toString();
@@ -1030,17 +989,51 @@ LocalizerHeartbeatResponse update(
             // list
             assoc.getResource().unlock();
             scheduled.remove(req);
-            
             break;
           default:
             LOG.info("Unknown status: " + stat.getStatus());
-            response.setLocalizerAction(LocalizerAction.DIE);
+            action = LocalizerAction.DIE;
             getLocalResourcesTracker(req.getVisibility(), user, applicationId)
               .handle(new ResourceFailedLocalizationEvent(
                   req, stat.getException().getMessage()));
             break;
         }
       }
+      if (action == LocalizerAction.DIE) {
+        response.setLocalizerAction(action);
+        return response;
+      }
+
+      // Give the localizer resources for remote-fetching.
+      List<ResourceLocalizationSpec> rsrcs =
+          new ArrayList<ResourceLocalizationSpec>();
+
+      /*
+       * TODO : It doesn't support multiple downloads per ContainerLocalizer
+       * at the same time. We need to think whether we should support this.
+       */
+      LocalResource next = findNextResource();
+      if (next != null) {
+        try {
+          ResourceLocalizationSpec resource =
+              NodeManagerBuilderUtils.newResourceLocalizationSpec(next,
+                getPathForLocalization(next));
+          rsrcs.add(resource);
+        } catch (IOException e) {
+          LOG.error("local path for PRIVATE localization could not be " +
+            "found. Disks might have failed.", e);
+        } catch (IllegalArgumentException e) {
+          LOG.error("Inorrect path for PRIVATE localization."
+              + next.getResource().getFile(), e);
+        } catch (URISyntaxException e) {
+            //TODO fail? Already translated several times...
+        }
+      } else if (pending.isEmpty()) {
+        // TODO: Synchronization
+        action = LocalizerAction.DIE;
+      }
+
+      response.setLocalizerAction(action);
       response.setResourceSpecs(rsrcs);
       return response;
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
new file mode 100644
index 0000000000000..7850688a6d2aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.metrics2.lib.MutableStat;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@InterfaceAudience.Private
+@Metrics(context="container")
+public class ContainerMetrics implements MetricsSource {
+
+  public static final String PMEM_LIMIT_METRIC_NAME = "pMemLimit";
+  public static final String VMEM_LIMIT_METRIC_NAME = "vMemLimit";
+  public static final String VCORE_LIMIT_METRIC_NAME = "vCoreLimit";
+  public static final String PMEM_USAGE_METRIC_NAME = "pMemUsage";
+
+  @Metric
+  public MutableStat pMemMBsStat;
+
+  @Metric
+  public MutableGaugeInt pMemLimitMbs;
+
+  @Metric
+  public MutableGaugeInt vMemLimitMbs;
+
+  @Metric
+  public MutableGaugeInt cpuVcores;
+
+  static final MetricsInfo RECORD_INFO =
+      info("ContainerResource", "Resource limit and usage by container");
+
+  public static final MetricsInfo PROCESSID_INFO =
+      info("ContainerPid", "Container Process Id");
+
+  final MetricsInfo recordInfo;
+  final MetricsRegistry registry;
+  final ContainerId containerId;
+  final MetricsSystem metricsSystem;
+
+  // Metrics publishing status
+  private long flushPeriodMs;
+  private boolean flushOnPeriod = false; // true if period elapsed
+  private boolean finished = false; // true if container finished
+  private boolean unregister = false; // unregister
+  private Timer timer; // lazily initialized
+
+  /**
+   * Simple metrics cache to help prevent re-registrations.
+   */
+  protected final static Map<ContainerId, ContainerMetrics>
+      usageMetrics = new HashMap<>();
+
+  ContainerMetrics(
+      MetricsSystem ms, ContainerId containerId, long flushPeriodMs) {
+    this.recordInfo =
+        info(sourceName(containerId), RECORD_INFO.description());
+    this.registry = new MetricsRegistry(recordInfo);
+    this.metricsSystem = ms;
+    this.containerId = containerId;
+    this.flushPeriodMs = flushPeriodMs;
+    scheduleTimerTaskIfRequired();
+
+    this.pMemMBsStat = registry.newStat(
+        PMEM_USAGE_METRIC_NAME, "Physical memory stats", "Usage", "MBs", true);
+    this.pMemLimitMbs = registry.newGauge(
+        PMEM_LIMIT_METRIC_NAME, "Physical memory limit in MBs", 0);
+    this.vMemLimitMbs = registry.newGauge(
+        VMEM_LIMIT_METRIC_NAME, "Virtual memory limit in MBs", 0);
+    this.cpuVcores = registry.newGauge(
+        VCORE_LIMIT_METRIC_NAME, "CPU limit in number of vcores", 0);
+  }
+
+  ContainerMetrics tag(MetricsInfo info, ContainerId containerId) {
+    registry.tag(info, containerId.toString());
+    return this;
+  }
+
+  static String sourceName(ContainerId containerId) {
+    return RECORD_INFO.name() + "_" + containerId.toString();
+  }
+
+  public static ContainerMetrics forContainer(
+      ContainerId containerId, long flushPeriodMs) {
+    return forContainer(
+        DefaultMetricsSystem.instance(), containerId, flushPeriodMs);
+  }
+
+  synchronized static ContainerMetrics forContainer(
+      MetricsSystem ms, ContainerId containerId, long flushPeriodMs) {
+    ContainerMetrics metrics = usageMetrics.get(containerId);
+    if (metrics == null) {
+      metrics = new ContainerMetrics(
+          ms, containerId, flushPeriodMs).tag(RECORD_INFO, containerId);
+
+      // Register with the MetricsSystems
+      if (ms != null) {
+        metrics =
+            ms.register(sourceName(containerId),
+                "Metrics for container: " + containerId, metrics);
+      }
+      usageMetrics.put(containerId, metrics);
+    }
+
+    return metrics;
+  }
+
+  @Override
+  public synchronized void getMetrics(MetricsCollector collector, boolean all) {
+    //Container goes through registered -> finished -> unregistered.
+    if (unregister) {
+      metricsSystem.unregisterSource(recordInfo.name());
+      usageMetrics.remove(containerId);
+      return;
+    }
+
+    if (finished || flushOnPeriod) {
+      registry.snapshot(collector.addRecord(registry.info()), all);
+    }
+
+    if (finished) {
+      this.unregister = true;
+    } else if (flushOnPeriod) {
+      flushOnPeriod = false;
+      scheduleTimerTaskIfRequired();
+    }
+  }
+
+  public synchronized void finished() {
+    this.finished = true;
+    if (timer != null) {
+      timer.cancel();
+      timer = null;
+    }
+  }
+
+  public void recordMemoryUsage(int memoryMBs) {
+    this.pMemMBsStat.add(memoryMBs);
+  }
+
+  public void recordProcessId(String processId) {
+    registry.tag(PROCESSID_INFO, processId);
+  }
+
+  public void recordResourceLimit(int vmemLimit, int pmemLimit, int cpuVcores) {
+    this.vMemLimitMbs.set(vmemLimit);
+    this.pMemLimitMbs.set(pmemLimit);
+    this.cpuVcores.set(cpuVcores);
+  }
+
+  private synchronized void scheduleTimerTaskIfRequired() {
+    if (flushPeriodMs > 0) {
+      // Lazily initialize timer
+      if (timer == null) {
+        this.timer = new Timer("Metrics flush checker", true);
+      }
+      TimerTask timerTask = new TimerTask() {
+        @Override
+        public void run() {
+          synchronized (ContainerMetrics.this) {
+            if (!finished) {
+              flushOnPeriod = true;
+            }
+          }
+        }
+      };
+      timer.schedule(timerTask, flushPeriodMs);
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
index 407ada56b53d9..56e2d8eed591f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
@@ -24,12 +24,14 @@ public class ContainerStartMonitoringEvent extends ContainersMonitorEvent {
 
   private final long vmemLimit;
   private final long pmemLimit;
+  private final int cpuVcores;
 
   public ContainerStartMonitoringEvent(ContainerId containerId,
-      long vmemLimit, long pmemLimit) {
+      long vmemLimit, long pmemLimit, int cpuVcores) {
     super(containerId, ContainersMonitorEventType.START_MONITORING_CONTAINER);
     this.vmemLimit = vmemLimit;
     this.pmemLimit = pmemLimit;
+    this.cpuVcores = cpuVcores;
   }
 
   public long getVmemLimit() {
@@ -40,4 +42,7 @@ public long getPmemLimit() {
     return this.pmemLimit;
   }
 
+  public int getCpuVcores() {
+    return this.cpuVcores;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 02a63acfb1a27..2cecda6cf2d4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -51,6 +51,8 @@ public class ContainersMonitorImpl extends AbstractService implements
 
   private long monitoringInterval;
   private MonitoringThread monitoringThread;
+  private boolean containerMetricsEnabled;
+  private long containerMetricsPeriodMs;
 
   final List<ContainerId> containersToBeRemoved;
   final Map<ContainerId, ProcessTreeInfo> containersToBeAdded;
@@ -106,6 +108,13 @@ protected void serviceInit(Configuration conf) throws Exception {
     LOG.info(" Using ResourceCalculatorProcessTree : "
         + this.processTreeClass);
 
+    this.containerMetricsEnabled =
+        conf.getBoolean(YarnConfiguration.NM_CONTAINER_METRICS_ENABLE,
+            YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_ENABLE);
+    this.containerMetricsPeriodMs =
+        conf.getLong(YarnConfiguration.NM_CONTAINER_METRICS_PERIOD_MS,
+            YarnConfiguration.DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS);
+
     long configuredPMemForContainers = conf.getLong(
         YarnConfiguration.NM_PMEM_MB,
         YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
@@ -210,14 +219,17 @@ private static class ProcessTreeInfo {
     private ResourceCalculatorProcessTree pTree;
     private long vmemLimit;
     private long pmemLimit;
+    private int cpuVcores;
 
     public ProcessTreeInfo(ContainerId containerId, String pid,
-        ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit) {
+        ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
+        int cpuVcores) {
       this.containerId = containerId;
       this.pid = pid;
       this.pTree = pTree;
       this.vmemLimit = vmemLimit;
       this.pmemLimit = pmemLimit;
+      this.cpuVcores = cpuVcores;
     }
 
     public ContainerId getContainerId() {
@@ -250,6 +262,14 @@ public long getVmemLimit() {
     public long getPmemLimit() {
       return this.pmemLimit;
     }
+
+    /**
+     * Return the number of cpu vcores assigned
+     * @return
+     */
+    public int getCpuVcores() {
+      return this.cpuVcores;
+    }
   }
 
 
@@ -352,6 +372,10 @@ public void run() {
         // Remove finished containers
         synchronized (containersToBeRemoved) {
           for (ContainerId containerId : containersToBeRemoved) {
+            if (containerMetricsEnabled) {
+              ContainerMetrics.forContainer(
+                  containerId, containerMetricsPeriodMs).finished();
+            }
             trackingContainers.remove(containerId);
             LOG.info("Stopping resource-monitoring for " + containerId);
           }
@@ -385,6 +409,17 @@ public void run() {
                     ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf);
                 ptInfo.setPid(pId);
                 ptInfo.setProcessTree(pt);
+
+                if (containerMetricsEnabled) {
+                  ContainerMetrics usageMetrics = ContainerMetrics
+                      .forContainer(containerId, containerMetricsPeriodMs);
+                  int cpuVcores = ptInfo.getCpuVcores();
+                  final int vmemLimit = (int) (ptInfo.getVmemLimit() >> 20);
+                  final int pmemLimit = (int) (ptInfo.getPmemLimit() >> 20);
+                  usageMetrics.recordResourceLimit(
+                      vmemLimit, pmemLimit, cpuVcores);
+                  usageMetrics.recordProcessId(pId);
+                }
               }
             }
             // End of initializing any uninitialized processTrees
@@ -408,7 +443,15 @@ public void run() {
             LOG.info(String.format(
                 "Memory usage of ProcessTree %s for container-id %s: ",
                      pId, containerId.toString()) +
-                formatUsageString(currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit));
+                formatUsageString(
+                    currentVmemUsage, vmemLimit, currentPmemUsage, pmemLimit));
+
+            // Add usage to container metrics
+            if (containerMetricsEnabled) {
+              ContainerMetrics.forContainer(
+                  containerId, containerMetricsPeriodMs).recordMemoryUsage(
+                  (int) (currentPmemUsage >> 20));
+            }
 
             boolean isMemoryOverLimit = false;
             String msg = "";
@@ -556,7 +599,8 @@ public void handle(ContainersMonitorEvent monitoringEvent) {
       synchronized (this.containersToBeAdded) {
         ProcessTreeInfo processTreeInfo =
             new ProcessTreeInfo(containerId, null, null,
-                startEvent.getVmemLimit(), startEvent.getPmemLimit());
+                startEvent.getVmemLimit(), startEvent.getPmemLimit(),
+                startEvent.getCpuVcores());
         this.containersToBeAdded.put(containerId, processTreeInfo);
       }
       break;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index beaafe192a40f..3615feefa9315 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -27,6 +27,8 @@
 import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.yarn.api.records.Resource;
 
+import com.google.common.annotations.VisibleForTesting;
+
 @Metrics(about="Metrics for node manager", context="yarn")
 public class NodeManagerMetrics {
   @Metric MutableCounterInt containersLaunched;
@@ -127,4 +129,18 @@ public int getRunningContainers() {
     return containersRunning.value();
   }
 
+  @VisibleForTesting
+  public int getKilledContainers() {
+    return containersKilled.value();
+  }
+
+  @VisibleForTesting
+  public int getFailedContainers() {
+    return containersFailed.value();
+  }
+
+  @VisibleForTesting
+  public int getCompletedContainers() {
+    return containersCompleted.value();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 9d5468845dac4..5f349dbe3590a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -907,9 +907,9 @@ public void log(String message) {
 
   Version loadVersion() throws IOException {
     byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
-    // if version is not stored previously, treat it as 1.0.
+    // if version is not stored previously, treat it as CURRENT_VERSION_INFO.
     if (data == null || data.length == 0) {
-      return Version.newInstance(1, 0);
+      return getCurrentVersion();
     }
     Version version =
         new VersionPBImpl(VersionProto.parseFrom(data));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index 63039d8ed2c43..a832a7aafcfba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -20,9 +20,13 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -38,6 +42,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -235,7 +240,6 @@ private void createCgroup(String controller, String groupName)
 
   private void updateCgroup(String controller, String groupName, String param,
                             String value) throws IOException {
-    FileWriter f = null;
     String path = pathForCgroup(controller, groupName);
     param = controller + "." + param;
 
@@ -243,19 +247,25 @@ private void updateCgroup(String controller, String groupName, String param,
       LOG.debug("updateCgroup: " + path + ": " + param + "=" + value);
     }
 
+    PrintWriter pw = null;
     try {
-      f = new FileWriter(path + "/" + param, false);
-      f.write(value);
+      File file = new File(path + "/" + param);
+      Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+      pw = new PrintWriter(w);
+      pw.write(value);
     } catch (IOException e) {
       throw new IOException("Unable to set " + param + "=" + value +
           " for cgroup at: " + path, e);
     } finally {
-      if (f != null) {
-        try {
-          f.close();
-        } catch (IOException e) {
-          LOG.warn("Unable to close cgroup file: " +
-              path, e);
+      if (pw != null) {
+        boolean hasError = pw.checkError();
+        pw.close();
+        if(hasError) {
+          throw new IOException("Unable to set " + param + "=" + value +
+                " for cgroup at: " + path);
+        }
+        if(pw.checkError()) {
+          throw new IOException("Error while closing cgroup file " + path);
         }
       }
     }
@@ -376,7 +386,8 @@ private Map<String, List<String>> parseMtab() throws IOException {
     BufferedReader in = null;
 
     try {
-      in = new BufferedReader(new FileReader(new File(getMtabFileName())));
+      FileInputStream fis = new FileInputStream(new File(getMtabFileName()));
+      in = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
 
       for (String str = in.readLine(); str != null;
           str = in.readLine()) {
@@ -396,12 +407,7 @@ private Map<String, List<String>> parseMtab() throws IOException {
     } catch (IOException e) {
       throw new IOException("Error while reading " + getMtabFileName(), e);
     } finally {
-      // Close the streams
-      try {
-        in.close();
-      } catch (IOException e2) {
-        LOG.warn("Error closing the stream: " + getMtabFileName(), e2);
-      }
+      IOUtils.cleanup(LOG, in);
     }
 
     return ret;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
index 0c7c2505237bc..52fcdec020e8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
@@ -19,8 +19,9 @@
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,14 +50,14 @@ public static String getProcessId(Path path) throws IOException {
     
     LOG.debug("Accessing pid from pid file " + path);
     String processId = null;
-    FileReader fileReader = null;
     BufferedReader bufReader = null;
 
     try {
       File file = new File(path.toString());
       if (file.exists()) {
-        fileReader = new FileReader(file);
-        bufReader = new BufferedReader(fileReader);
+        FileInputStream fis = new FileInputStream(file);
+        bufReader = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
+
         while (true) {
           String line = bufReader.readLine();
           if (line == null) {
@@ -91,9 +92,6 @@ public static String getProcessId(Path path) throws IOException {
         }
       }
     } finally {
-      if (fileReader != null) {
-        fileReader.close();
-      }
       if (bufReader != null) {
         bufReader.close();
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
index 7d2948ea834b7..48e0c87ac60ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
@@ -28,6 +28,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -151,7 +152,8 @@ private void printLogFile(Block html, File logFile) {
           }
           
           IOUtils.skipFully(logByteStream, start);
-          InputStreamReader reader = new InputStreamReader(logByteStream);
+          InputStreamReader reader =
+              new InputStreamReader(logByteStream, Charset.forName("UTF-8"));
           int bufferSize = 65536;
           char[] cbuf = new char[bufferSize];
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
index c198ae6b0fed7..1c3b63b4c8697 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
@@ -59,7 +59,7 @@ protected void render(Block html) {
           .li().a("/conf", "Configuration")._()
           .li().a("/logs", "Local logs")._()
           .li().a("/stacks", "Server stacks")._()
-          .li().a("/metrics", "Server metrics")._()._()._();
+          .li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 4fc78b6210c76..04d02326fe0fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -38,7 +38,7 @@
 
 static const int DEFAULT_MIN_USERID = 1000;
 
-static const char* DEFAULT_BANNED_USERS[] = {"mapred", "hdfs", "bin", 0};
+static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0};
 
 //struct to store the user details
 struct passwd *user_detail = NULL;
@@ -1354,21 +1354,37 @@ int delete_as_user(const char *user,
                    const char *subdir,
                    char* const* baseDirs) {
   int ret = 0;
-
+  int subDirEmptyStr = (subdir == NULL || subdir[0] == 0);
+  int needs_tt_user = subDirEmptyStr;
   char** ptr;
 
   // TODO: No switching user? !!!!
   if (baseDirs == NULL || *baseDirs == NULL) {
-    return delete_path(subdir, strlen(subdir) == 0);
+    return delete_path(subdir, needs_tt_user);
   }
   // do the delete
   for(ptr = (char**)baseDirs; *ptr != NULL; ++ptr) {
-    char* full_path = concatenate("%s/%s", "user subdir", 2,
-                              *ptr, subdir);
+    char* full_path = NULL;
+    struct stat sb;
+    if (stat(*ptr, &sb) != 0) {
+      fprintf(LOGFILE, "Could not stat %s\n", *ptr);
+      return -1;
+    }
+    if (!S_ISDIR(sb.st_mode)) {
+      if (!subDirEmptyStr) {
+        fprintf(LOGFILE, "baseDir \"%s\" is a file and cannot contain subdir \"%s\".\n", *ptr, subdir);
+        return -1;
+      }
+      full_path = strdup(*ptr);
+      needs_tt_user = 0;
+    } else {
+      full_path = concatenate("%s/%s", "user subdir", 2, *ptr, subdir);
+    }
+
     if (full_path == NULL) {
       return -1;
     }
-    int this_ret = delete_path(full_path, strlen(subdir) == 0);
+    int this_ret = delete_path(full_path, needs_tt_user);
     free(full_path);
     // delete as much as we can, but remember the error
     if (this_ret != 0) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index e9ac2342d18b7..7f08e069ab4ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -382,7 +382,28 @@ void test_delete_user() {
   if (mkdirs(app_dir, 0700) != 0) {
     exit(1);
   }
+
   char buffer[100000];
+  sprintf(buffer, "%s/test.cfg", app_dir);
+  if (write_config_file(buffer) != 0) {
+    exit(1);
+  }
+
+  char * dirs[] = {buffer, 0};
+  int ret = delete_as_user(yarn_username, "file1" , dirs);
+  if (ret == 0) {
+    printf("FAIL: if baseDir is a file, delete_as_user should fail if a subdir is also passed\n");
+    exit(1);
+  }
+
+  // Pass a file to delete_as_user in the baseDirs parameter. The file should
+  // be deleted.
+  ret = delete_as_user(yarn_username, "" , dirs);
+  if (ret != 0) {
+    printf("FAIL: delete_as_user could not delete baseDir when baseDir is a file: return code is %d\n", ret);
+    exit(1);
+  }
+
   sprintf(buffer, "%s/local-1/usercache/%s", TEST_ROOT, yarn_username);
   if (access(buffer, R_OK) != 0) {
     printf("FAIL: directory missing before test\n");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index d54367a8f138c..98ab8e0631add 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -319,10 +319,57 @@ public void testDeleteAsUser() throws IOException {
     String cmd = String.valueOf(
         LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue());
     Path dir = new Path("/tmp/testdir");
-    
+    Path testFile = new Path("testfile");
+    Path baseDir0 = new Path("/grid/0/BaseDir");
+    Path baseDir1 = new Path("/grid/1/BaseDir");
+
+    mockExec.deleteAsUser(appSubmitter, dir);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "/tmp/testdir"),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, ""),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    File f = new File("./src/test/resources/mock-container-executer-with-error");
+    if (!FileUtil.canExecute(f)) {
+      FileUtil.setExecutable(f, true);
+    }
+    String executorPath = f.getAbsolutePath();
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+    mockExec.setConf(conf);
+
     mockExec.deleteAsUser(appSubmitter, dir);
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, "/tmp/testdir"),
         readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, ""),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
index e22b7f9f3d4c0..84f2fad898c8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
@@ -31,9 +31,9 @@
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.AfterClass;
+import org.junit.After;
 import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestLocalDirsHandlerService {
@@ -41,14 +41,14 @@ public class TestLocalDirsHandlerService {
       TestDirectoryCollection.class.getName()).getAbsoluteFile();
   private static final File testFile = new File(testDir, "testfile");
 
-  @BeforeClass
-  public static void setup() throws IOException {
+  @Before
+  public void setup() throws IOException {
     testDir.mkdirs();
     testFile.createNewFile();
   }
 
-  @AfterClass
-  public static void teardown() {
+  @After
+  public void teardown() {
     FileUtil.fullyDelete(testDir);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index a58294fe48156..611e6713843fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -165,6 +165,7 @@ public void testBlockNewContainerRequestsOnStartAndResync()
       throws IOException, InterruptedException, YarnException {
     NodeManager nm = new TestNodeManager2();
     YarnConfiguration conf = createNMConfig();
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
     nm.init(conf);
     nm.start();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index e367085d8ee94..71a420eba4597 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import static org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils.newNodeHeartbeatResponse;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -37,7 +38,10 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -74,12 +78,14 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatResponsePBImpl;
 import org.apache.hadoop.yarn.server.api.records.MasterKey;
 import org.apache.hadoop.yarn.server.api.records.NodeAction;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -610,14 +616,14 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
           <ContainerId>();
       try {
         if (heartBeatID == 0) {
-          Assert.assertEquals(request.getNodeStatus().getContainersStatuses()
-            .size(), 0);
-          Assert.assertEquals(context.getContainers().size(), 0);
+          Assert.assertEquals(0, request.getNodeStatus().getContainersStatuses()
+            .size());
+          Assert.assertEquals(0, context.getContainers().size());
         } else if (heartBeatID == 1) {
           List<ContainerStatus> statuses =
               request.getNodeStatus().getContainersStatuses();
-          Assert.assertEquals(statuses.size(), 2);
-          Assert.assertEquals(context.getContainers().size(), 2);
+          Assert.assertEquals(2, statuses.size());
+          Assert.assertEquals(2, context.getContainers().size());
 
           boolean container2Exist = false, container3Exist = false;
           for (ContainerStatus status : statuses) {
@@ -643,8 +649,16 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
         } else if (heartBeatID == 2 || heartBeatID == 3) {
           List<ContainerStatus> statuses =
               request.getNodeStatus().getContainersStatuses();
-          Assert.assertEquals(statuses.size(), 4);
-          Assert.assertEquals(context.getContainers().size(), 4);
+          if (heartBeatID == 2) {
+            // NM should send completed containers again, since the last
+            // heartbeat is lost.
+            Assert.assertEquals(4, statuses.size());
+          } else {
+            // NM should not send completed containers again, since the last
+            // heartbeat is successful.
+            Assert.assertEquals(2, statuses.size());
+          }
+          Assert.assertEquals(4, context.getContainers().size());
 
           boolean container2Exist = false, container3Exist = false,
               container4Exist = false, container5Exist = false;
@@ -674,8 +688,14 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
               container5Exist = true;
             }
           }
-          Assert.assertTrue(container2Exist && container3Exist
-              && container4Exist && container5Exist);
+          if (heartBeatID == 2) {
+            Assert.assertTrue(container2Exist && container3Exist
+                && container4Exist && container5Exist);
+          } else {
+            // NM do not send completed containers again
+            Assert.assertTrue(container2Exist && !container3Exist
+                && container4Exist && !container5Exist);
+          }
 
           if (heartBeatID == 3) {
             finishedContainersPulledByAM.add(containerStatus3.getContainerId());
@@ -683,8 +703,9 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
         } else if (heartBeatID == 4) {
           List<ContainerStatus> statuses =
               request.getNodeStatus().getContainersStatuses();
-          Assert.assertEquals(statuses.size(), 3);
-          Assert.assertEquals(context.getContainers().size(), 3);
+          Assert.assertEquals(2, statuses.size());
+          // Container 3 is acked by AM, hence removed from context
+          Assert.assertEquals(3, context.getContainers().size());
 
           boolean container3Exist = false;
           for (ContainerStatus status : statuses) {
@@ -917,13 +938,14 @@ public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
     nodeStatusUpdater.removeOrTrackCompletedContainersFromContext(ackedContainers);
 
     Set<ContainerId> containerIdSet = new HashSet<ContainerId>();
-    for (ContainerStatus status : nodeStatusUpdater.getContainerStatuses()) {
+    List<ContainerStatus> containerStatuses = nodeStatusUpdater.getContainerStatuses();
+    for (ContainerStatus status : containerStatuses) {
       containerIdSet.add(status.getContainerId());
     }
 
-    Assert.assertTrue(nodeStatusUpdater.getContainerStatuses().size() == 1);
+    Assert.assertEquals(1, containerStatuses.size());
     // completed container is removed;
-    Assert.assertFalse(containerIdSet.contains(anyCompletedContainer));
+    Assert.assertFalse(containerIdSet.contains(cId));
     // running container is not removed;
     Assert.assertTrue(containerIdSet.contains(runningContainerId));
   }
@@ -967,15 +989,15 @@ public ContainerState getCurrentState() {
 
     when(application.getApplicationState()).thenReturn(
         ApplicationState.FINISHING_CONTAINERS_WAIT);
-    // The completed container will be sent one time. Then we will delete it.
+    // The completed container will be saved in case of lost heartbeat.
+    Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
     Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
-    Assert.assertEquals(0, nodeStatusUpdater.getContainerStatuses().size());
 
     nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
     nm.getNMContext().getApplications().remove(appId);
-    // The completed container will be sent one time. Then we will delete it.
+    // The completed container will be saved in case of lost heartbeat.
+    Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
     Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
-    Assert.assertEquals(0, nodeStatusUpdater.getContainerStatuses().size());
   }
 
   @Test
@@ -1447,6 +1469,63 @@ public void cleanUpApplicationsOnNMShutDown() {
     nm.stop();
   }
 
+  @Test
+  public void testConcurrentAccessToSystemCredentials(){
+    final Map<ApplicationId, ByteBuffer> testCredentials = new HashMap<>();
+    ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[300]);
+    ApplicationId applicationId = ApplicationId.newInstance(123456, 120);
+    testCredentials.put(applicationId, byteBuffer);
+
+    final List<Throwable> exceptions = Collections.synchronizedList(new
+        ArrayList<Throwable>());
+
+    final int NUM_THREADS = 10;
+    final CountDownLatch allDone = new CountDownLatch(NUM_THREADS);
+    final ExecutorService threadPool = Executors.newFixedThreadPool(
+        NUM_THREADS);
+
+    final AtomicBoolean stop = new AtomicBoolean(false);
+
+    try {
+      for (int i = 0; i < NUM_THREADS; i++) {
+        threadPool.submit(new Runnable() {
+          @Override
+          public void run() {
+            try {
+              for (int i = 0; i < 100 && !stop.get(); i++) {
+                NodeHeartbeatResponse nodeHeartBeatResponse =
+                    newNodeHeartbeatResponse(0, NodeAction.NORMAL,
+                        null, null, null, null, 0);
+                nodeHeartBeatResponse.setSystemCredentialsForApps(
+                    testCredentials);
+                NodeHeartbeatResponseProto proto =
+                    ((NodeHeartbeatResponsePBImpl)nodeHeartBeatResponse)
+                        .getProto();
+                Assert.assertNotNull(proto);
+              }
+            } catch (Throwable t) {
+              exceptions.add(t);
+              stop.set(true);
+            } finally {
+              allDone.countDown();
+            }
+          }
+        });
+      }
+
+      int testTimeout = 2;
+      Assert.assertTrue("Timeout waiting for more than " + testTimeout + " " +
+              "seconds",
+          allDone.await(testTimeout, TimeUnit.SECONDS));
+    } catch (InterruptedException ie) {
+      exceptions.add(ie);
+    } finally {
+      threadPool.shutdownNow();
+    }
+    Assert.assertTrue("Test failed with exception(s)" + exceptions,
+        exceptions.isEmpty());
+  }
+
   // Add new containers info into NM context each time node heart beats.
   private class MyNMContext extends NMContext {
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index c28d691a99dee..2834e30247e84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -173,13 +173,20 @@ public void testExternalKill() throws Exception {
       wc = new WrappedContainer(13, 314159265358979L, 4344, "yak");
       wc.initContainer();
       wc.localizeResources();
+      int running = metrics.getRunningContainers();
       wc.launchContainer();
+      assertEquals(running + 1, metrics.getRunningContainers());
       reset(wc.localizerBus);
       wc.containerKilledOnRequest();
       assertEquals(ContainerState.EXITED_WITH_FAILURE, 
           wc.c.getContainerState());
       assertNull(wc.c.getLocalizedResources());
       verifyCleanupCall(wc);
+      int failed = metrics.getFailedContainers();
+      wc.containerResourcesCleanup();
+      assertEquals(ContainerState.DONE, wc.c.getContainerState());
+      assertEquals(failed + 1, metrics.getFailedContainers());
+      assertEquals(running, metrics.getRunningContainers());
     }
     finally {
       if (wc != null) {
@@ -219,13 +226,20 @@ public void testCleanupOnSuccess() throws Exception {
       wc = new WrappedContainer(11, 314159265358979L, 4344, "yak");
       wc.initContainer();
       wc.localizeResources();
+      int running = metrics.getRunningContainers();
       wc.launchContainer();
+      assertEquals(running + 1, metrics.getRunningContainers());
       reset(wc.localizerBus);
       wc.containerSuccessful();
       assertEquals(ContainerState.EXITED_WITH_SUCCESS,
           wc.c.getContainerState());
       assertNull(wc.c.getLocalizedResources());
       verifyCleanupCall(wc);
+      int completed = metrics.getCompletedContainers();
+      wc.containerResourcesCleanup();
+      assertEquals(ContainerState.DONE, wc.c.getContainerState());
+      assertEquals(completed + 1, metrics.getCompletedContainers());
+      assertEquals(running, metrics.getRunningContainers());
     }
     finally {
       if (wc != null) {
@@ -319,12 +333,14 @@ public void testKillOnNew() throws Exception {
     try {
       wc = new WrappedContainer(13, 314159265358979L, 4344, "yak");
       assertEquals(ContainerState.NEW, wc.c.getContainerState());
+      int killed = metrics.getKilledContainers();
       wc.killContainer();
       assertEquals(ContainerState.DONE, wc.c.getContainerState());
       assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
           wc.c.cloneAndGetContainerStatus().getExitStatus());
       assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics()
           .contains("KillRequest"));
+      assertEquals(killed + 1, metrics.getKilledContainers());
     } finally {
       if (wc != null) {
         wc.finished();
@@ -345,6 +361,10 @@ public void testKillOnLocalizing() throws Exception {
           wc.c.cloneAndGetContainerStatus().getExitStatus());
       assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics()
           .contains("KillRequest"));
+      int killed = metrics.getKilledContainers();
+      wc.containerResourcesCleanup();
+      assertEquals(ContainerState.DONE, wc.c.getContainerState());
+      assertEquals(killed + 1, metrics.getKilledContainers());
     } finally {
       if (wc != null) {
         wc.finished();
@@ -365,6 +385,10 @@ public void testKillOnLocalizationFailed() throws Exception {
       assertEquals(ContainerState.LOCALIZATION_FAILED, wc.c.getContainerState());
       assertNull(wc.c.getLocalizedResources());
       verifyCleanupCall(wc);
+      int failed = metrics.getFailedContainers();
+      wc.containerResourcesCleanup();
+      assertEquals(ContainerState.DONE, wc.c.getContainerState());
+      assertEquals(failed + 1, metrics.getFailedContainers());
     } finally {
       if (wc != null) {
         wc.finished();
@@ -389,8 +413,11 @@ public void testKillOnLocalizedWhenContainerNotLaunched() throws Exception {
           wc.c.getContainerState());
       assertNull(wc.c.getLocalizedResources());
       verifyCleanupCall(wc);
+      int killed = metrics.getKilledContainers();
       wc.c.handle(new ContainerEvent(wc.c.getContainerId(),
           ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+      assertEquals(ContainerState.DONE, wc.c.getContainerState());
+      assertEquals(killed + 1, metrics.getKilledContainers());
       assertEquals(0, metrics.getRunningContainers());
     } finally {
       if (wc != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index 23a57d60fb24f..569525422b298 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -24,6 +24,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.timeout;
 
 import java.io.File;
 import java.io.IOException;
@@ -313,7 +314,7 @@ public void testLocalResourceCache() {
       // After receiving failed resource event; all waiting containers will be
       // notified with Container Resource Failed Event.
       Assert.assertEquals(0, localrsrc.size());
-      verify(containerEventHandler, times(2)).handle(
+      verify(containerEventHandler, timeout(1000).times(2)).handle(
         isA(ContainerResourceFailedEvent.class));
       Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
 
@@ -360,7 +361,7 @@ public void testLocalResourceCache() {
       dispatcher.await();
       
       // Verifying ContainerResourceLocalizedEvent .
-      verify(containerEventHandler, times(1)).handle(
+      verify(containerEventHandler, timeout(1000).times(1)).handle(
         isA(ContainerResourceLocalizedEvent.class));
       Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr)
         .getState());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 1051e7acfc1fe..30af5a435911c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -32,18 +32,16 @@
 import static org.mockito.Matchers.isA;
 import static org.mockito.Matchers.isNull;
 import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.timeout;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URI;
@@ -63,10 +61,7 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.Future;
 
-import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.UnresolvedLinkException;
-import org.apache.hadoop.security.AccessControlException;
 import org.junit.Assert;
 
 import org.apache.commons.io.FileUtils;
@@ -832,10 +827,16 @@ public boolean matches(Object o) {
       do {
         resource2 = getPrivateMockedResource(r);
       } while (resource2 == null || resource2.equals(resource1));
+      LocalResource resource3 = null;
+      do {
+        resource3 = getPrivateMockedResource(r);
+      } while (resource3 == null || resource3.equals(resource1)
+          || resource3.equals(resource2));
       // above call to make sure we don't get identical resources.
       
       final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
       final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
+      final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
       Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
         new HashMap<LocalResourceVisibility, 
                     Collection<LocalResourceRequest>>();
@@ -843,6 +844,7 @@ public boolean matches(Object o) {
           new ArrayList<LocalResourceRequest>();
       privateResourceList.add(req1);
       privateResourceList.add(req2);
+      privateResourceList.add(req3);
       rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
       spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
       // Sigh. Thread init of private localizer not accessible
@@ -857,30 +859,47 @@ public boolean matches(Object o) {
       Path localizationTokenPath = tokenPathCaptor.getValue();
 
       // heartbeat from localizer
-      LocalResourceStatus rsrcStat1 = mock(LocalResourceStatus.class);
-      LocalResourceStatus rsrcStat2 = mock(LocalResourceStatus.class);
+      LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
+      LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
+      LocalResourceStatus rsrc2success = mock(LocalResourceStatus.class);
+      LocalResourceStatus rsrc3success = mock(LocalResourceStatus.class);
       LocalizerStatus stat = mock(LocalizerStatus.class);
       when(stat.getLocalizerId()).thenReturn(ctnrStr);
-      when(rsrcStat1.getResource()).thenReturn(resource1);
-      when(rsrcStat2.getResource()).thenReturn(resource2);
-      when(rsrcStat1.getLocalSize()).thenReturn(4344L);
-      when(rsrcStat2.getLocalSize()).thenReturn(2342L);
+      when(rsrc1success.getResource()).thenReturn(resource1);
+      when(rsrc2pending.getResource()).thenReturn(resource2);
+      when(rsrc2success.getResource()).thenReturn(resource2);
+      when(rsrc3success.getResource()).thenReturn(resource3);
+      when(rsrc1success.getLocalSize()).thenReturn(4344L);
+      when(rsrc2success.getLocalSize()).thenReturn(2342L);
+      when(rsrc3success.getLocalSize()).thenReturn(5345L);
       URL locPath = getPath("/cache/private/blah");
-      when(rsrcStat1.getLocalPath()).thenReturn(locPath);
-      when(rsrcStat2.getLocalPath()).thenReturn(locPath);
-      when(rsrcStat1.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
-      when(rsrcStat2.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+      when(rsrc1success.getLocalPath()).thenReturn(locPath);
+      when(rsrc2success.getLocalPath()).thenReturn(locPath);
+      when(rsrc3success.getLocalPath()).thenReturn(locPath);
+      when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+      when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
+      when(rsrc2success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+      when(rsrc3success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+
+      // Four heartbeats with sending:
+      // 1 - empty
+      // 2 - resource1 FETCH_SUCCESS
+      // 3 - resource2 FETCH_PENDING
+      // 4 - resource2 FETCH_SUCCESS, resource3 FETCH_SUCCESS
+      List<LocalResourceStatus> rsrcs4 = new ArrayList<LocalResourceStatus>();
+      rsrcs4.add(rsrc2success);
+      rsrcs4.add(rsrc3success);
       when(stat.getResources())
         .thenReturn(Collections.<LocalResourceStatus>emptyList())
-        .thenReturn(Collections.singletonList(rsrcStat1))
-        .thenReturn(Collections.singletonList(rsrcStat2))
-        .thenReturn(Collections.<LocalResourceStatus>emptyList());
+        .thenReturn(Collections.singletonList(rsrc1success))
+        .thenReturn(Collections.singletonList(rsrc2pending))
+        .thenReturn(rsrcs4);
 
       String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE +
           Path.SEPARATOR + "user0" + Path.SEPARATOR +
           ContainerLocalizer.FILECACHE;
-      
-      // get first resource
+
+      // First heartbeat
       LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
       assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
       assertEquals(1, response.getResourceSpecs().size());
@@ -893,7 +912,7 @@ public boolean matches(Object o) {
       assertTrue(localizedPath.getFile().endsWith(
         localPath + Path.SEPARATOR + "10"));
 
-      // get second resource
+      // Second heartbeat
       response = spyService.heartbeat(stat);
       assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
       assertEquals(1, response.getResourceSpecs().size());
@@ -907,16 +926,21 @@ public boolean matches(Object o) {
       assertTrue(localizedPath.getFile().endsWith(
         localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11"));
 
-      // empty rsrc
+      // Third heartbeat
       response = spyService.heartbeat(stat);
       assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
-      assertEquals(0, response.getResourceSpecs().size());
+      assertEquals(1, response.getResourceSpecs().size());
+      assertEquals(req3, new LocalResourceRequest(response.getResourceSpecs()
+          .get(0).getResource()));
+      localizedPath =
+          response.getResourceSpecs().get(0).getDestinationDirectory();
+      assertTrue(localizedPath.getFile().endsWith(
+          localPath + Path.SEPARATOR + "1" + Path.SEPARATOR + "12"));
 
       // get shutdown
       response = spyService.heartbeat(stat);
       assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
 
-
       dispatcher.await();
       // verify container notification
       ArgumentMatcher<ContainerEvent> matchesContainerLoc =
@@ -928,8 +952,8 @@ public boolean matches(Object o) {
               && c.getContainerId() == evt.getContainerID();
           }
         };
-      // total 2 resource localzation calls. one for each resource.
-      verify(containerBus, times(2)).handle(argThat(matchesContainerLoc));
+      // total 3 resource localzation calls. one for each resource.
+      verify(containerBus, times(3)).handle(argThat(matchesContainerLoc));
         
       // Verify deletion of localization token.
       verify(delService).delete((String)isNull(), eq(localizationTokenPath));
@@ -940,6 +964,109 @@ public boolean matches(Object o) {
     }
   }
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testPublicResourceInitializesLocalDir() throws Exception {
+
+    // Setup state to simulate restart NM with existing state meaning no
+    // directory creation during initialization
+    NMStateStoreService spyStateStore = spy(nmContext.getNMStateStore());
+    when(spyStateStore.canRecover()).thenReturn(true);
+    NMContext spyContext = spy(nmContext);
+    when(spyContext.getNMStateStore()).thenReturn(spyStateStore);
+
+    List<Path> localDirs = new ArrayList<Path>();
+    String[] sDirs = new String[4];
+    for (int i = 0; i < 4; ++i) {
+      localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
+      sDirs[i] = localDirs.get(i).toString();
+    }
+    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
+
+
+    DrainDispatcher dispatcher = new DrainDispatcher();
+    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
+    dispatcher.register(ApplicationEventType.class, applicationBus);
+    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
+    dispatcher.register(ContainerEventType.class, containerBus);
+
+    ContainerExecutor exec = mock(ContainerExecutor.class);
+    DeletionService delService = mock(DeletionService.class);
+    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+    dirsHandler.init(conf);
+
+    dispatcher.init(conf);
+    dispatcher.start();
+
+    try {
+      ResourceLocalizationService rawService =
+          new ResourceLocalizationService(dispatcher, exec, delService,
+              dirsHandler, spyContext);
+      ResourceLocalizationService spyService = spy(rawService);
+      doReturn(mockServer).when(spyService).createServer();
+      doReturn(lfs).when(spyService).getLocalFileContext(
+          isA(Configuration.class));
+
+      spyService.init(conf);
+      spyService.start();
+
+      final FsPermission defaultPerm = new FsPermission((short)0755);
+
+      // verify directory is not created at initialization
+      for (Path p : localDirs) {
+        p = new Path((new URI(p.toString())).getPath());
+        Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
+        verify(spylfs, never())
+            .mkdir(eq(publicCache),eq(defaultPerm), eq(true));
+      }
+
+      final String user = "user0";
+      // init application
+      final Application app = mock(Application.class);
+      final ApplicationId appId =
+          BuilderUtils.newApplicationId(314159265358979L, 3);
+      when(app.getUser()).thenReturn(user);
+      when(app.getAppId()).thenReturn(appId);
+      spyService.handle(new ApplicationLocalizationEvent(
+          LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+      dispatcher.await();
+
+      // init container.
+      final Container c = getMockContainer(appId, 42, user);
+
+      // init resources
+      Random r = new Random();
+      long seed = r.nextLong();
+      System.out.println("SEED: " + seed);
+      r.setSeed(seed);
+
+      // Queue up public resource localization
+      final LocalResource pubResource = getPublicMockedResource(r);
+      final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
+
+      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+          new HashMap<LocalResourceVisibility,
+              Collection<LocalResourceRequest>>();
+      req.put(LocalResourceVisibility.PUBLIC,
+          Collections.singletonList(pubReq));
+
+      Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
+      pubRsrcs.add(pubReq);
+
+      spyService.handle(new ContainerLocalizationRequestEvent(c, req));
+      dispatcher.await();
+
+      // verify directory creation
+      for (Path p : localDirs) {
+        p = new Path((new URI(p.toString())).getPath());
+        Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
+        verify(spylfs).mkdir(eq(publicCache),eq(defaultPerm), eq(true));
+      }
+    } finally {
+      dispatcher.stop();
+    }
+  }
+
   @Test(timeout=20000)
   @SuppressWarnings("unchecked") // mocked generics
   public void testFailedPublicResource() throws Exception {
@@ -1126,14 +1253,33 @@ public void testPublicResourceAddResourceExceptions() throws Exception {
             user, appId);
       Assert.assertNull(tracker.getLocalizedResource(pubReq));
 
-      // test RejectedExecutionException
+      // test IllegalArgumentException
+      String name = Long.toHexString(r.nextLong());
+      URL url = getPath("/local/PRIVATE/" + name + "/");
+      final LocalResource rsrc =
+          BuilderUtils.newLocalResource(url, LocalResourceType.FILE,
+          LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L,
+          r.nextInt(1024) + 2048L, false);
+      final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc);
+      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 =
+          new HashMap<LocalResourceVisibility, 
+          Collection<LocalResourceRequest>>();
+      req1.put(LocalResourceVisibility.PUBLIC,
+          Collections.singletonList(pubReq1));
       Mockito
         .doCallRealMethod()
         .when(dirsHandlerSpy)
         .getLocalPathForWrite(isA(String.class), Mockito.anyLong(),
           Mockito.anyBoolean());
+      // send request
+      spyService.handle(new ContainerLocalizationRequestEvent(c, req1));
+      dispatcher.await();
+      tracker =
+          spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC,
+          user, appId);
+      Assert.assertNull(tracker.getLocalizedResource(pubReq));
 
-      // shutdown the thread pool
+      // test RejectedExecutionException by shutting down the thread pool
       PublicLocalizer publicLocalizer = spyService.getPublicLocalizer();
       publicLocalizer.threadPool.shutdown();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
new file mode 100644
index 0000000000000..c628648608324
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
+import org.apache.hadoop.metrics2.impl.MetricsRecords;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+public class TestContainerMetrics {
+
+  @Test
+  public void testContainerMetricsFlow() throws InterruptedException {
+    final String ERR = "Error in number of records";
+
+    // Create a dummy MetricsSystem
+    MetricsSystem system = mock(MetricsSystem.class);
+    doReturn(this).when(system).register(anyString(), anyString(), any());
+
+    MetricsCollectorImpl collector = new MetricsCollectorImpl();
+    ContainerId containerId = mock(ContainerId.class);
+    ContainerMetrics metrics = ContainerMetrics.forContainer(containerId, 100);
+
+    metrics.recordMemoryUsage(1024);
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 0, collector.getRecords().size());
+
+    Thread.sleep(110);
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 1, collector.getRecords().size());
+    collector.clear();
+
+    Thread.sleep(110);
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 1, collector.getRecords().size());
+    collector.clear();
+
+    metrics.finished();
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 1, collector.getRecords().size());
+    collector.clear();
+
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 0, collector.getRecords().size());
+
+    Thread.sleep(110);
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 0, collector.getRecords().size());
+  }
+
+  @Test
+  public void testContainerMetricsLimit() throws InterruptedException {
+    final String ERR = "Error in number of records";
+
+    MetricsSystem system = mock(MetricsSystem.class);
+    doReturn(this).when(system).register(anyString(), anyString(), any());
+
+    MetricsCollectorImpl collector = new MetricsCollectorImpl();
+    ContainerId containerId = mock(ContainerId.class);
+    ContainerMetrics metrics = ContainerMetrics.forContainer(containerId, 100);
+
+    int anyPmemLimit = 1024;
+    int anyVmemLimit = 2048;
+    int anyVcores = 10;
+    String anyProcessId = "1234";
+
+    metrics.recordResourceLimit(anyVmemLimit, anyPmemLimit, anyVcores);
+    metrics.recordProcessId(anyProcessId);
+
+    Thread.sleep(110);
+    metrics.getMetrics(collector, true);
+    assertEquals(ERR, 1, collector.getRecords().size());
+    MetricsRecord record = collector.getRecords().get(0);
+
+    MetricsRecords.assertTag(record, ContainerMetrics.PROCESSID_INFO.name(),
+        anyProcessId);
+
+    MetricsRecords.assertMetric(record, ContainerMetrics
+        .PMEM_LIMIT_METRIC_NAME, anyPmemLimit);
+    MetricsRecords.assertMetric(record, ContainerMetrics.VMEM_LIMIT_METRIC_NAME, anyVmemLimit);
+    MetricsRecords.assertMetric(record, ContainerMetrics.VCORE_LIMIT_METRIC_NAME, anyVcores);
+
+    collector.clear();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
index d2a069161713b..4dc4648cf41a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
@@ -17,17 +17,20 @@
  */
 package org.apache.hadoop.yarn.server.nodemanager.metrics;
 
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import static org.apache.hadoop.test.MetricsAsserts.*;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.Records;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestNodeManagerMetrics {
   static final int GiB = 1024; // MiB
 
   @Test public void testNames() {
+    DefaultMetricsSystem.initialize("NodeManager");
     NodeManagerMetrics metrics = NodeManagerMetrics.create();
     Resource total = Records.newRecord(Resource.class);
     total.setMemory(8*GiB);
@@ -61,7 +64,10 @@ public class TestNodeManagerMetrics {
 
     metrics.initingContainer();
     metrics.runningContainer();
+
+    Assert.assertTrue(!metrics.containerLaunchDuration.changed());
     metrics.addContainerLaunchDuration(1);
+    Assert.assertTrue(metrics.containerLaunchDuration.changed());
 
     // availableGB is expected to be floored,
     // while allocatedGB is expected to be ceiled.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 61bdf1036c467..7caad4ad1ea01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -57,6 +57,7 @@
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
@@ -78,13 +79,12 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
 /**
  * Test the nodemanager node info web services api's
  */
-public class TestNMWebServices extends JerseyTest {
+public class TestNMWebServices extends JerseyTestBase {
 
   private static Context nmContext;
   private static ResourceView resourceView;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
index 87aa85268942e..3e7aac8c8ae89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
@@ -50,6 +50,7 @@
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONArray;
@@ -73,10 +74,9 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestNMWebServicesApps extends JerseyTest {
+public class TestNMWebServicesApps extends JerseyTestBase {
 
   private static Context nmContext;
   private static ResourceView resourceView;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 62d9cb7b3b4b7..ceb1d571323d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONArray;
@@ -73,10 +74,9 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestNMWebServicesContainers extends JerseyTest {
+public class TestNMWebServicesContainers extends JerseyTestBase {
 
   private static Context nmContext;
   private static ResourceView resourceView;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 14109f90e07bc..d79de58b535be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -59,10 +59,6 @@
 import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
 import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
@@ -81,8 +77,6 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNodesToLabelsResponsePBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
@@ -348,14 +342,10 @@ public synchronized HAServiceStatus getServiceStatus() throws IOException {
   public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
       throws YarnException, StandbyException {
     String argName = "refreshQueues";
+    final String msg = "refresh queues.";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh queues.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, msg);
 
     RefreshQueuesResponse response =
         recordFactory.newRecordInstance(RefreshQueuesResponse.class);
@@ -370,11 +360,7 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
           "AdminService");
       return response;
     } catch (IOException ioe) {
-      LOG.info("Exception refreshing queues ", ioe);
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "Exception refreshing queues");
-      throw RPCUtil.getRemoteException(ioe);
+      throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
     }
   }
 
@@ -382,14 +368,10 @@ public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
   public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
       throws YarnException, StandbyException {
     String argName = "refreshNodes";
+    final String msg = "refresh nodes.";
     UserGroupInformation user = checkAcls("refreshNodes");
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh nodes.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, msg);
 
     try {
       Configuration conf =
@@ -400,10 +382,7 @@ public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
           "AdminService");
       return recordFactory.newRecordInstance(RefreshNodesResponse.class);
     } catch (IOException ioe) {
-      LOG.info("Exception refreshing nodes ", ioe);
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService", "Exception refreshing nodes");
-      throw RPCUtil.getRemoteException(ioe);
+      throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
     }
   }
 
@@ -414,12 +393,7 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu
     String argName = "refreshSuperUserGroupsConfiguration";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh super-user-groups.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, "refresh super-user-groups.");
 
     // Accept hadoop common configs in core-site.xml as well as RM specific
     // configurations in yarn-site.xml
@@ -443,12 +417,7 @@ public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
     String argName = "refreshUserToGroupsMappings";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh user-groups.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, "refresh user-groups.");
 
     Groups.getUserToGroupsMappingService(
         getConfiguration(new Configuration(false),
@@ -471,11 +440,8 @@ private RefreshAdminAclsResponse refreshAdminAcls(boolean checkRMHAState)
     String argName = "refreshAdminAcls";
     UserGroupInformation user = checkAcls(argName);
 
-    if (checkRMHAState && !isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh user-groups.");
-      throwStandbyException();
+    if (checkRMHAState) {
+      checkRMStatus(user.getShortUserName(), argName, "refresh Admin ACLs.");
     }
     Configuration conf =
         getConfiguration(new Configuration(false),
@@ -502,13 +468,9 @@ public RefreshServiceAclsResponse refreshServiceAcls(
     }
 
     String argName = "refreshServiceAcls";
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(UserGroupInformation.getCurrentUser()
-          .getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not refresh Service ACLs.");
-      throwStandbyException();
-    }
+    UserGroupInformation user = checkAcls(argName);
+
+    checkRMStatus(user.getShortUserName(), argName, "refresh Service ACLs.");
 
     PolicyProvider policyProvider = RMPolicyProvider.getInstance();
     Configuration conf =
@@ -543,12 +505,7 @@ public UpdateNodeResourceResponse updateNodeResource(
     String argName = "updateNodeResource";
     UserGroupInformation user = checkAcls(argName);
     
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not update node resource.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, "update node resource.");
     
     Map<NodeId, ResourceOption> nodeResourceMap = request.getNodeResourceMap();
     Set<NodeId> nodeIds = nodeResourceMap.keySet();
@@ -641,14 +598,10 @@ public Server getServer() {
   public AddToClusterNodeLabelsResponse addToClusterNodeLabels(AddToClusterNodeLabelsRequest request)
       throws YarnException, IOException {
     String argName = "addToClusterNodeLabels";
+    final String msg = "add labels.";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not add labels.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, msg);
 
     AddToClusterNodeLabelsResponse response =
         recordFactory.newRecordInstance(AddToClusterNodeLabelsResponse.class);
@@ -658,10 +611,7 @@ public AddToClusterNodeLabelsResponse addToClusterNodeLabels(AddToClusterNodeLab
           .logSuccess(user.getShortUserName(), argName, "AdminService");
       return response;
     } catch (IOException ioe) {
-      LOG.info("Exception add labels", ioe);
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService", "Exception add label");
-      throw RPCUtil.getRemoteException(ioe);
+      throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
     }
   }
 
@@ -669,14 +619,10 @@ public AddToClusterNodeLabelsResponse addToClusterNodeLabels(AddToClusterNodeLab
   public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
       RemoveFromClusterNodeLabelsRequest request) throws YarnException, IOException {
     String argName = "removeFromClusterNodeLabels";
+    final String msg = "remove labels.";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not remove labels.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, msg);
 
     RemoveFromClusterNodeLabelsResponse response =
         recordFactory.newRecordInstance(RemoveFromClusterNodeLabelsResponse.class);
@@ -686,10 +632,7 @@ public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
           .logSuccess(user.getShortUserName(), argName, "AdminService");
       return response;
     } catch (IOException ioe) {
-      LOG.info("Exception remove labels", ioe);
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService", "Exception remove label");
-      throw RPCUtil.getRemoteException(ioe);
+      throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
     }
   }
 
@@ -697,14 +640,10 @@ public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
   public ReplaceLabelsOnNodeResponse replaceLabelsOnNode(
       ReplaceLabelsOnNodeRequest request) throws YarnException, IOException {
     String argName = "replaceLabelsOnNode";
+    final String msg = "set node to labels.";
     UserGroupInformation user = checkAcls(argName);
 
-    if (!isRMActive()) {
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "ResourceManager is not active. Can not set node to labels.");
-      throwStandbyException();
-    }
+    checkRMStatus(user.getShortUserName(), argName, msg);
 
     ReplaceLabelsOnNodeResponse response =
         recordFactory.newRecordInstance(ReplaceLabelsOnNodeResponse.class);
@@ -715,11 +654,24 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode(
           .logSuccess(user.getShortUserName(), argName, "AdminService");
       return response;
     } catch (IOException ioe) {
-      LOG.info("Exception set node to labels. ", ioe);
-      RMAuditLogger.logFailure(user.getShortUserName(), argName,
-          adminAcl.toString(), "AdminService",
-          "Exception set node to labels.");
-      throw RPCUtil.getRemoteException(ioe);
+      throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
+    }
+  }
+
+  private void checkRMStatus(String user, String argName, String msg)
+      throws StandbyException {
+    if (!isRMActive()) {
+      RMAuditLogger.logFailure(user, argName, adminAcl.toString(), 
+          "AdminService", "ResourceManager is not active. Can not " + msg);
+      throwStandbyException();
     }
   }
+
+  private YarnException logAndWrapException(IOException ioe, String user,
+      String argName, String msg) throws YarnException {
+    LOG.info("Exception " + msg, ioe);
+    RMAuditLogger.logFailure(user, argName, adminAcl.toString(), 
+        "AdminService", "Exception " + msg);
+    return RPCUtil.getRemoteException(ioe);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index d0b199f23a63b..6650cf2737311 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -285,7 +285,7 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
       RegisterApplicationMasterResponse response = recordFactory
           .newRecordInstance(RegisterApplicationMasterResponse.class);
       response.setMaximumResourceCapability(rScheduler
-          .getMaximumResourceCapability());
+          .getMaximumResourceCapability(app.getQueue()));
       response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)
           .getSubmissionContext().getAMContainerSpec().getApplicationACLs());
       response.setQueue(app.getQueue());
@@ -488,10 +488,11 @@ public AllocateResponse allocate(AllocateRequest request)
       RMApp app =
           this.rmContext.getRMApps().get(applicationId);
       
-      // set label expression for Resource Requests
+      // set label expression for Resource Requests if resourceName=ANY 
       ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
       for (ResourceRequest req : ask) {
-        if (null == req.getNodeLabelExpression()) {
+        if (null == req.getNodeLabelExpression()
+            && ResourceRequest.ANY.equals(req.getResourceName())) {
           req.setNodeLabelExpression(asc.getNodeLabelExpression());
         }
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index bee6bf8ee8a14..d3ccb91546121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -366,8 +366,9 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport(
     if (allowAccess) {
       RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
       if (appAttempt == null) {
-        throw new ApplicationAttemptNotFoundException("ApplicationAttempt "
-            + appAttemptId + " Not Found in RM");
+        throw new ApplicationAttemptNotFoundException(
+            "ApplicationAttempt with id '" + appAttemptId +
+            "' doesn't exist in RM.");
       }
       ApplicationAttemptReport attemptReport = appAttempt
           .createApplicationAttemptReport();
@@ -451,14 +452,15 @@ public GetContainerReportResponse getContainerReport(
     if (allowAccess) {
       RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
       if (appAttempt == null) {
-        throw new ApplicationAttemptNotFoundException("ApplicationAttempt "
-            + appAttemptId + " Not Found in RM");
+        throw new ApplicationAttemptNotFoundException(
+            "ApplicationAttempt with id '" + appAttemptId +
+            "' doesn't exist in RM.");
       }
       RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer(
           containerId);
       if (rmConatiner == null) {
-        throw new ContainerNotFoundException("Container with id " + containerId
-            + " not found");
+        throw new ContainerNotFoundException("Container with id '" + containerId
+            + "' doesn't exist in RM.");
       }
       response = GetContainerReportResponse.newInstance(rmConatiner
           .createContainerReport());
@@ -500,8 +502,9 @@ public GetContainersResponse getContainers(GetContainersRequest request)
     if (allowAccess) {
       RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
       if (appAttempt == null) {
-        throw new ApplicationAttemptNotFoundException("ApplicationAttempt "
-            + appAttemptId + " Not Found in RM");
+        throw new ApplicationAttemptNotFoundException(
+            "ApplicationAttempt with id '" + appAttemptId +
+            "' doesn't exist in RM.");
       }
       Collection<RMContainer> rmContainers = Collections.emptyList();
       SchedulerAppReport schedulerAppReport =
@@ -827,6 +830,14 @@ public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
   @Override
   public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
       throws YarnException {
+    UserGroupInformation callerUGI;
+    try {
+      callerUGI = UserGroupInformation.getCurrentUser();
+    } catch (IOException ie) {
+      LOG.info("Error getting UGI ", ie);
+      throw RPCUtil.getRemoteException(ie);
+    }
+
     GetQueueInfoResponse response =
       recordFactory.newRecordInstance(GetQueueInfoResponse.class);
     try {
@@ -841,7 +852,16 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
         appReports = new ArrayList<ApplicationReport>(apps.size());
         for (ApplicationAttemptId app : apps) {
           RMApp rmApp = rmContext.getRMApps().get(app.getApplicationId());
-          appReports.add(rmApp.createAndGetApplicationReport(null, true));
+          if (rmApp != null) {
+            // Check if user is allowed access to this app
+            if (!checkAccess(callerUGI, rmApp.getUser(),
+                ApplicationAccessType.VIEW_APP, rmApp)) {
+              continue;
+            }
+            appReports.add(
+                rmApp.createAndGetApplicationReport(
+                    callerUGI.getUserName(), true));
+          }          
         }
       }
       queueInfo.setApplications(appReports);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 3bc2e9bbbf391..03fc40ed2c57c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -117,7 +117,8 @@ public RMActiveServiceContext(Dispatcher rmDispatcher,
       RMContainerTokenSecretManager containerTokenSecretManager,
       NMTokenSecretManagerInRM nmTokenSecretManager,
       ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
-      RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+      RMApplicationHistoryWriter rmApplicationHistoryWriter,
+      ResourceScheduler scheduler) {
     this();
     this.setContainerAllocationExpirer(containerAllocationExpirer);
     this.setAMLivelinessMonitor(amLivelinessMonitor);
@@ -128,6 +129,7 @@ public RMActiveServiceContext(Dispatcher rmDispatcher,
     this.setNMTokenSecretManager(nmTokenSecretManager);
     this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
     this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+    this.setScheduler(scheduler);
 
     RMStateStore nullStore = new NullRMStateStore();
     nullStore.setRMDispatcher(rmDispatcher);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index f38e128cfe290..8dcfe6799608c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -172,7 +172,8 @@ public static SummaryBuilder createAppSummary(RMApp app) {
           .add("vcoreSeconds", metrics.getVcoreSeconds())
           .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
           .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
-          .add("preemptedResources", metrics.getResourcePreempted());
+          .add("preemptedResources", metrics.getResourcePreempted())
+          .add("applicationType", app.getApplicationType());
       return summary;
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index ebf2fe48a8cbd..1d0d6c0bf8ab5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -87,18 +87,46 @@ public RMContextImpl(Dispatcher rmDispatcher,
       RMContainerTokenSecretManager containerTokenSecretManager,
       NMTokenSecretManagerInRM nmTokenSecretManager,
       ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
-      RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+      RMApplicationHistoryWriter rmApplicationHistoryWriter,
+      ResourceScheduler scheduler) {
     this();
     this.setDispatcher(rmDispatcher);
     setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
         containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
         delegationTokenRenewer, appTokenSecretManager,
         containerTokenSecretManager, nmTokenSecretManager,
-        clientToAMTokenSecretManager, rmApplicationHistoryWriter));
+        clientToAMTokenSecretManager, rmApplicationHistoryWriter,
+        scheduler));
 
     ConfigurationProvider provider = new LocalConfigurationProvider();
     setConfigurationProvider(provider);
   }
+  
+  @VisibleForTesting
+  // helper constructor for tests
+  public RMContextImpl(Dispatcher rmDispatcher,
+      ContainerAllocationExpirer containerAllocationExpirer,
+      AMLivelinessMonitor amLivelinessMonitor,
+      AMLivelinessMonitor amFinishingMonitor,
+      DelegationTokenRenewer delegationTokenRenewer,
+      AMRMTokenSecretManager appTokenSecretManager,
+      RMContainerTokenSecretManager containerTokenSecretManager,
+      NMTokenSecretManagerInRM nmTokenSecretManager,
+      ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
+      RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+    this(
+      rmDispatcher,
+      containerAllocationExpirer,
+      amLivelinessMonitor,
+      amFinishingMonitor,
+      delegationTokenRenewer,
+      appTokenSecretManager,
+      containerTokenSecretManager,
+      nmTokenSecretManager,
+      clientToAMTokenSecretManager,
+      rmApplicationHistoryWriter,
+      null);
+  }
 
   @Override
   public Dispatcher getDispatcher() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
index d0d7d16a276d1..f7fb7e631e888 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
@@ -127,14 +127,14 @@ protected ClientToAMTokenSecretManagerInRM createClientToAMTokenSecretManager()
   protected RMDelegationTokenSecretManager createRMDelegationTokenSecretManager(
       Configuration conf, RMContext rmContext) {
     long secretKeyInterval =
-        conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY,
-            YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+        conf.getLong(YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY,
+            YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
     long tokenMaxLifetime =
-        conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
-            YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+        conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+            YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
     long tokenRenewInterval =
-        conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
-            YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+        conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+            YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
 
     return new RMDelegationTokenSecretManager(secretKeyInterval,
         tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 775d08a46f302..4f242e93ae4e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -59,7 +59,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -334,10 +333,7 @@ protected AMLivelinessMonitor createAMLivelinessMonitor() {
   
   protected RMNodeLabelsManager createNodeLabelManager()
       throws InstantiationException, IllegalAccessException {
-    Class<? extends RMNodeLabelsManager> nlmCls =
-        conf.getClass(YarnConfiguration.RM_NODE_LABELS_MANAGER_CLASS,
-            MemoryRMNodeLabelsManager.class, RMNodeLabelsManager.class);
-    return nlmCls.newInstance();
+    return new RMNodeLabelsManager();
   }
   
   protected DelegationTokenRenewer createDelegationTokenRenewer() {
@@ -426,6 +422,7 @@ protected void serviceInit(Configuration configuration) throws Exception {
       rmContext.setAMFinishingMonitor(amFinishingMonitor);
       
       RMNodeLabelsManager nlm = createNodeLabelManager();
+      nlm.setRMContext(rmContext);
       addService(nlm);
       rmContext.setNodeLabelManager(nlm);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
index c457710db6b13..8d75f92e42681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 
 public class ApplicationFinishedEvent extends
     SystemMetricsEvent {
@@ -31,6 +32,7 @@ public class ApplicationFinishedEvent extends
   private FinalApplicationStatus appStatus;
   private YarnApplicationState state;
   private ApplicationAttemptId latestAppAttemptId;
+  private RMAppMetrics appMetrics;
 
   public ApplicationFinishedEvent(
       ApplicationId appId,
@@ -38,13 +40,15 @@ public ApplicationFinishedEvent(
       FinalApplicationStatus appStatus,
       YarnApplicationState state,
       ApplicationAttemptId latestAppAttemptId,
-      long finishedTime) {
+      long finishedTime,
+      RMAppMetrics appMetrics) {
     super(SystemMetricsEventType.APP_FINISHED, finishedTime);
     this.appId = appId;
     this.diagnosticsInfo = diagnosticsInfo;
     this.appStatus = appStatus;
     this.latestAppAttemptId = latestAppAttemptId;
     this.state = state;
+    this.appMetrics=appMetrics;
   }
 
   @Override
@@ -72,4 +76,7 @@ public ApplicationAttemptId getLatestApplicationAttemptId() {
     return latestAppAttemptId;
   }
 
+  public RMAppMetrics getAppMetrics() {
+    return appMetrics;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index d8e7ef6f2025c..3adf519a6c3cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -120,7 +121,8 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) {
               RMServerUtils.createApplicationState(state),
               app.getCurrentAppAttempt() == null ?
                   null : app.getCurrentAppAttempt().getAppAttemptId(),
-              finishedTime));
+              finishedTime,
+              app.getRMAppMetrics()));
     }
   }
 
@@ -276,6 +278,12 @@ private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
       eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
           event.getLatestApplicationAttemptId().toString());
     }
+    RMAppMetrics appMetrics = event.getAppMetrics();
+    entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+        appMetrics.getVcoreSeconds());
+    entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+        appMetrics.getMemorySeconds());
+    
     tEvent.setEventInfo(eventInfo);
     entity.addEvent(tEvent);
     putEntity(entity);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1a3f804b8314d..738f5272e2e45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -30,15 +30,19 @@
 import java.util.PriorityQueue;
 import java.util.Set;
 
+import org.apache.commons.collections.map.HashedMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType;
@@ -112,9 +116,6 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
   public static final String NATURAL_TERMINATION_FACTOR =
       "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor";
 
-  public static final String BASE_YARN_RM_PREEMPTION = "yarn.scheduler.capacity.";
-  public static final String SUFFIX_DISABLE_PREEMPTION = ".disable_preemption";
-
   // the dispatcher to send preempt and kill events
   public EventHandler<ContainerPreemptEvent> dispatcher;
 
@@ -129,6 +130,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
   private float percentageClusterPreemptionAllowed;
   private double naturalTerminationFactor;
   private boolean observeOnly;
+  private Map<NodeId, Set<String>> labels;
 
   public ProportionalCapacityPreemptionPolicy() {
     clock = new SystemClock();
@@ -168,6 +170,7 @@ public void init(Configuration config,
       config.getFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 0.1);
     observeOnly = config.getBoolean(OBSERVE_ONLY, false);
     rc = scheduler.getResourceCalculator();
+    labels = null;
   }
   
   @VisibleForTesting
@@ -176,13 +179,38 @@ public ResourceCalculator getResourceCalculator() {
   }
 
   @Override
-  public void editSchedule(){
+  public void editSchedule() {
     CSQueue root = scheduler.getRootQueue();
-    Resource clusterResources =
-      Resources.clone(scheduler.getClusterResource());
+    Resource clusterResources = Resources.clone(scheduler.getClusterResource());
+    clusterResources = getNonLabeledResources(clusterResources);
+    setNodeLabels(scheduler.getRMContext().getNodeLabelManager()
+        .getNodeLabels());
     containerBasedPreemptOrKill(root, clusterResources);
   }
 
+  /**
+   * Setting Node Labels
+   * 
+   * @param nodelabels
+   */
+  public void setNodeLabels(Map<NodeId, Set<String>> nodelabels) {
+    labels = nodelabels;
+  }
+
+  /**
+   * This method returns all non labeled resources.
+   * 
+   * @param clusterResources
+   * @return Resources
+   */
+  private Resource getNonLabeledResources(Resource clusterResources) {
+    RMContext rmcontext = scheduler.getRMContext();
+    RMNodeLabelsManager lm = rmcontext.getNodeLabelManager();
+    Resource res = lm.getResourceByLabel(RMNodeLabelsManager.NO_LABEL,
+        clusterResources);
+    return res == null ? clusterResources : res;
+  }
+  
   /**
    * This method selects and tracks containers to be preempted. If a container
    * is in the target list for more than maxWaitTime it is killed.
@@ -196,7 +224,7 @@ private void containerBasedPreemptOrKill(CSQueue root,
     // extract a summary of the queues from scheduler
     TempQueue tRoot;
     synchronized (scheduler) {
-      tRoot = cloneQueues(root, clusterResources, false);
+      tRoot = cloneQueues(root, clusterResources);
     }
 
     // compute the ideal distribution of resources among queues
@@ -593,7 +621,7 @@ private void preemptAMContainers(Resource clusterResource,
    * @param app
    * @param clusterResource
    * @param rsrcPreempt
-   * @return
+   * @return Set<RMContainer> Set of RMContainers
    */
   private Set<RMContainer> preemptFrom(FiCaSchedulerApp app,
       Resource clusterResource, Resource rsrcPreempt,
@@ -635,12 +663,26 @@ private Set<RMContainer> preemptFrom(FiCaSchedulerApp app,
         Resources.addTo(skippedAMSize, c.getContainer().getResource());
         continue;
       }
+      // skip Labeled resource
+      if(isLabeledContainer(c)){
+        continue;
+      }
       ret.add(c);
       Resources.subtractFrom(rsrcPreempt, c.getContainer().getResource());
     }
 
     return ret;
   }
+  
+  /**
+   * Checking if given container is a labeled container
+   * 
+   * @param c
+   * @return true/false
+   */
+  private boolean isLabeledContainer(RMContainer c) {
+    return labels.containsKey(c.getAllocatedNode());
+  }
 
   /**
    * Compare by reversed priority order first, and then reversed containerId
@@ -683,11 +725,9 @@ public String getPolicyName() {
    *
    * @param root the root of the CapacityScheduler queue hierarchy
    * @param clusterResources the total amount of resources in the cluster
-   * @param parentDisablePreempt true if disable preemption is set for parent
    * @return the root of the cloned queue hierarchy
    */
-  private TempQueue cloneQueues(CSQueue root, Resource clusterResources,
-      boolean parentDisablePreempt) {
+  private TempQueue cloneQueues(CSQueue root, Resource clusterResources) {
     TempQueue ret;
     synchronized (root) {
       String queueName = root.getQueueName();
@@ -699,12 +739,6 @@ private TempQueue cloneQueues(CSQueue root, Resource clusterResources,
       Resource guaranteed = Resources.multiply(clusterResources, absCap);
       Resource maxCapacity = Resources.multiply(clusterResources, absMaxCap);
 
-      boolean queueDisablePreemption = false;
-      String queuePropName = BASE_YARN_RM_PREEMPTION + root.getQueuePath()
-                               + SUFFIX_DISABLE_PREEMPTION;
-      queueDisablePreemption = scheduler.getConfiguration()
-                              .getBoolean(queuePropName, parentDisablePreempt);
-
       Resource extra = Resource.newInstance(0, 0);
       if (Resources.greaterThan(rc, clusterResources, current, guaranteed)) {
         extra = Resources.subtract(current, guaranteed);
@@ -714,7 +748,7 @@ private TempQueue cloneQueues(CSQueue root, Resource clusterResources,
         Resource pending = l.getTotalResourcePending();
         ret = new TempQueue(queueName, current, pending, guaranteed,
             maxCapacity);
-        if (queueDisablePreemption) {
+        if (root.getPreemptionDisabled()) {
           ret.untouchableExtra = extra;
         } else {
           ret.preemptableExtra = extra;
@@ -726,8 +760,7 @@ private TempQueue cloneQueues(CSQueue root, Resource clusterResources,
             maxCapacity);
         Resource childrensPreemptable = Resource.newInstance(0, 0);
         for (CSQueue c : root.getChildQueues()) {
-          TempQueue subq =
-                cloneQueues(c, clusterResources, queueDisablePreemption);
+          TempQueue subq = cloneQueues(c, clusterResources);
           Resources.addTo(childrensPreemptable, subq.preemptableExtra);
           ret.addChild(subq);
         }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index ba1727c2c1884..1555291cf1909 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -19,10 +19,12 @@
 package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -36,6 +38,9 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.collect.ImmutableSet;
@@ -57,6 +62,8 @@ protected Queue() {
       new ConcurrentHashMap<String, Queue>();
   protected AccessControlList adminAcl;
   
+  private RMContext rmContext = null;
+  
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
     super.serviceInit(conf);
@@ -193,6 +200,17 @@ public void activateNode(NodeId nodeId, Resource resource) {
       Node nm = getNMInNodeSet(nodeId);
       nm.resource = resource;
       nm.running = true;
+
+      // Add node in labelsCollection
+      Set<String> labelsForNode = getLabelsByNode(nodeId);
+      if (labelsForNode != null) {
+        for (String label : labelsForNode) {
+          NodeLabel labelInfo = labelCollections.get(label);
+          if(labelInfo != null) {
+            labelInfo.addNodeId(nodeId);
+          }
+        }
+      }
       
       // get the node after edition
       Map<String, Host> after = cloneNodeMap(ImmutableSet.of(nodeId));
@@ -331,6 +349,7 @@ private Map<String, Host> cloneNodeMap(Set<NodeId> nodesToCopy) {
     return map;
   }
 
+  @SuppressWarnings("unchecked")
   private void updateResourceMappings(Map<String, Host> before,
       Map<String, Host> after) {
     // Get NMs in before only
@@ -341,6 +360,10 @@ private void updateResourceMappings(Map<String, Host> before,
     for (Entry<String, Host> entry : after.entrySet()) {
       allNMs.addAll(entry.getValue().nms.keySet());
     }
+    
+    // Map used to notify RM
+    Map<NodeId, Set<String>> newNodeToLabelsMap =
+        new HashMap<NodeId, Set<String>>();
 
     // traverse all nms
     for (NodeId nodeId : allNMs) {
@@ -350,8 +373,8 @@ private void updateResourceMappings(Map<String, Host> before,
         // no label in the past
         if (oldLabels.isEmpty()) {
           // update labels
-          Label label = labelCollections.get(NO_LABEL);
-          Resources.subtractFrom(label.resource, oldNM.resource);
+          NodeLabel label = labelCollections.get(NO_LABEL);
+          label.removeNode(oldNM.resource);
 
           // update queues, all queue can access this node
           for (Queue q : queueCollections.values()) {
@@ -360,11 +383,11 @@ private void updateResourceMappings(Map<String, Host> before,
         } else {
           // update labels
           for (String labelName : oldLabels) {
-            Label label = labelCollections.get(labelName);
+            NodeLabel label = labelCollections.get(labelName);
             if (null == label) {
               continue;
             }
-            Resources.subtractFrom(label.resource, oldNM.resource);
+            label.removeNode(oldNM.resource);
           }
 
           // update queues, only queue can access this node will be subtract
@@ -379,11 +402,14 @@ private void updateResourceMappings(Map<String, Host> before,
       Node newNM;
       if ((newNM = getNMInNodeSet(nodeId, after, true)) != null) {
         Set<String> newLabels = getLabelsByNode(nodeId, after);
+        
+        newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels));
+        
         // no label in the past
         if (newLabels.isEmpty()) {
           // update labels
-          Label label = labelCollections.get(NO_LABEL);
-          Resources.addTo(label.resource, newNM.resource);
+          NodeLabel label = labelCollections.get(NO_LABEL);
+          label.addNode(newNM.resource);
 
           // update queues, all queue can access this node
           for (Queue q : queueCollections.values()) {
@@ -392,8 +418,8 @@ private void updateResourceMappings(Map<String, Host> before,
         } else {
           // update labels
           for (String labelName : newLabels) {
-            Label label = labelCollections.get(labelName);
-            Resources.addTo(label.resource, newNM.resource);
+            NodeLabel label = labelCollections.get(labelName);
+            label.addNode(newNM.resource);
           }
 
           // update queues, only queue can access this node will be subtract
@@ -405,6 +431,12 @@ private void updateResourceMappings(Map<String, Host> before,
         }
       }
     }
+    
+    // Notify RM
+    if (rmContext != null && rmContext.getDispatcher() != null) {
+      rmContext.getDispatcher().getEventHandler().handle(
+          new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap));
+    }
   }
   
   public Resource getResourceByLabel(String label, Resource clusterResource) {
@@ -414,7 +446,7 @@ public Resource getResourceByLabel(String label, Resource clusterResource) {
       if (null == labelCollections.get(label)) {
         return Resources.none();
       }
-      return labelCollections.get(label).resource;
+      return labelCollections.get(label).getResource();
     } finally {
       readLock.unlock();
     }
@@ -452,4 +484,25 @@ public boolean checkAccess(UserGroupInformation user) {
     }
     return false;
   }
+  
+  public void setRMContext(RMContext rmContext) {
+    this.rmContext = rmContext;
+  }
+
+  public List<NodeLabel> pullRMNodeLabelsInfo() {
+    try {
+      readLock.lock();
+      List<NodeLabel> infos = new ArrayList<NodeLabel>();
+
+      for (Entry<String, NodeLabel> entry : labelCollections.entrySet()) {
+        NodeLabel label = entry.getValue();
+        infos.add(label.getCopy());
+      }
+
+      Collections.sort(infos);
+      return infos;
+    } finally {
+      readLock.unlock();
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 299639225e839..6e830a028ab06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -60,8 +60,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-
 import com.google.common.annotations.VisibleForTesting;
 
 @Private
@@ -141,8 +139,8 @@ protected Version getCurrentVersion() {
   @Override
   protected synchronized Version loadVersion() throws Exception {
     Path versionNodePath = getNodePath(rootDirPath, VERSION_NODE);
-    if (fs.exists(versionNodePath)) {
-      FileStatus status = fs.getFileStatus(versionNodePath);
+    FileStatus status = getFileStatus(versionNodePath);
+    if (status != null) {
       byte[] data = readFile(versionNodePath, status.getLen());
       Version version =
           new VersionPBImpl(VersionProto.parseFrom(data));
@@ -167,9 +165,9 @@ protected synchronized void storeVersion() throws Exception {
   public synchronized long getAndIncrementEpoch() throws Exception {
     Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE);
     long currentEpoch = 0;
-    if (fs.exists(epochNodePath)) {
+    FileStatus status = getFileStatus(epochNodePath);
+    if (status != null) {
       // load current epoch
-      FileStatus status = fs.getFileStatus(epochNodePath);
       byte[] data = readFile(epochNodePath, status.getLen());
       Epoch epoch = new EpochPBImpl(EpochProto.parseFrom(data));
       currentEpoch = epoch.getEpoch();
@@ -203,13 +201,11 @@ private void loadAMRMTokenSecretManagerState(RMState rmState)
     checkAndResumeUpdateOperation(amrmTokenSecretManagerRoot);
     Path amrmTokenSecretManagerStateDataDir =
         new Path(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
-    FileStatus status;
-    try {
-      status = fs.getFileStatus(amrmTokenSecretManagerStateDataDir);
-      assert status.isFile();
-    } catch (FileNotFoundException ex) {
+    FileStatus status = getFileStatus(amrmTokenSecretManagerStateDataDir);
+    if (status == null) {
       return;
     }
+    assert status.isFile();
     byte[] data = readFile(amrmTokenSecretManagerStateDataDir, status.getLen());
     AMRMTokenSecretManagerStatePBImpl stateData =
         new AMRMTokenSecretManagerStatePBImpl(
@@ -452,11 +448,10 @@ public synchronized void removeApplicationStateInternal(
   }
 
   @Override
-  public synchronized void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier identifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
-    storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-        identifier, renewDate,latestSequenceNumber, false);
+  public synchronized void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier identifier, Long renewDate)
+      throws Exception {
+    storeOrUpdateRMDelegationTokenState(identifier, renewDate, false);
   }
 
   @Override
@@ -469,16 +464,15 @@ public synchronized void removeRMDelegationTokenState(
   }
 
   @Override
-  protected void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
-    storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-        rmDTIdentifier, renewDate,latestSequenceNumber, true);
+  protected synchronized void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
+    storeOrUpdateRMDelegationTokenState(rmDTIdentifier, renewDate, true);
   }
 
-  private void storeOrUpdateRMDelegationTokenAndSequenceNumberState(
+  private void storeOrUpdateRMDelegationTokenState(
       RMDelegationTokenIdentifier identifier, Long renewDate,
-      int latestSequenceNumber, boolean isUpdate) throws Exception {
+      boolean isUpdate) throws Exception {
     Path nodeCreatePath =
         getNodePath(rmDTSecretManagerRoot,
           DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
@@ -490,23 +484,24 @@ private void storeOrUpdateRMDelegationTokenAndSequenceNumberState(
     } else {
       LOG.info("Storing RMDelegationToken_" + identifier.getSequenceNumber());
       writeFile(nodeCreatePath, identifierData.toByteArray());
-    }
 
-    // store sequence number
-    Path latestSequenceNumberPath = getNodePath(rmDTSecretManagerRoot,
-          DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX + latestSequenceNumber);
-    LOG.info("Storing " + DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX
-        + latestSequenceNumber);
-    if (dtSequenceNumberPath == null) {
-      if (!createFile(latestSequenceNumberPath)) {
-        throw new Exception("Failed to create " + latestSequenceNumberPath);
-      }
-    } else {
-      if (!renameFile(dtSequenceNumberPath, latestSequenceNumberPath)) {
-        throw new Exception("Failed to rename " + dtSequenceNumberPath);
+      // store sequence number
+      Path latestSequenceNumberPath = getNodePath(rmDTSecretManagerRoot,
+            DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX
+            + identifier.getSequenceNumber());
+      LOG.info("Storing " + DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX
+          + identifier.getSequenceNumber());
+      if (dtSequenceNumberPath == null) {
+        if (!createFile(latestSequenceNumberPath)) {
+          throw new Exception("Failed to create " + latestSequenceNumberPath);
+        }
+      } else {
+        if (!renameFile(dtSequenceNumberPath, latestSequenceNumberPath)) {
+          throw new Exception("Failed to rename " + dtSequenceNumberPath);
+        }
       }
+      dtSequenceNumberPath = latestSequenceNumberPath;
     }
-    dtSequenceNumberPath = latestSequenceNumberPath;
   }
 
   @Override
@@ -563,6 +558,14 @@ private byte[] readFile(Path inputPath, long len) throws Exception {
     }
   }
 
+  private FileStatus getFileStatus(Path path) throws Exception {
+    try {
+      return fs.getFileStatus(path);
+    } catch (FileNotFoundException e) {
+      return null;
+    }
+  }
+
   /*
    * In order to make this write atomic as a part of write we will first write
    * data to .tmp file and then rename it. Here we are assuming that rename is
@@ -624,22 +627,17 @@ Path getNodePath(Path root, String nodeName) {
 
   @Override
   public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
-      AMRMTokenSecretManagerState amrmTokenSecretManagerState,
-      boolean isUpdate){
+      AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate)
+      throws Exception {
     Path nodeCreatePath =
         getNodePath(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE);
     AMRMTokenSecretManagerState data =
         AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
     byte[] stateData = data.getProto().toByteArray();
-    try {
-      if (isUpdate) {
-        updateFile(nodeCreatePath, stateData);
-      } else {
-        writeFile(nodeCreatePath, stateData);
-      }
-    } catch (Exception ex) {
-      LOG.info("Error storing info for AMRMTokenSecretManager", ex);
-      notifyStoreOperationFailed(ex);
+    if (isUpdate) {
+      updateFile(nodeCreatePath, stateData);
+    } else {
+      writeFile(nodeCreatePath, stateData);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 38ce37053c145..2c927146480e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -544,31 +544,30 @@ protected void removeApplicationStateInternal(ApplicationStateData appState)
       throw new IOException(e);
     }
   }
-
-  @Override
-  protected void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier tokenId, Long renewDate,
-      int latestSequenceNumber) throws IOException {
+  
+  private void storeOrUpdateRMDT(RMDelegationTokenIdentifier tokenId,
+      Long renewDate, boolean isUpdate) throws IOException {
     String tokenKey = getRMDTTokenNodeKey(tokenId);
     RMDelegationTokenIdentifierData tokenData =
         new RMDelegationTokenIdentifierData(tokenId, renewDate);
-    ByteArrayOutputStream bs = new ByteArrayOutputStream();
-    DataOutputStream ds = new DataOutputStream(bs);
-    try {
-      ds.writeInt(latestSequenceNumber);
-    } finally {
-      ds.close();
-    }
     if (LOG.isDebugEnabled()) {
       LOG.debug("Storing token to " + tokenKey);
-      LOG.debug("Storing " + latestSequenceNumber + " to "
-          + RM_DT_SEQUENCE_NUMBER_KEY);
     }
     try {
       WriteBatch batch = db.createWriteBatch();
       try {
         batch.put(bytes(tokenKey), tokenData.toByteArray());
-        batch.put(bytes(RM_DT_SEQUENCE_NUMBER_KEY), bs.toByteArray());
+        if(!isUpdate) {
+          ByteArrayOutputStream bs = new ByteArrayOutputStream();
+          try (DataOutputStream ds = new DataOutputStream(bs)) {
+            ds.writeInt(tokenId.getSequenceNumber());
+          }
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Storing " + tokenId.getSequenceNumber() + " to "
+                + RM_DT_SEQUENCE_NUMBER_KEY);   
+          }
+          batch.put(bytes(RM_DT_SEQUENCE_NUMBER_KEY), bs.toByteArray());
+        }
         db.write(batch);
       } finally {
         batch.close();
@@ -579,11 +578,17 @@ protected void storeRMDelegationTokenAndSequenceNumberState(
   }
 
   @Override
-  protected void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier tokenId, Long renewDate,
-      int latestSequenceNumber) throws IOException {
-    storeRMDelegationTokenAndSequenceNumberState(tokenId, renewDate,
-        latestSequenceNumber);
+  protected void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier tokenId, Long renewDate)
+      throws IOException {
+    storeOrUpdateRMDT(tokenId, renewDate, false);
+  }
+
+  @Override
+  protected void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier tokenId, Long renewDate)
+      throws IOException {
+    storeOrUpdateRMDT(tokenId, renewDate, true);
   }
 
   @Override
@@ -641,11 +646,7 @@ public void storeOrUpdateAMRMTokenSecretManagerState(
     AMRMTokenSecretManagerState data =
         AMRMTokenSecretManagerState.newInstance(state);
     byte[] stateData = data.getProto().toByteArray();
-    try {
-      db.put(bytes(AMRMTOKEN_SECRET_MANAGER_ROOT), stateData);
-    } catch (DBException e) {
-      notifyStoreOperationFailed(e);
-    }
+    db.put(bytes(AMRMTOKEN_SECRET_MANAGER_ROOT), stateData);
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
index 917fdc13a3816..8cd776e6044fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
@@ -91,15 +91,16 @@ protected synchronized void closeInternal() throws Exception {
   }
 
   @Override
-  public void storeApplicationStateInternal(
+  public synchronized void storeApplicationStateInternal(
       ApplicationId appId, ApplicationStateData appState)
       throws Exception {
     state.appState.put(appId, appState);
   }
 
   @Override
-  public void updateApplicationStateInternal(ApplicationId appId,
-      ApplicationStateData appState) throws Exception {
+  public synchronized void updateApplicationStateInternal(
+      ApplicationId appId, ApplicationStateData appState) 
+      throws Exception {
     LOG.info("Updating final state " + appState.getState() + " for app: "
         + appId);
     if (state.appState.get(appId) != null) {
@@ -149,23 +150,30 @@ public synchronized void removeApplicationStateInternal(
     }
   }
 
-  @Override
-  public synchronized void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  private void storeOrUpdateRMDT(RMDelegationTokenIdentifier rmDTIdentifier,
+      Long renewDate, boolean isUpdate) throws Exception {
     Map<RMDelegationTokenIdentifier, Long> rmDTState =
         state.rmSecretManagerState.getTokenState();
     if (rmDTState.containsKey(rmDTIdentifier)) {
       IOException e = new IOException("RMDelegationToken: " + rmDTIdentifier
-              + "is already stored.");
+          + "is already stored.");
       LOG.info("Error storing info for RMDelegationToken: " + rmDTIdentifier, e);
       throw e;
     }
     rmDTState.put(rmDTIdentifier, renewDate);
-    state.rmSecretManagerState.dtSequenceNumber = latestSequenceNumber;
+    if(!isUpdate) {
+      state.rmSecretManagerState.dtSequenceNumber = 
+          rmDTIdentifier.getSequenceNumber();
+    }
     LOG.info("Store RMDT with sequence number "
-        + rmDTIdentifier.getSequenceNumber()
-        + ". And the latest sequence number is " + latestSequenceNumber);
+             + rmDTIdentifier.getSequenceNumber());
+  }
+
+  @Override
+  public synchronized void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
+    storeOrUpdateRMDT(rmDTIdentifier, renewDate, false);
   }
 
   @Override
@@ -179,12 +187,11 @@ public synchronized void removeRMDelegationTokenState(
   }
 
   @Override
-  protected void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  protected synchronized void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
     removeRMDelegationTokenState(rmDTIdentifier);
-    storeRMDelegationTokenAndSequenceNumberState(
-        rmDTIdentifier, renewDate, latestSequenceNumber);
+    storeOrUpdateRMDT(rmDTIdentifier, renewDate, true);
     LOG.info("Update RMDT with sequence number "
         + rmDTIdentifier.getSequenceNumber());
   }
@@ -231,7 +238,7 @@ protected Version getCurrentVersion() {
   }
 
   @Override
-  public void storeOrUpdateAMRMTokenSecretManagerState(
+  public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
       AMRMTokenSecretManagerState amrmTokenSecretManagerState,
       boolean isUpdate) {
     if (amrmTokenSecretManagerState != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
index f80c497e80af5..d2c1e9d71f351 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
@@ -77,9 +77,9 @@ protected void removeApplicationStateInternal(ApplicationStateData appState)
   }
 
   @Override
-  public void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  public void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
     // Do nothing
   }
 
@@ -90,9 +90,9 @@ public void removeRMDelegationTokenState(RMDelegationTokenIdentifier rmDTIdentif
   }
 
   @Override
-  protected void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  protected void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
     // Do nothing
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 00e1dfc387a37..bccde53ddff39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -24,6 +24,9 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import javax.crypto.SecretKey;
 
@@ -86,6 +89,8 @@ public abstract class RMStateStore extends AbstractService {
   protected static final String VERSION_NODE = "RMVersionNode";
   protected static final String EPOCH_NODE = "EpochNode";
   private ResourceManager resourceManager;
+  private final ReadLock readLock;
+  private final WriteLock writeLock;
 
   public static final Log LOG = LogFactory.getLog(RMStateStore.class);
 
@@ -113,6 +118,24 @@ RMStateStoreEventType.REMOVE_APP, new RemoveAppTransition())
           RMStateStoreEventType.STORE_APP_ATTEMPT, new StoreAppAttemptTransition())
       .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
           RMStateStoreEventType.UPDATE_APP_ATTEMPT, new UpdateAppAttemptTransition())
+      .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+          RMStateStoreEventType.STORE_MASTERKEY,
+              new StoreRMDTMasterKeyTransition())
+      .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+          RMStateStoreEventType.REMOVE_MASTERKEY,
+              new RemoveRMDTMasterKeyTransition())
+      .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+          RMStateStoreEventType.STORE_DELEGATION_TOKEN,
+              new StoreRMDTTransition())
+      .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+          RMStateStoreEventType.REMOVE_DELEGATION_TOKEN,
+              new RemoveRMDTTransition())
+      .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+          RMStateStoreEventType.UPDATE_DELEGATION_TOKEN,
+              new UpdateRMDTTransition())
+       .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.ACTIVE,
+           RMStateStoreEventType.UPDATE_AMRM_TOKEN,
+              new StoreOrUpdateAMRMTokenTransition())
       .addTransition(RMStateStoreState.ACTIVE, RMStateStoreState.FENCED,
           RMStateStoreEventType.FENCED)
       .addTransition(RMStateStoreState.FENCED, RMStateStoreState.FENCED,
@@ -122,7 +145,13 @@ RMStateStoreEventType.UPDATE_APP_ATTEMPT, new UpdateAppAttemptTransition())
           RMStateStoreEventType.REMOVE_APP,
           RMStateStoreEventType.STORE_APP_ATTEMPT,
           RMStateStoreEventType.UPDATE_APP_ATTEMPT,
-          RMStateStoreEventType.FENCED));
+          RMStateStoreEventType.FENCED,
+          RMStateStoreEventType.STORE_MASTERKEY,
+          RMStateStoreEventType.REMOVE_MASTERKEY,
+          RMStateStoreEventType.STORE_DELEGATION_TOKEN,
+          RMStateStoreEventType.REMOVE_DELEGATION_TOKEN,
+          RMStateStoreEventType.UPDATE_DELEGATION_TOKEN,
+          RMStateStoreEventType.UPDATE_AMRM_TOKEN));
 
   private final StateMachine<RMStateStoreState,
                              RMStateStoreEventType,
@@ -255,8 +284,141 @@ public void transition(RMStateStore store, RMStateStoreEvent event) {
     };
   }
 
+  private static class StoreRMDTTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreRMDTEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+      RMStateStoreRMDTEvent dtEvent = (RMStateStoreRMDTEvent) event;
+      try {
+        LOG.info("Storing RMDelegationToken and SequenceNumber");
+        store.storeRMDelegationTokenState(
+            dtEvent.getRmDTIdentifier(), dtEvent.getRenewDate());
+      } catch (Exception e) {
+        LOG.error("Error While Storing RMDelegationToken and SequenceNumber ",
+            e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
+  private static class RemoveRMDTTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreRMDTEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+      RMStateStoreRMDTEvent dtEvent = (RMStateStoreRMDTEvent) event;
+      try {
+        LOG.info("Removing RMDelegationToken and SequenceNumber");
+        store.removeRMDelegationTokenState(dtEvent.getRmDTIdentifier());
+      } catch (Exception e) {
+        LOG.error("Error While Removing RMDelegationToken and SequenceNumber ",
+            e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
+  private static class UpdateRMDTTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreRMDTEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+
+      RMStateStoreRMDTEvent dtEvent = (RMStateStoreRMDTEvent) event;
+      try {
+        LOG.info("Updating RMDelegationToken and SequenceNumber");
+        store.updateRMDelegationTokenState(
+            dtEvent.getRmDTIdentifier(), dtEvent.getRenewDate());
+      } catch (Exception e) {
+        LOG.error("Error While Updating RMDelegationToken and SequenceNumber ",
+            e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
+  private static class StoreRMDTMasterKeyTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreRMDTMasterKeyEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+      RMStateStoreRMDTMasterKeyEvent dtEvent =
+          (RMStateStoreRMDTMasterKeyEvent) event;
+      try {
+        LOG.info("Storing RMDTMasterKey.");
+        store.storeRMDTMasterKeyState(dtEvent.getDelegationKey());
+      } catch (Exception e) {
+        LOG.error("Error While Storing RMDTMasterKey.", e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
+  private static class RemoveRMDTMasterKeyTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreRMDTMasterKeyEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+      RMStateStoreRMDTMasterKeyEvent dtEvent =
+          (RMStateStoreRMDTMasterKeyEvent) event;
+      try {
+        LOG.info("Removing RMDTMasterKey.");
+        store.removeRMDTMasterKeyState(dtEvent.getDelegationKey());
+      } catch (Exception e) {
+        LOG.error("Error While Removing RMDTMasterKey.", e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
+  private static class StoreOrUpdateAMRMTokenTransition implements
+      SingleArcTransition<RMStateStore, RMStateStoreEvent> {
+    @Override
+    public void transition(RMStateStore store, RMStateStoreEvent event) {
+      if (!(event instanceof RMStateStoreAMRMTokenEvent)) {
+        // should never happen
+        LOG.error("Illegal event type: " + event.getClass());
+        return;
+      }
+      RMStateStoreAMRMTokenEvent amrmEvent = (RMStateStoreAMRMTokenEvent) event;
+
+      try {
+        LOG.info("Updating AMRMToken");
+        store.storeOrUpdateAMRMTokenSecretManagerState(
+            amrmEvent.getAmrmTokenSecretManagerState(), amrmEvent.isUpdate());
+      } catch (Exception e) {
+        LOG.error("Error storing info for AMRMTokenSecretManager", e);
+        store.notifyStoreOperationFailed(e);
+      }
+    }
+  }
+
   public RMStateStore() {
     super(RMStateStore.class.getName());
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    this.readLock = lock.readLock();
+    this.writeLock = lock.writeLock();
     stateMachine = stateMachineFactory.make(this);
   }
 
@@ -365,7 +527,7 @@ protected void serviceStop() throws Exception {
    * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
    * 2) Any incompatible change of state-store is a major upgrade, and any
    *    compatible change of state-store is a minor upgrade.
-   * 3) If theres's no version, treat it as 1.0.
+   * 3) If theres's no version, treat it as CURRENT_VERSION_INFO.
    * 4) Within a minor upgrade, say 1.1 to 1.2:
    *    overwrite the version info and proceed as normal.
    * 5) Within a major upgrade, say 1.2 to 2.0:
@@ -378,9 +540,9 @@ public void checkVersion() throws Exception {
     if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
       return;
     }
-    // if there is no version info, treat it as 1.0;
+    // if there is no version info, treat it as CURRENT_VERSION_INFO;
     if (loadedVersion == null) {
-      loadedVersion = Version.newInstance(1, 0);
+      loadedVersion = getCurrentVersion();
     }
     if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
       LOG.info("Storing RM state version info " + getCurrentVersion());
@@ -445,9 +607,8 @@ public synchronized void updateApplicationState(
     dispatcher.getEventHandler().handle(new RMStateUpdateAppEvent(appState));
   }
 
-  public synchronized void updateFencedState() {
-    this.stateMachine.doTransition(RMStateStoreEventType.FENCED,
-         new RMStateStoreEvent(RMStateStoreEventType.FENCED));
+  public void updateFencedState() {
+    handleStoreEvent(new RMStateStoreEvent(RMStateStoreEventType.FENCED));
   }
 
   /**
@@ -509,19 +670,10 @@ protected abstract void updateApplicationAttemptStateInternal(
    * RMDTSecretManager call this to store the state of a delegation token
    * and sequence number
    */
-  public synchronized void storeRMDelegationTokenAndSequenceNumber(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) {
-    if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't store RM Delegation Token.");
-      return;
-    }
-    try {
-      storeRMDelegationTokenAndSequenceNumberState(rmDTIdentifier, renewDate,
-          latestSequenceNumber);
-    } catch (Exception e) {
-      notifyStoreOperationFailed(e);
-    }
+  public void storeRMDelegationToken(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
+    handleStoreEvent(new RMStateStoreRMDTEvent(rmDTIdentifier, renewDate,
+        RMStateStoreEventType.STORE_DELEGATION_TOKEN));
   }
 
   /**
@@ -529,24 +681,17 @@ public synchronized void storeRMDelegationTokenAndSequenceNumber(
    * Derived classes must implement this method to store the state of
    * RMDelegationToken and sequence number
    */
-  protected abstract void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception;
+  protected abstract void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception;
 
   /**
    * RMDTSecretManager call this to remove the state of a delegation token
    */
-  public synchronized void removeRMDelegationToken(
-      RMDelegationTokenIdentifier rmDTIdentifier, int sequenceNumber) {
-    if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't remove RM Delegation Token.");
-      return;
-    }
-    try {
-      removeRMDelegationTokenState(rmDTIdentifier);
-    } catch (Exception e) {
-      notifyStoreOperationFailed(e);
-    }
+  public void removeRMDelegationToken(
+      RMDelegationTokenIdentifier rmDTIdentifier) {
+    handleStoreEvent(new RMStateStoreRMDTEvent(rmDTIdentifier, null,
+        RMStateStoreEventType.REMOVE_DELEGATION_TOKEN));
   }
 
   /**
@@ -560,19 +705,10 @@ protected abstract void removeRMDelegationTokenState(
    * RMDTSecretManager call this to update the state of a delegation token
    * and sequence number
    */
-  public synchronized void updateRMDelegationTokenAndSequenceNumber(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) {
-    if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't update RM Delegation Token.");
-      return;
-    }
-    try {
-      updateRMDelegationTokenAndSequenceNumberInternal(rmDTIdentifier, renewDate,
-          latestSequenceNumber);
-    } catch (Exception e) {
-      notifyStoreOperationFailed(e);
-    }
+  public void updateRMDelegationToken(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
+    handleStoreEvent(new RMStateStoreRMDTEvent(rmDTIdentifier, renewDate,
+        RMStateStoreEventType.UPDATE_DELEGATION_TOKEN));
   }
 
   /**
@@ -580,24 +716,16 @@ public synchronized void updateRMDelegationTokenAndSequenceNumber(
    * Derived classes must implement this method to update the state of
    * RMDelegationToken and sequence number
    */
-  protected abstract void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception;
+  protected abstract void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception;
 
   /**
    * RMDTSecretManager call this to store the state of a master key
    */
-  public synchronized void storeRMDTMasterKey(DelegationKey delegationKey) {
-    if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't store RM Delegation " +
-               "Token Master key.");
-      return;
-    }
-    try {
-      storeRMDTMasterKeyState(delegationKey);
-    } catch (Exception e) {
-      notifyStoreOperationFailed(e);
-    }
+  public void storeRMDTMasterKey(DelegationKey delegationKey) {
+    handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
+        RMStateStoreEventType.STORE_MASTERKEY));
   }
 
   /**
@@ -611,17 +739,9 @@ protected abstract void storeRMDTMasterKeyState(DelegationKey delegationKey)
   /**
    * RMDTSecretManager call this to remove the state of a master key
    */
-  public synchronized void removeRMDTMasterKey(DelegationKey delegationKey) {
-   if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't remove RM Delegation " +
-               "Token Master key.");
-      return;
-    }
-    try {
-      removeRMDTMasterKeyState(delegationKey);
-    } catch (Exception e) {
-      notifyStoreOperationFailed(e);
-    }
+  public void removeRMDTMasterKey(DelegationKey delegationKey) {
+    handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
+        RMStateStoreEventType.REMOVE_MASTERKEY));
   }
 
   /**
@@ -636,9 +756,19 @@ protected abstract void removeRMDTMasterKeyState(DelegationKey delegationKey)
    * Blocking API Derived classes must implement this method to store or update
    * the state of AMRMToken Master Key
    */
-  public abstract void storeOrUpdateAMRMTokenSecretManagerState(
-      AMRMTokenSecretManagerState amrmTokenSecretManagerState,
-      boolean isUpdate);
+  protected abstract void storeOrUpdateAMRMTokenSecretManagerState(
+      AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate)
+      throws Exception;
+
+  /**
+   * Store or Update state of AMRMToken Master Key
+   */
+  public void storeOrUpdateAMRMTokenSecretManager(
+      AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate) {
+    handleStoreEvent(new RMStateStoreAMRMTokenEvent(
+        amrmTokenSecretManagerState, isUpdate,
+        RMStateStoreEventType.UPDATE_AMRM_TOKEN));
+  }
 
   /**
    * Non-blocking API
@@ -689,16 +819,32 @@ public Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
   }
   
   @VisibleForTesting
-  synchronized boolean isFencedState() {
-    return (RMStateStoreState.FENCED == this.stateMachine.getCurrentState());
+  protected boolean isFencedState() {
+    return (RMStateStoreState.FENCED == getRMStateStoreState());
   }
 
   // Dispatcher related code
   protected void handleStoreEvent(RMStateStoreEvent event) {
+    this.writeLock.lock();
     try {
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Processing event of type " + event.getType());
+      }
+
+      final RMStateStoreState oldState = getRMStateStoreState();
+
       this.stateMachine.doTransition(event.getType(), event);
+
+      if (oldState != getRMStateStoreState()) {
+        LOG.info("RMStateStore state change from " + oldState + " to "
+            + getRMStateStoreState());
+      }
+
     } catch (InvalidStateTransitonException e) {
       LOG.error("Can't handle this event at current state", e);
+    } finally {
+      this.writeLock.unlock();
     }
   }
 
@@ -772,4 +918,13 @@ public void run() {
       resourceManager.handleTransitionToStandBy();
     }
   }
+
+  public RMStateStoreState getRMStateStoreState() {
+    this.readLock.lock();
+    try {
+      return this.stateMachine.getCurrentState();
+    } finally {
+      this.readLock.unlock();
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAMRMTokenEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAMRMTokenEvent.java
new file mode 100644
index 0000000000000..befec06316363
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAMRMTokenEvent.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
+
+public class RMStateStoreAMRMTokenEvent extends RMStateStoreEvent {
+  private AMRMTokenSecretManagerState amrmTokenSecretManagerState;
+  private boolean isUpdate;
+
+  public RMStateStoreAMRMTokenEvent(RMStateStoreEventType type) {
+    super(type);
+  }
+
+  public RMStateStoreAMRMTokenEvent(
+      AMRMTokenSecretManagerState amrmTokenSecretManagerState,
+      boolean isUpdate, RMStateStoreEventType type) {
+    this(type);
+    this.amrmTokenSecretManagerState = amrmTokenSecretManagerState;
+    this.isUpdate = isUpdate;
+  }
+
+  public AMRMTokenSecretManagerState getAmrmTokenSecretManagerState() {
+    return amrmTokenSecretManagerState;
+  }
+
+  public boolean isUpdate() {
+    return isUpdate;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
index 9301bf9a12e92..beba5eb8f5aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
@@ -24,5 +24,13 @@ public enum RMStateStoreEventType {
   UPDATE_APP,
   UPDATE_APP_ATTEMPT,
   REMOVE_APP,
-  FENCED
+  FENCED,
+
+  // Below events should be called synchronously
+  STORE_MASTERKEY,
+  REMOVE_MASTERKEY,
+  STORE_DELEGATION_TOKEN,
+  REMOVE_DELEGATION_TOKEN,
+  UPDATE_DELEGATION_TOKEN,
+  UPDATE_AMRM_TOKEN
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTEvent.java
new file mode 100644
index 0000000000000..a3519ff6f8fe1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTEvent.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+
+public class RMStateStoreRMDTEvent extends RMStateStoreEvent {
+  private RMDelegationTokenIdentifier rmDTIdentifier;
+  private Long renewDate;
+
+  public RMStateStoreRMDTEvent(RMStateStoreEventType type) {
+    super(type);
+  }
+
+  public RMStateStoreRMDTEvent(RMDelegationTokenIdentifier rmDTIdentifier,
+      Long renewDate, RMStateStoreEventType type) {
+    this(type);
+    this.rmDTIdentifier = rmDTIdentifier;
+    this.renewDate = renewDate;
+  }
+
+  public RMDelegationTokenIdentifier getRmDTIdentifier() {
+    return rmDTIdentifier;
+  }
+
+  public Long getRenewDate() {
+    return renewDate;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTMasterKeyEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTMasterKeyEvent.java
new file mode 100644
index 0000000000000..c10b197ba7d31
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreRMDTMasterKeyEvent.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+
+public class RMStateStoreRMDTMasterKeyEvent extends RMStateStoreEvent {
+  private DelegationKey delegationKey;
+
+  public RMStateStoreRMDTMasterKeyEvent(RMStateStoreEventType type) {
+    super(type);
+  }
+
+  public RMStateStoreRMDTMasterKeyEvent(DelegationKey delegationKey,
+      RMStateStoreEventType type) {
+    this(type);
+    this.delegationKey = delegationKey;
+  }
+
+  public DelegationKey getDelegationKey() {
+    return delegationKey;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index a718fb5713cba..591a5511785d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -23,12 +23,14 @@
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -274,7 +276,7 @@ public synchronized void startInternal() throws Exception {
     createConnection();
 
     // ensure root dirs exist
-    createRootDir(znodeWorkingPath);
+    createRootDirRecursively(znodeWorkingPath);
     createRootDir(zkRootNodePath);
     if (HAUtil.isHAEnabled(getConfig())){
       fence();
@@ -697,12 +699,11 @@ public synchronized void removeApplicationStateInternal(
   }
 
   @Override
-  protected synchronized void storeRMDelegationTokenAndSequenceNumberState(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  protected synchronized void storeRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
     ArrayList<Op> opList = new ArrayList<Op>();
-    addStoreOrUpdateOps(
-        opList, rmDTIdentifier, renewDate, latestSequenceNumber, false);
+    addStoreOrUpdateOps(opList, rmDTIdentifier, renewDate, false);
     doMultiWithRetries(opList);
   }
 
@@ -726,29 +727,27 @@ protected synchronized void removeRMDelegationTokenState(
   }
 
   @Override
-  protected void updateRMDelegationTokenAndSequenceNumberInternal(
-      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber) throws Exception {
+  protected synchronized void updateRMDelegationTokenState(
+      RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
+      throws Exception {
     ArrayList<Op> opList = new ArrayList<Op>();
     String nodeRemovePath =
         getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
             + rmDTIdentifier.getSequenceNumber());
     if (existsWithRetries(nodeRemovePath, true) == null) {
       // in case znode doesn't exist
-      addStoreOrUpdateOps(
-          opList, rmDTIdentifier, renewDate, latestSequenceNumber, false);
+      addStoreOrUpdateOps(opList, rmDTIdentifier, renewDate, false);
       LOG.debug("Attempted to update a non-existing znode " + nodeRemovePath);
     } else {
       // in case znode exists
-      addStoreOrUpdateOps(
-          opList, rmDTIdentifier, renewDate, latestSequenceNumber, true);
+      addStoreOrUpdateOps(opList, rmDTIdentifier, renewDate, true);
     }
     doMultiWithRetries(opList);
   }
 
   private void addStoreOrUpdateOps(ArrayList<Op> opList,
       RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
-      int latestSequenceNumber, boolean isUpdate) throws Exception {
+      boolean isUpdate) throws Exception {
     // store RM delegation token
     String nodeCreatePath =
         getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
@@ -768,16 +767,15 @@ private void addStoreOrUpdateOps(ArrayList<Op> opList,
       } else {
         opList.add(Op.create(nodeCreatePath, identifierData.toByteArray(), zkAcl,
             CreateMode.PERSISTENT));
+        // Update Sequence number only while storing DT
+        seqOut.writeInt(rmDTIdentifier.getSequenceNumber());
+        if (LOG.isDebugEnabled()) {
+          LOG.debug((isUpdate ? "Storing " : "Updating ") +
+                    dtSequenceNumberPath + ". SequenceNumber: "
+                    + rmDTIdentifier.getSequenceNumber());
+        }
+        opList.add(Op.setData(dtSequenceNumberPath, seqOs.toByteArray(), -1));
       }
-
-
-     seqOut.writeInt(latestSequenceNumber);
-     if (LOG.isDebugEnabled()) {
-        LOG.debug((isUpdate ? "Storing " : "Updating ") + dtSequenceNumberPath +
-            ". SequenceNumber: " + latestSequenceNumber);
-      }
-
-     opList.add(Op.setData(dtSequenceNumberPath, seqOs.toByteArray(), -1));
     } finally {
       seqOs.close();
     }
@@ -1056,6 +1054,8 @@ private boolean shouldRetry(Code code) {
       switch (code) {
         case CONNECTIONLOSS:
         case OPERATIONTIMEOUT:
+        case SESSIONEXPIRED:
+        case SESSIONMOVED:
           return true;
         default:
           break;
@@ -1084,6 +1084,7 @@ T runWithRetries() throws Exception {
           if (shouldRetry(ke.code()) && ++retry < numRetries) {
             LOG.info("Retrying operation on ZK. Retry no. " + retry);
             Thread.sleep(zkRetryInterval);
+            createConnection();
             continue;
           }
           LOG.info("Maxed out ZK retries. Giving up!");
@@ -1105,7 +1106,7 @@ private synchronized void createConnection()
         }
         if (useDefaultFencingScheme) {
           zkClient.addAuthInfo(zkRootNodeAuthScheme,
-              (zkRootNodeUsername + ":" + zkRootNodePassword).getBytes());
+              (zkRootNodeUsername + ":" + zkRootNodePassword).getBytes(Charset.forName("UTF-8")));
         }
       } catch (IOException ioe) {
         // Retry in case of network failures
@@ -1135,22 +1136,26 @@ protected synchronized ZooKeeper getNewZooKeeper()
 
   @Override
   public synchronized void storeOrUpdateAMRMTokenSecretManagerState(
-      AMRMTokenSecretManagerState amrmTokenSecretManagerState,
-      boolean isUpdate) {
-    if(isFencedState()) {
-      LOG.info("State store is in Fenced state. Can't store/update " +
-               "AMRMToken Secret Manager state.");
-      return;
-    }	
+      AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate)
+      throws Exception {
     AMRMTokenSecretManagerState data =
         AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
     byte[] stateData = data.getProto().toByteArray();
-    try {
-      setDataWithRetries(amrmTokenSecretManagerRoot, stateData, -1);
-    } catch (Exception ex) {
-      LOG.info("Error storing info for AMRMTokenSecretManager", ex);
-      notifyStoreOperationFailed(ex);
-    }
+    setDataWithRetries(amrmTokenSecretManagerRoot, stateData, -1);
   }
 
+  /**
+   * Utility function to ensure that the configured base znode exists.
+   * This recursively creates the znode as well as all of its parents.
+   */
+  private void createRootDirRecursively(String path) throws Exception {
+    String pathParts[] = path.split("/");
+    Preconditions.checkArgument(pathParts.length >= 1 && pathParts[0].isEmpty(),
+        "Invalid path: %s", path);
+    StringBuilder sb = new StringBuilder();
+    for (int i = 1; i < pathParts.length; i++) {
+      sb.append("/").append(pathParts[i]);
+      createRootDir(sb.toString());
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 1539a6eb70f5d..8a15ac6a07b80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.UTCClock;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -203,6 +204,8 @@ private String getDefaultPlanFollower() {
     // currently only capacity scheduler is supported
     if (scheduler instanceof CapacityScheduler) {
       return CapacitySchedulerPlanFollower.class.getName();
+    } else if (scheduler instanceof FairScheduler) {
+      return FairSchedulerPlanFollower.class.getName();
     }
     return null;
   }
@@ -322,9 +325,10 @@ public Map<String, Plan> getAllPlans() {
    * @param scheduler the scheduler for which the reservation system is required
    */
   public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
-    // currently only capacity scheduler is supported
     if (scheduler instanceof CapacityScheduler) {
       return CapacityReservationSystem.class.getName();
+    } else if (scheduler instanceof FairScheduler) {
+      return FairReservationSystem.class.getName();
     }
     return null;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
new file mode 100644
index 0000000000000..ea7f27d2c035d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractSchedulerPlanFollower implements PlanFollower {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(CapacitySchedulerPlanFollower.class);
+
+  protected Collection<Plan> plans = new ArrayList<Plan>();
+  protected YarnScheduler scheduler;
+  protected Clock clock;
+
+  @Override
+  public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
+    this.clock = clock;
+    this.scheduler = sched;
+    this.plans.addAll(plans);
+  }
+
+  @Override
+  public synchronized void run() {
+    for (Plan plan : plans) {
+      synchronizePlan(plan);
+    }
+  }
+
+  @Override
+  public synchronized void setPlans(Collection<Plan> plans) {
+    this.plans.clear();
+    this.plans.addAll(plans);
+  }
+
+  @Override
+  public synchronized void synchronizePlan(Plan plan) {
+     String planQueueName = plan.getQueueName();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Running plan follower edit policy for plan: " + planQueueName);
+    }
+    // align with plan step
+    long step = plan.getStep();
+    long now = clock.getTime();
+    if (now % step != 0) {
+      now += step - (now % step);
+    }
+    Queue planQueue = getPlanQueue(planQueueName);
+    if (planQueue == null) return;
+
+    // first we publish to the plan the current availability of resources
+    Resource clusterResources = scheduler.getClusterResource();
+    Resource planResources = getPlanResources(plan, planQueue,
+        clusterResources);
+
+    Set<ReservationAllocation> currentReservations =
+        plan.getReservationsAtTime(now);
+    Set<String> curReservationNames = new HashSet<String>();
+    Resource reservedResources = Resource.newInstance(0, 0);
+    int numRes = getReservedResources(now, currentReservations,
+        curReservationNames, reservedResources);
+
+    // create the default reservation queue if it doesnt exist
+    String defReservationId = getReservationIdFromQueueName(planQueueName) +
+        ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+    String defReservationQueue = getReservationQueueName(planQueueName,
+        defReservationId);
+    createDefaultReservationQueue(planQueueName, planQueue,
+        defReservationId);
+    curReservationNames.add(defReservationId);
+
+    // if the resources dedicated to this plan has shrunk invoke replanner
+    if (arePlanResourcesLessThanReservations(clusterResources, planResources,
+        reservedResources)) {
+      try {
+        plan.getReplanner().plan(plan, null);
+      } catch (PlanningException e) {
+        LOG.warn("Exception while trying to replan: {}", planQueueName, e);
+      }
+    }
+    // identify the reservations that have expired and new reservations that
+    // have to be activated
+    List<? extends Queue> resQueues = getChildReservationQueues(planQueue);
+    Set<String> expired = new HashSet<String>();
+    for (Queue resQueue : resQueues) {
+      String resQueueName = resQueue.getQueueName();
+      String reservationId = getReservationIdFromQueueName(resQueueName);
+      if (curReservationNames.contains(reservationId)) {
+        // it is already existing reservation, so needed not create new
+        // reservation queue
+        curReservationNames.remove(reservationId);
+      } else {
+        // the reservation has termination, mark for cleanup
+        expired.add(reservationId);
+      }
+    }
+    // garbage collect expired reservations
+    cleanupExpiredQueues(planQueueName, plan.getMoveOnExpiry(), expired,
+        defReservationQueue);
+
+    // Add new reservations and update existing ones
+    float totalAssignedCapacity = 0f;
+    if (currentReservations != null) {
+      // first release all excess capacity in default queue
+      try {
+        setQueueEntitlement(planQueueName, defReservationQueue, 0f, 1.0f);
+      } catch (YarnException e) {
+        LOG.warn(
+            "Exception while trying to release default queue capacity for plan: {}",
+            planQueueName, e);
+      }
+      // sort allocations from the one giving up the most resources, to the
+      // one asking for the most
+      // avoid order-of-operation errors that temporarily violate 100%
+      // capacity bound
+      List<ReservationAllocation> sortedAllocations =
+          sortByDelta(
+              new ArrayList<ReservationAllocation>(currentReservations), now,
+              plan);
+      for (ReservationAllocation res : sortedAllocations) {
+        String currResId = res.getReservationId().toString();
+        if (curReservationNames.contains(currResId)) {
+          addReservationQueue(planQueueName, planQueue, currResId);
+        }
+        Resource capToAssign = res.getResourcesAtTime(now);
+        float targetCapacity = 0f;
+        if (planResources.getMemory() > 0
+            && planResources.getVirtualCores() > 0) {
+          targetCapacity =
+              calculateReservationToPlanRatio(clusterResources,
+                  planResources,
+                  capToAssign);
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(
+              "Assigning capacity of {} to queue {} with target capacity {}",
+              capToAssign, currResId, targetCapacity);
+        }
+        // set maxCapacity to 100% unless the job requires gang, in which
+        // case we stick to capacity (as running early/before is likely a
+        // waste of resources)
+        float maxCapacity = 1.0f;
+        if (res.containsGangs()) {
+          maxCapacity = targetCapacity;
+        }
+        try {
+          setQueueEntitlement(planQueueName, currResId, targetCapacity, maxCapacity);
+        } catch (YarnException e) {
+          LOG.warn("Exception while trying to size reservation for plan: {}",
+              currResId, planQueueName, e);
+        }
+        totalAssignedCapacity += targetCapacity;
+      }
+    }
+    // compute the default queue capacity
+    float defQCap = 1.0f - totalAssignedCapacity;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} "
+          + "currReservation: {} default-queue capacity: {}", planResources,
+          numRes, defQCap);
+    }
+    // set the default queue to eat-up all remaining capacity
+    try {
+      setQueueEntitlement(planQueueName, defReservationQueue, defQCap, 1.0f);
+    } catch (YarnException e) {
+      LOG.warn(
+          "Exception while trying to reclaim default queue capacity for plan: {}",
+          planQueueName, e);
+    }
+    // garbage collect finished reservations from plan
+    try {
+      plan.archiveCompletedReservations(now);
+    } catch (PlanningException e) {
+      LOG.error("Exception in archiving completed reservations: ", e);
+    }
+    LOG.info("Finished iteration of plan follower edit policy for plan: "
+        + planQueueName);
+
+    // Extension: update plan with app states,
+    // useful to support smart replanning
+  }
+
+  protected String getReservationIdFromQueueName(String resQueueName) {
+    return resQueueName;
+  }
+
+  protected void setQueueEntitlement(String planQueueName, String currResId,
+      float targetCapacity,
+      float maxCapacity) throws YarnException {
+    String reservationQueueName = getReservationQueueName(planQueueName,
+        currResId);
+    scheduler.setEntitlement(reservationQueueName, new QueueEntitlement(
+        targetCapacity, maxCapacity));
+  }
+
+  // Schedulers have different ways of naming queues. See YARN-2773
+  protected String getReservationQueueName(String planQueueName,
+      String reservationId) {
+    return reservationId;
+  }
+
+  /**
+   * First sets entitlement of queues to zero to prevent new app submission.
+   * Then move all apps in the set of queues to the parent plan queue's default
+   * reservation queue if move is enabled. Finally cleanups the queue by killing
+   * any apps (if move is disabled or move failed) and removing the queue
+   */
+  protected void cleanupExpiredQueues(String planQueueName,
+      boolean shouldMove, Set<String> toRemove, String defReservationQueue) {
+    for (String expiredReservationId : toRemove) {
+      try {
+        // reduce entitlement to 0
+        String expiredReservation = getReservationQueueName(planQueueName,
+            expiredReservationId);
+        setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f);
+        if (shouldMove) {
+          moveAppsInQueueSync(expiredReservation, defReservationQueue);
+        }
+        if (scheduler.getAppsInQueue(expiredReservation).size() > 0) {
+          scheduler.killAllAppsInQueue(expiredReservation);
+          LOG.info("Killing applications in queue: {}", expiredReservation);
+        } else {
+          scheduler.removeQueue(expiredReservation);
+          LOG.info("Queue: " + expiredReservation + " removed");
+        }
+      } catch (YarnException e) {
+        LOG.warn("Exception while trying to expire reservation: {}",
+            expiredReservationId, e);
+      }
+    }
+  }
+
+  /**
+   * Move all apps in the set of queues to the parent plan queue's default
+   * reservation queue in a synchronous fashion
+   */
+  private void moveAppsInQueueSync(String expiredReservation,
+                                   String defReservationQueue) {
+    List<ApplicationAttemptId> activeApps =
+        scheduler.getAppsInQueue(expiredReservation);
+    if (activeApps.isEmpty()) {
+      return;
+    }
+    for (ApplicationAttemptId app : activeApps) {
+      // fallback to parent's default queue
+      try {
+        scheduler.moveApplication(app.getApplicationId(), defReservationQueue);
+      } catch (YarnException e) {
+        LOG.warn(
+            "Encountered unexpected error during migration of application: {}" +
+                " from reservation: {}",
+            app, expiredReservation, e);
+      }
+    }
+  }
+
+  protected int getReservedResources(long now, Set<ReservationAllocation>
+      currentReservations, Set<String> curReservationNames,
+                                     Resource reservedResources) {
+    int numRes = 0;
+    if (currentReservations != null) {
+      numRes = currentReservations.size();
+      for (ReservationAllocation reservation : currentReservations) {
+        curReservationNames.add(reservation.getReservationId().toString());
+        Resources.addTo(reservedResources, reservation.getResourcesAtTime(now));
+      }
+    }
+    return numRes;
+  }
+
+  /**
+   * Sort in the order from the least new amount of resources asked (likely
+   * negative) to the highest. This prevents "order-of-operation" errors related
+   * to exceeding 100% capacity temporarily.
+   */
+  protected List<ReservationAllocation> sortByDelta(
+      List<ReservationAllocation> currentReservations, long now, Plan plan) {
+    Collections.sort(currentReservations, new ReservationAllocationComparator(
+        now, this, plan));
+    return currentReservations;
+  }
+
+  /**
+   * Get queue associated with reservable queue named
+   * @param planQueueName Name of the reservable queue
+   * @return queue associated with the reservable queue
+   */
+  protected abstract Queue getPlanQueue(String planQueueName);
+
+  /**
+   * Calculates ratio of reservationResources to planResources
+   */
+  protected abstract float calculateReservationToPlanRatio(
+      Resource clusterResources, Resource planResources,
+      Resource reservationResources);
+
+  /**
+   * Check if plan resources are less than expected reservation resources
+   */
+  protected abstract boolean arePlanResourcesLessThanReservations(
+      Resource clusterResources, Resource planResources,
+      Resource reservedResources);
+
+  /**
+   * Get a list of reservation queues for this planQueue
+   */
+  protected abstract List<? extends Queue> getChildReservationQueues(
+      Queue planQueue);
+
+  /**
+   * Add a new reservation queue for reservation currResId for this planQueue
+   */
+  protected abstract void addReservationQueue(
+      String planQueueName, Queue queue, String currResId);
+
+  /**
+   * Creates the default reservation queue for use when no reservation is
+   * used for applications submitted to this planQueue
+   */
+  protected abstract void createDefaultReservationQueue(
+      String planQueueName, Queue queue, String defReservationQueue);
+
+  /**
+   * Get plan resources for this planQueue
+   */
+  protected abstract Resource getPlanResources(
+      Plan plan, Queue queue, Resource clusterResources);
+
+  /**
+   * Get reservation queue resources if it exists otherwise return null
+   */
+  protected abstract Resource getReservationQueueResourceIfExists(Plan plan,
+      ReservationId reservationId);
+
+  private static class ReservationAllocationComparator implements
+      Comparator<ReservationAllocation> {
+    AbstractSchedulerPlanFollower planFollower;
+    long now;
+    Plan plan;
+
+    ReservationAllocationComparator(long now,
+        AbstractSchedulerPlanFollower planFollower, Plan plan) {
+      this.now = now;
+      this.planFollower = planFollower;
+      this.plan = plan;
+    }
+
+    private Resource getUnallocatedReservedResources(
+        ReservationAllocation reservation) {
+      Resource resResource;
+      Resource reservationResource = planFollower
+          .getReservationQueueResourceIfExists
+              (plan, reservation.getReservationId());
+      if (reservationResource != null) {
+        resResource =
+            Resources.subtract(
+                reservation.getResourcesAtTime(now),
+                reservationResource);
+      } else {
+        resResource = reservation.getResourcesAtTime(now);
+      }
+      return resResource;
+    }
+
+    @Override
+    public int compare(ReservationAllocation lhs, ReservationAllocation rhs) {
+      // compute delta between current and previous reservation, and compare
+      // based on that
+      Resource lhsRes = getUnallocatedReservedResources(lhs);
+      Resource rhsRes = getUnallocatedReservedResources(rhs);
+      return lhsRes.compareTo(rhsRes);
+    }
+  }
+}
+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
index 126560a388755..61772c9a37dd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
@@ -19,26 +19,19 @@
 package org.apache.hadoop.yarn.server.resourcemanager.reservation;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ReservationQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
@@ -58,319 +51,119 @@
  * differences among existing queues). This makes it resilient to frequency of
  * synchronization, and RM restart issues (no "catch up" is necessary).
  */
-public class CapacitySchedulerPlanFollower implements PlanFollower {
+public class CapacitySchedulerPlanFollower extends AbstractSchedulerPlanFollower {
 
   private static final Logger LOG = LoggerFactory
       .getLogger(CapacitySchedulerPlanFollower.class);
 
-  private Collection<Plan> plans = new ArrayList<Plan>();
-
-  private Clock clock;
-  private CapacityScheduler scheduler;
+  private CapacityScheduler cs;
 
   @Override
   public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
+    super.init(clock, sched, plans);
     LOG.info("Initializing Plan Follower Policy:"
         + this.getClass().getCanonicalName());
     if (!(sched instanceof CapacityScheduler)) {
       throw new YarnRuntimeException(
           "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
     }
-    this.clock = clock;
-    this.scheduler = (CapacityScheduler) sched;
-    this.plans.addAll(plans);
+    this.cs = (CapacityScheduler) sched;
   }
 
   @Override
-  public synchronized void run() {
-    for (Plan plan : plans) {
-      synchronizePlan(plan);
+  protected Queue getPlanQueue(String planQueueName) {
+    CSQueue queue = cs.getQueue(planQueueName);
+    if (!(queue instanceof PlanQueue)) {
+      LOG.error("The Plan is not an PlanQueue!");
+      return null;
     }
+    return queue;
   }
 
   @Override
-  public synchronized void synchronizePlan(Plan plan) {
-    String planQueueName = plan.getQueueName();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Running plan follower edit policy for plan: " + planQueueName);
-    }
-    // align with plan step
-    long step = plan.getStep();
-    long now = clock.getTime();
-    if (now % step != 0) {
-      now += step - (now % step);
-    }
-    CSQueue queue = scheduler.getQueue(planQueueName);
-    if (!(queue instanceof PlanQueue)) {
-      LOG.error("The Plan is not an PlanQueue!");
-      return;
-    }
-    PlanQueue planQueue = (PlanQueue) queue;
-    // first we publish to the plan the current availability of resources
-    Resource clusterResources = scheduler.getClusterResource();
-    float planAbsCap = planQueue.getAbsoluteCapacity();
-    Resource planResources = Resources.multiply(clusterResources, planAbsCap);
-    plan.setTotalCapacity(planResources);
+  protected float calculateReservationToPlanRatio(
+      Resource clusterResources, Resource planResources,
+      Resource reservationResources) {
+    return Resources.divide(cs.getResourceCalculator(),
+        clusterResources, reservationResources, planResources);
+  }
 
-    Set<ReservationAllocation> currentReservations =
-        plan.getReservationsAtTime(now);
-    Set<String> curReservationNames = new HashSet<String>();
-    Resource reservedResources = Resource.newInstance(0, 0);
-    int numRes = 0;
-    if (currentReservations != null) {
-      numRes = currentReservations.size();
-      for (ReservationAllocation reservation : currentReservations) {
-        curReservationNames.add(reservation.getReservationId().toString());
-        Resources.addTo(reservedResources, reservation.getResourcesAtTime(now));
-      }
+  @Override
+  protected boolean arePlanResourcesLessThanReservations(
+      Resource clusterResources, Resource planResources,
+      Resource reservedResources) {
+    return Resources.greaterThan(cs.getResourceCalculator(),
+        clusterResources, reservedResources, planResources);
+  }
+
+  @Override
+  protected List<? extends Queue> getChildReservationQueues(Queue queue) {
+    PlanQueue planQueue = (PlanQueue)queue;
+    List<CSQueue> childQueues = planQueue.getChildQueues();
+    return childQueues;
+  }
+
+  @Override
+  protected void addReservationQueue(
+      String planQueueName, Queue queue, String currResId) {
+    PlanQueue planQueue = (PlanQueue)queue;
+    try {
+      ReservationQueue resQueue =
+          new ReservationQueue(cs, currResId, planQueue);
+      cs.addQueue(resQueue);
+    } catch (SchedulerDynamicEditException e) {
+      LOG.warn(
+          "Exception while trying to activate reservation: {} for plan: {}",
+          currResId, planQueueName, e);
+    } catch (IOException e) {
+      LOG.warn(
+          "Exception while trying to activate reservation: {} for plan: {}",
+          currResId, planQueueName, e);
     }
-    // create the default reservation queue if it doesnt exist
-    String defReservationQueue = planQueueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
-    if (scheduler.getQueue(defReservationQueue) == null) {
+  }
+
+  @Override
+  protected void createDefaultReservationQueue(
+      String planQueueName, Queue queue, String defReservationId) {
+    PlanQueue planQueue = (PlanQueue)queue;
+    if (cs.getQueue(defReservationId) == null) {
       try {
         ReservationQueue defQueue =
-            new ReservationQueue(scheduler, defReservationQueue, planQueue);
-        scheduler.addQueue(defQueue);
+            new ReservationQueue(cs, defReservationId, planQueue);
+        cs.addQueue(defQueue);
       } catch (SchedulerDynamicEditException e) {
         LOG.warn(
             "Exception while trying to create default reservation queue for plan: {}",
             planQueueName, e);
       } catch (IOException e) {
         LOG.warn(
-            "Exception while trying to create default reservation queue for plan: {}",
-            planQueueName, e);
-      }
-    }
-    curReservationNames.add(defReservationQueue);
-    // if the resources dedicated to this plan has shrunk invoke replanner
-    if (Resources.greaterThan(scheduler.getResourceCalculator(),
-        clusterResources, reservedResources, planResources)) {
-      try {
-        plan.getReplanner().plan(plan, null);
-      } catch (PlanningException e) {
-        LOG.warn("Exception while trying to replan: {}", planQueueName, e);
-      }
-    }
-    // identify the reservations that have expired and new reservations that
-    // have to be activated
-    List<CSQueue> resQueues = planQueue.getChildQueues();
-    Set<String> expired = new HashSet<String>();
-    for (CSQueue resQueue : resQueues) {
-      String resQueueName = resQueue.getQueueName();
-      if (curReservationNames.contains(resQueueName)) {
-        // it is already existing reservation, so needed not create new
-        // reservation queue
-        curReservationNames.remove(resQueueName);
-      } else {
-        // the reservation has termination, mark for cleanup
-        expired.add(resQueueName);
-      }
-    }
-    // garbage collect expired reservations
-    cleanupExpiredQueues(plan.getMoveOnExpiry(), expired, defReservationQueue);
-
-    // Add new reservations and update existing ones
-    float totalAssignedCapacity = 0f;
-    if (currentReservations != null) {
-      // first release all excess capacity in default queue
-      try {
-        scheduler.setEntitlement(defReservationQueue, new QueueEntitlement(0f,
-            1.0f));
-      } catch (YarnException e) {
-        LOG.warn(
-            "Exception while trying to release default queue capacity for plan: {}",
+            "Exception while trying to create default reservation queue for " +
+                "plan: {}",
             planQueueName, e);
       }
-      // sort allocations from the one giving up the most resources, to the
-      // one asking for the most
-      // avoid order-of-operation errors that temporarily violate 100%
-      // capacity bound
-      List<ReservationAllocation> sortedAllocations =
-          sortByDelta(
-              new ArrayList<ReservationAllocation>(currentReservations), now);
-      for (ReservationAllocation res : sortedAllocations) {
-        String currResId = res.getReservationId().toString();
-        if (curReservationNames.contains(currResId)) {
-          try {
-            ReservationQueue resQueue =
-                new ReservationQueue(scheduler, currResId, planQueue);
-            scheduler.addQueue(resQueue);
-          } catch (SchedulerDynamicEditException e) {
-            LOG.warn(
-                "Exception while trying to activate reservation: {} for plan: {}",
-                currResId, planQueueName, e);
-          } catch (IOException e) {
-            LOG.warn(
-                "Exception while trying to activate reservation: {} for plan: {}",
-                currResId, planQueueName, e);
-          }
-        }
-        Resource capToAssign = res.getResourcesAtTime(now);
-        float targetCapacity = 0f;
-        if (planResources.getMemory() > 0
-            && planResources.getVirtualCores() > 0) {
-          targetCapacity =
-              Resources.divide(scheduler.getResourceCalculator(),
-                  clusterResources, capToAssign, planResources);
-        }
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(
-              "Assigning capacity of {} to queue {} with target capacity {}",
-              capToAssign, currResId, targetCapacity);
-        }
-        // set maxCapacity to 100% unless the job requires gang, in which
-        // case we stick to capacity (as running early/before is likely a
-        // waste of resources)
-        float maxCapacity = 1.0f;
-        if (res.containsGangs()) {
-          maxCapacity = targetCapacity;
-        }
-        try {
-          scheduler.setEntitlement(currResId, new QueueEntitlement(
-              targetCapacity, maxCapacity));
-        } catch (YarnException e) {
-          LOG.warn("Exception while trying to size reservation for plan: {}",
-              currResId, planQueueName, e);
-        }
-        totalAssignedCapacity += targetCapacity;
-      }
-    }
-    // compute the default queue capacity
-    float defQCap = 1.0f - totalAssignedCapacity;
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("PlanFollowerEditPolicyTask: total Plan Capacity: {} "
-          + "currReservation: {} default-queue capacity: {}", planResources,
-          numRes, defQCap);
-    }
-    // set the default queue to eat-up all remaining capacity
-    try {
-      scheduler.setEntitlement(defReservationQueue, new QueueEntitlement(
-          defQCap, 1.0f));
-    } catch (YarnException e) {
-      LOG.warn(
-          "Exception while trying to reclaim default queue capacity for plan: {}",
-          planQueueName, e);
-    }
-    // garbage collect finished reservations from plan
-    try {
-      plan.archiveCompletedReservations(now);
-    } catch (PlanningException e) {
-      LOG.error("Exception in archiving completed reservations: ", e);
-    }
-    LOG.info("Finished iteration of plan follower edit policy for plan: "
-        + planQueueName);
-
-    // Extension: update plan with app states,
-    // useful to support smart replanning
-  }
-
-  /**
-   * Move all apps in the set of queues to the parent plan queue's default
-   * reservation queue in a synchronous fashion
-   */
-  private void moveAppsInQueueSync(String expiredReservation,
-      String defReservationQueue) {
-    List<ApplicationAttemptId> activeApps =
-        scheduler.getAppsInQueue(expiredReservation);
-    if (activeApps.isEmpty()) {
-      return;
-    }
-    for (ApplicationAttemptId app : activeApps) {
-      // fallback to parent's default queue
-      try {
-        scheduler.moveApplication(app.getApplicationId(), defReservationQueue);
-      } catch (YarnException e) {
-        LOG.warn(
-            "Encountered unexpected error during migration of application: {} from reservation: {}",
-            app, expiredReservation, e);
-      }
-    }
-  }
-
-  /**
-   * First sets entitlement of queues to zero to prevent new app submission.
-   * Then move all apps in the set of queues to the parent plan queue's default
-   * reservation queue if move is enabled. Finally cleanups the queue by killing
-   * any apps (if move is disabled or move failed) and removing the queue
-   */
-  private void cleanupExpiredQueues(boolean shouldMove, Set<String> toRemove,
-      String defReservationQueue) {
-    for (String expiredReservation : toRemove) {
-      try {
-        // reduce entitlement to 0
-        scheduler.setEntitlement(expiredReservation, new QueueEntitlement(0.0f,
-            0.0f));
-        if (shouldMove) {
-          moveAppsInQueueSync(expiredReservation, defReservationQueue);
-        }
-        if (scheduler.getAppsInQueue(expiredReservation).size() > 0) {
-          scheduler.killAllAppsInQueue(expiredReservation);
-          LOG.info("Killing applications in queue: {}", expiredReservation);
-        } else {
-          scheduler.removeQueue(expiredReservation);
-          LOG.info("Queue: " + expiredReservation + " removed");
-        }
-      } catch (YarnException e) {
-        LOG.warn("Exception while trying to expire reservation: {}",
-            expiredReservation, e);
-      }
     }
   }
 
   @Override
-  public synchronized void setPlans(Collection<Plan> plans) {
-    this.plans.clear();
-    this.plans.addAll(plans);
-  }
-
-  /**
-   * Sort in the order from the least new amount of resources asked (likely
-   * negative) to the highest. This prevents "order-of-operation" errors related
-   * to exceeding 100% capacity temporarily.
-   */
-  private List<ReservationAllocation> sortByDelta(
-      List<ReservationAllocation> currentReservations, long now) {
-    Collections.sort(currentReservations, new ReservationAllocationComparator(
-        scheduler, now));
-    return currentReservations;
+  protected Resource getPlanResources(
+      Plan plan, Queue queue, Resource clusterResources) {
+    PlanQueue planQueue = (PlanQueue)queue;
+    float planAbsCap = planQueue.getAbsoluteCapacity();
+    Resource planResources = Resources.multiply(clusterResources, planAbsCap);
+    plan.setTotalCapacity(planResources);
+    return planResources;
   }
 
-  private static class ReservationAllocationComparator implements
-      Comparator<ReservationAllocation> {
-    CapacityScheduler scheduler;
-    long now;
-
-    ReservationAllocationComparator(CapacityScheduler scheduler, long now) {
-      this.scheduler = scheduler;
-      this.now = now;
-    }
-
-    private Resource getUnallocatedReservedResources(
-        ReservationAllocation reservation) {
-      Resource resResource;
-      CSQueue resQueue =
-          scheduler.getQueue(reservation.getReservationId().toString());
-      if (resQueue != null) {
-        resResource =
-            Resources.subtract(
-                reservation.getResourcesAtTime(now),
-                Resources.multiply(scheduler.getClusterResource(),
-                    resQueue.getAbsoluteCapacity()));
-      } else {
-        resResource = reservation.getResourcesAtTime(now);
-      }
-      return resResource;
-    }
-
-    @Override
-    public int compare(ReservationAllocation lhs, ReservationAllocation rhs) {
-      // compute delta between current and previous reservation, and compare
-      // based on that
-      Resource lhsRes = getUnallocatedReservedResources(lhs);
-      Resource rhsRes = getUnallocatedReservedResources(rhs);
-      return lhsRes.compareTo(rhsRes);
-    }
-
+  @Override
+  protected Resource getReservationQueueResourceIfExists(Plan plan,
+      ReservationId reservationId) {
+    CSQueue resQueue = cs.getQueue(reservationId.toString());
+    Resource reservationResource = null;
+    if (resQueue != null) {
+      reservationResource = Resources.multiply(cs.getClusterResource(),
+          resQueue.getAbsoluteCapacity());
+    }
+    return reservationResource;
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
new file mode 100644
index 0000000000000..9bf92c2ffa9f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+public class FairReservationSystem extends AbstractReservationSystem {
+
+  private FairScheduler fairScheduler;
+
+  public FairReservationSystem() {
+    super(FairReservationSystem.class.getName());
+  }
+
+  @Override
+  public void reinitialize(Configuration conf, RMContext rmContext)
+      throws YarnException {
+    // Validate if the scheduler is fair scheduler
+    ResourceScheduler scheduler = rmContext.getScheduler();
+    if (!(scheduler instanceof FairScheduler)) {
+      throw new YarnRuntimeException("Class "
+          + scheduler.getClass().getCanonicalName() + " not instance of "
+          + FairScheduler.class.getCanonicalName());
+    }
+    fairScheduler = (FairScheduler) scheduler;
+    this.conf = conf;
+    super.reinitialize(conf, rmContext);
+  }
+
+  @Override
+  protected ReservationSchedulerConfiguration
+      getReservationSchedulerConfiguration() {
+    return fairScheduler.getAllocationConfiguration();
+  }
+
+  @Override
+  protected ResourceCalculator getResourceCalculator() {
+    return fairScheduler.getResourceCalculator();
+  }
+
+  @Override
+  protected QueueMetrics getRootQueueMetrics() {
+    return fairScheduler.getRootQueueMetrics();
+  }
+
+  @Override
+  protected Resource getMinAllocation() {
+    return fairScheduler.getMinimumResourceCapability();
+  }
+
+  @Override
+  protected Resource getMaxAllocation() {
+    return fairScheduler.getMaximumResourceCapability();
+  }
+
+  @Override
+  protected String getPlanQueuePath(String planQueueName) {
+      return planQueueName; }
+
+  @Override
+  protected Resource getPlanQueueCapacity(String planQueueName) {
+    return fairScheduler.getQueueManager().getParentQueue(planQueueName, false)
+        .getSteadyFairShare();
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairSchedulerPlanFollower.java
new file mode 100644
index 0000000000000..7ca03c5f60872
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairSchedulerPlanFollower.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FairSchedulerPlanFollower extends AbstractSchedulerPlanFollower {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(FairSchedulerPlanFollower.class);
+
+  private FairScheduler fs;
+
+  @Override
+  public void init(Clock clock, ResourceScheduler sched,
+      Collection<Plan> plans) {
+    super.init(clock, sched, plans);
+    fs = (FairScheduler)sched;
+    LOG.info("Initializing Plan Follower Policy:"
+        + this.getClass().getCanonicalName());
+  }
+
+  @Override
+  protected Queue getPlanQueue(String planQueueName) {
+    Queue planQueue = fs.getQueueManager().getParentQueue(planQueueName, false);
+    if (planQueue == null) {
+      LOG.error("The queue " + planQueueName + " cannot be found or is not a " +
+          "ParentQueue");
+    }
+    return planQueue;
+  }
+
+  @Override
+  protected float calculateReservationToPlanRatio(Resource clusterResources,
+      Resource planResources, Resource capToAssign) {
+    return Resources.divide(fs.getResourceCalculator(),
+        clusterResources, capToAssign, planResources);
+  }
+
+  @Override
+  protected boolean arePlanResourcesLessThanReservations(Resource
+      clusterResources, Resource planResources, Resource reservedResources) {
+    return Resources.greaterThan(fs.getResourceCalculator(),
+        clusterResources, reservedResources, planResources);
+  }
+
+  @Override
+  protected List<? extends Queue> getChildReservationQueues(Queue queue) {
+    FSQueue planQueue = (FSQueue)queue;
+    List<FSQueue> childQueues = planQueue.getChildQueues();
+    return childQueues;
+  }
+
+
+  @Override
+  protected void addReservationQueue(String planQueueName, Queue queue,
+      String currResId) {
+    String leafQueueName = getReservationQueueName(planQueueName, currResId);
+    fs.getQueueManager().getLeafQueue(leafQueueName, true);
+  }
+
+  @Override
+  protected void createDefaultReservationQueue(String planQueueName,
+      Queue queue, String defReservationId) {
+    String defReservationQueueName = getReservationQueueName(planQueueName,
+        defReservationId);
+    if (!fs.getQueueManager().exists(defReservationQueueName)) {
+      fs.getQueueManager().getLeafQueue(defReservationQueueName, true);
+    }
+  }
+
+  @Override
+  protected Resource getPlanResources(Plan plan, Queue queue,
+      Resource clusterResources) {
+    FSParentQueue planQueue = (FSParentQueue)queue;
+    Resource planResources = planQueue.getSteadyFairShare();
+    return planResources;
+  }
+
+  @Override
+  protected Resource getReservationQueueResourceIfExists(Plan plan,
+      ReservationId reservationId) {
+    String reservationQueueName = getReservationQueueName(plan.getQueueName(),
+        reservationId.toString());
+    FSLeafQueue reservationQueue =
+        fs.getQueueManager().getLeafQueue(reservationQueueName, false);
+    Resource reservationResource = null;
+    if (reservationQueue != null) {
+      reservationResource = reservationQueue.getSteadyFairShare();
+    }
+    return reservationResource;
+  }
+
+  @Override
+  protected String getReservationQueueName(String planQueueName,
+      String reservationQueueName) {
+    String planQueueNameFullPath = fs.getQueueManager().getQueue
+        (planQueueName).getName();
+
+    if (!reservationQueueName.startsWith(planQueueNameFullPath)) {
+      // If name is not a path we need full path for FairScheduler. See
+      // YARN-2773 for the root cause
+      return planQueueNameFullPath + "." + reservationQueueName;
+    }
+    return reservationQueueName;
+  }
+
+  @Override
+  protected String getReservationIdFromQueueName(String resQueueName) {
+    return resQueueName.substring(resQueueName.lastIndexOf(".") + 1);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationConstants.java
new file mode 100644
index 0000000000000..1ff941f043204
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationConstants.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+public interface ReservationConstants {
+
+  /**
+   * The suffix used for a queue under a reservable queue that will be used
+   * as a default queue whenever no reservation is used
+   */
+  String DEFAULT_QUEUE_SUFFIX = "-default";
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 624aa18c6bf95..fbcaab9bf50c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -239,4 +240,6 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
   RMAppMetrics getRMAppMetrics();
 
   ReservationId getReservationId();
+  
+  ResourceRequest getAMResourceRequest();
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 33b62fe8ea224..2d1737af590fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1339,6 +1339,11 @@ public void setSystemClock(Clock clock) {
   public ReservationId getReservationId() {
     return submissionContext.getReservationID();
   }
+  
+  @Override
+  public ResourceRequest getAMResourceRequest() {
+    return this.amReq; 
+  }
 
   protected Credentials parseCredentials() throws IOException {
     Credentials credentials = new Credentials();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1774eb54e690d..b3020b7677b65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -54,6 +54,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -447,6 +448,8 @@ private void updateMetricsForRejoinedNode(NodeState previousNodeState) {
     case UNHEALTHY:
       metrics.decrNumUnhealthyNMs();
       break;
+    default:
+      LOG.debug("Unexpected previous node state");    
     }
   }
 
@@ -461,6 +464,8 @@ private void updateMetricsForDeactivatedNode(NodeState initialState,
       case UNHEALTHY:
         metrics.decrNumUnhealthyNMs();
         break;
+      default:
+        LOG.debug("Unexpected inital state");
     }
 
     switch (finalState) {
@@ -476,6 +481,8 @@ private void updateMetricsForDeactivatedNode(NodeState initialState,
     case UNHEALTHY:
       metrics.incrNumUnhealthyNMs();
       break;
+    default:
+      LOG.debug("Unexpected final state");
     }
   }
 
@@ -582,6 +589,8 @@ public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
             case UNHEALTHY:
               ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
               break;
+            default:
+              LOG.debug("Unexpected Rmnode state");
             }
             rmNode.context.getRMNodes().put(newNode.getNodeID(), newNode);
             rmNode.context.getDispatcher().getEventHandler().handle(
@@ -858,9 +867,10 @@ public Set<ContainerId> getLaunchedContainers() {
 
   @Override
   public Set<String> getNodeLabels() {
-    if (context.getNodeLabelManager() == null) {
+    RMNodeLabelsManager nlm = context.getNodeLabelManager();
+    if (nlm == null || nlm.getLabelsOnNode(nodeId) == null) {
       return CommonNodeLabelsManager.EMPTY_STRING_SET;
     }
-    return context.getNodeLabelManager().getLabelsOnNode(nodeId);
+    return nlm.getLabelsOnNode(nodeId);
   }
  }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index bf720ae51dc08..04b3452fe39fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -22,6 +22,8 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -82,8 +84,9 @@ public abstract class AbstractYarnScheduler
   private Resource configuredMaximumAllocation;
   private int maxNodeMemory = -1;
   private int maxNodeVCores = -1;
-  private ReentrantReadWriteLock maximumAllocationLock =
-      new ReentrantReadWriteLock();
+  private final ReadLock maxAllocReadLock;
+  private final WriteLock maxAllocWriteLock;
+
   private boolean useConfiguredMaximumAllocationOnly = true;
   private long configuredMaximumAllocationWaitTime;
 
@@ -103,6 +106,9 @@ public abstract class AbstractYarnScheduler
    */
   public AbstractYarnScheduler(String name) {
     super(name);
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    this.maxAllocReadLock = lock.readLock();
+    this.maxAllocWriteLock = lock.writeLock();
   }
 
   @Override
@@ -157,8 +163,7 @@ public Resource getMinimumResourceCapability() {
   @Override
   public Resource getMaximumResourceCapability() {
     Resource maxResource;
-    ReentrantReadWriteLock.ReadLock readLock = maximumAllocationLock.readLock();
-    readLock.lock();
+    maxAllocReadLock.lock();
     try {
       if (useConfiguredMaximumAllocationOnly) {
         if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
@@ -170,27 +175,30 @@ public Resource getMaximumResourceCapability() {
         maxResource = Resources.clone(maximumAllocation);
       }
     } finally {
-      readLock.unlock();
+      maxAllocReadLock.unlock();
     }
     return maxResource;
   }
 
+  @Override
+  public Resource getMaximumResourceCapability(String queueName) {
+    return getMaximumResourceCapability();
+  }
+
   protected void initMaximumResourceCapability(Resource maximumAllocation) {
-    ReentrantReadWriteLock.WriteLock writeLock =
-        maximumAllocationLock.writeLock();
-    writeLock.lock();
+    maxAllocWriteLock.lock();
     try {
       if (this.configuredMaximumAllocation == null) {
         this.configuredMaximumAllocation = Resources.clone(maximumAllocation);
         this.maximumAllocation = Resources.clone(maximumAllocation);
       }
     } finally {
-      writeLock.unlock();
+      maxAllocWriteLock.unlock();
     }
   }
 
-  protected void containerLaunchedOnNode(ContainerId containerId,
-                                         SchedulerNode node) {
+  protected synchronized void containerLaunchedOnNode(
+      ContainerId containerId, SchedulerNode node) {
     // Get the application for the finished container
     SchedulerApplicationAttempt application = getCurrentAttemptForContainer
         (containerId);
@@ -535,19 +543,24 @@ public synchronized void killAllAppsInQueue(String queueName)
    */
   public synchronized void updateNodeResource(RMNode nm, 
       ResourceOption resourceOption) {
-  
     SchedulerNode node = getSchedulerNode(nm.getNodeID());
     Resource newResource = resourceOption.getResource();
     Resource oldResource = node.getTotalResource();
     if(!oldResource.equals(newResource)) {
       // Log resource change
-      LOG.info("Update resource on node: " + node.getNodeName() 
+      LOG.info("Update resource on node: " + node.getNodeName()
           + " from: " + oldResource + ", to: "
           + newResource);
 
+      nodes.remove(nm.getNodeID());
+      updateMaximumAllocation(node, false);
+
       // update resource to node
       node.setTotalResource(newResource);
-    
+
+      nodes.put(nm.getNodeID(), (N)node);
+      updateMaximumAllocation(node, true);
+
       // update resource to clusterResource
       Resources.subtractFrom(clusterResource, oldResource);
       Resources.addTo(clusterResource, newResource);
@@ -571,28 +584,27 @@ public Set<String> getPlanQueues() throws YarnException {
   }
 
   protected void updateMaximumAllocation(SchedulerNode node, boolean add) {
-    ReentrantReadWriteLock.WriteLock writeLock =
-        maximumAllocationLock.writeLock();
-    writeLock.lock();
+    Resource totalResource = node.getTotalResource();
+    maxAllocWriteLock.lock();
     try {
       if (add) { // added node
-        int nodeMemory = node.getAvailableResource().getMemory();
+        int nodeMemory = totalResource.getMemory();
         if (nodeMemory > maxNodeMemory) {
           maxNodeMemory = nodeMemory;
           maximumAllocation.setMemory(Math.min(
               configuredMaximumAllocation.getMemory(), maxNodeMemory));
         }
-        int nodeVCores = node.getAvailableResource().getVirtualCores();
+        int nodeVCores = totalResource.getVirtualCores();
         if (nodeVCores > maxNodeVCores) {
           maxNodeVCores = nodeVCores;
           maximumAllocation.setVirtualCores(Math.min(
               configuredMaximumAllocation.getVirtualCores(), maxNodeVCores));
         }
       } else {  // removed node
-        if (maxNodeMemory == node.getAvailableResource().getMemory()) {
+        if (maxNodeMemory == totalResource.getMemory()) {
           maxNodeMemory = -1;
         }
-        if (maxNodeVCores == node.getAvailableResource().getVirtualCores()) {
+        if (maxNodeVCores == totalResource.getVirtualCores()) {
           maxNodeVCores = -1;
         }
         // We only have to iterate through the nodes if the current max memory
@@ -600,12 +612,12 @@ protected void updateMaximumAllocation(SchedulerNode node, boolean add) {
         if (maxNodeMemory == -1 || maxNodeVCores == -1) {
           for (Map.Entry<NodeId, N> nodeEntry : nodes.entrySet()) {
             int nodeMemory =
-                nodeEntry.getValue().getAvailableResource().getMemory();
+                nodeEntry.getValue().getTotalResource().getMemory();
             if (nodeMemory > maxNodeMemory) {
               maxNodeMemory = nodeMemory;
             }
             int nodeVCores =
-                nodeEntry.getValue().getAvailableResource().getVirtualCores();
+                nodeEntry.getValue().getTotalResource().getVirtualCores();
             if (nodeVCores > maxNodeVCores) {
               maxNodeVCores = nodeVCores;
             }
@@ -625,7 +637,25 @@ protected void updateMaximumAllocation(SchedulerNode node, boolean add) {
         }
       }
     } finally {
-      writeLock.unlock();
+      maxAllocWriteLock.unlock();
+    }
+  }
+
+  protected void refreshMaximumAllocation(Resource newMaxAlloc) {
+    maxAllocWriteLock.lock();
+    try {
+      configuredMaximumAllocation = Resources.clone(newMaxAlloc);
+      int maxMemory = newMaxAlloc.getMemory();
+      if (maxNodeMemory != -1) {
+        maxMemory = Math.min(maxMemory, maxNodeMemory);
+      }
+      int maxVcores = newMaxAlloc.getVirtualCores();
+      if (maxNodeVCores != -1) {
+        maxVcores = Math.min(maxVcores, maxNodeVCores);
+      }
+      maximumAllocation = Resources.createResource(maxMemory, maxVcores);
+    } finally {
+      maxAllocWriteLock.unlock();
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
new file mode 100644
index 0000000000000..c651878a9965d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -0,0 +1,312 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * Resource Usage by Labels for following fields by label - AM resource (to
+ * enforce max-am-resource-by-label after YARN-2637) - Used resource (includes
+ * AM resource usage) - Reserved resource - Pending resource - Headroom
+ * 
+ * This class can be used to track resource usage in queue/user/app.
+ * 
+ * And it is thread-safe
+ */
+public class ResourceUsage {
+  private ReadLock readLock;
+  private WriteLock writeLock;
+  private Map<String, UsageByLabel> usages;
+  // short for no-label :)
+  private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+  public ResourceUsage() {
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    readLock = lock.readLock();
+    writeLock = lock.writeLock();
+
+    usages = new HashMap<String, UsageByLabel>();
+  }
+
+  // Usage enum here to make implement cleaner
+  private enum ResourceType {
+    USED(0), PENDING(1), AMUSED(2), RESERVED(3), HEADROOM(4);
+
+    private int idx;
+
+    private ResourceType(int value) {
+      this.idx = value;
+    }
+  }
+
+  private static class UsageByLabel {
+    // usage by label, contains all UsageType
+    private Resource[] resArr;
+
+    public UsageByLabel(String label) {
+      resArr = new Resource[ResourceType.values().length];
+      for (int i = 0; i < resArr.length; i++) {
+        resArr[i] = Resource.newInstance(0, 0);
+      }
+    }
+  }
+
+  /*
+   * Used
+   */
+  public Resource getUsed() {
+    return getUsed(NL);
+  }
+
+  public Resource getUsed(String label) {
+    return _get(label, ResourceType.USED);
+  }
+
+  public void incUsed(String label, Resource res) {
+    _inc(label, ResourceType.USED, res);
+  }
+
+  public void incUsed(Resource res) {
+    incUsed(NL, res);
+  }
+
+  public void decUsed(Resource res) {
+    decUsed(NL, res);
+  }
+
+  public void decUsed(String label, Resource res) {
+    _dec(label, ResourceType.USED, res);
+  }
+
+  public void setUsed(Resource res) {
+    setUsed(NL, res);
+  }
+
+  public void setUsed(String label, Resource res) {
+    _set(label, ResourceType.USED, res);
+  }
+
+  /*
+   * Pending
+   */
+  public Resource getPending() {
+    return getPending(NL);
+  }
+
+  public Resource getPending(String label) {
+    return _get(label, ResourceType.PENDING);
+  }
+
+  public void incPending(String label, Resource res) {
+    _inc(label, ResourceType.PENDING, res);
+  }
+
+  public void incPending(Resource res) {
+    incPending(NL, res);
+  }
+
+  public void decPending(Resource res) {
+    decPending(NL, res);
+  }
+
+  public void decPending(String label, Resource res) {
+    _dec(label, ResourceType.PENDING, res);
+  }
+
+  public void setPending(Resource res) {
+    setPending(NL, res);
+  }
+
+  public void setPending(String label, Resource res) {
+    _set(label, ResourceType.PENDING, res);
+  }
+
+  /*
+   * Reserved
+   */
+  public Resource getReserved() {
+    return getReserved(NL);
+  }
+
+  public Resource getReserved(String label) {
+    return _get(label, ResourceType.RESERVED);
+  }
+
+  public void incReserved(String label, Resource res) {
+    _inc(label, ResourceType.RESERVED, res);
+  }
+
+  public void incReserved(Resource res) {
+    incReserved(NL, res);
+  }
+
+  public void decReserved(Resource res) {
+    decReserved(NL, res);
+  }
+
+  public void decReserved(String label, Resource res) {
+    _dec(label, ResourceType.RESERVED, res);
+  }
+
+  public void setReserved(Resource res) {
+    setReserved(NL, res);
+  }
+
+  public void setReserved(String label, Resource res) {
+    _set(label, ResourceType.RESERVED, res);
+  }
+
+  /*
+   * Headroom
+   */
+  public Resource getHeadroom() {
+    return getHeadroom(NL);
+  }
+
+  public Resource getHeadroom(String label) {
+    return _get(label, ResourceType.HEADROOM);
+  }
+
+  public void incHeadroom(String label, Resource res) {
+    _inc(label, ResourceType.HEADROOM, res);
+  }
+
+  public void incHeadroom(Resource res) {
+    incHeadroom(NL, res);
+  }
+
+  public void decHeadroom(Resource res) {
+    decHeadroom(NL, res);
+  }
+
+  public void decHeadroom(String label, Resource res) {
+    _dec(label, ResourceType.HEADROOM, res);
+  }
+
+  public void setHeadroom(Resource res) {
+    setHeadroom(NL, res);
+  }
+
+  public void setHeadroom(String label, Resource res) {
+    _set(label, ResourceType.HEADROOM, res);
+  }
+
+  /*
+   * AM-Used
+   */
+  public Resource getAMUsed() {
+    return getAMUsed(NL);
+  }
+
+  public Resource getAMUsed(String label) {
+    return _get(label, ResourceType.AMUSED);
+  }
+
+  public void incAMUsed(String label, Resource res) {
+    _inc(label, ResourceType.AMUSED, res);
+  }
+
+  public void incAMUsed(Resource res) {
+    incAMUsed(NL, res);
+  }
+
+  public void decAMUsed(Resource res) {
+    decAMUsed(NL, res);
+  }
+
+  public void decAMUsed(String label, Resource res) {
+    _dec(label, ResourceType.AMUSED, res);
+  }
+
+  public void setAMUsed(Resource res) {
+    setAMUsed(NL, res);
+  }
+
+  public void setAMUsed(String label, Resource res) {
+    _set(label, ResourceType.AMUSED, res);
+  }
+
+  private static Resource normalize(Resource res) {
+    if (res == null) {
+      return Resources.none();
+    }
+    return res;
+  }
+
+  private Resource _get(String label, ResourceType type) {
+    try {
+      readLock.lock();
+      UsageByLabel usage = usages.get(label);
+      if (null == usage) {
+        return Resources.none();
+      }
+      return normalize(usage.resArr[type.idx]);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private UsageByLabel getAndAddIfMissing(String label) {
+    if (!usages.containsKey(label)) {
+      UsageByLabel u = new UsageByLabel(label);
+      usages.put(label, u);
+      return u;
+    }
+
+    return usages.get(label);
+  }
+
+  private void _set(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      usage.resArr[type.idx] = res;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void _inc(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      Resources.addTo(usage.resArr[type.idx], res);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void _dec(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      Resources.subtractFrom(usage.resArr[type.idx], res);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 84975b6b84e93..d5b6ce6698736 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -125,7 +125,7 @@ public class SchedulerApplicationAttempt {
   public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, 
       String user, Queue queue, ActiveUsersManager activeUsersManager,
       RMContext rmContext) {
-    Preconditions.checkNotNull("RMContext should not be null", rmContext);
+    Preconditions.checkNotNull(rmContext, "RMContext should not be null");
     this.rmContext = rmContext;
     this.appSchedulingInfo = 
         new AppSchedulingInfo(applicationAttemptId, user, queue,  
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index b115fc896a2b1..957e8f650a402 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -33,11 +34,14 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import com.google.common.collect.ImmutableSet;
+
 
 /**
  * Represents a YARN Cluster Node from the viewpoint of the scheduler.
@@ -61,8 +65,11 @@ public abstract class SchedulerNode {
 
   private final RMNode rmNode;
   private final String nodeName;
-
-  public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+  
+  private volatile Set<String> labels = null;
+  
+  public SchedulerNode(RMNode node, boolean usePortForNodeName,
+      Set<String> labels) {
     this.rmNode = node;
     this.availableResource = Resources.clone(node.getTotalCapability());
     this.totalResourceCapability = Resources.clone(node.getTotalCapability());
@@ -71,6 +78,11 @@ public SchedulerNode(RMNode node, boolean usePortForNodeName) {
     } else {
       nodeName = rmNode.getHostName();
     }
+    this.labels = ImmutableSet.copyOf(labels);
+  }
+
+  public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+    this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
   }
 
   public RMNode getRMNode() {
@@ -274,4 +286,12 @@ public synchronized void recoverContainer(RMContainer rmContainer) {
     }
     allocateContainer(rmContainer);
   }
+  
+  public Set<String> getLabels() {
+    return labels;
+  }
+  
+  public void updateLabels(Set<String> labels) {
+    this.labels = labels;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 5d0000968a294..c4900c3976f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -231,11 +231,31 @@ public static void validateResourceRequest(ResourceRequest resReq,
     
     // if queue has default label expression, and RR doesn't have, use the
     // default label expression of queue
-    if (labelExp == null && queueInfo != null) {
+    if (labelExp == null && queueInfo != null
+        && ResourceRequest.ANY.equals(resReq.getResourceName())) {
       labelExp = queueInfo.getDefaultNodeLabelExpression();
       resReq.setNodeLabelExpression(labelExp);
     }
     
+    // we don't allow specify label expression other than resourceName=ANY now
+    if (!ResourceRequest.ANY.equals(resReq.getResourceName())
+        && labelExp != null && !labelExp.trim().isEmpty()) {
+      throw new InvalidResourceRequestException(
+          "Invailid resource request, queue=" + queueInfo.getQueueName()
+              + " specified node label expression in a "
+              + "resource request has resource name = "
+              + resReq.getResourceName());
+    }
+    
+    // we don't allow specify label expression with more than one node labels now
+    if (labelExp != null && labelExp.contains("&&")) {
+      throw new InvalidResourceRequestException(
+          "Invailid resource request, queue=" + queueInfo.getQueueName()
+              + " specified more than one node label "
+              + "in a node label expression, node label expression = "
+              + labelExp);
+    }
+    
     if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
       if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
           labelExp)) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index d1b5275a8f0e4..b99b2170d0a1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 
 /**
  * This interface is used by the components to talk to the
@@ -91,13 +92,26 @@ public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues,
   public Resource getMinimumResourceCapability();
   
   /**
-   * Get maximum allocatable {@link Resource}.
+   * Get maximum allocatable {@link Resource} at the cluster level.
    * @return maximum allocatable resource
    */
   @Public
   @Stable
   public Resource getMaximumResourceCapability();
 
+  /**
+   * Get maximum allocatable {@link Resource} for the queue specified.
+   * @param queueName queue name
+   * @return maximum allocatable resource
+   */
+  @Public
+  @Stable
+  public Resource getMaximumResourceCapability(String queueName);
+
+  @LimitedPrivate("yarn")
+  @Evolving
+  ResourceCalculator getResourceCalculator();
+
   /**
    * Get the number of nodes available in the cluster.
    * @return the number of available nodes.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fc0fbb42387b9..e4c26658b0bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -31,21 +31,22 @@
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
-
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Sets;
 
 public abstract class AbstractCSQueue implements CSQueue {
   
   CSQueue parent;
   final String queueName;
-  
   float capacity;
   float maximumCapacity;
   float absoluteCapacity;
@@ -56,7 +57,7 @@ public abstract class AbstractCSQueue implements CSQueue {
   volatile int numContainers;
   
   final Resource minimumAllocation;
-  final Resource maximumAllocation;
+  Resource maximumAllocation;
   QueueState state;
   final QueueMetrics metrics;
   
@@ -64,21 +65,23 @@ public abstract class AbstractCSQueue implements CSQueue {
   Set<String> accessibleLabels;
   RMNodeLabelsManager labelManager;
   String defaultLabelExpression;
-  Resource usedResources = Resources.createResource(0, 0);
-  QueueInfo queueInfo;
   Map<String, Float> absoluteCapacityByNodeLabels;
   Map<String, Float> capacitiyByNodeLabels;
-  Map<String, Resource> usedResourcesByNodeLabels = new HashMap<String, Resource>();
   Map<String, Float> absoluteMaxCapacityByNodeLabels;
   Map<String, Float> maxCapacityByNodeLabels;
   
   Map<QueueACL, AccessControlList> acls = 
       new HashMap<QueueACL, AccessControlList>();
   boolean reservationsContinueLooking;
+  private boolean preemptionDisabled;
+
+  // Track resource usage-by-label like used-resource/pending-resource, etc.
+  ResourceUsage queueUsage;
   
   private final RecordFactory recordFactory = 
       RecordFactoryProvider.getRecordFactory(null);
-  
+  private CapacitySchedulerContext csContext;
+
   public AbstractCSQueue(CapacitySchedulerContext cs, 
       String queueName, CSQueue parent, CSQueue old) throws IOException {
     this.minimumAllocation = cs.getMinimumResourceCapability();
@@ -87,7 +90,6 @@ public AbstractCSQueue(CapacitySchedulerContext cs,
     this.parent = parent;
     this.queueName = queueName;
     this.resourceCalculator = cs.getResourceCalculator();
-    this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
     
     // must be called after parent and queueName is set
     this.metrics = old != null ? old.getMetrics() :
@@ -99,9 +101,7 @@ public AbstractCSQueue(CapacitySchedulerContext cs,
     this.accessibleLabels = cs.getConfiguration().getAccessibleNodeLabels(getQueuePath());
     this.defaultLabelExpression = cs.getConfiguration()
         .getDefaultNodeLabelExpression(getQueuePath());
-    
-    this.queueInfo.setQueueName(queueName);
-    
+
     // inherit from parent if labels not set
     if (this.accessibleLabels == null && parent != null) {
       this.accessibleLabels = parent.getAccessibleNodeLabels();
@@ -124,6 +124,8 @@ public AbstractCSQueue(CapacitySchedulerContext cs,
     maxCapacityByNodeLabels =
         cs.getConfiguration().getMaximumNodeLabelCapacities(getQueuePath(),
             accessibleLabels, labelManager);
+    this.csContext = cs;
+    queueUsage = new ResourceUsage();
   }
   
   @Override
@@ -157,8 +159,8 @@ public synchronized float getUsedCapacity() {
   }
 
   @Override
-  public synchronized Resource getUsedResources() {
-    return usedResources;
+  public Resource getUsedResources() {
+    return queueUsage.getUsed();
   }
 
   public synchronized int getNumContainers() {
@@ -253,7 +255,7 @@ synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
       Set<String> labels, String defaultLabelExpression,
       Map<String, Float> nodeLabelCapacities,
       Map<String, Float> maximumNodeLabelCapacities,
-      boolean reservationContinueLooking)
+      boolean reservationContinueLooking, Resource maxAllocation)
       throws IOException {
     // Sanity check
     CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -280,12 +282,6 @@ synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
     this.capacitiyByNodeLabels = new HashMap<String, Float>(nodeLabelCapacities);
     this.maxCapacityByNodeLabels =
         new HashMap<String, Float>(maximumNodeLabelCapacities);
-    
-    this.queueInfo.setAccessibleNodeLabels(this.accessibleLabels);
-    this.queueInfo.setCapacity(this.capacity);
-    this.queueInfo.setMaximumCapacity(this.maximumCapacity);
-    this.queueInfo.setQueueState(this.state);
-    this.queueInfo.setDefaultNodeLabelExpression(this.defaultLabelExpression);
 
     // Update metrics
     CSQueueUtils.updateQueueStatistics(
@@ -328,10 +324,26 @@ synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
         absoluteCapacityByNodeLabels, absoluteCapacityByNodeLabels);
     
     this.reservationsContinueLooking = reservationContinueLooking;
+
+    this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
+
+    this.maximumAllocation = maxAllocation;
+  }
+  
+  protected QueueInfo getQueueInfo() {
+    QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+    queueInfo.setQueueName(queueName);
+    queueInfo.setAccessibleNodeLabels(accessibleLabels);
+    queueInfo.setCapacity(capacity);
+    queueInfo.setMaximumCapacity(maximumCapacity);
+    queueInfo.setQueueState(state);
+    queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
+    queueInfo.setCurrentCapacity(getUsedCapacity());
+    return queueInfo;
   }
   
   @Private
-  public Resource getMaximumAllocation() {
+  public synchronized Resource getMaximumAllocation() {
     return maximumAllocation;
   }
   
@@ -342,22 +354,13 @@ public Resource getMinimumAllocation() {
   
   synchronized void allocateResource(Resource clusterResource, 
       Resource resource, Set<String> nodeLabels) {
-    Resources.addTo(usedResources, resource);
     
     // Update usedResources by labels
     if (nodeLabels == null || nodeLabels.isEmpty()) {
-      if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
-        usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
-            Resources.createResource(0));
-      }
-      Resources.addTo(usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL),
-          resource);
+      queueUsage.incUsed(resource);
     } else {
       for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
-        if (!usedResourcesByNodeLabels.containsKey(label)) {
-          usedResourcesByNodeLabels.put(label, Resources.createResource(0));
-        }
-        Resources.addTo(usedResourcesByNodeLabels.get(label), resource);
+        queueUsage.incUsed(label, resource);
       }
     }
 
@@ -368,23 +371,12 @@ synchronized void allocateResource(Resource clusterResource,
   
   protected synchronized void releaseResource(Resource clusterResource,
       Resource resource, Set<String> nodeLabels) {
-    // Update queue metrics
-    Resources.subtractFrom(usedResources, resource);
-
     // Update usedResources by labels
     if (null == nodeLabels || nodeLabels.isEmpty()) {
-      if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
-        usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
-            Resources.createResource(0));
-      }
-      Resources.subtractFrom(
-          usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL), resource);
+      queueUsage.decUsed(resource);
     } else {
       for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
-        if (!usedResourcesByNodeLabels.containsKey(label)) {
-          usedResourcesByNodeLabels.put(label, Resources.createResource(0));
-        }
-        Resources.subtractFrom(usedResourcesByNodeLabels.get(label), resource);
+        queueUsage.decUsed(label, resource);
       }
     }
 
@@ -447,4 +439,53 @@ public boolean getReservationContinueLooking() {
   public Map<QueueACL, AccessControlList> getACLs() {
     return acls;
   }
+  
+  @Private
+  public Resource getUsedResourceByLabel(String nodeLabel) {
+    return queueUsage.getUsed(nodeLabel);
+  }
+  
+  @VisibleForTesting
+  public ResourceUsage getResourceUsage() {
+    return queueUsage;
+  }
+
+  @Private
+  public boolean getPreemptionDisabled() {
+    return preemptionDisabled;
+  }
+
+  /**
+   * The specified queue is preemptable if system-wide preemption is turned on
+   * unless any queue in the <em>qPath</em> hierarchy has explicitly turned
+   * preemption off.
+   * NOTE: Preemptability is inherited from a queue's parent.
+   * 
+   * @return true if queue has preemption disabled, false otherwise
+   */
+  private boolean isQueueHierarchyPreemptionDisabled(CSQueue q) {
+    CapacitySchedulerConfiguration csConf = csContext.getConfiguration();
+    boolean systemWidePreemption =
+        csConf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
+                       YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS);
+    CSQueue parentQ = q.getParent();
+
+    // If the system-wide preemption switch is turned off, all of the queues in
+    // the qPath hierarchy have preemption disabled, so return true.
+    if (!systemWidePreemption) return true;
+
+    // If q is the root queue and the system-wide preemption switch is turned
+    // on, then q does not have preemption disabled (default=false, below)
+    // unless the preemption_disabled property is explicitly set.
+    if (parentQ == null) {
+      return csConf.getPreemptionDisabled(q.getQueuePath(), false);
+    }
+
+    // If this is not the root queue, inherit the default value for the
+    // preemption_disabled property from the parent. Preemptability will be
+    // inherited from the parent's hierarchy unless explicitly overridden at
+    // this level.
+    return csConf.getPreemptionDisabled(q.getQueuePath(),
+                                        parentQ.getPreemptionDisabled());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 6438d6c83b8ea..46ee93c9f25b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -143,6 +143,14 @@ public interface CSQueue
    */
   public Resource getUsedResources();
   
+  /**
+   * Get the currently utilized resources which allocated at nodes with label
+   * specified
+   * 
+   * @return used resources by the queue and it's children
+   */
+  public Resource getUsedResourceByLabel(String nodeLabel);
+  
   /**
    * Get the current run-state of the queue
    * @return current run-state
@@ -292,4 +300,10 @@ public void attachContainer(Resource clusterResource,
    * @return capacity by node label
    */
   public float getCapacityByNodeLabel(String nodeLabel);
+
+  /**
+   * Check whether <em>disable_preemption</em> property is set for this queue
+   * @return true if <em>disable_preemption</em> is set, false if not
+   */
+  public boolean getPreemptionDisabled();
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 0a2fa3a8d02a1..f458057c35d6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -109,30 +109,6 @@ public static Map<String, Float> computeAbsoluteMaxCapacityByNodeLabels(
     }
     return absoluteMaxCapacityByNodeLabels;
   }
-
-  public static int computeMaxActiveApplications(
-      ResourceCalculator calculator,
-      Resource clusterResource, Resource minimumAllocation, 
-      float maxAMResourcePercent, float absoluteMaxCapacity) {
-    return
-        Math.max(
-            (int)Math.ceil(
-                Resources.ratio(
-                    calculator, 
-                    clusterResource, 
-                    minimumAllocation) * 
-                    maxAMResourcePercent * absoluteMaxCapacity
-                ), 
-            1);
-  }
-
-  public static int computeMaxActiveApplicationsPerUser(
-      int maxActiveApplications, int userLimit, float userLimitFactor) {
-    return Math.max(
-        (int)Math.ceil(
-            maxActiveApplications * (userLimit / 100.0f) * userLimitFactor),
-        1);
-  }
   
   @Lock(CSQueue.class)
   public static void updateQueueStatistics(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 28158c136f9e0..916a4db9101a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -27,6 +27,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -47,12 +48,15 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -62,6 +66,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -92,6 +97,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -244,7 +250,8 @@ public CapacitySchedulerConfiguration getConfiguration() {
   }
 
   @Override
-  public RMContainerTokenSecretManager getContainerTokenSecretManager() {
+  public synchronized RMContainerTokenSecretManager 
+  getContainerTokenSecretManager() {
     return this.rmContext.getContainerTokenSecretManager();
   }
 
@@ -350,9 +357,11 @@ public void serviceStop() throws Exception {
     validateConf(this.conf);
     try {
       LOG.info("Re-initializing queues...");
+      refreshMaximumAllocation(this.conf.getMaximumAllocation());
       reinitializeQueues(this.conf);
     } catch (Throwable t) {
       this.conf = oldConf;
+      refreshMaximumAllocation(this.conf.getMaximumAllocation());
       throw new IOException("Failed to re-init queues", t);
     }
   }
@@ -701,11 +710,14 @@ private synchronized void addApplication(ApplicationId applicationId,
     try {
       queue.submitApplication(applicationId, user, queueName);
     } catch (AccessControlException ace) {
-      LOG.info("Failed to submit application " + applicationId + " to queue "
-          + queueName + " from user " + user, ace);
-      this.rmContext.getDispatcher().getEventHandler()
-          .handle(new RMAppRejectedEvent(applicationId, ace.toString()));
-      return;
+      // Ignore the exception for recovered app as the app was previously accepted
+      if (!isAppRecovering) {
+        LOG.info("Failed to submit application " + applicationId + " to queue "
+            + queueName + " from user " + user, ace);
+        this.rmContext.getDispatcher().getEventHandler()
+            .handle(new RMAppRejectedEvent(applicationId, ace.toString()));
+        return;
+      }
     }
     // update the metrics
     queue.getMetrics().submitApp(user);
@@ -966,6 +978,51 @@ private synchronized void updateNodeAndQueueResource(RMNode nm,
     updateNodeResource(nm, resourceOption);
     root.updateClusterResource(clusterResource);
   }
+  
+  /**
+   * Process node labels update on a node.
+   * 
+   * TODO: Currently capacity scheduler will kill containers on a node when
+   * labels on the node changed. It is a simply solution to ensure guaranteed
+   * capacity on labels of queues. When YARN-2498 completed, we can let
+   * preemption policy to decide if such containers need to be killed or just
+   * keep them running.
+   */
+  private synchronized void updateLabelsOnNode(NodeId nodeId,
+      Set<String> newLabels) {
+    FiCaSchedulerNode node = nodes.get(nodeId);
+    if (null == node) {
+      return;
+    }
+    
+    // labels is same, we don't need do update
+    if (node.getLabels().size() == newLabels.size()
+        && node.getLabels().containsAll(newLabels)) {
+      return;
+    }
+    
+    // Kill running containers since label is changed
+    for (RMContainer rmContainer : node.getRunningContainers()) {
+      ContainerId containerId = rmContainer.getContainerId();
+      completedContainer(rmContainer, 
+          ContainerStatus.newInstance(containerId,
+              ContainerState.COMPLETE, 
+              String.format(
+                  "Container=%s killed since labels on the node=%s changed",
+                  containerId.toString(), nodeId.toString()),
+              ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+          RMContainerEventType.KILL);
+    }
+    
+    // Unreserve container on this node
+    RMContainer reservedContainer = node.getReservedContainer();
+    if (null != reservedContainer) {
+      dropContainerReservation(reservedContainer);
+    }
+    
+    // Update node labels after we've done this
+    node.updateLabels(newLabels);
+  }
 
   private synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
     if (rmContext.isWorkPreservingRecoveryEnabled()
@@ -1049,6 +1106,19 @@ public void handle(SchedulerEvent event) {
         nodeResourceUpdatedEvent.getResourceOption());
     }
     break;
+    case NODE_LABELS_UPDATE:
+    {
+      NodeLabelsUpdateSchedulerEvent labelUpdateEvent =
+          (NodeLabelsUpdateSchedulerEvent) event;
+      
+      for (Entry<NodeId, Set<String>> entry : labelUpdateEvent
+          .getUpdatedNodeToLabels().entrySet()) {
+        NodeId id = entry.getKey();
+        Set<String> labels = entry.getValue();
+        updateLabelsOnNode(id, labels);
+      }
+    }
+    break;
     case NODE_UPDATE:
     {
       NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
@@ -1117,13 +1187,8 @@ public void handle(SchedulerEvent event) {
   }
 
   private synchronized void addNode(RMNode nodeManager) {
-    // update this node to node label manager
-    if (labelManager != null) {
-      labelManager.activateNode(nodeManager.getNodeID(),
-          nodeManager.getTotalCapability());
-    }
     FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
-        usePortForNodeName);
+        usePortForNodeName, nodeManager.getNodeLabels());
     this.nodes.put(nodeManager.getNodeID(), schedulerNode);
     Resources.addTo(clusterResource, nodeManager.getTotalCapability());
     root.updateClusterResource(clusterResource);
@@ -1136,6 +1201,12 @@ private synchronized void addNode(RMNode nodeManager) {
     if (scheduleAsynchronously && numNodes == 1) {
       asyncSchedulerThread.beginSchedule();
     }
+    
+    // update this node to node label manager
+    if (labelManager != null) {
+      labelManager.activateNode(nodeManager.getNodeID(),
+          nodeManager.getTotalCapability());
+    }
   }
 
   private synchronized void removeNode(RMNode nodeInfo) {
@@ -1352,7 +1423,7 @@ private synchronized String resolveReservationQueueName(String queueName,
       queueName = resQName;
     } else {
       // use the default child queue of the plan for unreserved apps
-      queueName = queueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+      queueName = queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
     }
     return queueName;
   }
@@ -1512,11 +1583,25 @@ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
       .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
   }
   
+  @Override
+  public Resource getMaximumResourceCapability(String queueName) {
+    CSQueue queue = getQueue(queueName);
+    if (queue == null) {
+      LOG.error("Unknown queue: " + queueName);
+      return getMaximumResourceCapability();
+    }
+    if (!(queue instanceof LeafQueue)) {
+      LOG.error("queue " + queueName + " is not an leaf queue");
+      return getMaximumResourceCapability();
+    }
+    return ((LeafQueue)queue).getMaximumAllocation();
+  }
+
   private String handleMoveToPlanQueue(String targetQueueName) {
     CSQueue dest = getQueue(targetQueueName);
     if (dest != null && dest instanceof PlanQueue) {
       // use the default child reservation queue of the plan
-      targetQueueName = targetQueueName + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+      targetQueueName = targetQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
     }
     return targetQueueName;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 5bbb436c4386c..268cc6cb20a03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -108,6 +108,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   @Private
   public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES = true;
 
+  @Private
+  public static final String MAXIMUM_ALLOCATION_MB = "maximum-allocation-mb";
+
+  @Private
+  public static final String MAXIMUM_ALLOCATION_VCORES =
+      "maximum-allocation-vcores";
+
   @Private
   public static final int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS = 10000;
   
@@ -180,6 +187,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   @Private
   public static final boolean DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE = false;
 
+  @Private
+  public static final String QUEUE_PREEMPTION_DISABLED = "disable_preemption";
+
   @Private
   public static class QueueMapping {
 
@@ -577,6 +587,48 @@ public Resource getMaximumAllocation() {
     return Resources.createResource(maximumMemory, maximumCores);
   }
 
+  /**
+   * Get the per queue setting for the maximum limit to allocate to
+   * each container request.
+   *
+   * @param queue
+   *          name of the queue
+   * @return setting specified per queue else falls back to the cluster setting
+   */
+  public Resource getMaximumAllocationPerQueue(String queue) {
+    String queuePrefix = getQueuePrefix(queue);
+    int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
+        (int)UNDEFINED);
+    int maxAllocationVcoresPerQueue = getInt(
+        queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("max alloc mb per queue for " + queue + " is "
+          + maxAllocationMbPerQueue);
+      LOG.debug("max alloc vcores per queue for " + queue + " is "
+          + maxAllocationVcoresPerQueue);
+    }
+    Resource clusterMax = getMaximumAllocation();
+    if (maxAllocationMbPerQueue == (int)UNDEFINED) {
+      LOG.info("max alloc mb per queue for " + queue + " is undefined");
+      maxAllocationMbPerQueue = clusterMax.getMemory();
+    }
+    if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
+       LOG.info("max alloc vcore per queue for " + queue + " is undefined");
+      maxAllocationVcoresPerQueue = clusterMax.getVirtualCores();
+    }
+    Resource result = Resources.createResource(maxAllocationMbPerQueue,
+        maxAllocationVcoresPerQueue);
+    if (maxAllocationMbPerQueue > clusterMax.getMemory()
+        || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) {
+      throw new IllegalArgumentException(
+          "Queue maximum allocation cannot be larger than the cluster setting"
+          + " for queue " + queue
+          + " max allocation per queue: " + result
+          + " cluster setting: " + clusterMax);
+    }
+    return result;
+  }
+
   public boolean getEnableUserMetrics() {
     return getBoolean(ENABLE_USER_METRICS, DEFAULT_ENABLE_USER_METRICS);
   }
@@ -802,4 +854,32 @@ public long getEnforcementWindow(String queue) {
             DEFAULT_RESERVATION_ENFORCEMENT_WINDOW);
     return enforcementWindow;
   }
+
+  /**
+   * Sets the <em>disable_preemption</em> property in order to indicate
+   * whether or not container preemption will be disabled for the specified
+   * queue.
+   * 
+   * @param queue queue path
+   * @param preemptionDisabled true if preemption is disabled on queue
+   */
+  public void setPreemptionDisabled(String queue, boolean preemptionDisabled) {
+    setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED,
+               preemptionDisabled); 
+  }
+
+  /**
+   * Indicates whether preemption is disabled on the specified queue.
+   * 
+   * @param queue queue path to query
+   * @param defaultVal used as default if the <em>disable_preemption</em>
+   * is not set in the configuration
+   * @return true if preemption is disabled on <em>queue</em>, false otherwise
+   */
+  public boolean getPreemptionDisabled(String queue, boolean defaultVal) {
+    boolean preemptionDisabled =
+        getBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED,
+                   defaultVal);
+    return preemptionDisabled;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 03a1cb65aacbe..28dc98859d5cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -39,6 +39,8 @@ public interface CapacitySchedulerContext {
 
   Resource getMaximumResourceCapability();
 
+  Resource getMaximumResourceCapability(String queueName);
+
   RMContainerTokenSecretManager getContainerTokenSecretManager();
   
   int getNumClusterNodes();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index ffeec630858ff..c1432101510b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -61,6 +61,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -87,9 +88,6 @@ public class LeafQueue extends AbstractCSQueue {
   protected int maxApplicationsPerUser;
   
   private float maxAMResourcePerQueuePercent;
-  private int maxActiveApplications; // Based on absolute max capacity
-  private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
-  private int maxActiveApplicationsPerUser;
   
   private int nodeLocalityDelay;
 
@@ -99,7 +97,7 @@ public class LeafQueue extends AbstractCSQueue {
   
   Set<FiCaSchedulerApp> pendingApplications;
   
-  private final float minimumAllocationFactor;
+  private float minimumAllocationFactor;
 
   private Map<String, User> users = new HashMap<String, User>();
 
@@ -113,6 +111,9 @@ public class LeafQueue extends AbstractCSQueue {
   // cache last cluster resource to compute actual capacity
   private Resource lastClusterResource = Resources.none();
   
+  // absolute capacity as a resource (based on cluster resource)
+  private Resource absoluteCapacityResource = Resources.none();
+  
   private final QueueHeadroomInfo queueHeadroomInfo = new QueueHeadroomInfo();
   
   public LeafQueue(CapacitySchedulerContext cs, 
@@ -149,21 +150,6 @@ public LeafQueue(CapacitySchedulerContext cs,
 
     float maxAMResourcePerQueuePercent = cs.getConfiguration()
         .getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
-    int maxActiveApplications = 
-        CSQueueUtils.computeMaxActiveApplications(
-            resourceCalculator,
-            cs.getClusterResource(), this.minimumAllocation,
-            maxAMResourcePerQueuePercent, absoluteMaxCapacity);
-    this.maxActiveAppsUsingAbsCap = 
-            CSQueueUtils.computeMaxActiveApplications(
-                resourceCalculator,
-                cs.getClusterResource(), this.minimumAllocation,
-                maxAMResourcePerQueuePercent, absoluteCapacity);
-    int maxActiveApplicationsPerUser =
-        CSQueueUtils.computeMaxActiveApplicationsPerUser(
-            maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
-
-    this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
 
     QueueState state = cs.getConfiguration().getState(getQueuePath());
 
@@ -173,11 +159,11 @@ public LeafQueue(CapacitySchedulerContext cs,
     setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
         maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor,
         maxApplications, maxAMResourcePerQueuePercent, maxApplicationsPerUser,
-        maxActiveApplications, maxActiveApplicationsPerUser, state, acls, cs
-            .getConfiguration().getNodeLocalityDelay(), accessibleLabels,
+        state, acls, cs.getConfiguration().getNodeLocalityDelay(), accessibleLabels,
         defaultLabelExpression, this.capacitiyByNodeLabels,
         this.maxCapacityByNodeLabels,
-        cs.getConfiguration().getReservationContinueLook());
+        cs.getConfiguration().getReservationContinueLook(),
+        cs.getConfiguration().getMaximumAllocationPerQueue(getQueuePath()));
 
     if(LOG.isDebugEnabled()) {
       LOG.debug("LeafQueue:" + " name=" + queueName
@@ -202,22 +188,32 @@ protected synchronized void setupQueueConfigs(
       float maximumCapacity, float absoluteMaxCapacity,
       int userLimit, float userLimitFactor,
       int maxApplications, float maxAMResourcePerQueuePercent,
-      int maxApplicationsPerUser, int maxActiveApplications,
-      int maxActiveApplicationsPerUser, QueueState state,
+      int maxApplicationsPerUser, QueueState state,
       Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay,
       Set<String> labels, String defaultLabelExpression,
       Map<String, Float> capacitieByLabel,
       Map<String, Float> maximumCapacitiesByLabel, 
-      boolean revervationContinueLooking) throws IOException {
+      boolean revervationContinueLooking,
+      Resource maxAllocation) throws IOException {
     super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
         maximumCapacity, absoluteMaxCapacity, state, acls, labels,
         defaultLabelExpression, capacitieByLabel, maximumCapacitiesByLabel,
-        revervationContinueLooking);
+        revervationContinueLooking, maxAllocation);
     // Sanity check
     CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
     float absCapacity = getParent().getAbsoluteCapacity() * capacity;
     CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absCapacity,
         absoluteMaxCapacity);
+    
+    this.lastClusterResource = clusterResource;
+    updateAbsoluteCapacityResource(clusterResource);
+    
+    // Initialize headroom info, also used for calculating application 
+    // master resource limits.  Since this happens during queue initialization
+    // and all queues may not be realized yet, we'll use (optimistic) 
+    // absoluteMaxCapacity (it will be replaced with the more accurate 
+    // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
+    updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
 
     this.absoluteCapacity = absCapacity;
 
@@ -228,25 +224,28 @@ protected synchronized void setupQueueConfigs(
     this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
     this.maxApplicationsPerUser = maxApplicationsPerUser;
 
-    this.maxActiveApplications = maxActiveApplications;
-    this.maxActiveApplicationsPerUser = maxActiveApplicationsPerUser;
-
     if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
         this.defaultLabelExpression)) {
       throw new IOException("Invalid default label expression of "
           + " queue="
-          + queueInfo.getQueueName()
+          + getQueueName()
           + " doesn't have permission to access all labels "
           + "in default label expression. labelExpression of resource request="
           + (this.defaultLabelExpression == null ? ""
               : this.defaultLabelExpression)
           + ". Queue labels="
-          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
-              .getAccessibleNodeLabels().iterator(), ',')));
+          + (getAccessibleNodeLabels() == null ? "" : StringUtils.join(
+              getAccessibleNodeLabels().iterator(), ',')));
     }
     
     this.nodeLocalityDelay = nodeLocalityDelay;
 
+    // re-init this since max allocation could have changed
+    this.minimumAllocationFactor =
+        Resources.ratio(resourceCalculator,
+            Resources.subtract(maximumAllocation, minimumAllocation),
+            maximumAllocation);
+
     StringBuilder aclsString = new StringBuilder();
     for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
       aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
@@ -282,21 +281,6 @@ protected synchronized void setupQueueConfigs(
         "maxApplicationsPerUser = " + maxApplicationsPerUser +
         " [= (int)(maxApplications * (userLimit / 100.0f) * " +
         "userLimitFactor) ]" + "\n" +
-        "maxActiveApplications = " + maxActiveApplications +
-        " [= max(" + 
-        "(int)ceil((clusterResourceMemory / minimumAllocation) * " + 
-        "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," + 
-        "1) ]" + "\n" +
-        "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
-        " [= max(" + 
-        "(int)ceil((clusterResourceMemory / minimumAllocation) *" + 
-        "maxAMResourcePercent * absoluteCapacity)," + 
-        "1) ]" + "\n" +
-        "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
-        " [= max(" +
-        "(int)(maxActiveApplications * (userLimit / 100.0f) * " +
-        "userLimitFactor)," +
-        "1) ]" + "\n" +
         "usedCapacity = " + usedCapacity +
         " [= usedResourcesMemory / " +
         "(clusterResourceMemory * absoluteCapacity)]" + "\n" +
@@ -307,6 +291,8 @@ protected synchronized void setupQueueConfigs(
         "minimumAllocationFactor = " + minimumAllocationFactor +
         " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
         "maximumAllocationMemory ]" + "\n" +
+        "maximumAllocation = " + maximumAllocation +
+        " [= configuredMaxAllocation ]" + "\n" +
         "numContainers = " + numContainers +
         " [= currentNumContainers ]" + "\n" +
         "state = " + state +
@@ -317,7 +303,8 @@ protected synchronized void setupQueueConfigs(
         "labels=" + labelStrBuilder.toString() + "\n" +
         "nodeLocalityDelay = " +  nodeLocalityDelay + "\n" +
         "reservationsContinueLooking = " +
-        reservationsContinueLooking + "\n");
+        reservationsContinueLooking + "\n" +
+        "preemptionDisabled = " + getPreemptionDisabled() + "\n");
   }
 
   @Override
@@ -349,14 +336,6 @@ public synchronized int getMaxApplicationsPerUser() {
     return maxApplicationsPerUser;
   }
 
-  public synchronized int getMaximumActiveApplications() {
-    return maxActiveApplications;
-  }
-
-  public synchronized int getMaximumActiveApplicationsPerUser() {
-    return maxActiveApplicationsPerUser;
-  }
-
   @Override
   public ActiveUsersManager getActiveUsersManager() {
     return activeUsersManager;
@@ -433,7 +412,7 @@ public synchronized float getUserLimitFactor() {
   @Override
   public synchronized QueueInfo getQueueInfo(
       boolean includeChildQueues, boolean recursive) {
-    queueInfo.setCurrentCapacity(usedCapacity);
+    QueueInfo queueInfo = getQueueInfo();
     return queueInfo;
   }
 
@@ -463,7 +442,7 @@ public String toString() {
     return queueName + ": " + 
         "capacity=" + capacity + ", " + 
         "absoluteCapacity=" + absoluteCapacity + ", " + 
-        "usedResources=" + usedResources +  ", " +
+        "usedResources=" + queueUsage.getUsed() +  ", " +
         "usedCapacity=" + getUsedCapacity() + ", " + 
         "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
         "numApps=" + getNumApplications() + ", " + 
@@ -492,7 +471,7 @@ public synchronized ArrayList<UserInfo> getUsers() {
     ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
     for (Map.Entry<String, User> entry: users.entrySet()) {
       usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(
-        entry.getValue().consumed), entry.getValue().getActiveApplications(),
+        entry.getValue().getUsed()), entry.getValue().getActiveApplications(),
         entry.getValue().getPendingApplications()));
     }
     return usersToReturn;
@@ -510,6 +489,21 @@ public synchronized void reinitialize(
     }
 
     LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
+
+    // don't allow the maximum allocation to be decreased in size
+    // since we have already told running AM's the size
+    Resource oldMax = getMaximumAllocation();
+    Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
+    if (newMax.getMemory() < oldMax.getMemory()
+        || newMax.getVirtualCores() < oldMax.getVirtualCores()) {
+      throw new IOException(
+          "Trying to reinitialize "
+              + getQueuePath()
+              + " the maximum allocation size can not be decreased!"
+              + " Current setting: " + oldMax
+              + ", trying to set it to: " + newMax);
+    }
+
     setupQueueConfigs(
         clusterResource,
         newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity, 
@@ -519,15 +513,14 @@ public synchronized void reinitialize(
         newlyParsedLeafQueue.maxApplications,
         newlyParsedLeafQueue.maxAMResourcePerQueuePercent,
         newlyParsedLeafQueue.getMaxApplicationsPerUser(),
-        newlyParsedLeafQueue.getMaximumActiveApplications(), 
-        newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
         newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
         newlyParsedLeafQueue.getNodeLocalityDelay(),
         newlyParsedLeafQueue.accessibleLabels,
         newlyParsedLeafQueue.defaultLabelExpression,
         newlyParsedLeafQueue.capacitiyByNodeLabels,
         newlyParsedLeafQueue.maxCapacityByNodeLabels,
-        newlyParsedLeafQueue.reservationsContinueLooking);
+        newlyParsedLeafQueue.reservationsContinueLooking,
+        newlyParsedLeafQueue.getMaximumAllocation());
 
     // queue metrics are updated, more resource may be available
     // activate the pending applications if possible
@@ -606,27 +599,114 @@ public void submitApplication(ApplicationId applicationId, String userName,
     }
 
   }
+  
+  public synchronized Resource getAMResourceLimit() {
+     /* 
+      * The limit to the amount of resources which can be consumed by
+      * application masters for applications running in the queue
+      * is calculated by taking the greater of the max resources currently
+      * available to the queue (see absoluteMaxAvailCapacity) and the absolute
+      * resources guaranteed for the queue and multiplying it by the am
+      * resource percent.
+      *
+      * This is to allow a queue to grow its (proportional) application 
+      * master resource use up to its max capacity when other queues are 
+      * idle but to scale back down to it's guaranteed capacity as they 
+      * become busy.
+      *
+      */
+     Resource queueMaxCap;
+     synchronized (queueHeadroomInfo) {
+       queueMaxCap = queueHeadroomInfo.getQueueMaxCap();
+     }
+     Resource queueCap = Resources.max(resourceCalculator, lastClusterResource,
+       absoluteCapacityResource, queueMaxCap);
+     return Resources.multiplyAndNormalizeUp( 
+          resourceCalculator,
+          queueCap, 
+          maxAMResourcePerQueuePercent, minimumAllocation);
+  }
+  
+  public synchronized Resource getUserAMResourceLimit() {
+     /*
+      * The user amresource limit is based on the same approach as the 
+      * user limit (as it should represent a subset of that).  This means that
+      * it uses the absolute queue capacity instead of the max and is modified
+      * by the userlimit and the userlimit factor as is the userlimit
+      *
+      */ 
+     float effectiveUserLimit = Math.max(userLimit / 100.0f, 1.0f /    
+       Math.max(getActiveUsersManager().getNumActiveUsers(), 1));
+     
+     return Resources.multiplyAndNormalizeUp( 
+          resourceCalculator,
+          absoluteCapacityResource, 
+          maxAMResourcePerQueuePercent * effectiveUserLimit  *
+            userLimitFactor, minimumAllocation);
+  }
 
   private synchronized void activateApplications() {
+    //limit of allowed resource usage for application masters
+    Resource amLimit = getAMResourceLimit();
+    Resource userAMLimit = getUserAMResourceLimit();
+        
     for (Iterator<FiCaSchedulerApp> i=pendingApplications.iterator(); 
          i.hasNext(); ) {
       FiCaSchedulerApp application = i.next();
       
-      // Check queue limit
-      if (getNumActiveApplications() >= getMaximumActiveApplications()) {
-        break;
+      // Check am resource limit
+      Resource amIfStarted = 
+        Resources.add(application.getAMResource(), queueUsage.getAMUsed());
+      
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("application AMResource " + application.getAMResource() +
+          " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent +
+          " amLimit " + amLimit +
+          " lastClusterResource " + lastClusterResource +
+          " amIfStarted " + amIfStarted);
+      }
+      
+      if (!Resources.lessThanOrEqual(
+        resourceCalculator, lastClusterResource, amIfStarted, amLimit)) {
+        if (getNumActiveApplications() < 1) {
+          LOG.warn("maximum-am-resource-percent is insufficient to start a" +
+            " single application in queue, it is likely set too low." +
+            " skipping enforcement to allow at least one application to start"); 
+        } else {
+          LOG.info("not starting application as amIfStarted exceeds amLimit");
+          continue;
+        }
       }
       
-      // Check user limit
+      // Check user am resource limit
+      
       User user = getUser(application.getUser());
-      if (user.getActiveApplications() < getMaximumActiveApplicationsPerUser()) {
-        user.activateApplication();
-        activeApplications.add(application);
-        i.remove();
-        LOG.info("Application " + application.getApplicationId() +
-            " from user: " + application.getUser() + 
-            " activated in queue: " + getQueueName());
+      
+      Resource userAmIfStarted = 
+        Resources.add(application.getAMResource(),
+          user.getConsumedAMResources());
+        
+      if (!Resources.lessThanOrEqual(
+          resourceCalculator, lastClusterResource, userAmIfStarted, 
+          userAMLimit)) {
+        if (getNumActiveApplications() < 1) {
+          LOG.warn("maximum-am-resource-percent is insufficient to start a" +
+            " single application in queue for user, it is likely set too low." +
+            " skipping enforcement to allow at least one application to start"); 
+        } else {
+          LOG.info("not starting application as amIfStarted exceeds " +
+            "userAmLimit");
+          continue;
+        }
       }
+      user.activateApplication();
+      activeApplications.add(application);
+      queueUsage.incAMUsed(application.getAMResource());
+      user.getResourceUsage().incAMUsed(application.getAMResource());
+      i.remove();
+      LOG.info("Application " + application.getApplicationId() +
+          " from user: " + application.getUser() + 
+          " activated in queue: " + getQueueName());
     }
   }
   
@@ -672,6 +752,9 @@ public synchronized void removeApplicationAttempt(
     boolean wasActive = activeApplications.remove(application);
     if (!wasActive) {
       pendingApplications.remove(application);
+    } else {
+      queueUsage.decAMUsed(application.getAMResource());
+      user.getResourceUsage().decAMUsed(application.getAMResource());
     }
     applicationAttemptMap.remove(application.getApplicationAttemptId());
 
@@ -730,7 +813,7 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
     
     // if our queue cannot access this node, just return
     if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
-        labelManager.getLabelsOnNode(node.getNodeID()))) {
+        node.getLabels())) {
       return NULL_ASSIGNMENT;
     }
     
@@ -799,7 +882,7 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
           
           // Check queue max-capacity limit
           if (!canAssignToThisQueue(clusterResource, required,
-              labelManager.getLabelsOnNode(node.getNodeID()), application, true)) {
+              node.getLabels(), application, true)) {
             return NULL_ASSIGNMENT;
           }
 
@@ -832,7 +915,7 @@ public synchronized CSAssignment assignContainers(Resource clusterResource,
             // Book-keeping 
             // Note: Update headroom to account for current allocation too...
             allocateResource(clusterResource, application, assigned,
-                labelManager.getLabelsOnNode(node.getNodeID()));
+                node.getLabels());
             
             // Don't reset scheduling opportunities for non-local assignments
             // otherwise the app will be delayed for each non-local assignment.
@@ -910,8 +993,8 @@ private Resource getHeadroom(User user, Resource queueMaxCap,
      */
     Resource headroom = 
       Resources.min(resourceCalculator, clusterResource,
-        Resources.subtract(userLimit, user.getTotalConsumedResources()),
-        Resources.subtract(queueMaxCap, usedResources)
+        Resources.subtract(userLimit, user.getUsed()),
+        Resources.subtract(queueMaxCap, queueUsage.getUsed())
         );
     return headroom;
   }
@@ -931,12 +1014,8 @@ synchronized boolean canAssignToThisQueue(Resource clusterResource,
     
     boolean canAssign = true;
     for (String label : labelCanAccess) {
-      if (!usedResourcesByNodeLabels.containsKey(label)) {
-        usedResourcesByNodeLabels.put(label, Resources.createResource(0));
-      }
-      
       Resource potentialTotalCapacity =
-          Resources.add(usedResourcesByNodeLabels.get(label), required);
+          Resources.add(queueUsage.getUsed(label), required);
       
       float potentialNewCapacity =
           Resources.divide(resourceCalculator, clusterResource,
@@ -959,14 +1038,14 @@ synchronized boolean canAssignToThisQueue(Resource clusterResource,
             LOG.debug("try to use reserved: "
                 + getQueueName()
                 + " usedResources: "
-                + usedResources
+                + queueUsage.getUsed()
                 + " clusterResources: "
                 + clusterResource
                 + " reservedResources: "
                 + application.getCurrentReservation()
                 + " currentCapacity "
                 + Resources.divide(resourceCalculator, clusterResource,
-                    usedResources, clusterResource) + " required " + required
+                    queueUsage.getUsed(), clusterResource) + " required " + required
                 + " potentialNewWithoutReservedCapacity: "
                 + potentialNewWithoutReservedCapacity + " ( " + " max-capacity: "
                 + absoluteMaxCapacity + ")");
@@ -986,11 +1065,11 @@ synchronized boolean canAssignToThisQueue(Resource clusterResource,
       if (LOG.isDebugEnabled()) {
         LOG.debug(getQueueName()
             + "Check assign to queue, label=" + label
-            + " usedResources: " + usedResourcesByNodeLabels.get(label)
+            + " usedResources: " + queueUsage.getUsed(label)
             + " clusterResources: " + clusterResource
             + " currentCapacity "
             + Resources.divide(resourceCalculator, clusterResource,
-                usedResourcesByNodeLabels.get(label),
+                queueUsage.getUsed(label),
                 labelManager.getResourceByLabel(label, clusterResource))
             + " potentialNewCapacity: " + potentialNewCapacity + " ( "
             + " max-capacity: " + absoluteMaxCapacity + ")");
@@ -999,6 +1078,25 @@ synchronized boolean canAssignToThisQueue(Resource clusterResource,
     
     return canAssign;
   }
+  
+  private Resource updateHeadroomInfo(Resource clusterResource, 
+      float absoluteMaxAvailCapacity) {
+  
+    Resource queueMaxCap = 
+      Resources.multiplyAndNormalizeDown(
+          resourceCalculator, 
+          clusterResource, 
+          absoluteMaxAvailCapacity,
+          minimumAllocation);
+
+    synchronized (queueHeadroomInfo) {
+      queueHeadroomInfo.setQueueMaxCap(queueMaxCap);
+      queueHeadroomInfo.setClusterResource(clusterResource);
+    }
+    
+    return queueMaxCap;
+    
+  }
 
   @Lock({LeafQueue.class, FiCaSchedulerApp.class})
   Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
@@ -1017,18 +1115,9 @@ Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
     //capacity
     float absoluteMaxAvailCapacity = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, this);
-
-    Resource queueMaxCap =                        // Queue Max-Capacity
-        Resources.multiplyAndNormalizeDown(
-            resourceCalculator, 
-            clusterResource, 
-            absoluteMaxAvailCapacity,
-            minimumAllocation);
-	
-    synchronized (queueHeadroomInfo) {
-      queueHeadroomInfo.setQueueMaxCap(queueMaxCap);
-      queueHeadroomInfo.setClusterResource(clusterResource);
-    }
+    
+    Resource queueMaxCap = 
+      updateHeadroomInfo(clusterResource, absoluteMaxAvailCapacity);
     
     Resource headroom =
         getHeadroom(queueUser, queueMaxCap, clusterResource, userLimit);
@@ -1037,7 +1126,7 @@ Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
       LOG.debug("Headroom calculation for user " + user + ": " + 
           " userLimit=" + userLimit + 
           " queueMaxCap=" + queueMaxCap + 
-          " consumed=" + queueUser.getTotalConsumedResources() + 
+          " consumed=" + queueUser.getUsed() + 
           " headroom=" + headroom);
     }
     
@@ -1092,8 +1181,8 @@ private Resource computeUserLimit(FiCaSchedulerApp application,
 
     Resource currentCapacity =
         Resources.lessThan(resourceCalculator, clusterResource, 
-            usedResources, queueCapacity) ?
-            queueCapacity : Resources.add(usedResources, required);
+            queueUsage.getUsed(), queueCapacity) ?
+            queueCapacity : Resources.add(queueUsage.getUsed(), required);
     
     // Never allow a single user to take more than the 
     // queue's configured capacity * user-limit-factor.
@@ -1128,10 +1217,10 @@ private Resource computeUserLimit(FiCaSchedulerApp application,
           " userLimit=" + userLimit +
           " userLimitFactor=" + userLimitFactor +
           " required: " + required + 
-          " consumed: " + user.getTotalConsumedResources() + 
+          " consumed: " + user.getUsed() + 
           " limit: " + limit +
           " queueCapacity: " + queueCapacity + 
-          " qconsumed: " + usedResources +
+          " qconsumed: " + queueUsage.getUsed() +
           " currentCapacity: " + currentCapacity +
           " activeUsers: " + activeUsers +
           " clusterCapacity: " + clusterResource
@@ -1156,7 +1245,7 @@ protected synchronized boolean assignToUser(Resource clusterResource,
     // overhead of the AM, but it's a > check, not a >= check, so...
     if (Resources
         .greaterThan(resourceCalculator, clusterResource,
-            user.getConsumedResourceByLabel(label),
+            user.getUsed(label),
             limit)) {
       // if enabled, check to see if could we potentially use this node instead
       // of a reserved node if the application has reserved containers
@@ -1164,13 +1253,13 @@ protected synchronized boolean assignToUser(Resource clusterResource,
         if (Resources.lessThanOrEqual(
             resourceCalculator,
             clusterResource,
-            Resources.subtract(user.getTotalConsumedResources(),
+            Resources.subtract(user.getUsed(),
                 application.getCurrentReservation()), limit)) {
 
           if (LOG.isDebugEnabled()) {
             LOG.debug("User " + userName + " in queue " + getQueueName()
                 + " will exceed limit based on reservations - " + " consumed: "
-                + user.getTotalConsumedResources() + " reserved: "
+                + user.getUsed() + " reserved: "
                 + application.getCurrentReservation() + " limit: " + limit);
           }
           return true;
@@ -1179,7 +1268,7 @@ protected synchronized boolean assignToUser(Resource clusterResource,
       if (LOG.isDebugEnabled()) {
         LOG.debug("User " + userName + " in queue " + getQueueName()
             + " will exceed limit - " + " consumed: "
-            + user.getTotalConsumedResources() + " limit: " + limit);
+            + user.getUsed() + " limit: " + limit);
       }
       return false;
     }
@@ -1478,7 +1567,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod
     
     // check if the resource request can access the label
     if (!SchedulerUtils.checkNodeLabelExpression(
-        labelManager.getLabelsOnNode(node.getNodeID()),
+        node.getLabels(),
         request.getNodeLabelExpression())) {
       // this is a reserved container, but we cannot allocate it now according
       // to label not match. This can be caused by node label changed
@@ -1601,7 +1690,7 @@ private Resource assignContainer(Resource clusterResource, FiCaSchedulerNode nod
             " queue=" + this.toString() + 
             " usedCapacity=" + getUsedCapacity() + 
             " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + 
-            " used=" + usedResources +
+            " used=" + queueUsage.getUsed() +
             " cluster=" + clusterResource);
 
         return request.getCapability();
@@ -1669,8 +1758,7 @@ public void completedContainer(Resource clusterResource,
         // Book-keeping
         if (removed) {
           releaseResource(clusterResource, application,
-              container.getResource(),
-              labelManager.getLabelsOnNode(node.getNodeID()));
+              container.getResource(), node.getLabels());
           LOG.info("completedContainer" +
               " container=" + container +
               " queue=" + this +
@@ -1703,9 +1791,9 @@ synchronized void allocateResource(Resource clusterResource,
     if (LOG.isDebugEnabled()) {
       LOG.info(getQueueName() + 
           " user=" + userName + 
-          " used=" + usedResources + " numContainers=" + numContainers +
+          " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
           " headroom = " + application.getHeadroom() +
-          " user-resources=" + user.getTotalConsumedResources()
+          " user-resources=" + user.getUsed()
           );
     }
   }
@@ -1721,28 +1809,28 @@ synchronized void releaseResource(Resource clusterResource,
     metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
       
     LOG.info(getQueueName() + 
-        " used=" + usedResources + " numContainers=" + numContainers + 
-        " user=" + userName + " user-resources=" + user.getTotalConsumedResources());
+        " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + 
+        " user=" + userName + " user-resources=" + user.getUsed());
+  }
+  
+  private void updateAbsoluteCapacityResource(Resource clusterResource) {
+    
+       absoluteCapacityResource = Resources.multiplyAndNormalizeUp( 
+          resourceCalculator,
+          clusterResource, 
+          absoluteCapacity, minimumAllocation);
+       
   }
 
   @Override
   public synchronized void updateClusterResource(Resource clusterResource) {
     lastClusterResource = clusterResource;
+    updateAbsoluteCapacityResource(clusterResource);
     
-    // Update queue properties
-    maxActiveApplications = 
-        CSQueueUtils.computeMaxActiveApplications(
-            resourceCalculator,
-            clusterResource, minimumAllocation, 
-            maxAMResourcePerQueuePercent, absoluteMaxCapacity);
-    maxActiveAppsUsingAbsCap = 
-        CSQueueUtils.computeMaxActiveApplications(
-            resourceCalculator,
-            clusterResource, minimumAllocation, 
-            maxAMResourcePerQueuePercent, absoluteCapacity);
-    maxActiveApplicationsPerUser = 
-        CSQueueUtils.computeMaxActiveApplicationsPerUser(
-            maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
+    // Update headroom info based on new cluster resource value
+    // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
+    // during allocation
+    updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
     
     // Update metrics
     CSQueueUtils.updateQueueStatistics(
@@ -1764,21 +1852,20 @@ resourceCalculator, this, getParent(), clusterResource,
 
   @VisibleForTesting
   public static class User {
-    Resource consumed = Resources.createResource(0, 0);
-    Map<String, Resource> consumedByLabel = new HashMap<String, Resource>();
+    ResourceUsage userResourceUsage = new ResourceUsage();
     int pendingApplications = 0;
     int activeApplications = 0;
 
-    public Resource getTotalConsumedResources() {
-      return consumed;
+    public ResourceUsage getResourceUsage() {
+      return userResourceUsage;
     }
     
-    public Resource getConsumedResourceByLabel(String label) {
-      Resource r = consumedByLabel.get(label);
-      if (null != r) {
-        return r;
-      }
-      return Resources.none();
+    public Resource getUsed() {
+      return userResourceUsage.getUsed();
+    }
+    
+    public Resource getUsed(String label) {
+      return userResourceUsage.getUsed(label);
     }
 
     public int getPendingApplications() {
@@ -1788,6 +1875,10 @@ public int getPendingApplications() {
     public int getActiveApplications() {
       return activeApplications;
     }
+    
+    public Resource getConsumedAMResources() {
+      return userResourceUsage.getAMUsed();
+    }
 
     public int getTotalApplications() {
       return getPendingApplications() + getActiveApplications();
@@ -1811,47 +1902,26 @@ public synchronized void finishApplication(boolean wasActive) {
       }
     }
 
-    public synchronized void assignContainer(Resource resource,
+    public void assignContainer(Resource resource,
         Set<String> nodeLabels) {
-      Resources.addTo(consumed, resource);
-      
       if (nodeLabels == null || nodeLabels.isEmpty()) {
-        if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
-          consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
-              Resources.createResource(0));
-        }
-        Resources.addTo(consumedByLabel.get(RMNodeLabelsManager.NO_LABEL),
-            resource);
+        userResourceUsage.incUsed(resource);
       } else {
         for (String label : nodeLabels) {
-          if (!consumedByLabel.containsKey(label)) {
-            consumedByLabel.put(label, Resources.createResource(0));
-          }
-          Resources.addTo(consumedByLabel.get(label), resource);
+          userResourceUsage.incUsed(label, resource);
         }
       }
     }
 
-    public synchronized void releaseContainer(Resource resource, Set<String> nodeLabels) {
-      Resources.subtractFrom(consumed, resource);
-      
-      // Update usedResources by labels
+    public void releaseContainer(Resource resource, Set<String> nodeLabels) {
       if (nodeLabels == null || nodeLabels.isEmpty()) {
-        if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
-          consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
-              Resources.createResource(0));
-        }
-        Resources.subtractFrom(
-            consumedByLabel.get(RMNodeLabelsManager.NO_LABEL), resource);
+        userResourceUsage.decUsed(resource);
       } else {
         for (String label : nodeLabels) {
-          if (!consumedByLabel.containsKey(label)) {
-            consumedByLabel.put(label, Resources.createResource(0));
-          }
-          Resources.subtractFrom(consumedByLabel.get(label), resource);
+          userResourceUsage.decUsed(label, resource);
         }
       }
-    }  
+    }
   }
 
   @Override
@@ -1862,9 +1932,10 @@ public void recoverContainer(Resource clusterResource,
     }
     // Careful! Locking order is important! 
     synchronized (this) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       allocateResource(clusterResource, attempt, rmContainer.getContainer()
-          .getResource(), labelManager.getLabelsOnNode(rmContainer
-          .getContainer().getNodeId()));
+          .getResource(), node.getLabels());
     }
     getParent().recoverContainer(clusterResource, attempt, rmContainer);
   }
@@ -1878,7 +1949,7 @@ public Set<FiCaSchedulerApp> getApplications() {
   }
 
   // return a single Resource capturing the overal amount of pending resources
-  public Resource getTotalResourcePending() {
+  public synchronized Resource getTotalResourcePending() {
     Resource ret = BuilderUtils.newResource(0, 0);
     for (FiCaSchedulerApp f : activeApplications) {
       Resources.addTo(ret, f.getTotalPendingRequests());
@@ -1887,7 +1958,7 @@ public Resource getTotalResourcePending() {
   }
 
   @Override
-  public void collectSchedulerApplications(
+  public synchronized void collectSchedulerApplications(
       Collection<ApplicationAttemptId> apps) {
     for (FiCaSchedulerApp pendingApp : pendingApplications) {
       apps.add(pendingApp.getApplicationAttemptId());
@@ -1901,14 +1972,15 @@ public void collectSchedulerApplications(
   public void attachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       allocateResource(clusterResource, application, rmContainer.getContainer()
-          .getResource(), labelManager.getLabelsOnNode(rmContainer
-          .getContainer().getNodeId()));
+          .getResource(), node.getLabels());
       LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
           + " resource=" + rmContainer.getContainer().getResource()
           + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
           + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
-          + usedResources + " cluster=" + clusterResource);
+          + queueUsage.getUsed() + " cluster=" + clusterResource);
       // Inform the parent queue
       getParent().attachContainer(clusterResource, application, rmContainer);
     }
@@ -1918,14 +1990,15 @@ public void attachContainer(Resource clusterResource,
   public void detachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       releaseResource(clusterResource, application, rmContainer.getContainer()
-          .getResource(), labelManager.getLabelsOnNode(rmContainer.getContainer()
-          .getNodeId()));
+          .getResource(), node.getLabels());
       LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
           + " resource=" + rmContainer.getContainer().getResource()
           + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
           + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
-          + usedResources + " cluster=" + clusterResource);
+          + queueUsage.getUsed() + " cluster=" + clusterResource);
       // Inform the parent queue
       getParent().detachContainer(clusterResource, application, rmContainer);
     }
@@ -1933,6 +2006,10 @@ public void detachContainer(Resource clusterResource,
 
   @Override
   public float getAbsActualCapacity() {
+    //? Is this actually used by anything at present?
+    //  There is a findbugs warning -re lastClusterResource (now excluded),
+    //  when this is used, verify that the access is mt correct and remove
+    //  the findbugs exclusion if possible
     if (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
         lastClusterResource, Resources.none())) {
       return absoluteCapacity;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 6ffaf4c32b7f9..5a2e234436382 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -73,6 +73,7 @@ public class ParentQueue extends AbstractCSQueue {
   private final boolean rootQueue;
   final Comparator<CSQueue> queueComparator;
   volatile int numApplications;
+  private final CapacitySchedulerContext scheduler;
 
   private final RecordFactory recordFactory = 
     RecordFactoryProvider.getRecordFactory(null);
@@ -80,7 +81,7 @@ public class ParentQueue extends AbstractCSQueue {
   public ParentQueue(CapacitySchedulerContext cs, 
       String queueName, CSQueue parent, CSQueue old) throws IOException {
     super(cs, queueName, parent, old);
-    
+    this.scheduler = cs;
     this.queueComparator = cs.getQueueComparator();
 
     this.rootQueue = (parent == null);
@@ -109,8 +110,6 @@ public ParentQueue(CapacitySchedulerContext cs,
     Map<QueueACL, AccessControlList> acls = 
       cs.getConfiguration().getAcls(getQueuePath());
 
-    this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
-
     setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
         maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
         defaultLabelExpression, capacitiyByNodeLabels, maxCapacityByNodeLabels, 
@@ -133,7 +132,7 @@ synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
     super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
         maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
         defaultLabelExpression, nodeLabelCapacities, maximumCapacitiesByLabel,
-        reservationContinueLooking);
+        reservationContinueLooking, maximumAllocation);
    StringBuilder aclsString = new StringBuilder();
     for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
       aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
@@ -159,7 +158,7 @@ synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
   }
 
   private static float PRECISION = 0.0005f; // 0.05% precision
-  void setChildQueues(Collection<CSQueue> childQueues) {
+  synchronized void setChildQueues(Collection<CSQueue> childQueues) {
     // Validate
     float childCapacities = 0;
     for (CSQueue queue : childQueues) {
@@ -205,7 +204,7 @@ public String getQueuePath() {
   @Override
   public synchronized QueueInfo getQueueInfo( 
       boolean includeChildQueues, boolean recursive) {
-    queueInfo.setCurrentCapacity(usedCapacity);
+    QueueInfo queueInfo = getQueueInfo();
 
     List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
     if (includeChildQueues) {
@@ -257,7 +256,7 @@ public String toString() {
         "numChildQueue= " + childQueues.size() + ", " + 
         "capacity=" + capacity + ", " +  
         "absoluteCapacity=" + absoluteCapacity + ", " +
-        "usedResources=" + usedResources + 
+        "usedResources=" + queueUsage.getUsed() + 
         "usedCapacity=" + getUsedCapacity() + ", " + 
         "numApps=" + getNumApplications() + ", " + 
         "numContainers=" + getNumContainers();
@@ -420,10 +419,10 @@ public synchronized CSAssignment assignContainers(
       Resource clusterResource, FiCaSchedulerNode node, boolean needToUnreserve) {
     CSAssignment assignment = 
         new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+    Set<String> nodeLabels = node.getLabels();
     
     // if our queue cannot access this node, just return
-    if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
-        labelManager.getLabelsOnNode(node.getNodeID()))) {
+    if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
       return assignment;
     }
     
@@ -434,7 +433,6 @@ public synchronized CSAssignment assignContainers(
       }
       
       boolean localNeedToUnreserve = false;
-      Set<String> nodeLabels = labelManager.getLabelsOnNode(node.getNodeID()); 
       
       // Are we over maximum-capacity for this queue?
       if (!canAssignToThisQueue(clusterResource, nodeLabels)) {
@@ -465,7 +463,7 @@ public synchronized CSAssignment assignContainers(
             " queue=" + getQueueName() + 
             " usedCapacity=" + getUsedCapacity() +
             " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
-            " used=" + usedResources + 
+            " used=" + queueUsage.getUsed() + 
             " cluster=" + clusterResource);
 
       } else {
@@ -508,19 +506,16 @@ private synchronized boolean canAssignToThisQueue(Resource clusterResource,
     
     boolean canAssign = true;
     for (String label : labelCanAccess) {
-      if (!usedResourcesByNodeLabels.containsKey(label)) {
-        usedResourcesByNodeLabels.put(label, Resources.createResource(0));
-      }
       float currentAbsoluteLabelUsedCapacity =
           Resources.divide(resourceCalculator, clusterResource,
-              usedResourcesByNodeLabels.get(label),
+              queueUsage.getUsed(label),
               labelManager.getResourceByLabel(label, clusterResource));
       // if any of the label doesn't beyond limit, we can allocate on this node
       if (currentAbsoluteLabelUsedCapacity >= 
             getAbsoluteMaximumCapacityByNodeLabel(label)) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug(getQueueName() + " used=" + usedResources
-              + " current-capacity (" + usedResourcesByNodeLabels.get(label) + ") "
+          LOG.debug(getQueueName() + " used=" + queueUsage.getUsed()
+              + " current-capacity (" + queueUsage.getUsed(label) + ") "
               + " >= max-capacity ("
               + labelManager.getResourceByLabel(label, clusterResource) + ")");
         }
@@ -542,16 +537,16 @@ private synchronized boolean assignToQueueIfUnreserve(Resource clusterResource)
           .getReservedMB(), getMetrics().getReservedVirtualCores());
       float capacityWithoutReservedCapacity = Resources.divide(
           resourceCalculator, clusterResource,
-          Resources.subtract(usedResources, reservedResources),
+          Resources.subtract(queueUsage.getUsed(), reservedResources),
           clusterResource);
 
       if (capacityWithoutReservedCapacity <= absoluteMaxCapacity) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("parent: try to use reserved: " + getQueueName()
-            + " usedResources: " + usedResources.getMemory()
+            + " usedResources: " + queueUsage.getUsed().getMemory()
             + " clusterResources: " + clusterResource.getMemory()
             + " reservedResources: " + reservedResources.getMemory()
-            + " currentCapacity " + ((float) usedResources.getMemory())
+            + " currentCapacity " + ((float) queueUsage.getUsed().getMemory())
             / clusterResource.getMemory()
             + " potentialNewWithoutReservedCapacity: "
             + capacityWithoutReservedCapacity + " ( " + " max-capacity: "
@@ -579,7 +574,7 @@ private synchronized CSAssignment assignContainersToChildQueues(Resource cluster
     printChildQueues();
 
     // Try to assign to most 'under-served' sub-queue
-    for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
+    for (Iterator<CSQueue> iter = childQueues.iterator(); iter.hasNext();) {
       CSQueue childQueue = iter.next();
       if(LOG.isDebugEnabled()) {
         LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
@@ -641,34 +636,34 @@ public void completedContainer(Resource clusterResource,
       // Book keeping
       synchronized (this) {
         super.releaseResource(clusterResource, rmContainer.getContainer()
-            .getResource(), labelManager.getLabelsOnNode(node.getNodeID()));
+            .getResource(), node.getLabels());
 
         LOG.info("completedContainer" +
             " queue=" + getQueueName() + 
             " usedCapacity=" + getUsedCapacity() +
             " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
-            " used=" + usedResources + 
+            " used=" + queueUsage.getUsed() + 
             " cluster=" + clusterResource);
-      }
 
-      // Note that this is using an iterator on the childQueues so this can't be
-      // called if already within an iterator for the childQueues. Like  
-      // from assignContainersToChildQueues.
-      if (sortQueues) {
-        // reinsert the updated queue
-        for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
-          CSQueue csqueue = iter.next();
-          if(csqueue.equals(completedChildQueue))
-          {
-            iter.remove();
-            LOG.info("Re-sorting completed queue: " + csqueue.getQueuePath() + 
-                " stats: " + csqueue);
-            childQueues.add(csqueue);
-            break;
+        // Note that this is using an iterator on the childQueues so this can't
+        // be called if already within an iterator for the childQueues. Like
+        // from assignContainersToChildQueues.
+        if (sortQueues) {
+          // reinsert the updated queue
+          for (Iterator<CSQueue> iter = childQueues.iterator();
+               iter.hasNext();) {
+            CSQueue csqueue = iter.next();
+            if(csqueue.equals(completedChildQueue)) {
+              iter.remove();
+              LOG.info("Re-sorting completed queue: " + csqueue.getQueuePath() +
+                  " stats: " + csqueue);
+              childQueues.add(csqueue);
+              break;
+            }
           }
         }
       }
-      
+
       // Inform the parent
       if (parent != null) {
         // complete my parent
@@ -703,9 +698,10 @@ public void recoverContainer(Resource clusterResource,
     }
     // Careful! Locking order is important! 
     synchronized (this) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       super.allocateResource(clusterResource, rmContainer.getContainer()
-          .getResource(), labelManager.getLabelsOnNode(rmContainer
-          .getContainer().getNodeId()));
+          .getResource(), node.getLabels());
     }
     if (parent != null) {
       parent.recoverContainer(clusterResource, attempt, rmContainer);
@@ -719,7 +715,7 @@ public ActiveUsersManager getActiveUsersManager() {
   }
 
   @Override
-  public void collectSchedulerApplications(
+  public synchronized void collectSchedulerApplications(
       Collection<ApplicationAttemptId> apps) {
     for (CSQueue queue : childQueues) {
       queue.collectSchedulerApplications(apps);
@@ -730,12 +726,13 @@ public void collectSchedulerApplications(
   public void attachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       super.allocateResource(clusterResource, rmContainer.getContainer()
-          .getResource(), labelManager.getLabelsOnNode(rmContainer
-          .getContainer().getNodeId()));
+          .getResource(), node.getLabels());
       LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
           + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
-          + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+          + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
           + clusterResource);
       // Inform the parent
       if (parent != null) {
@@ -748,12 +745,14 @@ public void attachContainer(Resource clusterResource,
   public void detachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
+      FiCaSchedulerNode node =
+          scheduler.getNode(rmContainer.getContainer().getNodeId());
       super.releaseResource(clusterResource,
           rmContainer.getContainer().getResource(),
-          labelManager.getLabelsOnNode(rmContainer.getContainer().getNodeId()));
+          node.getLabels());
       LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
           + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
-          + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+          + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
           + clusterResource);
       // Inform the parent
       if (parent != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 0725959ccaa45..f8b11ebf0dcd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -37,8 +37,6 @@
  */
 public class PlanQueue extends ParentQueue {
 
-  public static final String DEFAULT_QUEUE_SUFFIX = "-default";
-
   private static final Logger LOG = LoggerFactory.getLogger(PlanQueue.class);
 
   private int maxAppsForReservation;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
new file mode 100644
index 0000000000000..a0e6d8c9b17d7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+public class QueueCapacities {
+  private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+  private static final float LABEL_DOESNT_EXIST_CAP = 0f;
+  private Map<String, Capacities> capacitiesMap;
+  private ReadLock readLock;
+  private WriteLock writeLock;
+
+  public QueueCapacities() {
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    readLock = lock.readLock();
+    writeLock = lock.writeLock();
+
+    capacitiesMap = new HashMap<String, Capacities>();
+  }
+  
+  // Usage enum here to make implement cleaner
+  private enum CapacityType {
+    USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5);
+
+    private int idx;
+
+    private CapacityType(int idx) {
+      this.idx = idx;
+    }
+  }
+
+  private static class Capacities {
+    private float[] capacitiesArr;
+    
+    public Capacities() {
+      capacitiesArr = new float[CapacityType.values().length];
+    }
+  }
+  
+  private float _get(String label, CapacityType type) {
+    try {
+      readLock.lock();
+      Capacities cap = capacitiesMap.get(label);
+      if (null == cap) {
+        return LABEL_DOESNT_EXIST_CAP;
+      }
+      return cap.capacitiesArr[type.idx];
+    } finally {
+      readLock.unlock();
+    }
+  }
+  
+  private void _set(String label, CapacityType type, float value) {
+    try {
+      writeLock.lock();
+      Capacities cap = capacitiesMap.get(label);
+      if (null == cap) {
+        cap = new Capacities();
+        capacitiesMap.put(label, cap);
+      }
+      cap.capacitiesArr[type.idx] = value;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  /* Used Capacity Getter and Setter */
+  public float getUsedCapacity() {
+    return _get(NL, CapacityType.USED_CAP);
+  }
+
+  public float getUsedCapacity(String label) {
+    return _get(label, CapacityType.USED_CAP);
+  }
+
+  public void setUsedCapacity(float value) {
+    _set(NL, CapacityType.USED_CAP, value);
+  }
+
+  public void setUsedCapacity(String label, float value) {
+    _set(label, CapacityType.USED_CAP, value);
+  }
+
+  /* Absolute Used Capacity Getter and Setter */
+  public float getAbsoluteUsedCapacity() {
+    return _get(NL, CapacityType.ABS_USED_CAP);
+  }
+
+  public float getAbsoluteUsedCapacity(String label) {
+    return _get(label, CapacityType.ABS_USED_CAP);
+  }
+
+  public void setAbsoluteUsedCapacity(float value) {
+    _set(NL, CapacityType.ABS_USED_CAP, value);
+  }
+
+  public void setAbsoluteUsedCapacity(String label, float value) {
+    _set(label, CapacityType.ABS_USED_CAP, value);
+  }
+
+  /* Capacity Getter and Setter */
+  public float getCapacity() {
+    return _get(NL, CapacityType.CAP);
+  }
+
+  public float getCapacity(String label) {
+    return _get(label, CapacityType.CAP);
+  }
+
+  public void setCapacity(float value) {
+    _set(NL, CapacityType.CAP, value);
+  }
+
+  public void setCapacity(String label, float value) {
+    _set(label, CapacityType.CAP, value);
+  }
+
+  /* Absolute Capacity Getter and Setter */
+  public float getAbsoluteCapacity() {
+    return _get(NL, CapacityType.ABS_CAP);
+  }
+
+  public float getAbsoluteCapacity(String label) {
+    return _get(label, CapacityType.ABS_CAP);
+  }
+
+  public void setAbsoluteCapacity(float value) {
+    _set(NL, CapacityType.ABS_CAP, value);
+  }
+
+  public void setAbsoluteCapacity(String label, float value) {
+    _set(label, CapacityType.ABS_CAP, value);
+  }
+
+  /* Maximum Capacity Getter and Setter */
+  public float getMaximumCapacity() {
+    return _get(NL, CapacityType.MAX_CAP);
+  }
+
+  public float getMaximumCapacity(String label) {
+    return _get(label, CapacityType.MAX_CAP);
+  }
+
+  public void setMaximumCapacity(float value) {
+    _set(NL, CapacityType.MAX_CAP, value);
+  }
+
+  public void setMaximumCapacity(String label, float value) {
+    _set(label, CapacityType.MAX_CAP, value);
+  }
+
+  /* Absolute Maximum Capacity Getter and Setter */
+  public float getAbsoluteMaximumCapacity() {
+    return _get(NL, CapacityType.ABS_MAX_CAP);
+  }
+
+  public float getAbsoluteMaximumCapacity(String label) {
+    return _get(label, CapacityType.ABS_MAX_CAP);
+  }
+
+  public void setAbsoluteMaximumCapacity(float value) {
+    _set(NL, CapacityType.ABS_MAX_CAP, value);
+  }
+
+  public void setAbsoluteMaximumCapacity(String label, float value) {
+    _set(label, CapacityType.ABS_MAX_CAP, value);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 2f9569c6cbc34..9f97b137f2ddb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 
 /**
  * Represents an application attempt from the viewpoint of the FIFO or Capacity
@@ -72,6 +73,20 @@ public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
       String user, Queue queue, ActiveUsersManager activeUsersManager,
       RMContext rmContext) {
     super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
+    
+    RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
+    
+    Resource amResource;
+    if (rmApp == null || rmApp.getAMResourceRequest() == null) {
+      //the rmApp may be undefined (the resource manager checks for this too)
+      //and unmanaged applications do not provide an amResource request
+      //in these cases, provide a default using the scheduler
+      amResource = rmContext.getScheduler().getMinimumResourceCapability();
+    } else {
+      amResource = rmApp.getAMResourceRequest().getCapability();
+    }
+    
+    setAMResource(amResource);
   }
 
   synchronized public boolean containerCompleted(RMContainer rmContainer,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 5227aac195dd4..fe6db47d283a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -19,12 +19,14 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica;
 
 
+import java.util.Set;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -32,9 +34,14 @@
 public class FiCaSchedulerNode extends SchedulerNode {
 
   private static final Log LOG = LogFactory.getLog(FiCaSchedulerNode.class);
+  
+  public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName,
+      Set<String> nodeLabels) {
+    super(node, usePortForNodeName, nodeLabels);
+  }
 
   public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) {
-    super(node, usePortForNodeName);
+    this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
new file mode 100644
index 0000000000000..7723e25c6ce4a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+public class NodeLabelsUpdateSchedulerEvent extends SchedulerEvent {
+  private Map<NodeId, Set<String>> nodeToLabels;
+
+  public NodeLabelsUpdateSchedulerEvent(Map<NodeId, Set<String>> nodeToLabels) {
+    super(SchedulerEventType.NODE_LABELS_UPDATE);
+    this.nodeToLabels = nodeToLabels;
+  }
+  
+  public Map<NodeId, Set<String>> getUpdatedNodeToLabels() {
+    return nodeToLabels;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 062f831c4ca7a..13aecb3f2de18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -25,6 +25,7 @@ public enum SchedulerEventType {
   NODE_REMOVED,
   NODE_UPDATE,
   NODE_RESOURCE_UPDATE,
+  NODE_LABELS_UPDATE,
 
   // Source: RMApp
   APP_ADDED,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 70a6496a4a9e2..0ea731403029e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -27,12 +27,13 @@
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.annotations.VisibleForTesting;
 
-public class AllocationConfiguration {
+public class AllocationConfiguration extends ReservationSchedulerConfiguration {
   private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
   private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
   
@@ -76,6 +77,8 @@ public class AllocationConfiguration {
   // preempt other queues' tasks.
   private final Map<String, Float> fairSharePreemptionThresholds;
 
+  private final Set<String> reservableQueues;
+
   private final Map<String, SchedulingPolicy> schedulingPolicies;
   
   private final SchedulingPolicy defaultSchedulingPolicy;
@@ -87,7 +90,10 @@ public class AllocationConfiguration {
   //Configured queues in the alloc xml
   @VisibleForTesting
   Map<FSQueueType, Set<String>> configuredQueues;
-  
+
+  // Reservation system configuration
+  private ReservationQueueConfiguration globalReservationQueueConfig;
+
   public AllocationConfiguration(Map<String, Resource> minQueueResources,
       Map<String, Resource> maxQueueResources,
       Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
@@ -101,7 +107,9 @@ public AllocationConfiguration(Map<String, Resource> minQueueResources,
       Map<String, Float> fairSharePreemptionThresholds,
       Map<String, Map<QueueACL, AccessControlList>> queueAcls,
       QueuePlacementPolicy placementPolicy,
-      Map<FSQueueType, Set<String>> configuredQueues) {
+      Map<FSQueueType, Set<String>> configuredQueues,
+      ReservationQueueConfiguration globalReservationQueueConfig,
+      Set<String> reservableQueues) {
     this.minQueueResources = minQueueResources;
     this.maxQueueResources = maxQueueResources;
     this.queueMaxApps = queueMaxApps;
@@ -117,6 +125,8 @@ public AllocationConfiguration(Map<String, Resource> minQueueResources,
     this.fairSharePreemptionTimeouts = fairSharePreemptionTimeouts;
     this.fairSharePreemptionThresholds = fairSharePreemptionThresholds;
     this.queueAcls = queueAcls;
+    this.reservableQueues = reservableQueues;
+    this.globalReservationQueueConfig = globalReservationQueueConfig;
     this.placementPolicy = placementPolicy;
     this.configuredQueues = configuredQueues;
   }
@@ -137,6 +147,7 @@ public AllocationConfiguration(Configuration conf) {
     fairSharePreemptionThresholds = new HashMap<String, Float>();
     schedulingPolicies = new HashMap<String, SchedulingPolicy>();
     defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
+    reservableQueues = new HashSet<>();
     configuredQueues = new HashMap<FSQueueType, Set<String>>();
     for (FSQueueType queueType : FSQueueType.values()) {
       configuredQueues.put(queueType, new HashSet<String>());
@@ -196,6 +207,10 @@ public ResourceWeights getQueueWeight(String queue) {
     ResourceWeights weight = queueWeights.get(queue);
     return (weight == null) ? ResourceWeights.NEUTRAL : weight;
   }
+
+  public void setQueueWeight(String queue, ResourceWeights weight) {
+    queueWeights.put(queue, weight);
+  }
   
   public int getUserMaxApps(String user) {
     Integer maxApps = userMaxApps.get(user);
@@ -262,4 +277,64 @@ public Map<FSQueueType, Set<String>> getConfiguredQueues() {
   public QueuePlacementPolicy getPlacementPolicy() {
     return placementPolicy;
   }
+
+  @Override
+  public boolean isReservable(String queue) {
+    return reservableQueues.contains(queue);
+  }
+
+  @Override
+  public long getReservationWindow(String queue) {
+    return globalReservationQueueConfig.getReservationWindowMsec();
+  }
+
+  @Override
+  public float getAverageCapacity(String queue) {
+    return globalReservationQueueConfig.getAvgOverTimeMultiplier() * 100;
+  }
+
+  @Override
+  public float getInstantaneousMaxCapacity(String queue) {
+    return globalReservationQueueConfig.getMaxOverTimeMultiplier() * 100;
+  }
+
+  @Override
+  public String getReservationAdmissionPolicy(String queue) {
+    return globalReservationQueueConfig.getReservationAdmissionPolicy();
+  }
+
+  @Override
+  public String getReservationAgent(String queue) {
+    return globalReservationQueueConfig.getReservationAgent();
+  }
+
+  @Override
+  public boolean getShowReservationAsQueues(String queue) {
+    return globalReservationQueueConfig.shouldShowReservationAsQueues();
+  }
+
+  @Override
+  public String getReplanner(String queue) {
+    return globalReservationQueueConfig.getPlanner();
+  }
+
+  @Override
+  public boolean getMoveOnExpiry(String queue) {
+    return globalReservationQueueConfig.shouldMoveOnExpiry();
+  }
+
+  @Override
+  public long getEnforcementWindow(String queue) {
+    return globalReservationQueueConfig.getEnforcementWindowMsec();
+  }
+
+  @VisibleForTesting
+  public void setReservationWindow(long window) {
+    globalReservationQueueConfig.setReservationWindow(window);
+  }
+
+  @VisibleForTesting
+  public void setAverageCapacity(int avgCapacity) {
+    globalReservationQueueConfig.setAverageCapacity(avgCapacity);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index e0e23e0cb1c6c..76fa588fc767f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -86,7 +86,7 @@ public class AllocationFileLoaderService extends AbstractService {
   
   private Thread reloadThread;
   private volatile boolean running = true;
-  
+
   public AllocationFileLoaderService() {
     this(new SystemClock());
   }
@@ -222,6 +222,7 @@ public synchronized void reloadAllocations() throws IOException,
         new HashMap<String, Float>();
     Map<String, Map<QueueACL, AccessControlList>> queueAcls =
         new HashMap<String, Map<QueueACL, AccessControlList>>();
+    Set<String> reservableQueues = new HashSet<String>();
     int userMaxAppsDefault = Integer.MAX_VALUE;
     int queueMaxAppsDefault = Integer.MAX_VALUE;
     float queueMaxAMShareDefault = 0.5f;
@@ -230,6 +231,11 @@ public synchronized void reloadAllocations() throws IOException,
     float defaultFairSharePreemptionThreshold = 0.5f;
     SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY;
 
+    // Reservation global configuration knobs
+    String planner = null;
+    String reservationAgent = null;
+    String reservationAdmissionPolicy = null;
+
     QueuePlacementPolicy newPlacementPolicy = null;
 
     // Remember all queue names so we can display them on web UI, etc.
@@ -317,6 +323,15 @@ public synchronized void reloadAllocations() throws IOException,
           defaultSchedPolicy = SchedulingPolicy.parse(text);
         } else if ("queuePlacementPolicy".equals(element.getTagName())) {
           placementPolicyElement = element;
+        } else if ("reservation-planner".equals(element.getTagName())) {
+          String text = ((Text) element.getFirstChild()).getData().trim();
+          planner = text;
+        } else if ("reservation-agent".equals(element.getTagName())) {
+          String text = ((Text) element.getFirstChild()).getData().trim();
+          reservationAgent = text;
+        } else if ("reservation-policy".equals(element.getTagName())) {
+          String text = ((Text) element.getFirstChild()).getData().trim();
+          reservationAdmissionPolicy = text;
         } else {
           LOG.warn("Bad element in allocations file: " + element.getTagName());
         }
@@ -337,7 +352,8 @@ public synchronized void reloadAllocations() throws IOException,
       loadQueue(parent, element, minQueueResources, maxQueueResources,
           queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights,
           queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts,
-          fairSharePreemptionThresholds, queueAcls, configuredQueues);
+          fairSharePreemptionThresholds, queueAcls, configuredQueues,
+          reservableQueues);
     }
 
     // Load placement policy and pass it configured queues
@@ -366,13 +382,27 @@ public synchronized void reloadAllocations() throws IOException,
           defaultFairSharePreemptionThreshold);
     }
 
+    ReservationQueueConfiguration globalReservationQueueConfig = new
+        ReservationQueueConfiguration();
+    if (planner != null) {
+      globalReservationQueueConfig.setPlanner(planner);
+    }
+    if (reservationAdmissionPolicy != null) {
+      globalReservationQueueConfig.setReservationAdmissionPolicy
+          (reservationAdmissionPolicy);
+    }
+    if (reservationAgent != null) {
+      globalReservationQueueConfig.setReservationAgent(reservationAgent);
+    }
+
     AllocationConfiguration info = new AllocationConfiguration(minQueueResources,
         maxQueueResources, queueMaxApps, userMaxApps, queueWeights,
         queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault,
         queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy,
         minSharePreemptionTimeouts, fairSharePreemptionTimeouts,
         fairSharePreemptionThresholds, queueAcls,
-        newPlacementPolicy, configuredQueues);
+        newPlacementPolicy, configuredQueues, globalReservationQueueConfig,
+        reservableQueues);
     
     lastSuccessfulReload = clock.getTime();
     lastReloadAttemptFailed = false;
@@ -392,8 +422,9 @@ private void loadQueue(String parentName, Element element,
       Map<String, Long> minSharePreemptionTimeouts,
       Map<String, Long> fairSharePreemptionTimeouts,
       Map<String, Float> fairSharePreemptionThresholds,
-      Map<String, Map<QueueACL, AccessControlList>> queueAcls, 
-      Map<FSQueueType, Set<String>> configuredQueues) 
+      Map<String, Map<QueueACL, AccessControlList>> queueAcls,
+      Map<FSQueueType, Set<String>> configuredQueues,
+      Set<String> reservableQueues)
       throws AllocationConfigurationException {
     String queueName = element.getAttribute("name");
 
@@ -460,14 +491,17 @@ private void loadQueue(String parentName, Element element,
       } else if ("aclAdministerApps".equals(field.getTagName())) {
         String text = ((Text)field.getFirstChild()).getData();
         acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text));
+      } else if ("reservation".equals(field.getTagName())) {
+        isLeaf = false;
+        reservableQueues.add(queueName);
+        configuredQueues.get(FSQueueType.PARENT).add(queueName);
       } else if ("queue".endsWith(field.getTagName()) || 
           "pool".equals(field.getTagName())) {
         loadQueue(queueName, field, minQueueResources, maxQueueResources,
             queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights,
             queuePolicies, minSharePreemptionTimeouts,
             fairSharePreemptionTimeouts, fairSharePreemptionThresholds,
-            queueAcls, configuredQueues);
-        configuredQueues.get(FSQueueType.PARENT).add(queueName);
+            queueAcls, configuredQueues, reservableQueues);
         isLeaf = false;
       }
     }
@@ -479,6 +513,13 @@ private void loadQueue(String parentName, Element element,
       } else {
         configuredQueues.get(FSQueueType.LEAF).add(queueName);
       }
+    } else {
+      if ("parent".equals(element.getAttribute("type"))) {
+        throw new AllocationConfigurationException("Both <reservation> and " +
+            "type=\"parent\" found for queue " + queueName + " which is " +
+            "unsupported");
+      }
+      configuredQueues.get(FSQueueType.PARENT).add(queueName);
     }
     queueAcls.put(queueName, acls);
     if (maxQueueResources.containsKey(queueName) &&
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index b9966e7f5511a..2cb0f0bb9eadb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -172,7 +172,7 @@ private synchronized void unreserveInternal(
   }
 
   @Override
-  public synchronized Resource getHeadroom() {
+  public Resource getHeadroom() {
     final FSQueue queue = (FSQueue) this.queue;
     SchedulingPolicy policy = queue.getPolicy();
 
@@ -530,6 +530,10 @@ private Resource assignContainer(
 
       return container.getResource();
     } else {
+      if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
+        return Resources.none();
+      }
+
       // The desired container won't fit here, so reserve
       reserve(request.getPriority(), node, container, reserved);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 345ea8b7c365a..3c975356dfe4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -23,6 +23,9 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
@@ -34,6 +37,7 @@
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
@@ -50,6 +54,10 @@ public class FSLeafQueue extends FSQueue {
       new ArrayList<FSAppAttempt>();
   private final List<FSAppAttempt> nonRunnableApps =
       new ArrayList<FSAppAttempt>();
+  // get a lock with fair distribution for app list updates
+  private final ReadWriteLock rwl = new ReentrantReadWriteLock(true);
+  private final Lock readLock = rwl.readLock();
+  private final Lock writeLock = rwl.writeLock();
   
   private Resource demand = Resources.createResource(0);
   
@@ -72,16 +80,26 @@ public FSLeafQueue(String name, FairScheduler scheduler,
   }
   
   public void addApp(FSAppAttempt app, boolean runnable) {
-    if (runnable) {
-      runnableApps.add(app);
-    } else {
-      nonRunnableApps.add(app);
+    writeLock.lock();
+    try {
+      if (runnable) {
+        runnableApps.add(app);
+      } else {
+        nonRunnableApps.add(app);
+      }
+    } finally {
+      writeLock.unlock();
     }
   }
   
   // for testing
   void addAppSchedulable(FSAppAttempt appSched) {
-    runnableApps.add(appSched);
+    writeLock.lock();
+    try {
+      runnableApps.add(appSched);
+    } finally {
+      writeLock.unlock();
+    }
   }
   
   /**
@@ -89,36 +107,108 @@ void addAppSchedulable(FSAppAttempt appSched) {
    * @return whether or not the app was runnable
    */
   public boolean removeApp(FSAppAttempt app) {
-    if (runnableApps.remove(app)) {
-      // Update AM resource usage
-      if (app.isAmRunning() && app.getAMResource() != null) {
-        Resources.subtractFrom(amResourceUsage, app.getAMResource());
+    boolean runnable = false;
+
+    // Remove app from runnable/nonRunnable list while holding the write lock
+    writeLock.lock();
+    try {
+      runnable = runnableApps.remove(app);
+      if (!runnable) {
+        // removeNonRunnableApp acquires the write lock again, which is fine
+        if (!removeNonRunnableApp(app)) {
+          throw new IllegalStateException("Given app to remove " + app +
+              " does not exist in queue " + this);
+        }
       }
-      return true;
-    } else if (nonRunnableApps.remove(app)) {
-      return false;
-    } else {
-      throw new IllegalStateException("Given app to remove " + app +
-          " does not exist in queue " + this);
+    } finally {
+      writeLock.unlock();
+    }
+
+    // Update AM resource usage if needed
+    if (runnable && app.isAmRunning() && app.getAMResource() != null) {
+      Resources.subtractFrom(amResourceUsage, app.getAMResource());
     }
+
+    return runnable;
   }
-  
-  public Collection<FSAppAttempt> getRunnableAppSchedulables() {
-    return runnableApps;
+
+  /**
+   * Removes the given app if it is non-runnable and belongs to this queue
+   * @return true if the app is removed, false otherwise
+   */
+  public boolean removeNonRunnableApp(FSAppAttempt app) {
+    writeLock.lock();
+    try {
+      return nonRunnableApps.remove(app);
+    } finally {
+      writeLock.unlock();
+    }
   }
-  
-  public List<FSAppAttempt> getNonRunnableAppSchedulables() {
-    return nonRunnableApps;
+
+  public boolean isRunnableApp(FSAppAttempt attempt) {
+    readLock.lock();
+    try {
+      return runnableApps.contains(attempt);
+    } finally {
+      readLock.unlock();
+    }
   }
-  
+
+  public boolean isNonRunnableApp(FSAppAttempt attempt) {
+    readLock.lock();
+    try {
+      return nonRunnableApps.contains(attempt);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public void resetPreemptedResources() {
+    readLock.lock();
+    try {
+      for (FSAppAttempt attempt : runnableApps) {
+        attempt.resetPreemptedResources();
+      }
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public void clearPreemptedResources() {
+    readLock.lock();
+    try {
+      for (FSAppAttempt attempt : runnableApps) {
+        attempt.clearPreemptedResources();
+      }
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public List<FSAppAttempt> getCopyOfNonRunnableAppSchedulables() {
+    List<FSAppAttempt> appsToReturn = new ArrayList<FSAppAttempt>();
+    readLock.lock();
+    try {
+      appsToReturn.addAll(nonRunnableApps);
+    } finally {
+      readLock.unlock();
+    }
+    return appsToReturn;
+  }
+
   @Override
   public void collectSchedulerApplications(
       Collection<ApplicationAttemptId> apps) {
-    for (FSAppAttempt appSched : runnableApps) {
-      apps.add(appSched.getApplicationAttemptId());
-    }
-    for (FSAppAttempt appSched : nonRunnableApps) {
-      apps.add(appSched.getApplicationAttemptId());
+    readLock.lock();
+    try {
+      for (FSAppAttempt appSched : runnableApps) {
+        apps.add(appSched.getApplicationAttemptId());
+      }
+      for (FSAppAttempt appSched : nonRunnableApps) {
+        apps.add(appSched.getApplicationAttemptId());
+      }
+    } finally {
+      readLock.unlock();
     }
   }
 
@@ -133,7 +223,12 @@ public void setPolicy(SchedulingPolicy policy)
   
   @Override
   public void recomputeShares() {
-    policy.computeShares(getRunnableAppSchedulables(), getFairShare());
+    readLock.lock();
+    try {
+      policy.computeShares(runnableApps, getFairShare());
+    } finally {
+      readLock.unlock();
+    }
   }
 
   @Override
@@ -144,11 +239,16 @@ public Resource getDemand() {
   @Override
   public Resource getResourceUsage() {
     Resource usage = Resources.createResource(0);
-    for (FSAppAttempt app : runnableApps) {
-      Resources.addTo(usage, app.getResourceUsage());
-    }
-    for (FSAppAttempt app : nonRunnableApps) {
-      Resources.addTo(usage, app.getResourceUsage());
+    readLock.lock();
+    try {
+      for (FSAppAttempt app : runnableApps) {
+        Resources.addTo(usage, app.getResourceUsage());
+      }
+      for (FSAppAttempt app : nonRunnableApps) {
+        Resources.addTo(usage, app.getResourceUsage());
+      }
+    } finally {
+      readLock.unlock();
     }
     return usage;
   }
@@ -164,17 +264,22 @@ public void updateDemand() {
     Resource maxRes = scheduler.getAllocationConfiguration()
         .getMaxResources(getName());
     demand = Resources.createResource(0);
-    for (FSAppAttempt sched : runnableApps) {
-      if (Resources.equals(demand, maxRes)) {
-        break;
+    readLock.lock();
+    try {
+      for (FSAppAttempt sched : runnableApps) {
+        if (Resources.equals(demand, maxRes)) {
+          break;
+        }
+        updateDemandForApp(sched, maxRes);
       }
-      updateDemandForApp(sched, maxRes);
-    }
-    for (FSAppAttempt sched : nonRunnableApps) {
-      if (Resources.equals(demand, maxRes)) {
-        break;
+      for (FSAppAttempt sched : nonRunnableApps) {
+        if (Resources.equals(demand, maxRes)) {
+          break;
+        }
+        updateDemandForApp(sched, maxRes);
       }
-      updateDemandForApp(sched, maxRes);
+    } finally {
+      readLock.unlock();
     }
     if (LOG.isDebugEnabled()) {
       LOG.debug("The updated demand for " + getName() + " is " + demand
@@ -198,7 +303,8 @@ private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) {
   public Resource assignContainer(FSSchedulerNode node) {
     Resource assigned = Resources.none();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Node " + node.getNodeName() + " offered to queue: " + getName());
+      LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
+          getName());
     }
 
     if (!assignContainerPreCheck(node)) {
@@ -206,16 +312,29 @@ public Resource assignContainer(FSSchedulerNode node) {
     }
 
     Comparator<Schedulable> comparator = policy.getComparator();
-    Collections.sort(runnableApps, comparator);
-    for (FSAppAttempt sched : runnableApps) {
-      if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
-        continue;
-      }
-
-      assigned = sched.assignContainer(node);
-      if (!assigned.equals(Resources.none())) {
-        break;
+    writeLock.lock();
+    try {
+      Collections.sort(runnableApps, comparator);
+    } finally {
+      writeLock.unlock();
+    }
+    // Release write lock here for better performance and avoiding deadlocks.
+    // runnableApps can be in unsorted state because of this section,
+    // but we can accept it in practice since the probability is low.
+    readLock.lock();
+    try {
+      for (FSAppAttempt sched : runnableApps) {
+        if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
+          continue;
+        }
+
+        assigned = sched.assignContainer(node);
+        if (!assigned.equals(Resources.none())) {
+          break;
+        }
       }
+    } finally {
+      readLock.unlock();
     }
     return assigned;
   }
@@ -237,11 +356,16 @@ public RMContainer preemptContainer() {
     // Choose the app that is most over fair share
     Comparator<Schedulable> comparator = policy.getComparator();
     FSAppAttempt candidateSched = null;
-    for (FSAppAttempt sched : runnableApps) {
-      if (candidateSched == null ||
-          comparator.compare(sched, candidateSched) > 0) {
-        candidateSched = sched;
+    readLock.lock();
+    try {
+      for (FSAppAttempt sched : runnableApps) {
+        if (candidateSched == null ||
+            comparator.compare(sched, candidateSched) > 0) {
+          candidateSched = sched;
+        }
       }
+    } finally {
+      readLock.unlock();
     }
 
     // Preempt from the selected app
@@ -291,9 +415,58 @@ private void setLastTimeAtFairShareThreshold(
 
   @Override
   public int getNumRunnableApps() {
-    return runnableApps.size();
+    readLock.lock();
+    try {
+      return runnableApps.size();
+    } finally {
+      readLock.unlock();
+    }
   }
-  
+
+  public int getNumNonRunnableApps() {
+    readLock.lock();
+    try {
+      return nonRunnableApps.size();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public int getNumPendingApps() {
+    int numPendingApps = 0;
+    readLock.lock();
+    try {
+      for (FSAppAttempt attempt : runnableApps) {
+        if (attempt.isPending()) {
+          numPendingApps++;
+        }
+      }
+      numPendingApps += nonRunnableApps.size();
+    } finally {
+      readLock.unlock();
+    }
+    return numPendingApps;
+  }
+
+  /**
+   * TODO: Based on how frequently this is called, we might want to club
+   * counting pending and active apps in the same method.
+   */
+  public int getNumActiveApps() {
+    int numActiveApps = 0;
+    readLock.lock();
+    try {
+      for (FSAppAttempt attempt : runnableApps) {
+        if (!attempt.isPending()) {
+          numActiveApps++;
+        }
+      }
+    } finally {
+      readLock.unlock();
+    }
+    return numActiveApps;
+  }
+
   @Override
   public ActiveUsersManager getActiveUsersManager() {
     return activeUsersManager;
@@ -344,6 +517,15 @@ public void updateStarvationStats() {
     }
   }
 
+  /** Allows setting weight for a dynamically created queue
+   * Currently only used for reservation based queues
+   * @param weight queue weight
+   */
+  public void setWeights(float weight) {
+    scheduler.getAllocationConfiguration().setQueueWeight(getName(),
+        new ResourceWeights(weight));
+  }
+
   /**
    * Helper method to check if the queue should preempt containers
    *
@@ -372,9 +554,9 @@ boolean isStarvedForFairShare() {
   }
 
   private boolean isStarved(Resource share) {
-    Resource desiredShare = Resources.min(FairScheduler.getResourceCalculator(),
+    Resource desiredShare = Resources.min(scheduler.getResourceCalculator(),
         scheduler.getClusterResource(), share, getDemand());
-    return Resources.lessThan(FairScheduler.getResourceCalculator(),
+    return Resources.lessThan(scheduler.getResourceCalculator(),
         scheduler.getClusterResource(), getResourceUsage(), desiredShare);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a687e71807272..2b597164baf70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -68,6 +70,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.ContainersAndNMTokensAllocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -124,6 +127,8 @@ public class FairScheduler extends
   
   private static final ResourceCalculator RESOURCE_CALCULATOR =
       new DefaultResourceCalculator();
+  private static final ResourceCalculator DOMINANT_RESOURCE_CALCULATOR =
+      new DominantResourceCalculator();
   
   // Value that container assignment methods return when a container is
   // reserved
@@ -400,9 +405,7 @@ protected void preemptResources(Resource toPreempt) {
     try {
       // Reset preemptedResource for each app
       for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
-        for (FSAppAttempt app : queue.getRunnableAppSchedulables()) {
-          app.resetPreemptedResources();
-        }
+        queue.resetPreemptedResources();
       }
 
       while (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
@@ -421,9 +424,7 @@ protected void preemptResources(Resource toPreempt) {
     } finally {
       // Clear preemptedResources for each app
       for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
-        for (FSAppAttempt app : queue.getRunnableAppSchedulables()) {
-          app.clearPreemptedResources();
-        }
+        queue.clearPreemptedResources();
       }
     }
 
@@ -879,7 +880,8 @@ private synchronized void removeNode(RMNode rmNode) {
 
   @Override
   public Allocation allocate(ApplicationAttemptId appAttemptId,
-      List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
+      List<ResourceRequest> ask, List<ContainerId> release,
+      List<String> blacklistAdditions, List<String> blacklistRemovals) {
 
     // Make sure this application exists
     FSAppAttempt application = getSchedulerApp(appAttemptId);
@@ -890,7 +892,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId,
     }
 
     // Sanity check
-    SchedulerUtils.normalizeRequests(ask, new DominantResourceCalculator(),
+    SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR,
         clusterResource, minimumAllocation, getMaximumResourceCapability(),
         incrAllocation);
 
@@ -1047,7 +1049,8 @@ private synchronized void attemptScheduling(FSSchedulerNode node) {
       FSQueue queue = reservedAppSchedulable.getQueue();
 
       if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)
-          || !fitInMaxShare(queue)) {
+          || !fitsInMaxShare(queue,
+          node.getReservedContainer().getReservedResource())) {
         // Don't hold the reservation if app can no longer use it
         LOG.info("Releasing reservation that cannot be satisfied for application "
             + reservedAppSchedulable.getApplicationAttemptId()
@@ -1082,14 +1085,18 @@ private synchronized void attemptScheduling(FSSchedulerNode node) {
     updateRootQueueMetrics();
   }
 
-  private boolean fitInMaxShare(FSQueue queue) {
-    if (Resources.fitsIn(queue.getResourceUsage(), queue.getMaxShare())) {
+  static boolean fitsInMaxShare(FSQueue queue, Resource
+      additionalResource) {
+    Resource usagePlusAddition =
+        Resources.add(queue.getResourceUsage(), additionalResource);
+
+    if (!Resources.fitsIn(usagePlusAddition, queue.getMaxShare())) {
       return false;
     }
     
     FSQueue parentQueue = queue.getParent();
     if (parentQueue != null) {
-      return fitInMaxShare(parentQueue);
+      return fitsInMaxShare(parentQueue, additionalResource);
     }
     return true;
   }
@@ -1098,7 +1105,8 @@ public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
     return super.getApplicationAttempt(appAttemptId);
   }
 
-  public static ResourceCalculator getResourceCalculator() {
+  @Override
+  public ResourceCalculator getResourceCalculator() {
     return RESOURCE_CALCULATOR;
   }
 
@@ -1166,9 +1174,15 @@ public void handle(SchedulerEvent event) {
         throw new RuntimeException("Unexpected event type: " + event);
       }
       AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
-      addApplication(appAddedEvent.getApplicationId(),
-        appAddedEvent.getQueue(), appAddedEvent.getUser(),
-        appAddedEvent.getIsAppRecovering());
+      String queueName =
+          resolveReservationQueueName(appAddedEvent.getQueue(),
+              appAddedEvent.getApplicationId(),
+              appAddedEvent.getReservationID());
+      if (queueName != null) {
+        addApplication(appAddedEvent.getApplicationId(),
+            queueName, appAddedEvent.getUser(),
+            appAddedEvent.getIsAppRecovering());
+      }
       break;
     case APP_REMOVED:
       if (!(event instanceof AppRemovedSchedulerEvent)) {
@@ -1226,6 +1240,51 @@ public void handle(SchedulerEvent event) {
     }
   }
 
+  private synchronized String resolveReservationQueueName(String queueName,
+      ApplicationId applicationId, ReservationId reservationID) {
+    FSQueue queue = queueMgr.getQueue(queueName);
+    if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) {
+      return queueName;
+    }
+    // Use fully specified name from now on (including root. prefix)
+    queueName = queue.getQueueName();
+    if (reservationID != null) {
+      String resQName = queueName + "." + reservationID.toString();
+      queue = queueMgr.getQueue(resQName);
+      if (queue == null) {
+        String message =
+            "Application "
+                + applicationId
+                + " submitted to a reservation which is not yet currently active: "
+                + resQName;
+        this.rmContext.getDispatcher().getEventHandler()
+            .handle(new RMAppRejectedEvent(applicationId, message));
+        return null;
+      }
+      if (!queue.getParent().getQueueName().equals(queueName)) {
+        String message =
+            "Application: " + applicationId + " submitted to a reservation "
+                + resQName + " which does not belong to the specified queue: "
+                + queueName;
+        this.rmContext.getDispatcher().getEventHandler()
+            .handle(new RMAppRejectedEvent(applicationId, message));
+        return null;
+      }
+      // use the reservation queue to run the app
+      queueName = resQName;
+    } else {
+      // use the default child queue of the plan for unreserved apps
+      queueName = getDefaultQueueForPlanQueue(queueName);
+    }
+    return queueName;
+  }
+
+  private String getDefaultQueueForPlanQueue(String queueName) {
+    String planName = queueName.substring(queueName.lastIndexOf(".") + 1);
+    queueName = queueName + "." + planName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+    return queueName;
+  }
+
   @Override
   public void recover(RMState state) throws Exception {
     // NOT IMPLEMENTED
@@ -1444,7 +1503,8 @@ public synchronized String moveApplication(ApplicationId appId,
     // To serialize with FairScheduler#allocate, synchronize on app attempt
     synchronized (attempt) {
       FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
-      FSLeafQueue targetQueue = queueMgr.getLeafQueue(queueName, false);
+      String destQueueName = handleMoveToPlanQueue(queueName);
+      FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false);
       if (targetQueue == null) {
         throw new YarnException("Target queue " + queueName
             + " not found or is not a leaf queue.");
@@ -1453,7 +1513,7 @@ public synchronized String moveApplication(ApplicationId appId,
         return oldQueue.getQueueName();
       }
       
-      if (oldQueue.getRunnableAppSchedulables().contains(attempt)) {
+      if (oldQueue.isRunnableApp(attempt)) {
         verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue);
       }
       
@@ -1568,4 +1628,57 @@ public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
     return EnumSet
       .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
   }
+
+  @Override
+  public Set<String> getPlanQueues() throws YarnException {
+    Set<String> planQueues = new HashSet<String>();
+    for (FSQueue fsQueue : queueMgr.getQueues()) {
+      String queueName = fsQueue.getName();
+      if (allocConf.isReservable(queueName)) {
+        planQueues.add(queueName);
+      }
+    }
+    return planQueues;
+  }
+
+  @Override
+  public void setEntitlement(String queueName,
+      QueueEntitlement entitlement) throws YarnException {
+
+    FSLeafQueue reservationQueue = queueMgr.getLeafQueue(queueName, false);
+    if (reservationQueue == null) {
+      throw new YarnException("Target queue " + queueName
+          + " not found or is not a leaf queue.");
+    }
+
+    reservationQueue.setWeights(entitlement.getCapacity());
+
+    // TODO Does MaxCapacity need to be set for fairScheduler ?
+  }
+
+  /**
+   * Only supports removing empty leaf queues
+   * @param queueName name of queue to remove
+   * @throws YarnException if queue to remove is either not a leaf or if its
+   * not empty
+   */
+  @Override
+  public void removeQueue(String queueName) throws YarnException {
+    FSLeafQueue reservationQueue = queueMgr.getLeafQueue(queueName, false);
+    if (reservationQueue != null) {
+      if (!queueMgr.removeLeafQueue(queueName)) {
+        throw new YarnException("Could not remove queue " + queueName + " as " +
+            "its either not a leaf queue or its not empty");
+      }
+    }
+  }
+
+  private String handleMoveToPlanQueue(String targetQueueName) {
+    FSQueue dest = queueMgr.getQueue(targetQueueName);
+    if (dest != null && allocConf.isReservable(dest.getQueueName())) {
+      // use the default child reservation queue of the plan
+      targetQueueName = getDefaultQueueForPlanQueue(targetQueueName);
+    }
+    return targetQueueName;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
index feeda1e90c8d4..2c90edd400a47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
@@ -170,7 +170,7 @@ public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) {
       if (canAppBeRunnable(next.getQueue(), next.getUser())) {
         trackRunnableApp(next);
         FSAppAttempt appSched = next;
-        next.getQueue().getRunnableAppSchedulables().add(appSched);
+        next.getQueue().addApp(appSched, true);
         noLongerPendingApps.add(appSched);
 
         // No more than one app per list will be able to be made runnable, so
@@ -187,8 +187,7 @@ public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) {
     // pull them out from under the iterator.  If they are not in these lists
     // in the first place, there is a bug.
     for (FSAppAttempt appSched : noLongerPendingApps) {
-      if (!appSched.getQueue().getNonRunnableAppSchedulables()
-          .remove(appSched)) {
+      if (!appSched.getQueue().removeNonRunnableApp(appSched)) {
         LOG.error("Can't make app runnable that does not already exist in queue"
             + " as non-runnable: " + appSched + ". This should never happen.");
       }
@@ -239,7 +238,8 @@ private void gatherPossiblyRunnableAppLists(FSQueue queue,
     if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration()
         .getQueueMaxApps(queue.getName())) {
       if (queue instanceof FSLeafQueue) {
-        appLists.add(((FSLeafQueue)queue).getNonRunnableAppSchedulables());
+        appLists.add(
+            ((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
       } else {
         for (FSQueue child : queue.getChildQueues()) {
           gatherPossiblyRunnableAppLists(child, appLists);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 61b3b6c325f4a..27e571e4f6691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -91,7 +91,18 @@ public FSLeafQueue getLeafQueue(String name, boolean create) {
     }
     return (FSLeafQueue) queue;
   }
-  
+
+  /**
+   * Remove a leaf queue if empty
+   * @param name name of the queue
+   * @return true if queue was removed or false otherwise
+   */
+  public boolean removeLeafQueue(String name) {
+    name = ensureRootPrefix(name);
+    return removeEmptyIncompatibleQueues(name, FSQueueType.PARENT);
+  }
+
+
   /**
    * Get a parent queue by name, creating it if the create param is true and is necessary.
    * If the queue is not or can not be a parent queue, i.e. it already exists as a
@@ -297,7 +308,7 @@ protected boolean isEmpty(FSQueue queue) {
     if (queue instanceof FSLeafQueue) {
       FSLeafQueue leafQueue = (FSLeafQueue)queue;
       return queue.getNumRunnableApps() == 0 &&
-          leafQueue.getNonRunnableAppSchedulables().isEmpty();
+          leafQueue.getNumNonRunnableApps() == 0;
     } else {
       for (FSQueue child : queue.getChildQueues()) {
         if (!isEmpty(child)) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ReservationQueueConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ReservationQueueConfiguration.java
new file mode 100644
index 0000000000000..cf7f84e27abaa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ReservationQueueConfiguration.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ReservationQueueConfiguration {
+  private long reservationWindow;
+  private long enforcementWindow;
+  private String reservationAdmissionPolicy;
+  private String reservationAgent;
+  private String planner;
+  private boolean showReservationAsQueues;
+  private boolean moveOnExpiry;
+  private float avgOverTimeMultiplier;
+  private float maxOverTimeMultiplier;
+
+  public ReservationQueueConfiguration() {
+    this.reservationWindow = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_WINDOW;
+    this.enforcementWindow = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_ENFORCEMENT_WINDOW;
+    this.reservationAdmissionPolicy = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_ADMISSION_POLICY;
+    this.reservationAgent = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_AGENT_NAME;
+    this.planner = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_PLANNER_NAME;
+    this.showReservationAsQueues = ReservationSchedulerConfiguration
+        .DEFAULT_SHOW_RESERVATIONS_AS_QUEUES;
+    this.moveOnExpiry = ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_MOVE_ON_EXPIRY;
+    this.avgOverTimeMultiplier = ReservationSchedulerConfiguration
+        .DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
+    this.maxOverTimeMultiplier = ReservationSchedulerConfiguration
+        .DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
+  }
+
+  public long getReservationWindowMsec() {
+    return reservationWindow;
+  }
+
+  public long getEnforcementWindowMsec() {
+    return enforcementWindow;
+  }
+
+  public boolean shouldShowReservationAsQueues() {
+    return showReservationAsQueues;
+  }
+
+  public boolean shouldMoveOnExpiry() {
+    return moveOnExpiry;
+  }
+
+  public String getReservationAdmissionPolicy() {
+    return reservationAdmissionPolicy;
+  }
+
+  public String getReservationAgent() {
+    return reservationAgent;
+  }
+
+  public String getPlanner() {
+    return planner;
+  }
+
+  public float getAvgOverTimeMultiplier() {
+    return avgOverTimeMultiplier;
+  }
+
+  public float getMaxOverTimeMultiplier() {
+    return maxOverTimeMultiplier;
+  }
+
+  public void setPlanner(String planner) {
+    this.planner = planner;
+  }
+
+  public void setReservationAdmissionPolicy(String reservationAdmissionPolicy) {
+    this.reservationAdmissionPolicy = reservationAdmissionPolicy;
+  }
+
+  public void setReservationAgent(String reservationAgent) {
+    this.reservationAgent = reservationAgent;
+  }
+
+  @VisibleForTesting
+  public void setReservationWindow(long reservationWindow) {
+    this.reservationWindow = reservationWindow;
+  }
+
+  @VisibleForTesting
+  public void setAverageCapacity(int averageCapacity) {
+    this.avgOverTimeMultiplier = averageCapacity;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index 4f3123dffdd67..cc28afc1e494c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -48,10 +48,10 @@ public abstract class SchedulingPolicy {
    * Returns a {@link SchedulingPolicy} instance corresponding to the passed clazz
    */
   public static SchedulingPolicy getInstance(Class<? extends SchedulingPolicy> clazz) {
-    SchedulingPolicy policy = instances.get(clazz);
-    if (policy == null) {
-      policy = ReflectionUtils.newInstance(clazz, null);
-      instances.put(clazz, policy);
+    SchedulingPolicy policy = ReflectionUtils.newInstance(clazz, null);
+    SchedulingPolicy policyRet = instances.putIfAbsent(clazz, policy);
+    if(policyRet != null) {
+      return policyRet;
     }
     return policy;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 3d4c9dd9b5bf8..e00671536500b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -919,6 +919,11 @@ public List<QueueUserACLInfo> getQueueUserAclInfo() {
     return DEFAULT_QUEUE.getQueueUserAclInfo(null); 
   }
 
+  @Override
+  public ResourceCalculator getResourceCalculator() {
+    return resourceCalculator;
+  }
+
   private synchronized void addNode(RMNode nodeManager) {
     FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
         usePortForNodeName);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
index 56143386b520f..8adb2ed1ecadb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
@@ -96,7 +96,7 @@ public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
     if (rollingInterval <= activationDelay * 2) {
       throw new IllegalArgumentException(
           YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
-              + " should be more than 2 X "
+              + " should be more than 3 X "
               + YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS);
     }
   }
@@ -107,8 +107,8 @@ public void start() {
       AMRMTokenSecretManagerState state =
           AMRMTokenSecretManagerState.newInstance(
             this.currentMasterKey.getMasterKey(), null);
-      rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
-        false);
+      rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state,
+          false);
     }
     this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval,
       rollingInterval);
@@ -145,8 +145,8 @@ void rollMasterKey() {
           AMRMTokenSecretManagerState.newInstance(
             this.currentMasterKey.getMasterKey(),
             this.nextMasterKey.getMasterKey());
-      rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
-        true);
+      rmContext.getStateStore()
+          .storeOrUpdateAMRMTokenSecretManager(state, true);
       this.timer.schedule(new NextKeyActivator(), this.activationDelay);
     } finally {
       this.writeLock.unlock();
@@ -170,8 +170,8 @@ public void activateNextMasterKey() {
       AMRMTokenSecretManagerState state =
           AMRMTokenSecretManagerState.newInstance(
             this.currentMasterKey.getMasterKey(), null);
-      rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state,
-        true);
+      rmContext.getStateStore()
+          .storeOrUpdateAMRMTokenSecretManager(state, true);
     } finally {
       this.writeLock.unlock();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index cca6e8df0da6b..dfcceb8612268 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -69,7 +69,6 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
 /**
  * Service to renew application delegation tokens.
  */
@@ -94,6 +93,9 @@ public class DelegationTokenRenewer extends AbstractService {
   private ConcurrentMap<ApplicationId, Set<DelegationTokenToRenew>> appTokens =
       new ConcurrentHashMap<ApplicationId, Set<DelegationTokenToRenew>>();
 
+  private ConcurrentMap<Token<?>, DelegationTokenToRenew> allTokens =
+      new ConcurrentHashMap<Token<?>, DelegationTokenToRenew>();
+
   private final ConcurrentMap<ApplicationId, Long> delayedRemovalMap =
       new ConcurrentHashMap<ApplicationId, Long>();
 
@@ -148,7 +150,7 @@ protected ThreadPoolExecutor createNewThreadPoolService(Configuration conf) {
         .setNameFormat("DelegationTokenRenewer #%d")
         .build();
     ThreadPoolExecutor pool =
-        new ThreadPoolExecutor((5 < nThreads ? 5 : nThreads), nThreads, 3L,
+        new ThreadPoolExecutor(nThreads, nThreads, 3L,
             TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
     pool.setThreadFactory(tf);
     pool.allowCoreThreadTimeOut(true);
@@ -202,6 +204,7 @@ protected void serviceStop() {
       renewalTimer.cancel();
     }
     appTokens.clear();
+    allTokens.clear();
     this.renewerService.shutdown();
     dtCancelThread.interrupt();
     try {
@@ -230,7 +233,7 @@ protected static class DelegationTokenToRenew {
     public final Configuration conf;
     public long expirationDate;
     public TimerTask timerTask;
-    public final boolean shouldCancelAtEnd;
+    public volatile boolean shouldCancelAtEnd;
     public long maxDate;
     public String user;
 
@@ -407,12 +410,25 @@ private void handleAppSubmitEvent(DelegationTokenRenewerAppSubmitEvent evt)
     boolean hasHdfsToken = false;
     for (Token<?> token : tokens) {
       if (token.isManaged()) {
-        tokenList.add(new DelegationTokenToRenew(applicationId,
-            token, getConfig(), now, shouldCancelAtEnd, evt.getUser()));
         if (token.getKind().equals(new Text("HDFS_DELEGATION_TOKEN"))) {
           LOG.info(applicationId + " found existing hdfs token " + token);
           hasHdfsToken = true;
         }
+
+        DelegationTokenToRenew dttr = allTokens.get(token);
+        if (dttr != null) {
+          // If any of the jobs sharing the same token doesn't want to cancel
+          // the token, we should not cancel the token.
+          if (!evt.shouldCancelAtEnd) {
+            dttr.shouldCancelAtEnd = evt.shouldCancelAtEnd;
+            LOG.info("Set shouldCancelAtEnd=" + shouldCancelAtEnd
+                + " for token " + dttr.token);
+          }
+          continue;
+        }
+
+        tokenList.add(new DelegationTokenToRenew(applicationId, token,
+          getConfig(), now, shouldCancelAtEnd, evt.getUser()));
       }
     }
 
@@ -429,6 +445,7 @@ private void handleAppSubmitEvent(DelegationTokenRenewerAppSubmitEvent evt)
       }
       for (DelegationTokenToRenew dtr : tokenList) {
         appTokens.get(applicationId).add(dtr);
+        allTokens.put(dtr.token, dtr);
         setTimerForTokenRenewal(dtr);
       }
     }
@@ -496,7 +513,6 @@ protected void setTimerForTokenRenewal(DelegationTokenToRenew token)
     token.setTimerTask(tTask); // keep reference to the timer
 
     renewalTimer.schedule(token.timerTask, new Date(renewIn));
-
     LOG.info("Renew " + token + " in " + expiresIn + " ms, appId = "
         + token.applicationId);
   }
@@ -559,6 +575,10 @@ private void requestNewHdfsDelegationTokenIfNeeded(
   private void requestNewHdfsDelegationToken(ApplicationId applicationId,
       String user, boolean shouldCancelAtEnd) throws IOException,
       InterruptedException {
+    if (!hasProxyUserPrivileges) {
+      LOG.info("RM proxy-user privilege is not enabled. Skip requesting hdfs tokens.");
+      return;
+    }
     // Get new hdfs tokens for this user
     Credentials credentials = new Credentials();
     Token<?>[] newTokens = obtainSystemTokensForUser(user, credentials);
@@ -621,6 +641,8 @@ private void removeFailedDelegationToken(DelegationTokenToRenew t) {
     LOG.error("removing failed delegation token for appid=" + applicationId
         + ";t=" + t.token.getService());
     appTokens.get(applicationId).remove(t);
+    allTokens.remove(t.token);
+
     // cancel the timer
     if (t.timerTask != null) {
       t.timerTask.cancel();
@@ -685,9 +707,14 @@ private void removeApplicationFromRenewal(ApplicationId applicationId) {
           cancelToken(dttr);
 
           it.remove();
+          allTokens.remove(dttr.token);
         }
       }
     }
+
+    if(tokens != null && tokens.isEmpty()) {
+      appTokens.remove(applicationId);
+    }
   }
 
   /**
@@ -842,4 +869,9 @@ public ApplicationId getApplicationId() {
       return appId;
     }
   }
+
+  // only for testing
+  protected ConcurrentMap<Token<?>, DelegationTokenToRenew> getAllTokens() {
+    return allTokens;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
index b068a60e0e3f1..1471f780b56da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java
@@ -78,7 +78,7 @@ public NMTokenSecretManagerInRM(Configuration conf) {
     if (rollingInterval <= activationDelay * 2) {
       throw new IllegalArgumentException(
           YarnConfiguration.RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
-              + " should be more than 2 X "
+              + " should be more than 3 X "
               + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS);
     }
     appAttemptToNodeKeyMap =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
index 15dd1a9219e80..1595d17926acd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
@@ -80,8 +80,8 @@ public RMContainerTokenSecretManager(Configuration conf) {
     if (rollingInterval <= activationDelay * 2) {
       throw new IllegalArgumentException(
           YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
-              + " should be more than 2 X "
-              + YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS);
+              + " should be more than 3 X "
+              + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java
index 90706ff8c94b5..83defc5424707 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java
@@ -29,10 +29,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -109,8 +107,7 @@ protected void storeNewToken(RMDelegationTokenIdentifier identifier,
     try {
       LOG.info("storing RMDelegation token with sequence number: "
           + identifier.getSequenceNumber());
-      rmContext.getStateStore().storeRMDelegationTokenAndSequenceNumber(
-        identifier, renewDate, identifier.getSequenceNumber());
+      rmContext.getStateStore().storeRMDelegationToken(identifier, renewDate);
     } catch (Exception e) {
       LOG.error("Error in storing RMDelegationToken with sequence number: "
           + identifier.getSequenceNumber());
@@ -124,11 +121,10 @@ protected void updateStoredToken(RMDelegationTokenIdentifier id,
     try {
       LOG.info("updating RMDelegation token with sequence number: "
           + id.getSequenceNumber());
-      rmContext.getStateStore().updateRMDelegationTokenAndSequenceNumber(id,
-        renewDate, id.getSequenceNumber());
+      rmContext.getStateStore().updateRMDelegationToken(id, renewDate);
     } catch (Exception e) {
-      LOG.error("Error in updating persisted RMDelegationToken with sequence number: "
-            + id.getSequenceNumber());
+      LOG.error("Error in updating persisted RMDelegationToken" +
+                " with sequence number: " + id.getSequenceNumber());
       ExitUtil.terminate(1, e);
     }
   }
@@ -139,8 +135,7 @@ protected void removeStoredToken(RMDelegationTokenIdentifier ident)
     try {
       LOG.info("removing RMDelegation token with sequence number: "
           + ident.getSequenceNumber());
-      rmContext.getStateStore().removeRMDelegationToken(ident,
-        delegationTokenSequenceNumber);
+      rmContext.getStateStore().removeRMDelegationToken(ident);
     } catch (Exception e) {
       LOG.error("Error in removing RMDelegationToken with sequence number: "
           + ident.getSequenceNumber());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 89b4a78430cef..83df72b0376ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -114,14 +114,15 @@ protected void render(Block html) {
           _("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
           _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
           _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
-          _("Max Schedulable Applications:", Integer.toString(lqinfo.getMaxActiveApplications())).
-          _("Max Schedulable Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())).
+          _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()).
+          _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()).
           _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)).
           _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
           _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
           _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())).
           _r("Active Users: ", activeUserList.toString()).
-          _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels()));
+          _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())).
+          _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled");
 
       html._(InfoBlock.class);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index ca7edb7dd8f4a..8d901b1ecce46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -58,7 +58,7 @@ public JAXBContextResolver() throws Exception {
     final Class[] rootUnwrappedTypes =
         { NewApplication.class, ApplicationSubmissionContextInfo.class,
             ContainerLaunchContextInfo.class, LocalResourceInfo.class,
-            DelegationToken.class };
+            DelegationToken.class, AppQueue.class };
 
     this.typesContextMap = new HashMap<Class, JAXBContext>();
     context =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
index ce8fd9e2263ce..48df39134b610 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
@@ -33,7 +33,8 @@ public class NavBlock extends HtmlBlock {
         h3("Cluster").
         ul().
           li().a(url("cluster"), "About")._().
-          li().a(url("nodes"), "Nodes")._();
+          li().a(url("nodes"), "Nodes")._().
+          li().a(url("nodelabels"), "Node Labels")._();
     UL<LI<UL<DIV<Hamlet>>>> subAppsList = mainList.
           li().a(url("apps"), "Applications").
             ul();
@@ -50,6 +51,6 @@ public class NavBlock extends HtmlBlock {
           li().a("/conf", "Configuration")._().
           li().a("/logs", "Local logs")._().
           li().a("/stacks", "Server stacks")._().
-          li().a("/metrics", "Server metrics")._()._()._();
+          li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java
new file mode 100644
index 0000000000000..5e8c1ed5bf2ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+public class NodeLabelsPage extends RmView {
+  static class NodeLabelsBlock extends HtmlBlock {
+    final ResourceManager rm;
+
+    @Inject
+    NodeLabelsBlock(ResourceManager rm, ViewContext ctx) {
+      super(ctx);
+      this.rm = rm;
+    }
+
+    @Override
+    protected void render(Block html) {
+      TBODY<TABLE<Hamlet>> tbody = html.table("#nodelabels").
+          thead().
+          tr().
+          th(".name", "Label Name").
+          th(".numOfActiveNMs", "Num Of Active NMs").
+          th(".totalResource", "Total Resource").
+          _()._().
+          tbody();
+  
+      RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
+      for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) {
+        TR<TBODY<TABLE<Hamlet>>> row =
+            tbody.tr().td(
+                info.getLabelName().isEmpty() ? "<NO_LABEL>" : info
+                    .getLabelName());
+        int nActiveNMs = info.getNumActiveNMs();
+        if (nActiveNMs > 0) {
+          row = row.td()
+          .a(url("nodes",
+              "?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()),
+              String.valueOf(nActiveNMs))
+           ._();
+        } else {
+          row = row.td(String.valueOf(nActiveNMs));
+        }
+        row.td(info.getResource().toString())._();
+      }
+      tbody._()._();
+    }
+  }
+
+  @Override protected void preHead(Page.HTML<_> html) {
+    commonPreHead(html);
+    String title = "Node labels of the cluster";
+    setTitle(title);
+    set(DATATABLES_ID, "nodelabels");
+    setTableStyles(html, "nodelabels", ".healthStatus {width:10em}",
+                   ".healthReport {width:10em}");
+  }
+
+  @Override protected Class<? extends SubView> content() {
+    return NodeLabelsBlock.class;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 57119d218745b..f28a9a88bc3be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -1,24 +1,25 @@
 /**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
@@ -28,7 +29,9 @@
 
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
@@ -60,26 +63,20 @@ protected void render(Block html) {
 
       ResourceScheduler sched = rm.getResourceScheduler();
       String type = $(NODE_STATE);
-      TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").
-          thead().
-          tr().
-          th(".nodelabels", "Node Labels").
-          th(".rack", "Rack").
-          th(".state", "Node State").
-          th(".nodeaddress", "Node Address").
-          th(".nodehttpaddress", "Node HTTP Address").
-          th(".lastHealthUpdate", "Last health-update").
-          th(".healthReport", "Health-report").
-          th(".containers", "Containers").
-          th(".mem", "Mem Used").
-          th(".mem", "Mem Avail").
-          th(".vcores", "VCores Used").
-          th(".vcores", "VCores Avail").
-          th(".nodeManagerVersion", "Version").
-          _()._().
-          tbody();
+      String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
+      TBODY<TABLE<Hamlet>> tbody =
+          html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels")
+              .th(".rack", "Rack").th(".state", "Node State")
+              .th(".nodeaddress", "Node Address")
+              .th(".nodehttpaddress", "Node HTTP Address")
+              .th(".lastHealthUpdate", "Last health-update")
+              .th(".healthReport", "Health-report")
+              .th(".containers", "Containers").th(".mem", "Mem Used")
+              .th(".mem", "Mem Avail").th(".vcores", "VCores Used")
+              .th(".vcores", "VCores Avail")
+              .th(".nodeManagerVersion", "Version")._()._().tbody();
       NodeState stateFilter = null;
-      if(type != null && !type.isEmpty()) {
+      if (type != null && !type.isEmpty()) {
         stateFilter = NodeState.valueOf(type.toUpperCase());
       }
       Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
@@ -92,12 +89,14 @@ protected void render(Block html) {
           rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
           isInactive = true;
           break;
+        default:
+          LOG.debug("Unexpected state filter for inactive RM node");
         }
       }
       for (RMNode ni : rmNodes) {
-        if(stateFilter != null) {
+        if (stateFilter != null) {
           NodeState state = ni.getState();
-          if(!stateFilter.equals(state)) {
+          if (!stateFilter.equals(state)) {
             continue;
           }
         } else {
@@ -107,61 +106,71 @@ protected void render(Block html) {
             continue;
           }
         }
+        // Besides state, we need to filter label as well.
+        if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
+          if (labelFilter.isEmpty()) {
+            // Empty label filter means only shows nodes without label
+            if (!ni.getNodeLabels().isEmpty()) {
+              continue;
+            }
+          } else if (!ni.getNodeLabels().contains(labelFilter)) {
+            // Only nodes have given label can show on web page.
+            continue;
+          }
+        }
         NodeInfo info = new NodeInfo(ni, sched);
-        int usedMemory = (int)info.getUsedMemory();
-        int availableMemory = (int)info.getAvailableMemory();
-        TR<TBODY<TABLE<Hamlet>>> row = tbody.tr().
-            td(StringUtils.join(",", info.getNodeLabels())).
-            td(info.getRack()).
-            td(info.getState()).
-            td(info.getNodeId());
+        int usedMemory = (int) info.getUsedMemory();
+        int availableMemory = (int) info.getAvailableMemory();
+        TR<TBODY<TABLE<Hamlet>>> row =
+            tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
+                .td(info.getRack()).td(info.getState()).td(info.getNodeId());
         if (isInactive) {
           row.td()._("N/A")._();
         } else {
           String httpAddress = info.getNodeHTTPAddress();
-          row.td().a("//" + httpAddress,
-              httpAddress)._();
+          row.td().a("//" + httpAddress, httpAddress)._();
         }
-        row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
-              _(Times.format(info.getLastHealthUpdate()))._().
-            td(info.getHealthReport()).
-            td(String.valueOf(info.getNumContainers())).
-            td().br().$title(String.valueOf(usedMemory))._().
-              _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
-            td().br().$title(String.valueOf(availableMemory))._().
-              _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
-              td(String.valueOf(info.getUsedVirtualCores())).
-              td(String.valueOf(info.getAvailableVirtualCores())).
-            td(ni.getNodeManagerVersion()).
-            _();
+        row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
+            ._(Times.format(info.getLastHealthUpdate()))._()
+            .td(info.getHealthReport())
+            .td(String.valueOf(info.getNumContainers())).td().br()
+            .$title(String.valueOf(usedMemory))._()
+            ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
+            .$title(String.valueOf(availableMemory))._()
+            ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
+            .td(String.valueOf(info.getUsedVirtualCores()))
+            .td(String.valueOf(info.getAvailableVirtualCores()))
+            .td(ni.getNodeManagerVersion())._();
       }
       tbody._()._();
     }
   }
 
-  @Override protected void preHead(Page.HTML<_> html) {
+  @Override
+  protected void preHead(Page.HTML<_> html) {
     commonPreHead(html);
     String type = $(NODE_STATE);
     String title = "Nodes of the cluster";
-    if(type != null && !type.isEmpty()) {
-      title = title+" ("+type+")";
+    if (type != null && !type.isEmpty()) {
+      title = title + " (" + type + ")";
     }
     setTitle(title);
     set(DATATABLES_ID, "nodes");
     set(initID(DATATABLES, "nodes"), nodesTableInit());
     setTableStyles(html, "nodes", ".healthStatus {width:10em}",
-                   ".healthReport {width:10em}");
+        ".healthReport {width:10em}");
   }
 
-  @Override protected Class<? extends SubView> content() {
+  @Override
+  protected Class<? extends SubView> content() {
     return NodesBlock.class;
   }
 
   private String nodesTableInit() {
     StringBuilder b = tableInit().append(", aoColumnDefs: [");
     b.append("{'bSearchable': false, 'aTargets': [ 6 ]}");
-    b.append(", {'sType': 'title-numeric', 'bSearchable': false, " +
-        "'aTargets': [ 7, 8 ] }");
+    b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+        + "'aTargets': [ 7, 8 ] }");
     b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}");
     b.append("]}");
     return b.toString();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 67c73b812737a..c0e68349e4ab5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -61,6 +61,7 @@ public void setup() {
     route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
     route("/scheduler", RmController.class, "scheduler");
     route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
+    route("/nodelabels", RmController.class, "nodelabels");
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 27417a96633d6..1834b6a1a7dca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -24,7 +24,6 @@
 import java.nio.ByteBuffer;
 import java.security.Principal;
 import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
@@ -81,6 +80,7 @@
 import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -117,6 +117,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
@@ -167,6 +168,12 @@ public RMWebServices(final ResourceManager rm, Configuration conf) {
     this.conf = conf;
   }
 
+  RMWebServices(ResourceManager rm, Configuration conf,
+      HttpServletResponse response) {
+    this(rm, conf);
+    this.response = response;
+  }
+
   protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
     // Check for the authorization.
     UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -457,6 +464,9 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
     AppsInfo allApps = new AppsInfo();
     for (ApplicationReport report : appReports) {
       RMApp rmapp = apps.get(report.getApplicationId());
+      if (rmapp == null) {
+        continue;
+      }
 
       if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
         FinalApplicationStatus.valueOf(finalStatusQuery);
@@ -696,7 +706,7 @@ public Response updateAppState(AppState targetState,
       app = getRMAppForAppId(appId);
     } catch (NotFoundException e) {
       RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
-        "UNKNOWN", "RMWebService", "Trying to kill/move an absent application "
+        "UNKNOWN", "RMWebService", "Trying to kill an absent application "
             + appId);
       throw e;
     }
@@ -945,6 +955,126 @@ public KillApplicationResponse run() throws IOException,
     return Response.status(Status.OK).entity(ret).build();
   }
 
+  @GET
+  @Path("/apps/{appid}/queue")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public AppQueue getAppQueue(@Context HttpServletRequest hsr,
+      @PathParam("appid") String appId) throws AuthorizationException {
+    init();
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    String userName = "UNKNOWN-USER";
+    if (callerUGI != null) {
+      userName = callerUGI.getUserName();
+    }
+    RMApp app = null;
+    try {
+      app = getRMAppForAppId(appId);
+    } catch (NotFoundException e) {
+      RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+        "UNKNOWN", "RMWebService",
+        "Trying to get state of an absent application " + appId);
+      throw e;
+    }
+
+    AppQueue ret = new AppQueue();
+    ret.setQueue(app.getQueue());
+
+    return ret;
+  }
+
+  @PUT
+  @Path("/apps/{appid}/queue")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response updateAppQueue(AppQueue targetQueue,
+      @Context HttpServletRequest hsr, @PathParam("appid") String appId)
+      throws AuthorizationException, YarnException, InterruptedException,
+      IOException {
+
+    init();
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated";
+      throw new AuthorizationException(msg);
+    }
+
+    if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+      String msg = "The default static user cannot carry out this operation.";
+      return Response.status(Status.FORBIDDEN).entity(msg).build();
+    }
+
+    String userName = callerUGI.getUserName();
+    RMApp app = null;
+    try {
+      app = getRMAppForAppId(appId);
+    } catch (NotFoundException e) {
+      RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+        "UNKNOWN", "RMWebService", "Trying to move an absent application "
+            + appId);
+      throw e;
+    }
+
+    if (!app.getQueue().equals(targetQueue.getQueue())) {
+      // user is attempting to change queue.
+      return moveApp(app, callerUGI, targetQueue.getQueue());
+    }
+
+    AppQueue ret = new AppQueue();
+    ret.setQueue(app.getQueue());
+
+    return Response.status(Status.OK).entity(ret).build();
+  }
+
+  protected Response moveApp(RMApp app, UserGroupInformation callerUGI,
+      String targetQueue) throws IOException, InterruptedException {
+
+    if (app == null) {
+      throw new IllegalArgumentException("app cannot be null");
+    }
+    String userName = callerUGI.getUserName();
+    final ApplicationId appid = app.getApplicationId();
+    final String reqTargetQueue = targetQueue;
+    try {
+      callerUGI
+        .doAs(new PrivilegedExceptionAction<Void>() {
+          @Override
+          public Void run() throws IOException,
+              YarnException {
+            MoveApplicationAcrossQueuesRequest req =
+                MoveApplicationAcrossQueuesRequest.newInstance(appid,
+                  reqTargetQueue);
+            rm.getClientRMService().moveApplicationAcrossQueues(req);
+            return null;
+          }
+        });
+    } catch (UndeclaredThrowableException ue) {
+      // if the root cause is a permissions issue
+      // bubble that up to the user
+      if (ue.getCause() instanceof YarnException) {
+        YarnException ye = (YarnException) ue.getCause();
+        if (ye.getCause() instanceof AccessControlException) {
+          String appId = app.getApplicationId().toString();
+          String msg =
+              "Unauthorized attempt to move appid " + appId
+                  + " by remote user " + userName;
+          return Response.status(Status.FORBIDDEN).entity(msg).build();
+        } else if (ye.getMessage().startsWith("App in")
+            && ye.getMessage().endsWith("state cannot be moved.")) {
+          return Response.status(Status.BAD_REQUEST).entity(ye.getMessage())
+            .build();
+        } else {
+          throw ue;
+        }
+      } else {
+        throw ue;
+      }
+    }
+
+    AppQueue ret = new AppQueue();
+    ret.setQueue(app.getQueue());
+    return Response.status(Status.OK).entity(ret).build();
+  }
+
   private RMApp getRMAppForAppId(String appId) {
 
     if (appId == null || appId.isEmpty()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
index f186bf498b0f0..972432b9882d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.util.StringHelper;
 import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.WebAppException;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 
 import com.google.inject.Inject;
@@ -93,4 +92,9 @@ public void queue() {
   public void submit() {
     setTitle("Application Submission Not Allowed");
   }
+  
+  public void nodelabels() {
+    setTitle("Node Labels");
+    render(NodeLabelsPage.class);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppQueue.java
new file mode 100644
index 0000000000000..5dab6ce187c1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppQueue.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "appqueue")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AppQueue {
+
+  String queue;
+
+  public AppQueue() {
+  }
+
+  public AppQueue(String queue) {
+    this.queue = queue;
+  }
+
+  public void setQueue(String queue) {
+    this.queue = queue;
+  }
+
+  public String getQueue() {
+    return this.queue;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index d90e9631b95a8..a8b0d32343e7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -32,11 +32,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   protected int numContainers;
   protected int maxApplications;
   protected int maxApplicationsPerUser;
-  protected int maxActiveApplications;
-  protected int maxActiveApplicationsPerUser;
   protected int userLimit;
   protected UsersInfo users; // To add another level in the XML
   protected float userLimitFactor;
+  protected ResourceInfo aMResourceLimit;
+  protected ResourceInfo userAMResourceLimit;
+  protected boolean preemptionDisabled;
 
   CapacitySchedulerLeafQueueInfo() {
   };
@@ -48,11 +49,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
     numContainers = q.getNumContainers();
     maxApplications = q.getMaxApplications();
     maxApplicationsPerUser = q.getMaxApplicationsPerUser();
-    maxActiveApplications = q.getMaximumActiveApplications();
-    maxActiveApplicationsPerUser = q.getMaximumActiveApplicationsPerUser();
     userLimit = q.getUserLimit();
     users = new UsersInfo(q.getUsers());
     userLimitFactor = q.getUserLimitFactor();
+    aMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
+    userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
+    preemptionDisabled = q.getPreemptionDisabled();
   }
 
   public int getNumActiveApplications() {
@@ -75,14 +77,6 @@ public int getMaxApplicationsPerUser() {
     return maxApplicationsPerUser;
   }
 
-  public int getMaxActiveApplications() {
-    return maxActiveApplications;
-  }
-
-  public int getMaxActiveApplicationsPerUser() {
-    return maxActiveApplicationsPerUser;
-  }
-
   public int getUserLimit() {
     return userLimit;
   }
@@ -95,4 +89,16 @@ public UsersInfo getUsers() {
   public float getUserLimitFactor() {
     return userLimitFactor;
   }
+  
+  public ResourceInfo getAMResourceLimit() {
+    return aMResourceLimit;
+  }
+  
+  public ResourceInfo getUserAMResourceLimit() {
+    return userAMResourceLimit; 
+  }
+
+  public boolean getPreemptionDisabled() {
+    return preemptionDisabled;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
index d389b9f07658d..189c877b9b21f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
@@ -18,14 +18,11 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
-import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
-    .FSAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
 
@@ -40,15 +37,8 @@ public FairSchedulerLeafQueueInfo() {
   
   public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
     super(queue, scheduler);
-    Collection<FSAppAttempt> apps = queue.getRunnableAppSchedulables();
-    for (FSAppAttempt app : apps) {
-      if (app.isPending()) {
-        numPendingApps++;
-      } else {
-        numActiveApps++;
-      }
-    }
-    numPendingApps += queue.getNonRunnableAppSchedulables().size();
+    numPendingApps = queue.getNumPendingApps();
+    numActiveApps = queue.getNumActiveApps();
   }
   
   public int getNumActiveApplications() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index c62aaf08c64d2..5fbfe5152951b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -94,9 +94,14 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
     fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
     
     maxApps = allocConf.getQueueMaxApps(queueName);
-    
-    Collection<FSQueue> children = queue.getChildQueues();
+
     childQueues = new ArrayList<FairSchedulerQueueInfo>();
+    if (allocConf.isReservable(queueName) &&
+        !allocConf.getShowReservationAsQueues(queueName)) {
+      return;
+    }
+
+    Collection<FSQueue> children = queue.getChildQueues();
     for (FSQueue child : children) {
       if (child instanceof FSLeafQueue) {
         childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index e2e3cc13da943..a3968f86c9035 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -165,12 +165,17 @@ public List<ResourceRequest> createReq(String[] hosts, int memory, int priority,
       int containers, String labelExpression) throws Exception {
     List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
     for (String host : hosts) {
-      ResourceRequest hostReq = createResourceReq(host, memory, priority,
-          containers, labelExpression);
-      reqs.add(hostReq);
-      ResourceRequest rackReq = createResourceReq("/default-rack", memory,
-          priority, containers, labelExpression);
-      reqs.add(rackReq);
+      // only add host/rack request when asked host isn't ANY
+      if (!host.equals(ResourceRequest.ANY)) {
+        ResourceRequest hostReq =
+            createResourceReq(host, memory, priority, containers,
+                labelExpression);
+        reqs.add(hostReq);
+        ResourceRequest rackReq =
+            createResourceReq("/default-rack", memory, priority, containers,
+                labelExpression);
+        reqs.add(rackReq);
+      }
     }
 
     ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 228f2006fdaad..2d863d1af5c09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -30,10 +30,13 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 
 /**
@@ -52,7 +55,12 @@ public static List<RMNode> newNodes(int racks, int nodesPerRack,
           // One unhealthy node per rack.
           list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
         }
-        list.add(newNodeInfo(i, perNode));
+        if (j == 0) {
+          // One node with label
+          list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
+        } else {
+          list.add(newNodeInfo(i, perNode));
+        }
       }
     }
     return list;
@@ -99,10 +107,12 @@ private static class MockRMNodeImpl implements RMNode {
     private String healthReport;
     private long lastHealthReportTime;
     private NodeState state;
+    private Set<String> labels;
 
     public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
         Resource perNode, String rackName, String healthReport,
-        long lastHealthReportTime, int cmdPort, String hostName, NodeState state) {
+        long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
+        Set<String> labels) {
       this.nodeId = nodeId;
       this.nodeAddr = nodeAddr;
       this.httpAddress = httpAddress;
@@ -113,6 +123,7 @@ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
       this.cmdPort = cmdPort;
       this.hostName = hostName;
       this.state = state;
+      this.labels = labels;
     }
 
     @Override
@@ -206,16 +217,33 @@ public long getLastHealthReportTime() {
 
     @Override
     public Set<String> getNodeLabels() {
-      return null;
+      if (labels != null) {
+        return labels;
+      }
+      return CommonNodeLabelsManager.EMPTY_STRING_SET;
     }
   };
 
-  private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
-    return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
+  private static RMNode buildRMNode(int rack, final Resource perNode,
+      NodeState state, String httpAddr) {
+    return buildRMNode(rack, perNode, state, httpAddr, null);
   }
-
+  
+  private static RMNode buildRMNode(int rack, final Resource perNode,
+      NodeState state, String httpAddr, Set<String> labels) {
+    return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123,
+        labels);
+  }
+  
   private static RMNode buildRMNode(int rack, final Resource perNode,
       NodeState state, String httpAddr, int hostnum, String hostName, int port) {
+    return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
+        null);
+  }
+
+  private static RMNode buildRMNode(int rack, final Resource perNode,
+      NodeState state, String httpAddr, int hostnum, String hostName, int port,
+      Set<String> labels) {
     final String rackName = "rack"+ rack;
     final int nid = hostnum;
     final String nodeAddr = hostName + ":" + nid;
@@ -227,13 +255,18 @@ private static RMNode buildRMNode(int rack, final Resource perNode,
     final String httpAddress = httpAddr;
     String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
     return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
-        rackName, healthReport, 0, nid, hostName, state);
+        rackName, healthReport, 0, nid, hostName, state, labels);
   }
 
   public static RMNode nodeInfo(int rack, final Resource perNode,
       NodeState state) {
     return buildRMNode(rack, perNode, state, "N/A");
   }
+  
+  public static RMNode nodeInfo(int rack, final Resource perNode,
+      NodeState state, Set<String> labels) {
+    return buildRMNode(rack, perNode, state, "N/A", labels);
+  }
 
   public static RMNode newNodeInfo(int rack, final Resource perNode) {
     return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 9d0ac2739bc66..06c6b3275e334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -59,7 +59,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -115,7 +115,7 @@ public MockRM(Configuration conf, RMStateStore store) {
   
   @Override
   protected RMNodeLabelsManager createNodeLabelManager() {
-    RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
     mgr.init(getConfig());
     return mgr;
   }
@@ -313,7 +313,7 @@ public RMApp submitApp(int masterMemory, String name, String user,
       boolean waitForAccepted, boolean keepContainers) throws Exception {
     return submitApp(masterMemory, name, user, acls, unmanaged, queue,
         maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
-        false, null, 0, null);
+        false, null, 0, null, true);
   }
 
   public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
@@ -322,7 +322,7 @@ public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
       .getShortUserName(), null, false, null,
       super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
       YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
-      false, null, attemptFailuresValidityInterval, null);
+      false, null, attemptFailuresValidityInterval, null, true);
   }
 
   public RMApp submitApp(int masterMemory, String name, String user,
@@ -332,26 +332,24 @@ public RMApp submitApp(int masterMemory, String name, String user,
       ApplicationId applicationId) throws Exception {
     return submitApp(masterMemory, name, user, acls, unmanaged, queue,
       maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
-      isAppIdProvided, applicationId, 0, null);
+      isAppIdProvided, applicationId, 0, null, true);
   }
 
-  @SuppressWarnings("deprecation")
   public RMApp submitApp(int masterMemory,
       LogAggregationContext logAggregationContext) throws Exception {
     return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
       .getShortUserName(), null, false, null,
       super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
       YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
-      false, null, 0, logAggregationContext);
+      false, null, 0, logAggregationContext, true);
    }
 
-  @SuppressWarnings("deprecation")
   public RMApp submitApp(int masterMemory, String name, String user,
       Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
       int maxAppAttempts, Credentials ts, String appType,
       boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
       ApplicationId applicationId, long attemptFailuresValidityInterval,
-      LogAggregationContext logAggregationContext)
+      LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete)
       throws Exception {
     ApplicationId appId = isAppIdProvided ? applicationId : null;
     ApplicationClientProtocol client = getClientRMService();
@@ -392,6 +390,7 @@ public RMApp submitApp(int masterMemory, String name, String user,
     if (logAggregationContext != null) {
       sub.setLogAggregationContext(logAggregationContext);
     }
+    sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
     req.setApplicationSubmissionContext(sub);
     UserGroupInformation fakeUser =
       UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 6b3eea20ff265..d2ac4efb9572b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -572,7 +572,7 @@ public void testEscapeApplicationSummary() {
     when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
     when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
     when(app.getState()).thenReturn(RMAppState.RUNNING);
-
+    when(app.getApplicationType()).thenReturn("MAPREDUCE");
     RMAppMetrics metrics =
         new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
     when(app.getRMAppMetrics()).thenReturn(metrics);
@@ -593,6 +593,7 @@ public void testEscapeApplicationSummary() {
     Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
     Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
     Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+    Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
  }
 
   private static ResourceScheduler mockResourceScheduler() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index a344e9a91ac39..a68434664bef3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -553,8 +553,17 @@ public void testGetQueueInfo() throws Exception {
     YarnScheduler yarnScheduler = mock(YarnScheduler.class);
     RMContext rmContext = mock(RMContext.class);
     mockRMContext(yarnScheduler, rmContext);
+
+    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
+    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
+    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
+        any(QueueACL.class), anyString())).thenReturn(true);
+    when(mockAclsManager.checkAccess(any(UserGroupInformation.class),
+        any(ApplicationAccessType.class), anyString(),
+        any(ApplicationId.class))).thenReturn(true);
+
     ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
-        null, null, null, null);
+        null, mockAclsManager, mockQueueACLsManager, null);
     GetQueueInfoRequest request = recordFactory
         .newRecordInstance(GetQueueInfoRequest.class);
     request.setQueueName("testqueue");
@@ -567,6 +576,26 @@ public void testGetQueueInfo() throws Exception {
     request.setIncludeApplications(true);
     // should not throw exception on nonexistent queue
     queueInfo = rmService.getQueueInfo(request);
+
+    // Case where user does not have application access
+    ApplicationACLsManager mockAclsManager1 =
+        mock(ApplicationACLsManager.class);
+    QueueACLsManager mockQueueACLsManager1 =
+        mock(QueueACLsManager.class);
+    when(mockQueueACLsManager1.checkAccess(any(UserGroupInformation.class),
+        any(QueueACL.class), anyString())).thenReturn(false);
+    when(mockAclsManager1.checkAccess(any(UserGroupInformation.class),
+        any(ApplicationAccessType.class), anyString(),
+        any(ApplicationId.class))).thenReturn(false);
+
+    ClientRMService rmService1 = new ClientRMService(rmContext, yarnScheduler,
+        null, mockAclsManager1, mockQueueACLsManager1, null);
+    request.setQueueName("testqueue");
+    request.setIncludeApplications(true);
+    GetQueueInfoResponse queueInfo1 = rmService1.getQueueInfo(request);
+    List<ApplicationReport> applications1 = queueInfo1.getQueueInfo()
+        .getApplications();
+    Assert.assertEquals(0, applications1.size());
   }
 
   private static final UserGroupInformation owner =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index b9397bf070f20..fcb48a05ec04d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -61,7 +61,6 @@ public void setup() throws UnknownHostException {
     rootLogger.setLevel(Level.DEBUG);
     conf = new YarnConfiguration();
     UserGroupInformation.setConfiguration(conf);
-    conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
         YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
   }
@@ -137,6 +136,8 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
     // Set max attempts to 1 so that when the first attempt fails, the app
     // won't try to start a new one.
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
     MemoryRMStateStore memStore = new MemoryRMStateStore();
     memStore.init(conf);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
index 225e225a8cfdd..95aa856fcefa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
@@ -130,6 +130,7 @@ public void testAllocateContainerOnNodeWithoutOffSwitchSpecified()
       Assert.fail("NPE when allocating container on node but "
           + "forget to set off-switch request should be handled");
     }
+    rm.stop();
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 29f0208f28e79..4caf7e3833008 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -39,6 +39,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -87,7 +88,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreAMRMTokenEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreRMDTEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreRMDTMasterKeyEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -130,7 +134,8 @@ public void setup() throws IOException {
     Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
     UserGroupInformation.setConfiguration(conf);
-    conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
     conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
     rmAddr = new InetSocketAddress("localhost", 8032);
     Assert.assertTrue(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS > 1);
@@ -1457,7 +1462,12 @@ public void serviceStop() throws Exception {
       @Override
       protected void handleStoreEvent(RMStateStoreEvent event) {
         // Block app saving request.
-        while (wait);
+        // Skip if synchronous updation of DTToken
+        if (!(event instanceof RMStateStoreAMRMTokenEvent)
+            && !(event instanceof RMStateStoreRMDTEvent)
+            && !(event instanceof RMStateStoreRMDTMasterKeyEvent)) {
+          while (wait);
+        }
         super.handleStoreEvent(event);
       }
     };
@@ -2048,8 +2058,22 @@ protected void doSecureLogin() throws IOException {
   // 4. Get cluster and node lobel, it should be present by recovering it
   @Test(timeout = 20000)
   public void testRMRestartRecoveringNodeLabelManager() throws Exception {
+    // Initial FS node label store root dir to a random tmp dir
+    File nodeLabelFsStoreDir =
+        new File("target", this.getClass().getSimpleName()
+            + "-testRMRestartRecoveringNodeLabelManager");
+    if (nodeLabelFsStoreDir.exists()) {
+      FileUtils.deleteDirectory(nodeLabelFsStoreDir);
+    }
+    nodeLabelFsStoreDir.deleteOnExit();
+    
+    String nodeLabelFsStoreDirURI = nodeLabelFsStoreDir.toURI().toString(); 
+    conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+        nodeLabelFsStoreDirURI);
+    
     MemoryRMStateStore memStore = new MemoryRMStateStore();
     memStore.init(conf);
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     MockRM rm1 = new MockRM(conf, memStore) {
       @Override
       protected RMNodeLabelsManager createNodeLabelManager() {
@@ -2080,7 +2104,7 @@ protected RMNodeLabelsManager createNodeLabelManager() {
     nodeLabelManager.removeFromClusterNodeLabels(toSet("z"));
     
     // Replace nodelabel h1->x,y
-    nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("x", "y")));
+    nodeLabelManager.replaceLabelsOnNode(ImmutableMap.of(n1, toSet("y")));
 
     // Wait for updating store.It is expected NodeStore update should happen
     // very fast since it has separate dispatcher. So waiting for max 5 seconds,
@@ -2098,7 +2122,7 @@ protected RMNodeLabelsManager createNodeLabelManager() {
 
     Map<NodeId, Set<String>> nodeLabels = nodeLabelManager.getNodeLabels();
     Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size());
-    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("x", "y")));
+    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y")));
 
     MockRM rm2 = new MockRM(conf, memStore) {
       @Override
@@ -2117,7 +2141,7 @@ protected RMNodeLabelsManager createNodeLabelManager() {
 
     nodeLabels = nodeLabelManager.getNodeLabels();
     Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size());
-    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("x", "y")));
+    Assert.assertTrue(nodeLabels.get(n1).equals(toSet("y")));
     rm1.stop();
     rm2.stop();
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 853e0a57e0b8e..a9caf77d36ba7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -321,7 +321,7 @@ private void checkCSLeafQueue(MockRM rm,
       1e-8);
     // assert user consumed resources.
     assertEquals(usedResource, leafQueue.getUser(app.getUser())
-      .getTotalConsumedResources());
+      .getUsed());
   }
 
   private void checkFifoQueue(ResourceManager rm,
@@ -500,6 +500,8 @@ public void testCapacitySchedulerRecovery() throws Exception {
     rm1.clearQueueMetrics(app1_2);
     rm1.clearQueueMetrics(app2);
 
+    csConf.set("yarn.scheduler.capacity.root.Default.QueueB.state", "STOPPED");
+
     // Re-start RM
     rm2 = new MockRM(csConf, memStore);
     rm2.start();
@@ -837,7 +839,7 @@ public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception {
   // Test if RM on recovery receives the container release request from AM
   // before it receives the container status reported by NM for recovery. this
   // container should not be recovered.
-  @Test (timeout = 30000)
+  @Test (timeout = 50000)
   public void testReleasedContainerNotRecovered() throws Exception {
     MemoryRMStateStore memStore = new MemoryRMStateStore();
     memStore.init(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 62e3e5c8b9d0d..f8d92aa2f1741 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -51,6 +52,7 @@
 public abstract class MockAsm extends MockApps {
 
   public static class ApplicationBase implements RMApp {
+    ResourceRequest amReq;
     @Override
     public String getUser() {
       throw new UnsupportedOperationException("Not supported yet.");
@@ -183,6 +185,11 @@ public RMAppMetrics getRMAppMetrics() {
     public ReservationId getReservationId() {
       throw new UnsupportedOperationException("Not supported yet.");
     }
+    
+    @Override
+    public ResourceRequest getAMResourceRequest() {
+      return this.amReq; 
+    }
   }
 
   public static RMApp newApplication(int i) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
index e93d3510a3316..f4cb3b346c599 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
@@ -23,6 +23,7 @@
 
 import org.junit.Assert;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -41,6 +42,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -54,6 +56,13 @@ public class TestAMRMRPCNodeUpdates {
   public void setUp() {
     dispatcher = new DrainDispatcher();
     this.rm = new MockRM() {
+      @Override
+      public void init(Configuration conf) {
+        conf.set(
+          CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+          "1.0");
+        super.init(conf);
+      }
       @Override
       protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
         return new SchedulerEventDispatcher(this.scheduler) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index 49b18418c4b37..7befba4a14423 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -470,6 +470,8 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception {
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
     conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
+
     conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
     // explicitly set max-am-retry count as 1.
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
@@ -535,6 +537,8 @@ public void testRMRestartOrFailoverNotCountedForAMFailures()
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
     conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
+
     conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
     // explicitly set max-am-retry count as 1.
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
@@ -594,6 +598,8 @@ public void testRMAppAttemptFailuresValidityInterval() throws Exception {
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
     conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
+
     conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
     // explicitly set max-am-retry count as 2.
     conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 65c8547218097..9f02721399a33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -50,6 +51,8 @@
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -68,6 +71,8 @@ public static void setup() throws Exception {
     conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
     conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
         MemoryTimelineStore.class, TimelineStore.class);
+    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+        MemoryTimelineStateStore.class, TimelineStateStore.class);
     conf.setInt(
         YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
         2);
@@ -144,8 +149,18 @@ public void testPublishApplicationMetrics() throws Exception {
             entity.getOtherInfo().get(
                 ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
       } else {
-        Assert.assertEquals("", entity.getOtherInfo().get(
-            ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+        Assert.assertEquals(
+            "",
+            entity.getOtherInfo().get(
+                ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+        Assert.assertEquals(
+            app.getRMAppMetrics().getMemorySeconds(),
+            Long.parseLong(entity.getOtherInfo()
+                .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+        Assert.assertEquals(
+            app.getRMAppMetrics().getVcoreSeconds(),
+            Long.parseLong(entity.getOtherInfo()
+                .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
       }
       boolean hasCreatedEvent = false;
       boolean hasFinishedEvent = false;
@@ -336,6 +351,8 @@ private static RMApp createRMApp(ApplicationId appId) {
     when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
     when(app.getFinalApplicationStatus()).thenReturn(
         FinalApplicationStatus.UNDEFINED);
+    when(app.getRMAppMetrics()).thenReturn(
+        new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
     return app;
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index ca67ef0d6e01b..696b9bb77cf7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -17,12 +17,10 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.BASE_YARN_RM_PREEMPTION;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MAX_IGNORED_OVER_CAPACITY;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.MONITORING_INTERVAL;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.NATURAL_TERMINATION_FACTOR;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.OBSERVE_ONLY;
-import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.SUFFIX_DISABLE_PREEMPTION;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TOTAL_PREEMPTION_PER_ROUND;
 import static org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.WAIT_TIME_BEFORE_KILL;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType.KILL_CONTAINER;
@@ -38,27 +36,38 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
 
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Random;
+import java.util.Set;
+import java.util.StringTokenizer;
 import java.util.TreeSet;
 
+import org.apache.commons.collections.map.HashedMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEvent;
@@ -72,12 +81,14 @@
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatcher;
+import org.mortbay.log.Log;
 
 public class TestProportionalCapacityPreemptionPolicy {
 
@@ -85,14 +96,18 @@ public class TestProportionalCapacityPreemptionPolicy {
 
   int appAlloc = 0;
   boolean setAMContainer = false;
+  boolean setLabeledContainer = false;
   float setAMResourcePercent = 0.0f;
   Random rand = null;
   Clock mClock = null;
   Configuration conf = null;
   CapacityScheduler mCS = null;
+  RMContext rmContext = null;
+  RMNodeLabelsManager lm = null;
   CapacitySchedulerConfiguration schedConf = null;
   EventHandler<ContainerPreemptEvent> mDisp = null;
   ResourceCalculator rc = new DefaultResourceCalculator();
+  Resource clusterResources = null;
   final ApplicationAttemptId appA = ApplicationAttemptId.newInstance(
       ApplicationId.newInstance(TS, 0), 0);
   final ApplicationAttemptId appB = ApplicationAttemptId.newInstance(
@@ -108,6 +123,19 @@ public class TestProportionalCapacityPreemptionPolicy {
   final ArgumentCaptor<ContainerPreemptEvent> evtCaptor =
     ArgumentCaptor.forClass(ContainerPreemptEvent.class);
 
+  public enum priority {
+    AMCONTAINER(0), CONTAINER(1), LABELEDCONTAINER(2);
+    int value;
+
+    private priority(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return this.value;
+    }
+  };  
+
   @Rule public TestName name = new TestName();
 
   @Before
@@ -130,8 +158,12 @@ public void setup() {
     mClock = mock(Clock.class);
     mCS = mock(CapacityScheduler.class);
     when(mCS.getResourceCalculator()).thenReturn(rc);
+    lm = mock(RMNodeLabelsManager.class);
     schedConf = new CapacitySchedulerConfiguration();
     when(mCS.getConfiguration()).thenReturn(schedConf);
+    rmContext = mock(RMContext.class);
+    when(mCS.getRMContext()).thenReturn(rmContext);
+    when(rmContext.getNodeLabelManager()).thenReturn(lm);
     mDisp = mock(EventHandler.class);
     rand = new Random();
     long seed = rand.nextLong();
@@ -289,24 +321,22 @@ public void testPerQueueDisablePreemption() {
         {   3,   0,   0,  0 },  // subqueues
       };
 
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueB" + SUFFIX_DISABLE_PREEMPTION, true);
+    schedConf.setPreemptionDisabled("root.queueB", true);
 
     ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
     policy.editSchedule();
-    // With PREEMPTION_DISABLED set for queueB, get resources from queueC
+    // Since queueB is not preemptable, get resources from queueC
     verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appC)));
     verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appB)));
 
-    // With no PREEMPTION_DISABLED set for queueB, resources will be preempted
-    // from both queueB and queueC. Test must be reset for so that the mDisp
+    // Since queueB is preemptable, resources will be preempted
+    // from both queueB and queueC. Test must be reset so that the mDisp
     // event handler will count only events from the following test and not the
     // previous one.
     setup();
+    schedConf.setPreemptionDisabled("root.queueB", false);
     ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
-    
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueB" + SUFFIX_DISABLE_PREEMPTION, false);
+
     policy2.editSchedule();
 
     verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appB)));
@@ -342,9 +372,8 @@ public void testPerQueueDisablePreemptionHierarchical() {
 
     // Need to call setup() again to reset mDisp
     setup();
-    // Disable preemption for queueB and it's children
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueA.queueB" + SUFFIX_DISABLE_PREEMPTION, true);
+    // Turn off preemption for queueB and it's children
+    schedConf.setPreemptionDisabled("root.queueA.queueB", true);
     ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
     policy2.editSchedule();
     ApplicationAttemptId expectedAttemptOnQueueC = 
@@ -390,9 +419,8 @@ public void testPerQueueDisablePreemptionBroadHierarchical() {
 
     // Need to call setup() again to reset mDisp
     setup();
-    // Disable preemption for queueB(appA)
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueA.queueB" + SUFFIX_DISABLE_PREEMPTION, true);
+    // Turn off preemption for queueB(appA)
+    schedConf.setPreemptionDisabled("root.queueA.queueB", true);
     ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
     policy2.editSchedule();
     // Now that queueB(appA) is not preemptable, verify that resources come
@@ -401,11 +429,9 @@ public void testPerQueueDisablePreemptionBroadHierarchical() {
     verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
 
     setup();
-    // Disable preemption for two of the 3 queues with over-capacity.
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueD.queueE" + SUFFIX_DISABLE_PREEMPTION, true);
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueA.queueB" + SUFFIX_DISABLE_PREEMPTION, true);
+    // Turn off preemption for two of the 3 queues with over-capacity.
+    schedConf.setPreemptionDisabled("root.queueD.queueE", true);
+    schedConf.setPreemptionDisabled("root.queueA.queueB", true);
     ProportionalCapacityPreemptionPolicy policy3 = buildPolicy(qData);
     policy3.editSchedule();
 
@@ -443,11 +469,10 @@ public void testPerQueueDisablePreemptionInheritParent() {
     verify(mDisp, times(16)).handle(argThat(new IsPreemptionRequestFor(appA)));
     verify(mDisp, times(182)).handle(argThat(new IsPreemptionRequestFor(appB)));
 
-    // Disable preemption for queueA and it's children. queueF(appC)'s request
+    // Turn off preemption for queueA and it's children. queueF(appC)'s request
     // should starve.
     setup(); // Call setup() to reset mDisp
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueA" + SUFFIX_DISABLE_PREEMPTION, true);
+    schedConf.setPreemptionDisabled("root.queueA", true);
     ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
     policy2.editSchedule();
     verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA))); // queueC
@@ -471,8 +496,7 @@ public void testPerQueuePreemptionNotAllUntouchable() {
       {   -1,   -1,    1,    1,    1,   -1,    1,    1,    1 },  // req granularity
       {    2,    3,    0,    0,    0,    3,    0,    0,    0 },  // subqueues
     };
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root.queueA.queueC" + SUFFIX_DISABLE_PREEMPTION, true);
+    schedConf.setPreemptionDisabled("root.queueA.queueC", true);
     ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
     policy.editSchedule();
     // Although queueC(appB) is way over capacity and is untouchable,
@@ -496,9 +520,8 @@ public void testPerQueueDisablePreemptionRootDisablesAll() {
         {   3,   2,   0,   0,   2,   0,   0,   2,   0,   0 },  // subqueues
    };
 
+    schedConf.setPreemptionDisabled("root", true);
     ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
-    schedConf.setBoolean(BASE_YARN_RM_PREEMPTION
-        + "root" + SUFFIX_DISABLE_PREEMPTION, true);
     policy.editSchedule();
     // All queues should be non-preemptable, so request should starve.
     verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appB))); // queueC
@@ -746,7 +769,51 @@ public void testSkipAMContainer() {
     verify(mDisp, times(2)).handle(argThat(new IsPreemptionRequestFor(appB)));
     setAMContainer = false;
   }
-  
+
+  @Test
+  public void testIdealAllocationForLabels() {
+    int[][] qData = new int[][] {
+    // / A B
+        { 80, 40, 40 }, // abs
+        { 80, 80, 80 }, // maxcap
+        { 80, 80, 0 }, // used
+        { 70, 20, 50 }, // pending
+        { 0, 0, 0 }, // reserved
+        { 5, 4, 1 }, // apps
+        { -1, 1, 1 }, // req granularity
+        { 2, 0, 0 }, // subqueues
+    };
+    setAMContainer = true;
+    setLabeledContainer = true;
+    Map<NodeId, Set<String>> labels = new HashMap<NodeId, Set<String>>();
+    NodeId node = NodeId.newInstance("node1", 0);
+    Set<String> labelSet = new HashSet<String>();
+    labelSet.add("x");
+    labels.put(node, labelSet);
+    when(lm.getNodeLabels()).thenReturn(labels);
+    ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+    // Subtracting Label X resources from cluster resources
+    when(lm.getResourceByLabel(anyString(), any(Resource.class))).thenReturn(
+        Resources.clone(Resource.newInstance(80, 0)));
+    clusterResources.setMemory(100);
+    policy.editSchedule();
+
+    // By skipping AM Container and Labeled container, all other 18 containers
+    // of appD will be
+    // preempted
+    verify(mDisp, times(19)).handle(argThat(new IsPreemptionRequestFor(appD)));
+
+    // By skipping AM Container and Labeled container, all other 18 containers
+    // of appC will be
+    // preempted
+    verify(mDisp, times(19)).handle(argThat(new IsPreemptionRequestFor(appC)));
+
+    // rest 4 containers from appB will be preempted
+    verify(mDisp, times(2)).handle(argThat(new IsPreemptionRequestFor(appB)));
+    setAMContainer = false;
+    setLabeledContainer = false;
+  }
+
   @Test
   public void testPreemptSkippedAMContainers() {
     int[][] qData = new int[][] {
@@ -816,7 +883,7 @@ public void testAMResourcePercentForSkippedAMContainers() {
     verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appA)));
     setAMContainer = false;
   }
-  
+
   static class IsPreemptionRequestFor
       extends ArgumentMatcher<ContainerPreemptEvent> {
     private final ApplicationAttemptId appAttId;
@@ -846,7 +913,7 @@ ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) {
     ParentQueue mRoot = buildMockRootQueue(rand, qData);
     when(mCS.getRootQueue()).thenReturn(mRoot);
 
-    Resource clusterResources =
+    clusterResources =
       Resource.newInstance(leafAbsCapacities(qData[0], qData[7]), 0);
     when(mCS.getClusterResource()).thenReturn(clusterResources);
     return policy;
@@ -875,6 +942,8 @@ ParentQueue mockNested(int[] abs, int[] maxCap, int[] used,
     when(root.getAbsoluteCapacity()).thenReturn(abs[0] / tot);
     when(root.getAbsoluteMaximumCapacity()).thenReturn(maxCap[0] / tot);
     when(root.getQueuePath()).thenReturn("root");
+    boolean preemptionDisabled = mockPreemptionStatus("root");
+    when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
 
     for (int i = 1; i < queues.length; ++i) {
       final CSQueue q;
@@ -894,11 +963,29 @@ ParentQueue mockNested(int[] abs, int[] maxCap, int[] used,
       parentPathName = (parentPathName == null) ? "root" : parentPathName;
       String queuePathName = (parentPathName+"."+queueName).replace("/","root");
       when(q.getQueuePath()).thenReturn(queuePathName);
+      preemptionDisabled = mockPreemptionStatus(queuePathName);
+      when(q.getPreemptionDisabled()).thenReturn(preemptionDisabled);
     }
     assert 0 == pqs.size();
     return root;
   }
 
+  // Determine if any of the elements in the queupath have preemption disabled.
+  // Also must handle the case where preemption disabled property is explicitly
+  // set to something other than the default. Assumes system-wide preemption
+  // property is true.
+  private boolean mockPreemptionStatus(String queuePathName) {
+    boolean preemptionDisabled = false;
+    StringTokenizer tokenizer = new StringTokenizer(queuePathName, ".");
+    String qName = "";
+    while(tokenizer.hasMoreTokens()) {
+      qName += tokenizer.nextToken();
+      preemptionDisabled = schedConf.getPreemptionDisabled(qName, preemptionDisabled);
+      qName += ".";
+    }
+    return preemptionDisabled;
+  }
+
   ParentQueue mockParentQueue(ParentQueue p, int subqueues,
       Deque<ParentQueue> pqs) {
     ParentQueue pq = mock(ParentQueue.class);
@@ -965,7 +1052,8 @@ FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved,
     Resource unit = Resource.newInstance(gran, 0);
     List<RMContainer> cReserved = new ArrayList<RMContainer>();
     for (int i = 0; i < reserved; i += gran) {
-      cReserved.add(mockContainer(appAttId, cAlloc, unit, 1));
+      cReserved.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
+          .getValue()));
       ++cAlloc;
     }
     when(app.getReservedContainers()).thenReturn(cReserved);
@@ -973,9 +1061,16 @@ FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved,
     List<RMContainer> cLive = new ArrayList<RMContainer>();
     for (int i = 0; i < used; i += gran) {
       if(setAMContainer && i == 0){
-        cLive.add(mockContainer(appAttId, cAlloc, unit, 0));
-      }else{
-        cLive.add(mockContainer(appAttId, cAlloc, unit, 1));
+        cLive.add(mockContainer(appAttId, cAlloc, unit, priority.AMCONTAINER
+            .getValue()));
+      }else if(setLabeledContainer && i ==1){
+        cLive.add(mockContainer(appAttId, cAlloc, unit,
+            priority.LABELEDCONTAINER.getValue()));
+        ++used;
+      }
+      else{
+        cLive.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
+            .getValue()));
       }
       ++cAlloc;
     }
@@ -984,18 +1079,21 @@ FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved,
   }
 
   RMContainer mockContainer(ApplicationAttemptId appAttId, int id,
-      Resource r, int priority) {
+      Resource r, int cpriority) {
     ContainerId cId = ContainerId.newContainerId(appAttId, id);
     Container c = mock(Container.class);
     when(c.getResource()).thenReturn(r);
-    when(c.getPriority()).thenReturn(Priority.create(priority));
+    when(c.getPriority()).thenReturn(Priority.create(cpriority));
     RMContainer mC = mock(RMContainer.class);
     when(mC.getContainerId()).thenReturn(cId);
     when(mC.getContainer()).thenReturn(c);
     when(mC.getApplicationAttemptId()).thenReturn(appAttId);
-    if(0 == priority){
+    if (priority.AMCONTAINER.getValue() == cpriority) {
       when(mC.isAMContainer()).thenReturn(true);
     }
+    if (priority.LABELEDCONTAINER.getValue() == cpriority) {
+      when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
+    }
     return mC;
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
similarity index 85%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
index 89053ca9baa4a..b1be525ae8574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/MemoryRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
@@ -25,9 +25,10 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
 
-public class MemoryRMNodeLabelsManager extends RMNodeLabelsManager {
+public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
   Map<NodeId, Set<String>> lastNodeToLabels = null;
   Collection<String> lastAddedlabels = null;
   Collection<String> lastRemovedlabels = null;
@@ -79,4 +80,11 @@ protected void startDispatcher() {
   protected void stopDispatcher() {
     // do nothing
   }
+  
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+    // always enable node labels while using MemoryRMNodeLabelsManager
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    super.serviceInit(conf);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index ed675f3736324..a91947fa58b56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -20,13 +20,16 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
@@ -42,12 +45,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
   private final Resource SMALL_RESOURCE = Resource.newInstance(100, 0);
   private final Resource LARGE_NODE = Resource.newInstance(1000, 0);
   
-  MemoryRMNodeLabelsManager mgr = null;
+  NullRMNodeLabelsManager mgr = null;
 
   @Before
   public void before() {
-    mgr = new MemoryRMNodeLabelsManager();
-    mgr.init(new Configuration());
+    mgr = new NullRMNodeLabelsManager();
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    mgr.init(conf);
     mgr.start();
   }
 
@@ -206,16 +211,14 @@ public void testGetQueueResource() throws Exception {
     
     /*
      * Node->Labels:
-     *   host1 : red, blue
-     *   host2 : blue, yellow
+     *   host1 : red
+     *   host2 : blue
      *   host3 : yellow
      *   host4 :
      */
     mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow"));
-    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"),
-        toSet("red", "blue")));
-    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"),
-        toSet("blue", "yellow")));
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red")));
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow")));
     
     // active two NM to n1, one large and one small
@@ -243,31 +246,29 @@ public void testGetQueueResource() throws Exception {
     // check resource
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
-        mgr.getQueueResource("Q2", q2Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+        mgr.getQueueResource("Q2", q2Label, clusterResource));
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q3", q3Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
         mgr.getQueueResource("Q4", q4Label, clusterResource));
     Assert.assertEquals(clusterResource,
         mgr.getQueueResource("Q5", q5Label, clusterResource));
     
-    mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host1"), toSet("red"),
-        toNodeId("host2"), toSet("blue", "yellow")));
-    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("host3"), toSet("red")));
+    mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
     /*
      * Check resource after changes some labels
      * Node->Labels:
-     *   host1 : blue (was: red, blue)
-     *   host2 : (was: blue, yellow)
-     *   host3 : red, yellow (was: yellow)
+     *   host1 : red
+     *   host2 : (was: blue)
+     *   host3 : yellow
      *   host4 :
      */
     
     // check resource
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q2", q2Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q3", q3Label, clusterResource));
@@ -279,9 +280,9 @@ public void testGetQueueResource() throws Exception {
     /*
      * Check resource after deactive/active some nodes 
      * Node->Labels:
-     *   (deactived) host1 : blue
+     *   (deactived) host1 : red
      *   host2 :
-     *   (deactived and then actived) host3 : red, yellow
+     *   (deactived and then actived) host3 : yellow
      *   host4 :
      */
     mgr.deactivateNode(NodeId.newInstance("host1", 1));
@@ -289,7 +290,7 @@ public void testGetQueueResource() throws Exception {
     mgr.activateNode(NodeId.newInstance("host3", 1), SMALL_RESOURCE);
     
     // check resource
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q2", q2Label, clusterResource));
@@ -326,9 +327,9 @@ public void testGetQueueResource() throws Exception {
     // check resource
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q2", q2Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q3", q3Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
         mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -339,7 +340,7 @@ public void testGetQueueResource() throws Exception {
      * Active NMs in nodes already have NM
      * Node->Labels:
      *   host2 :
-     *   host3 : red, yellow (3 NMs)
+     *   host3 : yellow (3 NMs)
      *   host4 : (2 NMs)
      */
     mgr.activateNode(NodeId.newInstance("host3", 2), SMALL_RESOURCE);
@@ -349,9 +350,9 @@ public void testGetQueueResource() throws Exception {
     // check resource
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q2", q2Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q3", q3Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
         mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -362,7 +363,7 @@ public void testGetQueueResource() throws Exception {
      * Deactive NMs in nodes already have NMs
      * Node->Labels:
      *   host2 :
-     *   host3 : red, yellow (2 NMs)
+     *   host3 : yellow (2 NMs)
      *   host4 : (0 NMs)
      */
     mgr.deactivateNode(NodeId.newInstance("host3", 3));
@@ -372,9 +373,9 @@ public void testGetQueueResource() throws Exception {
     // check resource
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
         mgr.getQueueResource("Q1", q1Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
         mgr.getQueueResource("Q2", q2Label, clusterResource));
-    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+    Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
         mgr.getQueueResource("Q3", q3Label, clusterResource));
     Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
         mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -428,4 +429,62 @@ public void testRemoveLabelsFromNode() throws Exception {
       Assert.fail("IOException from removeLabelsFromNode " + e);
     }
   }
+  
+  private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) {
+    for (NodeLabel info : infos) {
+      if (info.getLabelName().equals(labelName)) {
+        Assert.assertEquals(activeNMs, info.getNumActiveNMs());
+        Assert.assertEquals(memory, info.getResource().getMemory());
+        return;
+      }
+    }
+    Assert.fail("Failed to find info has label=" + labelName);
+  }
+  
+  @Test(timeout = 5000)
+  public void testPullRMNodeLabelsInfo() throws IOException {
+    mgr.addToCluserNodeLabels(toSet("x", "y", "z"));
+    mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
+    mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0));
+    mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0));
+    mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0));
+    mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0));
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"),
+        toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y")));
+    
+    // x, y, z and ""
+    List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo();
+    Assert.assertEquals(4, infos.size());
+    checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20);
+    checkNodeLabelInfo(infos, "x", 2, 20);
+    checkNodeLabelInfo(infos, "y", 1, 10);
+    checkNodeLabelInfo(infos, "z", 0, 0);
+  }
+  
+  @Test(timeout = 5000)
+  public void testLabelsToNodesOnNodeActiveDeactive() throws Exception {
+    // Activate a node without assigning any labels
+    mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
+    Assert.assertTrue(mgr.getLabelsToNodes().isEmpty());
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Add labels and replace labels on node
+    mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
+        toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3")));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Activate a node for which host to label mapping exists
+    mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
+
+    // Deactivate a node. Label mapping should still exist.
+    mgr.deactivateNode(NodeId.newInstance("n1", 1));
+    assertLabelsToNodesEquals(
+        mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 3d07b377f29d4..b01969bd08525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -411,16 +411,15 @@ public void testRMDTSecretManagerStateStore(
     RMDelegationTokenIdentifier dtId1 =
         new RMDelegationTokenIdentifier(new Text("owner1"),
           new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1111;
+    dtId1.setSequenceNumber(sequenceNumber);
     byte[] tokenBeforeStore = dtId1.getBytes();
     Long renewDate1 = new Long(System.currentTimeMillis());
-    int sequenceNumber = 1111;
-    store.storeRMDelegationTokenAndSequenceNumber(dtId1, renewDate1,
-      sequenceNumber);
+    store.storeRMDelegationToken(dtId1, renewDate1);
     modifyRMDelegationTokenState();
     Map<RMDelegationTokenIdentifier, Long> token1 =
         new HashMap<RMDelegationTokenIdentifier, Long>();
     token1.put(dtId1, renewDate1);
-
     // store delegation key;
     DelegationKey key = new DelegationKey(1234, 4321 , "keyBytes".getBytes());
     HashSet<DelegationKey> keySet = new HashSet<DelegationKey>();
@@ -440,9 +439,7 @@ public void testRMDTSecretManagerStateStore(
 
     // update RM delegation token;
     renewDate1 = new Long(System.currentTimeMillis());
-    ++sequenceNumber;
-    store.updateRMDelegationTokenAndSequenceNumber(
-        dtId1, renewDate1, sequenceNumber);
+    store.updateRMDelegationToken(dtId1, renewDate1);
     token1.put(dtId1, renewDate1);
 
     RMDTSecretManagerState updateSecretManagerState =
@@ -463,7 +460,7 @@ public void testRMDTSecretManagerStateStore(
         noKeySecretManagerState.getDTSequenceNumber());
 
     // check to delete delegationToken
-    store.removeRMDelegationToken(dtId1, sequenceNumber);
+    store.removeRMDelegationToken(dtId1);
     RMDTSecretManagerState noKeyAndTokenSecretManagerState =
         store.loadState().getRMDTSecretManagerState();
     token1.clear();
@@ -616,7 +613,8 @@ public void testAMRMTokenSecretManagerStateStore(
     AMRMTokenSecretManagerState state1 =
         AMRMTokenSecretManagerState.newInstance(
           firstMasterKeyData.getMasterKey(), null);
-    rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state1,
+    rmContext.getStateStore()
+        .storeOrUpdateAMRMTokenSecretManager(state1,
       false);
 
     // load state
@@ -636,7 +634,7 @@ public void testAMRMTokenSecretManagerStateStore(
         AMRMTokenSecretManagerState
           .newInstance(firstMasterKeyData.getMasterKey(),
             secondMasterKeyData.getMasterKey());
-    rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state2,
+    rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2,
       true);
 
     // load state
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 88e5393f14df5..d0d19e310317e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.TestZKRMStateStore.TestZKRMStateStoreTester;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -166,6 +167,59 @@ public void testFSRMStateStore() throws Exception {
     }
   }
 
+  @Test(timeout = 60000)
+  public void testCheckMajorVersionChange() throws Exception {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    try {
+      fsTester = new TestFSRMStateStoreTester(cluster) {
+        Version VERSION_INFO = Version.newInstance(Integer.MAX_VALUE, 0);
+
+        @Override
+        public Version getCurrentVersion() throws Exception {
+          return VERSION_INFO;
+        }
+
+        @Override
+        public RMStateStore getRMStateStore() throws Exception {
+          YarnConfiguration conf = new YarnConfiguration();
+          conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
+              workingDirPathURI.toString());
+          conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
+              "100,6000");
+          this.store = new TestFileSystemRMStore(conf) {
+            Version storedVersion = null;
+
+            @Override
+            public Version getCurrentVersion() {
+              return VERSION_INFO;
+            }
+
+            @Override
+            protected synchronized Version loadVersion() throws Exception {
+              return storedVersion;
+            }
+
+            @Override
+            protected synchronized void storeVersion() throws Exception {
+              storedVersion = VERSION_INFO;
+            }
+          };
+          return store;
+        }
+      };
+
+      // default version
+      RMStateStore store = fsTester.getRMStateStore();
+      Version defaultVersion = fsTester.getCurrentVersion();
+      store.checkVersion();
+      Assert.assertEquals(defaultVersion, store.loadVersion());
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   @Override
   protected void modifyAppState() throws Exception {
     // imitate appAttemptFile1 is still .new, but old one is deleted
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index e936677868e62..bbbf5a12ca369 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -33,6 +33,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -42,6 +44,7 @@
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -56,6 +59,7 @@
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestZKRMStateStore extends RMStateStoreTestBase {
@@ -99,7 +103,7 @@ public String getAppNode(String appId) {
 
     public RMStateStore getRMStateStore() throws Exception {
       YarnConfiguration conf = new YarnConfiguration();
-      workingZnode = "/Test";
+      workingZnode = "/jira/issue/3077/rmstore";
       conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
       conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
       this.client = createClient();
@@ -144,6 +148,52 @@ public void testZKRMStateStoreRealZK() throws Exception {
     testAMRMTokenSecretManagerStateStore(zkTester);
   }
 
+  @Test (timeout = 60000)
+  public void testCheckMajorVersionChange() throws Exception {
+    TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester() {
+      Version VERSION_INFO = Version.newInstance(Integer.MAX_VALUE, 0);
+
+      @Override
+      public Version getCurrentVersion() throws Exception {
+        return VERSION_INFO;
+      }
+
+      @Override
+      public RMStateStore getRMStateStore() throws Exception {
+        YarnConfiguration conf = new YarnConfiguration();
+        workingZnode = "/jira/issue/3077/rmstore";
+        conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
+        conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
+        this.client = createClient();
+        this.store = new TestZKRMStateStoreInternal(conf, workingZnode) {
+          Version storedVersion = null;
+
+          @Override
+          public Version getCurrentVersion() {
+            return VERSION_INFO;
+          }
+
+          @Override
+          protected synchronized Version loadVersion() throws Exception {
+            return storedVersion;
+          }
+
+          @Override
+          protected synchronized void storeVersion() throws Exception {
+            storedVersion = VERSION_INFO;
+          }
+        };
+        return this.store;
+      }
+
+    };
+    // default version
+    RMStateStore store = zkTester.getRMStateStore();
+    Version defaultVersion = zkTester.getCurrentVersion();
+    store.checkVersion();
+    Assert.assertEquals(defaultVersion, store.loadVersion());
+  }
+
   private Configuration createHARMConf(
       String rmIds, String rmId, int adminPort) {
     Configuration conf = new YarnConfiguration();
@@ -282,7 +332,42 @@ public void testFencedState() throws Exception {
     store.removeApplication(mockApp);
     assertEquals("RMStateStore should have been in fenced state",
             true, store.isFencedState());
- 
+
+    // store RM delegation token;
+    RMDelegationTokenIdentifier dtId1 =
+        new RMDelegationTokenIdentifier(new Text("owner1"),
+            new Text("renewer1"), new Text("realuser1"));
+    Long renewDate1 = new Long(System.currentTimeMillis()); 
+    dtId1.setSequenceNumber(1111);
+    store.storeRMDelegationToken(dtId1, renewDate1);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
+    store.updateRMDelegationToken(dtId1, renewDate1);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
+    // remove delegation key;
+    store.removeRMDelegationToken(dtId1);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
+    // store delegation master key;
+    DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    store.storeRMDTMasterKey(key);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
+    // remove delegation master key;
+    store.removeRMDTMasterKey(key);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
+    // store or update AMRMToken;
+    store.storeOrUpdateAMRMTokenSecretManager(null, false);
+    assertEquals("RMStateStore should have been in fenced state", true,
+        store.isFencedState());
+
     store.close();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index 90e71bf942480..bfaf06bddc545 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -23,11 +23,12 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Random;
-import java.util.Set;
 import java.util.TreeMap;
 
 import org.apache.hadoop.conf.Configuration;
@@ -40,12 +41,16 @@
 import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@@ -102,6 +107,83 @@ public static void validateNewReservationQueue(
         .assertTrue(newPlan.getSharingPolicy() instanceof CapacityOverTimePolicy);
   }
 
+  public static void setupFSAllocationFile(String allocationFile)
+      throws IOException {
+    PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"default\">");
+    out.println("<weight>1</weight>");
+    out.println("</queue>");
+    out.println("<queue name=\"a\">");
+    out.println("<weight>1</weight>");
+    out.println("<queue name=\"a1\">");
+    out.println("<weight>3</weight>");
+    out.println("</queue>");
+    out.println("<queue name=\"a2\">");
+    out.println("<weight>7</weight>");
+    out.println("</queue>");
+    out.println("</queue>");
+    out.println("<queue name=\"dedicated\">");
+    out.println("<reservation></reservation>");
+    out.println("<weight>8</weight>");
+    out.println("</queue>");
+    out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
+    out.println("</allocations>");
+    out.close();
+  }
+
+  public static void updateFSAllocationFile(String allocationFile)
+      throws IOException {
+    PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"default\">");
+    out.println("<weight>5</weight>");
+    out.println("</queue>");
+    out.println("<queue name=\"a\">");
+    out.println("<weight>5</weight>");
+    out.println("<queue name=\"a1\">");
+    out.println("<weight>3</weight>");
+    out.println("</queue>");
+    out.println("<queue name=\"a2\">");
+    out.println("<weight>7</weight>");
+    out.println("</queue>");
+    out.println("</queue>");
+    out.println("<queue name=\"dedicated\">");
+    out.println("<reservation></reservation>");
+    out.println("<weight>80</weight>");
+    out.println("</queue>");
+    out.println("<queue name=\"reservation\">");
+    out.println("<reservation></reservation>");
+    out.println("<weight>10</weight>");
+    out.println("</queue>");
+    out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
+    out.println("</allocations>");
+    out.close();
+  }
+
+  public static FairScheduler setupFairScheduler(
+      ReservationSystemTestUtil testUtil,
+      RMContext rmContext, Configuration conf, int numContainers) throws
+      IOException {
+    FairScheduler scheduler = new FairScheduler();
+    scheduler.setRMContext(rmContext);
+
+    when(rmContext.getScheduler()).thenReturn(scheduler);
+
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, rmContext);
+
+
+    Resource resource = testUtil.calculateClusterResource(numContainers);
+    RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+    return scheduler;
+  }
+
   @SuppressWarnings("unchecked")
   public CapacityScheduler mockCapacityScheduler(int numContainers)
       throws IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index 4eedd42cc66f9..b8663f660d424 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -33,25 +33,19 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
@@ -62,21 +56,12 @@
 import org.mockito.Matchers;
 import org.mockito.Mockito;
 
-public class TestCapacitySchedulerPlanFollower {
+public class TestCapacitySchedulerPlanFollower extends TestSchedulerPlanFollowerBase {
 
-  final static int GB = 1024;
-
-  private Clock mClock = null;
-  private CapacityScheduler scheduler = null;
   private RMContext rmContext;
   private RMContext spyRMContext;
   private CapacitySchedulerContext csContext;
-  private ReservationAgent mAgent;
-  private Plan plan;
-  private Resource minAlloc = Resource.newInstance(GB, 1);
-  private Resource maxAlloc = Resource.newInstance(GB * 8, 8);
-  private ResourceCalculator res = new DefaultResourceCalculator();
-  private CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+  private CapacityScheduler cs;
 
   @Rule
   public TestName name = new TestName();
@@ -84,7 +69,9 @@ public class TestCapacitySchedulerPlanFollower {
   @Before
   public void setUp() throws Exception {
     CapacityScheduler spyCs = new CapacityScheduler();
-    scheduler = spy(spyCs);
+    cs = spy(spyCs);
+    scheduler = cs;
+
     rmContext = TestUtils.getMockRMContext();
     spyRMContext = spy(rmContext);
 
@@ -95,12 +82,13 @@ public void setUp() throws Exception {
         .thenReturn(null);
     Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
     when(spyRMContext.getRMApps()).thenReturn(spyApps);
+    when(spyRMContext.getScheduler()).thenReturn(scheduler);
 
     CapacitySchedulerConfiguration csConf =
         new CapacitySchedulerConfiguration();
     ReservationSystemTestUtil.setupQueueConfiguration(csConf);
 
-    scheduler.setConf(csConf);
+    cs.setConf(csConf);
 
     csContext = mock(CapacitySchedulerContext.class);
     when(csContext.getConfiguration()).thenReturn(csConf);
@@ -119,9 +107,9 @@ public void setUp() throws Exception {
     when(csContext.getContainerTokenSecretManager()).thenReturn(
         containerTokenSecretManager);
 
-    scheduler.setRMContext(spyRMContext);
-    scheduler.init(csConf);
-    scheduler.start();
+    cs.setRMContext(spyRMContext);
+    cs.init(csConf);
+    cs.start();
 
     setupPlanFollower();
   }
@@ -132,7 +120,7 @@ private void setupPlanFollower() throws Exception {
     mAgent = mock(ReservationAgent.class);
 
     String reservationQ = testUtil.getFullReservationQueueName();
-    CapacitySchedulerConfiguration csConf = scheduler.getConfiguration();
+    CapacitySchedulerConfiguration csConf = cs.getConfiguration();
     csConf.setReservationWindow(reservationQ, 20L);
     csConf.setMaximumCapacity(reservationQ, 40);
     csConf.setAverageCapacity(reservationQ, 20);
@@ -153,155 +141,51 @@ public void testWithKillOnExpiry() throws PlanningException,
     testPlanFollower(false);
   }
 
-  private void testPlanFollower(boolean isMove) throws PlanningException,
-      InterruptedException, AccessControlException {
-    // Initialize plan based on move flag
-    plan =
-        new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
-            scheduler.getClusterResource(), 1L, res,
-            scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated",
-            null, isMove);
-
-    // add a few reservations to the plan
-    long ts = System.currentTimeMillis();
-    ReservationId r1 = ReservationId.newInstance(ts, 1);
-    int[] f1 = { 10, 10, 10, 10, 10 };
-    assertTrue(plan.toString(),
-        plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3",
-            "dedicated", 0, 0 + f1.length, ReservationSystemTestUtil
-                .generateAllocation(0L, 1L, f1), res, minAlloc)));
+  @Override
+  protected void verifyCapacity(Queue defQ) {
+    CSQueue csQueue = (CSQueue)defQ;
+    assertTrue(csQueue.getCapacity() > 0.9);
+  }
 
-    ReservationId r2 = ReservationId.newInstance(ts, 2);
-    assertTrue(plan.toString(),
-        plan.addReservation(new InMemoryReservationAllocation(r2, null, "u3",
-            "dedicated", 3, 3 + f1.length, ReservationSystemTestUtil
-                .generateAllocation(3L, 1L, f1), res, minAlloc)));
+  @Override
+  protected Queue getDefaultQueue() {
+    return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX);
+  }
 
-    ReservationId r3 = ReservationId.newInstance(ts, 3);
-    int[] f2 = { 0, 10, 20, 10, 0 };
-    assertTrue(plan.toString(),
-        plan.addReservation(new InMemoryReservationAllocation(r3, null, "u4",
-            "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil
-                .generateAllocation(10L, 1L, f2), res, minAlloc)));
+  @Override
+  protected int getNumberOfApplications(Queue queue) {
+    CSQueue csQueue = (CSQueue)queue;
+    int numberOfApplications = csQueue.getNumApplications();
+    return numberOfApplications;
+  }
 
+  @Override
+  protected CapacitySchedulerPlanFollower createPlanFollower() {
     CapacitySchedulerPlanFollower planFollower =
         new CapacitySchedulerPlanFollower();
     planFollower.init(mClock, scheduler, Collections.singletonList(plan));
+    return planFollower;
+  }
 
-    when(mClock.getTime()).thenReturn(0L);
-    planFollower.run();
-
-    CSQueue defQ =
-        scheduler.getQueue("dedicated" + PlanQueue.DEFAULT_QUEUE_SUFFIX);
-    CSQueue q = scheduler.getQueue(r1.toString());
+  @Override
+  protected void assertReservationQueueExists(ReservationId r) {
+    CSQueue q = cs.getQueue(r.toString());
     assertNotNull(q);
-    // submit an app to r1
-    String user_0 = "test-user";
-    ApplicationId appId = ApplicationId.newInstance(0, 1);
-    ApplicationAttemptId appAttemptId_0 =
-        ApplicationAttemptId.newInstance(appId, 0);
-    AppAddedSchedulerEvent addAppEvent =
-        new AppAddedSchedulerEvent(appId, q.getQueueName(), user_0);
-    scheduler.handle(addAppEvent);
-    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
-        new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
-    scheduler.handle(appAttemptAddedEvent);
-
-    // initial default reservation queue should have no apps
-    Assert.assertEquals(0, defQ.getNumApplications());
-
-    Assert.assertEquals(0.1, q.getCapacity(), 0.01);
-    Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
-    Assert.assertEquals(1, q.getNumApplications());
-
-    CSQueue q2 = scheduler.getQueue(r2.toString());
-    assertNull(q2);
-    CSQueue q3 = scheduler.getQueue(r3.toString());
-    assertNull(q3);
-
-    when(mClock.getTime()).thenReturn(3L);
-    planFollower.run();
+  }
 
-    Assert.assertEquals(0, defQ.getNumApplications());
-    q = scheduler.getQueue(r1.toString());
+  @Override
+  protected void assertReservationQueueExists(ReservationId r2,
+      double expectedCapacity, double expectedMaxCapacity) {
+    CSQueue q = cs.getQueue(r2.toString());
     assertNotNull(q);
-    Assert.assertEquals(0.1, q.getCapacity(), 0.01);
-    Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
-    Assert.assertEquals(1, q.getNumApplications());
-    q2 = scheduler.getQueue(r2.toString());
-    assertNotNull(q2);
-    Assert.assertEquals(0.1, q.getCapacity(), 0.01);
-    Assert.assertEquals(0.1, q.getMaximumCapacity(), 1.0);
-    q3 = scheduler.getQueue(r3.toString());
-    assertNull(q3);
-
-    when(mClock.getTime()).thenReturn(10L);
-    planFollower.run();
-
-    q = scheduler.getQueue(r1.toString());
-    if (isMove) {
-      // app should have been moved to default reservation queue
-      Assert.assertEquals(1, defQ.getNumApplications());
-      assertNull(q);
-    } else {
-      // app should be killed
-      Assert.assertEquals(0, defQ.getNumApplications());
-      assertNotNull(q);
-      AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
-          new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
-              RMAppAttemptState.KILLED, false);
-      scheduler.handle(appAttemptRemovedEvent);
-    }
-    q2 = scheduler.getQueue(r2.toString());
-    assertNull(q2);
-    q3 = scheduler.getQueue(r3.toString());
-    assertNotNull(q3);
-    Assert.assertEquals(0, q3.getCapacity(), 0.01);
-    Assert.assertEquals(1.0, q3.getMaximumCapacity(), 1.0);
-
-    when(mClock.getTime()).thenReturn(11L);
-    planFollower.run();
-
-    if (isMove) {
-      // app should have been moved to default reservation queue
-      Assert.assertEquals(1, defQ.getNumApplications());
-    } else {
-      // app should be killed
-      Assert.assertEquals(0, defQ.getNumApplications());
-    }
-    q = scheduler.getQueue(r1.toString());
-    assertNull(q);
-    q2 = scheduler.getQueue(r2.toString());
-    assertNull(q2);
-    q3 = scheduler.getQueue(r3.toString());
-    assertNotNull(q3);
-    Assert.assertEquals(0.1, q3.getCapacity(), 0.01);
-    Assert.assertEquals(0.1, q3.getMaximumCapacity(), 1.0);
-
-    when(mClock.getTime()).thenReturn(12L);
-    planFollower.run();
-
-    q = scheduler.getQueue(r1.toString());
-    assertNull(q);
-    q2 = scheduler.getQueue(r2.toString());
-    assertNull(q2);
-    q3 = scheduler.getQueue(r3.toString());
-    assertNotNull(q3);
-    Assert.assertEquals(0.2, q3.getCapacity(), 0.01);
-    Assert.assertEquals(0.2, q3.getMaximumCapacity(), 1.0);
-
-    when(mClock.getTime()).thenReturn(16L);
-    planFollower.run();
+    Assert.assertEquals(expectedCapacity, q.getCapacity(), 0.01);
+    Assert.assertEquals(expectedMaxCapacity, q.getMaximumCapacity(), 1.0);
+  }
 
-    q = scheduler.getQueue(r1.toString());
-    assertNull(q);
-    q2 = scheduler.getQueue(r2.toString());
+  @Override
+  protected void assertReservationQueueDoesNotExist(ReservationId r2) {
+    CSQueue q2 = cs.getQueue(r2.toString());
     assertNull(q2);
-    q3 = scheduler.getQueue(r3.toString());
-    assertNull(q3);
-
-    assertTrue(defQ.getCapacity() > 0.9);
-
   }
 
   public static ApplicationACLsManager mockAppACLsManager() {
@@ -312,8 +196,11 @@ public static ApplicationACLsManager mockAppACLsManager() {
   @After
   public void tearDown() throws Exception {
     if (scheduler != null) {
-      scheduler.stop();
+      cs.stop();
     }
   }
 
+  protected Queue getReservationQueue(String reservationId) {
+    return cs.getQueue(reservationId);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java
new file mode 100644
index 0000000000000..f294eaf09452f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairReservationSystem.java
@@ -0,0 +1,128 @@
+/*******************************************************************************
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *******************************************************************************/
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerTestBase;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+public class TestFairReservationSystem {
+  private final static String ALLOC_FILE = new File(FairSchedulerTestBase.
+    TEST_DIR,
+      TestFairReservationSystem.class.getName() + ".xml").getAbsolutePath();
+  private Configuration conf;
+  private FairScheduler scheduler;
+  private FairSchedulerTestBase testHelper = new FairSchedulerTestBase();
+
+  protected Configuration createConfiguration() {
+    Configuration conf = testHelper.createConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+        ResourceScheduler.class);
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+    return conf;
+  }
+
+  @Before
+  public void setup() throws IOException {
+    conf = createConfiguration();
+  }
+
+  @After
+  public void teardown() {
+    conf = null;
+  }
+
+  @Test
+  public void testFairReservationSystemInitialize() throws IOException {
+    ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE);
+
+    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+    
+    // Setup
+    RMContext mockRMContext = testUtil.createRMContext(conf);
+    scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil,
+        mockRMContext, conf, 10);
+
+    FairReservationSystem reservationSystem = new FairReservationSystem();
+    reservationSystem.setRMContext(mockRMContext);
+
+    try {
+      reservationSystem.reinitialize(scheduler.getConf(), mockRMContext);
+    } catch (YarnException e) {
+      Assert.fail(e.getMessage());
+    }
+
+    ReservationSystemTestUtil.validateReservationQueue(reservationSystem,
+        testUtil.getFullReservationQueueName());
+  }
+
+  @Test
+  public void testFairReservationSystemReinitialize() throws IOException {
+    ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE);
+
+    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+
+    // Setup
+    RMContext mockRMContext = testUtil.createRMContext(conf);
+    scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil,
+        mockRMContext, conf, 10);
+
+    FairReservationSystem reservationSystem = new FairReservationSystem();
+    reservationSystem.setRMContext(mockRMContext);
+
+    try {
+      reservationSystem.reinitialize(scheduler.getConf(), mockRMContext);
+    } catch (YarnException e) {
+      Assert.fail(e.getMessage());
+    }
+
+    // Assert queue in original config
+    final String planQNam = testUtil.getFullReservationQueueName();
+    ReservationSystemTestUtil.validateReservationQueue(reservationSystem,
+        planQNam);
+
+    // Dynamically add a plan
+    ReservationSystemTestUtil.updateFSAllocationFile(ALLOC_FILE);
+    scheduler.reinitialize(conf, mockRMContext);
+
+    try {
+      reservationSystem.reinitialize(conf, mockRMContext);
+    } catch (YarnException e) {
+      Assert.fail(e.getMessage());
+    }
+
+    String newQueue = "root.reservation";
+    ReservationSystemTestUtil.validateNewReservationQueue
+        (reservationSystem, newQueue);
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
new file mode 100644
index 0000000000000..e9a4f50bac52a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
@@ -0,0 +1,203 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerTestBase;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.Clock;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+public class TestFairSchedulerPlanFollower extends
+    TestSchedulerPlanFollowerBase {
+  private final static String ALLOC_FILE = new File(FairSchedulerTestBase.
+      TEST_DIR,
+      TestFairReservationSystem.class.getName() + ".xml").getAbsolutePath();
+  private RMContext rmContext;
+  private RMContext spyRMContext;
+  private FairScheduler fs;
+  private Configuration conf;
+  private FairSchedulerTestBase testHelper = new FairSchedulerTestBase();
+
+  @Rule
+  public TestName name = new TestName();
+
+  protected Configuration createConfiguration() {
+    Configuration conf = testHelper.createConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+        ResourceScheduler.class);
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+    return conf;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    conf = createConfiguration();
+    ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE);
+    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+
+    // Setup
+    rmContext = TestUtils.getMockRMContext();
+    spyRMContext = spy(rmContext);
+    fs = ReservationSystemTestUtil.setupFairScheduler(testUtil,
+        spyRMContext, conf, 125);
+    scheduler = fs;
+
+    ConcurrentMap<ApplicationId, RMApp> spyApps =
+        spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+    RMApp rmApp = mock(RMApp.class);
+    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
+        .thenReturn(null);
+    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
+    when(spyRMContext.getRMApps()).thenReturn(spyApps);
+
+    ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE);
+    setupPlanFollower();
+  }
+
+  private void setupPlanFollower() throws Exception {
+    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+    mClock = mock(Clock.class);
+    mAgent = mock(ReservationAgent.class);
+
+    String reservationQ = testUtil.getFullReservationQueueName();
+    AllocationConfiguration allocConf = fs.getAllocationConfiguration();
+    allocConf.setReservationWindow(20L);
+    allocConf.setAverageCapacity(20);
+    policy.init(reservationQ, allocConf);
+  }
+
+  @Test
+  public void testWithMoveOnExpiry() throws PlanningException,
+      InterruptedException, AccessControlException {
+    // invoke plan follower test with move
+    testPlanFollower(true);
+  }
+
+  @Test
+  public void testWithKillOnExpiry() throws PlanningException,
+      InterruptedException, AccessControlException {
+    // invoke plan follower test with kill
+    testPlanFollower(false);
+  }
+
+  @Override
+  protected void verifyCapacity(Queue defQ) {
+    assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) >
+        0.9);
+  }
+
+  @Override
+  protected Queue getDefaultQueue() {
+    return getReservationQueue("dedicated" +
+        ReservationConstants.DEFAULT_QUEUE_SUFFIX);
+  }
+
+  @Override
+  protected int getNumberOfApplications(Queue queue) {
+    int numberOfApplications = fs.getAppsInQueue(queue.getQueueName()).size();
+    return numberOfApplications;
+  }
+
+  @Override
+  protected AbstractSchedulerPlanFollower createPlanFollower() {
+    FairSchedulerPlanFollower planFollower =
+        new FairSchedulerPlanFollower();
+    planFollower.init(mClock, scheduler, Collections.singletonList(plan));
+    return planFollower;
+  }
+
+  @Override
+  protected void assertReservationQueueExists(ReservationId r) {
+    Queue q = getReservationQueue(r.toString());
+    assertNotNull(q);
+  }
+
+  @Override
+  protected void assertReservationQueueExists(ReservationId r,
+      double expectedCapacity, double expectedMaxCapacity) {
+    FSLeafQueue q = fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" +
+        "." +
+        r, false);
+    assertNotNull(q);
+    // For now we are setting both to same weight
+    Assert.assertEquals(expectedCapacity, q.getWeights().getWeight
+        (ResourceType.MEMORY), 0.01);
+  }
+
+  @Override
+  protected void assertReservationQueueDoesNotExist(ReservationId r) {
+    Queue q = getReservationQueue(r.toString());
+    assertNull(q);
+  }
+
+  @Override
+  protected Queue getReservationQueue(String r) {
+    return fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" +
+        "." +
+        r, false);
+  }
+
+  public static ApplicationACLsManager mockAppACLsManager() {
+    Configuration conf = new Configuration();
+    return new ApplicationACLsManager(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (scheduler != null) {
+      fs.stop();
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
new file mode 100644
index 0000000000000..50df8fe091f36
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Assert;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+public abstract class TestSchedulerPlanFollowerBase {
+  final static int GB = 1024;
+  protected Clock mClock = null;
+  protected ResourceScheduler scheduler = null;
+  protected ReservationAgent mAgent;
+  protected Resource minAlloc = Resource.newInstance(GB, 1);
+  protected Resource maxAlloc = Resource.newInstance(GB * 8, 8);
+  protected CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+  protected Plan plan;
+  private ResourceCalculator res = new DefaultResourceCalculator();
+
+  protected void testPlanFollower(boolean isMove) throws PlanningException,
+      InterruptedException, AccessControlException {
+    // Initialize plan based on move flag
+    plan =
+        new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
+            scheduler.getClusterResource(), 1L, res,
+            scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated",
+            null, isMove);
+
+    // add a few reservations to the plan
+    long ts = System.currentTimeMillis();
+    ReservationId r1 = ReservationId.newInstance(ts, 1);
+    int[] f1 = { 10, 10, 10, 10, 10 };
+    assertTrue(plan.toString(),
+        plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3",
+            "dedicated", 0, 0 + f1.length, ReservationSystemTestUtil
+                .generateAllocation(0L, 1L, f1), res, minAlloc)));
+
+    ReservationId r2 = ReservationId.newInstance(ts, 2);
+    assertTrue(plan.toString(),
+        plan.addReservation(new InMemoryReservationAllocation(r2, null, "u3",
+            "dedicated", 3, 3 + f1.length, ReservationSystemTestUtil
+                .generateAllocation(3L, 1L, f1), res, minAlloc)));
+
+    ReservationId r3 = ReservationId.newInstance(ts, 3);
+    int[] f2 = { 0, 10, 20, 10, 0 };
+    assertTrue(plan.toString(),
+        plan.addReservation(new InMemoryReservationAllocation(r3, null, "u4",
+            "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil
+                .generateAllocation(10L, 1L, f2), res, minAlloc)));
+
+    AbstractSchedulerPlanFollower planFollower = createPlanFollower();
+
+    when(mClock.getTime()).thenReturn(0L);
+    planFollower.run();
+
+    Queue q = getReservationQueue(r1.toString());
+    assertReservationQueueExists(r1);
+    // submit an app to r1
+    String user_0 = "test-user";
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId_0 =
+        ApplicationAttemptId.newInstance(appId, 0);
+    AppAddedSchedulerEvent addAppEvent =
+        new AppAddedSchedulerEvent(appId, q.getQueueName(), user_0);
+    scheduler.handle(addAppEvent);
+    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
+        new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
+    scheduler.handle(appAttemptAddedEvent);
+
+    // initial default reservation queue should have no apps
+
+    Queue defQ = getDefaultQueue();
+    Assert.assertEquals(0, getNumberOfApplications(defQ));
+
+    assertReservationQueueExists(r1, 0.1, 0.1);
+    Assert.assertEquals(1, getNumberOfApplications(q));
+
+    assertReservationQueueDoesNotExist(r2);
+    assertReservationQueueDoesNotExist(r3);
+
+    when(mClock.getTime()).thenReturn(3L);
+    planFollower.run();
+
+    Assert.assertEquals(0, getNumberOfApplications(defQ));
+    assertReservationQueueExists(r1, 0.1, 0.1);
+    Assert.assertEquals(1, getNumberOfApplications(q));
+    assertReservationQueueExists(r2, 0.1, 0.1);
+    assertReservationQueueDoesNotExist(r3);
+
+    when(mClock.getTime()).thenReturn(10L);
+    planFollower.run();
+
+    q = getReservationQueue(r1.toString());
+    if (isMove) {
+      // app should have been moved to default reservation queue
+      Assert.assertEquals(1, getNumberOfApplications(defQ));
+      assertNull(q);
+    } else {
+      // app should be killed
+      Assert.assertEquals(0, getNumberOfApplications(defQ));
+      assertNotNull(q);
+      AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
+          new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
+              RMAppAttemptState.KILLED, false);
+      scheduler.handle(appAttemptRemovedEvent);
+    }
+    assertReservationQueueDoesNotExist(r2);
+    assertReservationQueueExists(r3, 0, 1.0);
+
+    when(mClock.getTime()).thenReturn(11L);
+    planFollower.run();
+
+    if (isMove) {
+      // app should have been moved to default reservation queue
+      Assert.assertEquals(1, getNumberOfApplications(defQ));
+    } else {
+      // app should be killed
+      Assert.assertEquals(0, getNumberOfApplications(defQ));
+    }
+    assertReservationQueueDoesNotExist(r1);
+    assertReservationQueueDoesNotExist(r2);
+    assertReservationQueueExists(r3, 0.1, 0.1);
+
+    when(mClock.getTime()).thenReturn(12L);
+    planFollower.run();
+
+    assertReservationQueueDoesNotExist(r1);
+    assertReservationQueueDoesNotExist(r2);
+    assertReservationQueueExists(r3, 0.2, 0.2);
+
+    when(mClock.getTime()).thenReturn(16L);
+    planFollower.run();
+
+    assertReservationQueueDoesNotExist(r1);
+    assertReservationQueueDoesNotExist(r2);
+    assertReservationQueueDoesNotExist(r3);
+
+    verifyCapacity(defQ);
+  }
+
+  protected abstract Queue getReservationQueue(String reservationId);
+
+  protected abstract void verifyCapacity(Queue defQ);
+
+  protected abstract Queue getDefaultQueue();
+
+  protected abstract int getNumberOfApplications(Queue queue);
+
+  protected abstract AbstractSchedulerPlanFollower createPlanFollower();
+
+  protected abstract void assertReservationQueueExists(ReservationId r);
+
+  protected abstract void assertReservationQueueExists(ReservationId r2,
+      double expectedCapacity, double expectedMaxCapacity);
+
+  protected abstract void assertReservationQueueDoesNotExist(ReservationId r2);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 787b5d7b39194..ec990f95b8067 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -55,6 +56,7 @@ public class MockRMApp implements RMApp {
   StringBuilder diagnostics = new StringBuilder();
   RMAppAttempt attempt;
   int maxAppAttempts = 1;
+  ResourceRequest amReq;
 
   public MockRMApp(int newid, long time, RMAppState newState) {
     finish = time;
@@ -264,4 +266,9 @@ public RMAppMetrics getRMAppMetrics() {
   public ReservationId getReservationId() {
     throw new UnsupportedOperationException("Not supported yet.");
   }
+  
+  @Override
+  public ResourceRequest getAMResourceRequest() {
+    return this.amReq; 
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 67bdae2c6cc45..48ce822fdc8d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -25,11 +28,17 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
-
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("unchecked")
 public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
 
   public TestAbstractYarnScheduler(SchedulerType type) {
@@ -210,4 +219,136 @@ private void testMaximumAllocationVCoresHelper(
     Assert.assertEquals(0, scheduler.getNumClusterNodes());
   }
 
+  @Test
+  public void testUpdateMaxAllocationUsesTotal() throws IOException {
+    final int configuredMaxVCores = 20;
+    final int configuredMaxMemory = 10 * 1024;
+    Resource configuredMaximumResource = Resource.newInstance
+        (configuredMaxMemory, configuredMaxVCores);
+
+    configureScheduler();
+    YarnConfiguration conf = getConf();
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        configuredMaxVCores);
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        configuredMaxMemory);
+    conf.setLong(
+        YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
+        0);
+
+    MockRM rm = new MockRM(conf);
+    try {
+      rm.start();
+      AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
+          .getResourceScheduler();
+
+      Resource emptyResource = Resource.newInstance(0, 0);
+      Resource fullResource1 = Resource.newInstance(1024, 5);
+      Resource fullResource2 = Resource.newInstance(2048, 10);
+
+      SchedulerNode mockNode1 = mock(SchedulerNode.class);
+      when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
+      when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
+      when(mockNode1.getTotalResource()).thenReturn(fullResource1);
+
+      SchedulerNode mockNode2 = mock(SchedulerNode.class);
+      when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
+      when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
+      when(mockNode2.getTotalResource()).thenReturn(fullResource2);
+
+      verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
+
+      scheduler.nodes = new HashMap<NodeId, SchedulerNode>();
+
+      scheduler.nodes.put(mockNode1.getNodeID(), mockNode1);
+      scheduler.updateMaximumAllocation(mockNode1, true);
+      verifyMaximumResourceCapability(fullResource1, scheduler);
+
+      scheduler.nodes.put(mockNode2.getNodeID(), mockNode2);
+      scheduler.updateMaximumAllocation(mockNode2, true);
+      verifyMaximumResourceCapability(fullResource2, scheduler);
+
+      scheduler.nodes.remove(mockNode2.getNodeID());
+      scheduler.updateMaximumAllocation(mockNode2, false);
+      verifyMaximumResourceCapability(fullResource1, scheduler);
+
+      scheduler.nodes.remove(mockNode1.getNodeID());
+      scheduler.updateMaximumAllocation(mockNode1, false);
+      verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
+    } finally {
+      rm.stop();
+    }
+  }
+
+  @Test
+  public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
+    final int configuredMaxVCores = 20;
+    final int configuredMaxMemory = 10 * 1024;
+    Resource configuredMaximumResource = Resource.newInstance
+        (configuredMaxMemory, configuredMaxVCores);
+
+    configureScheduler();
+    YarnConfiguration conf = getConf();
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        configuredMaxVCores);
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        configuredMaxMemory);
+    conf.setLong(
+        YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
+        0);
+
+    MockRM rm = new MockRM(conf);
+    try {
+      rm.start();
+      AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
+          .getResourceScheduler();
+      verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
+
+      Resource resource1 = Resource.newInstance(2048, 5);
+      Resource resource2 = Resource.newInstance(4096, 10);
+      Resource resource3 = Resource.newInstance(512, 1);
+      Resource resource4 = Resource.newInstance(1024, 2);
+
+      RMNode node1 = MockNodes.newNodeInfo(
+          0, resource1, 1, "127.0.0.2");
+      scheduler.handle(new NodeAddedSchedulerEvent(node1));
+      RMNode node2 = MockNodes.newNodeInfo(
+          0, resource3, 2, "127.0.0.3");
+      scheduler.handle(new NodeAddedSchedulerEvent(node2));
+      verifyMaximumResourceCapability(resource1, scheduler);
+
+      // increase node1 resource
+      scheduler.updateNodeResource(node1, ResourceOption.newInstance(
+          resource2, 0));
+      verifyMaximumResourceCapability(resource2, scheduler);
+
+      // decrease node1 resource
+      scheduler.updateNodeResource(node1, ResourceOption.newInstance(
+          resource1, 0));
+      verifyMaximumResourceCapability(resource1, scheduler);
+
+      // increase node2 resource
+      scheduler.updateNodeResource(node2, ResourceOption.newInstance(
+          resource4, 0));
+      verifyMaximumResourceCapability(resource1, scheduler);
+
+      // decrease node2 resource
+      scheduler.updateNodeResource(node2, ResourceOption.newInstance(
+          resource3, 0));
+      verifyMaximumResourceCapability(resource1, scheduler);
+    } finally {
+      rm.stop();
+    }
+  }
+
+  private void verifyMaximumResourceCapability(
+      Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
+
+    final Resource schedulerMaximumResourceCapability = scheduler
+        .getMaximumResourceCapability();
+    Assert.assertEquals(expectedMaximumResource.getMemory(),
+        schedulerMaximumResourceCapability.getMemory());
+    Assert.assertEquals(expectedMaximumResource.getVirtualCores(),
+        schedulerMaximumResourceCapability.getVirtualCores());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
new file mode 100644
index 0000000000000..b6dfacb7028af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestResourceUsage {
+  private static final Log LOG = LogFactory.getLog(TestResourceUsage.class);
+  private String suffix;
+
+  @Parameterized.Parameters
+  public static Collection<String[]> getParameters() {
+    return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
+        { "Headroom" }, { "Reserved" }, { "AMUsed" } });
+  }
+
+  public TestResourceUsage(String suffix) {
+    this.suffix = suffix;
+  }
+
+  private static void dec(ResourceUsage obj, String suffix, Resource res,
+      String label) throws Exception {
+    executeByName(obj, "dec" + suffix, res, label);
+  }
+
+  private static void inc(ResourceUsage obj, String suffix, Resource res,
+      String label) throws Exception {
+    executeByName(obj, "inc" + suffix, res, label);
+  }
+
+  private static void set(ResourceUsage obj, String suffix, Resource res,
+      String label) throws Exception {
+    executeByName(obj, "set" + suffix, res, label);
+  }
+
+  private static Resource get(ResourceUsage obj, String suffix, String label)
+      throws Exception {
+    return executeByName(obj, "get" + suffix, null, label);
+  }
+
+  // Use reflection to avoid too much avoid code
+  private static Resource executeByName(ResourceUsage obj, String methodName,
+      Resource arg, String label) throws Exception {
+    // We have 4 kinds of method
+    // 1. getXXX() : Resource
+    // 2. getXXX(label) : Resource
+    // 3. set/inc/decXXX(res) : void
+    // 4. set/inc/decXXX(label, res) : void
+    if (methodName.startsWith("get")) {
+      Resource result;
+      if (label == null) {
+        // 1.
+        Method method = ResourceUsage.class.getDeclaredMethod(methodName);
+        result = (Resource) method.invoke(obj);
+      } else {
+        // 2.
+        Method method =
+            ResourceUsage.class.getDeclaredMethod(methodName, String.class);
+        result = (Resource) method.invoke(obj, label);
+      }
+      return result;
+    } else {
+      if (label == null) {
+        // 3.
+        Method method =
+            ResourceUsage.class.getDeclaredMethod(methodName, Resource.class);
+        method.invoke(obj, arg);
+      } else {
+        // 4.
+        Method method =
+            ResourceUsage.class.getDeclaredMethod(methodName, String.class,
+                Resource.class);
+        method.invoke(obj, label, arg);
+      }
+      return null;
+    }
+  }
+
+  private void internalTestModifyAndRead(String label) throws Exception {
+    ResourceUsage usage = new ResourceUsage();
+    Resource res;
+
+    // First get returns 0 always
+    res = get(usage, suffix, label);
+    check(0, 0, res);
+
+    // Add 1,1 should returns 1,1
+    inc(usage, suffix, Resource.newInstance(1, 1), label);
+    check(1, 1, get(usage, suffix, label));
+
+    // Set 2,2
+    set(usage, suffix, Resource.newInstance(2, 2), label);
+    check(2, 2, get(usage, suffix, label));
+
+    // dec 2,2
+    dec(usage, suffix, Resource.newInstance(2, 2), label);
+    check(0, 0, get(usage, suffix, label));
+  }
+
+  void check(int mem, int cpu, Resource res) {
+    Assert.assertEquals(mem, res.getMemory());
+    Assert.assertEquals(cpu, res.getVirtualCores());
+  }
+
+  @Test
+  public void testModifyAndRead() throws Exception {
+    LOG.info("Test - " + suffix);
+    internalTestModifyAndRead(null);
+    internalTestModifyAndRead("label");
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index c9e81eebb9714..6e2b56f5fd259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -62,6 +62,8 @@
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
@@ -213,11 +215,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
       resReq.setNodeLabelExpression("x");
       SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
           scheduler);
-      
-      resReq.setNodeLabelExpression("x && y");
-      SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
-          scheduler);
-      
+
       resReq.setNodeLabelExpression("y");
       SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
           scheduler);
@@ -252,6 +250,8 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
     } catch (InvalidResourceRequestException e) {
     }
     
+    // we don't allow specify more than two node labels in a single expression
+    // now
     try {
       // set queue accessible node labesl to [x, y]
       queueAccessibleNodeLabels.clear();
@@ -262,7 +262,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
           YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
       ResourceRequest resReq = BuilderUtils.newResourceRequest(
           mock(Priority.class), ResourceRequest.ANY, resource, 1);
-      resReq.setNodeLabelExpression("x && y && z");
+      resReq.setNodeLabelExpression("x && y");
       SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
           scheduler);
       fail("Should fail");
@@ -327,7 +327,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
       SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
           scheduler);
       
-      resReq.setNodeLabelExpression("x && y && z");
+      resReq.setNodeLabelExpression("y");
       SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
           scheduler);
       
@@ -336,7 +336,45 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
           scheduler);
     } catch (InvalidResourceRequestException e) {
       e.printStackTrace();
-      fail("Should be valid when request labels is empty");
+      fail("Should be valid when queue can access any labels");
+    }
+    
+    // we don't allow resource name other than ANY and specify label
+    try {
+      // set queue accessible node labesl to [x, y]
+      queueAccessibleNodeLabels.clear();
+      queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+      
+      Resource resource = Resources.createResource(
+          0,
+          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+      ResourceRequest resReq = BuilderUtils.newResourceRequest(
+          mock(Priority.class), "rack", resource, 1);
+      resReq.setNodeLabelExpression("x");
+      SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
+          scheduler);
+      fail("Should fail");
+    } catch (InvalidResourceRequestException e) {
+    }
+    
+    // we don't allow resource name other than ANY and specify label even if
+    // queue has accessible label = *
+    try {
+      // set queue accessible node labesl to *
+      queueAccessibleNodeLabels.clear();
+      queueAccessibleNodeLabels.addAll(Arrays
+          .asList(CommonNodeLabelsManager.ANY));
+      
+      Resource resource = Resources.createResource(
+          0,
+          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+      ResourceRequest resReq = BuilderUtils.newResourceRequest(
+          mock(Priority.class), "rack", resource, 1);
+      resReq.setNodeLabelExpression("x");
+      SchedulerUtils.validateResourceRequest(resReq, maxResource, "queue",
+          scheduler);
+      fail("Should fail");
+    } catch (InvalidResourceRequestException e) {
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 0cd74d03b4cf2..81a5aad955b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,16 +28,21 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -47,8 +52,10 @@
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -56,6 +63,7 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.Ignore;
 
 public class TestApplicationLimits {
   
@@ -119,8 +127,6 @@ public void setUp() throws IOException {
     // Some default values
     doReturn(100).when(queue).getMaxApplications();
     doReturn(25).when(queue).getMaxApplicationsPerUser();
-    doReturn(10).when(queue).getMaximumActiveApplications();
-    doReturn(2).when(queue).getMaximumActiveApplicationsPerUser();
   }
   
   private static final String A = "a";
@@ -136,10 +142,14 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
     final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
     conf.setCapacity(Q_B, 90);
     
+    conf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 50);
+    conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + "." + A, 5.0f);
+    
     LOG.info("Setup top-level queues a and b");
   }
 
-  private FiCaSchedulerApp getMockApplication(int appId, String user) {
+  private FiCaSchedulerApp getMockApplication(int appId, String user,
+    Resource amResource) {
     FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
     ApplicationAttemptId applicationAttemptId =
         TestUtils.getMockApplicationAttemptId(appId, 0);
@@ -147,9 +157,89 @@ private FiCaSchedulerApp getMockApplication(int appId, String user) {
         when(application).getApplicationId();
     doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
     doReturn(user).when(application).getUser();
+    doReturn(amResource).when(application).getAMResource();
     return application;
   }
   
+  @Test
+  public void testAMResourceLimit() throws Exception {
+    final String user_0 = "user_0";
+    final String user_1 = "user_1";
+    
+    // This uses the default 10% of cluster value for the max am resources
+    // which are allowed, at 80GB = 8GB for AM's at the queue level.  The user
+    // am limit is 4G initially (based on the queue absolute capacity)
+    // when there is only 1 user, and drops to 2G (the userlimit) when there
+    // is a second user
+    queue.updateClusterResource(Resource.newInstance(80 * GB, 40));
+    
+    ActiveUsersManager activeUsersManager = mock(ActiveUsersManager.class);
+    when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
+    
+    assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+    assertEquals(Resource.newInstance(4 * GB, 1),
+      queue.getUserAMResourceLimit());
+    
+    // Two apps for user_0, both start
+    int APPLICATION_ID = 0;
+    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0, 
+      Resource.newInstance(2 * GB, 1));
+    queue.submitApplicationAttempt(app_0, user_0);
+    assertEquals(1, queue.getNumActiveApplications());
+    assertEquals(0, queue.getNumPendingApplications());
+    assertEquals(1, queue.getNumActiveApplications(user_0));
+    assertEquals(0, queue.getNumPendingApplications(user_0));
+    
+    when(activeUsersManager.getNumActiveUsers()).thenReturn(1);
+
+    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0, 
+      Resource.newInstance(2 * GB, 1));
+    queue.submitApplicationAttempt(app_1, user_0);
+    assertEquals(2, queue.getNumActiveApplications());
+    assertEquals(0, queue.getNumPendingApplications());
+    assertEquals(2, queue.getNumActiveApplications(user_0));
+    assertEquals(0, queue.getNumPendingApplications(user_0));
+    
+    // AMLimits unchanged
+    assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+    assertEquals(Resource.newInstance(4 * GB, 1),
+      queue.getUserAMResourceLimit());
+    
+    // One app for user_1, starts
+    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_1, 
+      Resource.newInstance(2 * GB, 1));
+    queue.submitApplicationAttempt(app_2, user_1);
+    assertEquals(3, queue.getNumActiveApplications());
+    assertEquals(0, queue.getNumPendingApplications());
+    assertEquals(1, queue.getNumActiveApplications(user_1));
+    assertEquals(0, queue.getNumPendingApplications(user_1));
+    
+    when(activeUsersManager.getNumActiveUsers()).thenReturn(2);
+    
+    // Now userAMResourceLimit drops to the queue configured 50% as there is
+    // another user active
+    assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+    assertEquals(Resource.newInstance(2 * GB, 1),
+      queue.getUserAMResourceLimit());
+    
+    // Second user_1 app cannot start
+    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_1, 
+      Resource.newInstance(2 * GB, 1));
+    queue.submitApplicationAttempt(app_3, user_1);
+    assertEquals(3, queue.getNumActiveApplications());
+    assertEquals(1, queue.getNumPendingApplications());
+    assertEquals(1, queue.getNumActiveApplications(user_1));
+    assertEquals(1, queue.getNumPendingApplications(user_1));
+
+    // Now finish app so another should be activated
+    queue.finishApplicationAttempt(app_2, A);
+    assertEquals(3, queue.getNumActiveApplications());
+    assertEquals(0, queue.getNumPendingApplications());
+    assertEquals(1, queue.getNumActiveApplications(user_1));
+    assertEquals(0, queue.getNumPendingApplications(user_1));
+    
+  }
+  
   @Test
   public void testLimitsComputation() throws Exception {
     CapacitySchedulerConfiguration csConf = 
@@ -172,7 +262,8 @@ public void testLimitsComputation() throws Exception {
     when(csContext.getRMContext()).thenReturn(rmContext);
     
     // Say cluster has 100 nodes of 16G each
-    Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16);
+    Resource clusterResource = 
+      Resources.createResource(100 * 16 * GB, 100 * 16);
     when(csContext.getClusterResource()).thenReturn(clusterResource);
     
     Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
@@ -183,28 +274,14 @@ public void testLimitsComputation() throws Exception {
     LeafQueue queue = (LeafQueue)queues.get(A);
     
     LOG.info("Queue 'A' -" +
-    		" maxActiveApplications=" + queue.getMaximumActiveApplications() + 
-    		" maxActiveApplicationsPerUser=" + 
-    		queue.getMaximumActiveApplicationsPerUser());
-    int expectedMaxActiveApps = 
-        Math.max(1, 
-            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                   csConf.
-                     getMaximumApplicationMasterResourcePerQueuePercent(
-                                                        queue.getQueuePath()) *
-                   queue.getAbsoluteMaximumCapacity()));
-    assertEquals(expectedMaxActiveApps, 
-                 queue.getMaximumActiveApplications());
-    int expectedMaxActiveAppsUsingAbsCap = 
-            Math.max(1, 
-                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                       csConf.getMaximumApplicationMasterResourcePercent() *
-                       queue.getAbsoluteCapacity()));
-    assertEquals(
-        (int)Math.ceil(
-        		expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) * 
-            queue.getUserLimitFactor()), 
-        queue.getMaximumActiveApplicationsPerUser());
+    		" aMResourceLimit=" + queue.getAMResourceLimit() + 
+    		" UserAMResourceLimit=" + 
+    		queue.getUserAMResourceLimit());
+    
+    assertEquals(queue.getAMResourceLimit(), Resource.newInstance(160*GB, 1));
+    assertEquals(queue.getUserAMResourceLimit(), 
+      Resource.newInstance(80*GB, 1));
+    
     assertEquals(
         (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
         queue.getMetrics().getAvailableMB()
@@ -213,24 +290,11 @@ public void testLimitsComputation() throws Exception {
     // Add some nodes to the cluster & test new limits
     clusterResource = Resources.createResource(120 * 16 * GB);
     root.updateClusterResource(clusterResource);
-    expectedMaxActiveApps = 
-        Math.max(1, 
-            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                   csConf.
-                     getMaximumApplicationMasterResourcePerQueuePercent(
-                                                        queue.getQueuePath()) *
-                   queue.getAbsoluteMaximumCapacity()));
-    assertEquals(expectedMaxActiveApps, 
-                 queue.getMaximumActiveApplications());
-    expectedMaxActiveAppsUsingAbsCap = 
-            Math.max(1, 
-                (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                       csConf.getMaximumApplicationMasterResourcePercent() *
-                       queue.getAbsoluteCapacity()));
-    assertEquals(
-        (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap * 
-            (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), 
-        queue.getMaximumActiveApplicationsPerUser());
+    
+    assertEquals(queue.getAMResourceLimit(), Resource.newInstance(192*GB, 1));
+    assertEquals(queue.getUserAMResourceLimit(), 
+      Resource.newInstance(96*GB, 1));
+    
     assertEquals(
         (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
         queue.getMetrics().getAvailableMB()
@@ -271,18 +335,15 @@ public void testLimitsComputation() throws Exception {
     clusterResource = Resources.createResource(100 * 16 * GB);
 
     queue = (LeafQueue)queues.get(A);
-    expectedMaxActiveApps = 
-        Math.max(1, 
-            (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) * 
-                   csConf.
-                     getMaximumApplicationMasterResourcePerQueuePercent(
-                                                        queue.getQueuePath()) *
-                   queue.getAbsoluteMaximumCapacity()));
 
     assertEquals((long) 0.5, 
-        (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
-    assertEquals(expectedMaxActiveApps, 
-        queue.getMaximumActiveApplications());
+        (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+          queue.getQueuePath())
+        );
+    
+    assertEquals(queue.getAMResourceLimit(), Resource.newInstance(800*GB, 1));
+    assertEquals(queue.getUserAMResourceLimit(), 
+      Resource.newInstance(400*GB, 1));
 
     // Change the per-queue max applications.
     csConf.setInt(
@@ -308,10 +369,16 @@ public void testLimitsComputation() throws Exception {
   public void testActiveApplicationLimits() throws Exception {
     final String user_0 = "user_0";
     final String user_1 = "user_1";
+    final String user_2 = "user_2";
+    
+    assertEquals(Resource.newInstance(16 * GB, 1), queue.getAMResourceLimit());
+    assertEquals(Resource.newInstance(8 * GB, 1),
+      queue.getUserAMResourceLimit());
     
     int APPLICATION_ID = 0;
     // Submit first application
-    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_0, user_0);
     assertEquals(1, queue.getNumActiveApplications());
     assertEquals(0, queue.getNumPendingApplications());
@@ -319,15 +386,17 @@ public void testActiveApplicationLimits() throws Exception {
     assertEquals(0, queue.getNumPendingApplications(user_0));
 
     // Submit second application
-    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_1, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(0, queue.getNumPendingApplications());
     assertEquals(2, queue.getNumActiveApplications(user_0));
     assertEquals(0, queue.getNumPendingApplications(user_0));
     
-    // Submit third application, should remain pending
-    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0);
+    // Submit third application, should remain pending due to user amlimit
+    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_2, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(1, queue.getNumPendingApplications());
@@ -342,18 +411,17 @@ public void testActiveApplicationLimits() throws Exception {
     assertEquals(0, queue.getNumPendingApplications(user_0));
     
     // Submit another one for user_0
-    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_3, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(1, queue.getNumPendingApplications());
     assertEquals(2, queue.getNumActiveApplications(user_0));
     assertEquals(1, queue.getNumPendingApplications(user_0));
     
-    // Change queue limit to be smaller so 2 users can fill it up
-    doReturn(3).when(queue).getMaximumActiveApplications();
-    
     // Submit first app for user_1
-    FiCaSchedulerApp app_4 = getMockApplication(APPLICATION_ID++, user_1);
+    FiCaSchedulerApp app_4 = getMockApplication(APPLICATION_ID++, user_1,
+      Resources.createResource(8 * GB, 0));
     queue.submitApplicationAttempt(app_4, user_1);
     assertEquals(3, queue.getNumActiveApplications());
     assertEquals(1, queue.getNumPendingApplications());
@@ -362,15 +430,17 @@ public void testActiveApplicationLimits() throws Exception {
     assertEquals(1, queue.getNumActiveApplications(user_1));
     assertEquals(0, queue.getNumPendingApplications(user_1));
 
-    // Submit second app for user_1, should block due to queue-limit
-    FiCaSchedulerApp app_5 = getMockApplication(APPLICATION_ID++, user_1);
-    queue.submitApplicationAttempt(app_5, user_1);
+    // Submit first app for user_2, should block due to queue amlimit
+    FiCaSchedulerApp app_5 = getMockApplication(APPLICATION_ID++, user_2,
+      Resources.createResource(8 * GB, 0));
+    queue.submitApplicationAttempt(app_5, user_2);
     assertEquals(3, queue.getNumActiveApplications());
     assertEquals(2, queue.getNumPendingApplications());
     assertEquals(2, queue.getNumActiveApplications(user_0));
     assertEquals(1, queue.getNumPendingApplications(user_0));
     assertEquals(1, queue.getNumActiveApplications(user_1));
-    assertEquals(1, queue.getNumPendingApplications(user_1));
+    assertEquals(0, queue.getNumPendingApplications(user_1));
+    assertEquals(1, queue.getNumPendingApplications(user_2));
 
     // Now finish one app of user_1 so app_5 should be activated
     queue.finishApplicationAttempt(app_4, A);
@@ -378,21 +448,22 @@ public void testActiveApplicationLimits() throws Exception {
     assertEquals(1, queue.getNumPendingApplications());
     assertEquals(2, queue.getNumActiveApplications(user_0));
     assertEquals(1, queue.getNumPendingApplications(user_0));
-    assertEquals(1, queue.getNumActiveApplications(user_1));
+    assertEquals(0, queue.getNumActiveApplications(user_1));
     assertEquals(0, queue.getNumPendingApplications(user_1));
+    assertEquals(1, queue.getNumActiveApplications(user_2));
+    assertEquals(0, queue.getNumPendingApplications(user_2));
+    
   }
-
+  
   @Test
   public void testActiveLimitsWithKilledApps() throws Exception {
     final String user_0 = "user_0";
 
     int APPLICATION_ID = 0;
 
-    // set max active to 2
-    doReturn(2).when(queue).getMaximumActiveApplications();
-
     // Submit first application
-    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_0, user_0);
     assertEquals(1, queue.getNumActiveApplications());
     assertEquals(0, queue.getNumPendingApplications());
@@ -401,7 +472,8 @@ public void testActiveLimitsWithKilledApps() throws Exception {
     assertTrue(queue.activeApplications.contains(app_0));
 
     // Submit second application
-    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_1, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(0, queue.getNumPendingApplications());
@@ -410,7 +482,8 @@ public void testActiveLimitsWithKilledApps() throws Exception {
     assertTrue(queue.activeApplications.contains(app_1));
 
     // Submit third application, should remain pending
-    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_2, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(1, queue.getNumPendingApplications());
@@ -419,7 +492,8 @@ public void testActiveLimitsWithKilledApps() throws Exception {
     assertTrue(queue.pendingApplications.contains(app_2));
 
     // Submit fourth application, should remain pending
-    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0);
+    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0,
+      Resources.createResource(4 * GB, 0));
     queue.submitApplicationAttempt(app_3, user_0);
     assertEquals(2, queue.getNumActiveApplications());
     assertEquals(2, queue.getNumPendingApplications());
@@ -506,6 +580,18 @@ public void testHeadroom() throws Exception {
     RecordFactory recordFactory = 
         RecordFactoryProvider.getRecordFactory(null);
     RMContext rmContext = TestUtils.getMockRMContext();
+    RMContext spyRMContext = spy(rmContext);
+    
+    ConcurrentMap<ApplicationId, RMApp> spyApps = 
+        spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+    RMApp rmApp = mock(RMApp.class);
+    ResourceRequest amResourceRequest = mock(ResourceRequest.class);
+    Resource amResource = Resources.createResource(0, 0);
+    when(amResourceRequest.getCapability()).thenReturn(amResource);
+    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
+    when(spyRMContext.getRMApps()).thenReturn(spyApps);
+    
 
     Priority priority_1 = TestUtils.createMockPriority(1);
 
@@ -513,9 +599,9 @@ public void testHeadroom() throws Exception {
     // and check headroom
     final ApplicationAttemptId appAttemptId_0_0 = 
         TestUtils.getMockApplicationAttemptId(0, 0); 
-    FiCaSchedulerApp app_0_0 = 
-        spy(new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue, 
-            queue.getActiveUsersManager(), rmContext));
+    FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(
+      appAttemptId_0_0, user_0, queue, 
+            queue.getActiveUsersManager(), spyRMContext);
     queue.submitApplicationAttempt(app_0_0, user_0);
 
     List<ResourceRequest> app_0_0_requests = new ArrayList<ResourceRequest>();
@@ -532,9 +618,9 @@ public void testHeadroom() throws Exception {
     // Submit second application from user_0, check headroom
     final ApplicationAttemptId appAttemptId_0_1 = 
         TestUtils.getMockApplicationAttemptId(1, 0); 
-    FiCaSchedulerApp app_0_1 = 
-        spy(new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue, 
-            queue.getActiveUsersManager(), rmContext));
+    FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(
+      appAttemptId_0_1, user_0, queue, 
+            queue.getActiveUsersManager(), spyRMContext);
     queue.submitApplicationAttempt(app_0_1, user_0);
     
     List<ResourceRequest> app_0_1_requests = new ArrayList<ResourceRequest>();
@@ -551,9 +637,9 @@ public void testHeadroom() throws Exception {
     // Submit first application from user_1, check  for new headroom
     final ApplicationAttemptId appAttemptId_1_0 = 
         TestUtils.getMockApplicationAttemptId(2, 0); 
-    FiCaSchedulerApp app_1_0 = 
-        spy(new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue, 
-            queue.getActiveUsersManager(), rmContext));
+    FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(
+      appAttemptId_1_0, user_1, queue, 
+            queue.getActiveUsersManager(), spyRMContext);
     queue.submitApplicationAttempt(app_1_0, user_1);
 
     List<ResourceRequest> app_1_0_requests = new ArrayList<ResourceRequest>();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
index a62889b3d59b0..d643c9dc0ef03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
@@ -202,33 +202,33 @@ public void testAbsoluteMaxAvailCapacityWithUse() throws Exception {
     LOG.info("t2 l2q2 " + result);
     
     //some usage, but below the base capacity
-    Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
-    Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+    root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+    l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
     result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, l2q2);
     assertEquals( 0.4f, result, 0.000001f);
     LOG.info("t2 l2q2 " + result);
     
     //usage gt base on parent sibling
-    Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
-    Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
+    root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+    l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
     result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, l2q2);
     assertEquals( 0.3f, result, 0.000001f);
     LOG.info("t2 l2q2 " + result);
     
     //same as last, but with usage also on direct parent
-    Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
-    Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+    root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+    l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
     result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, l2q2);
     assertEquals( 0.3f, result, 0.000001f);
     LOG.info("t2 l2q2 " + result);
     
     //add to direct sibling, below the threshold of effect at present
-    Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
-    Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
-    Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+    root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+    l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+    l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
     result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, l2q2);
     assertEquals( 0.3f, result, 0.000001f);
@@ -236,9 +236,9 @@ public void testAbsoluteMaxAvailCapacityWithUse() throws Exception {
     
     //add to direct sibling, now above the threshold of effect
     //(it's cumulative with prior tests)
-    Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
-    Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
-    Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+    root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+    l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+    l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
     result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
       resourceCalculator, clusterResource, l2q2);
     assertEquals( 0.1f, result, 0.000001f);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2aa57a0d79524..38d9d27f3c537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -84,7 +85,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.Task;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
 import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -154,7 +155,7 @@ public void setUp() throws Exception {
     resourceManager = new ResourceManager() {
       @Override
       protected RMNodeLabelsManager createNodeLabelManager() {
-        RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
         mgr.init(getConfig());
         return mgr;
       }
@@ -499,9 +500,12 @@ private void checkNodeResourceUsage(int expected,
   public void testParseQueue() throws IOException {
     CapacityScheduler cs = new CapacityScheduler();
     cs.setConf(new YarnConfiguration());
-
+    cs.setRMContext(resourceManager.getRMContext());
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
     setupQueueConfiguration(conf);
+    cs.init(conf);
+    cs.start();
+
     conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
     conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
     conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
@@ -1485,7 +1489,7 @@ public void testMoveAppViolateQueueState() throws Exception {
     resourceManager = new ResourceManager() {
        @Override
         protected RMNodeLabelsManager createNodeLabelManager() {
-          RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+          RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
           mgr.init(getConfig());
           return mgr;
         }
@@ -2071,4 +2075,353 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception
     Assert.assertEquals(0, report.getNumReservedContainers());
     rm.stop();
   }
+
+  @Test
+  public void testPreemptionDisabled() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+    RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueB2 = findQueue(queueB, B2);
+
+    // When preemption turned on for the whole system
+    // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other 
+    // preemption properties set, queue root.b.b2 should be preemptable.
+    assertFalse("queue " + B2 + " should default to preemptable",
+               queueB2.getPreemptionDisabled());
+
+    // Disable preemption at the root queue level.
+    // The preemption property should be inherited from root all the
+    // way down so that root.b.b2 should NOT be preemptable.
+    conf.setPreemptionDisabled(rootQueue.getQueuePath(), true);
+    cs.reinitialize(conf, rmContext);
+    assertTrue(
+        "queue " + B2 + " should have inherited non-preemptability from root",
+        queueB2.getPreemptionDisabled());
+
+    // Enable preemption for root (grandparent) but disable for root.b (parent).
+    // root.b.b2 should inherit property from parent and NOT be preemptable
+    conf.setPreemptionDisabled(rootQueue.getQueuePath(), false);
+    conf.setPreemptionDisabled(queueB.getQueuePath(), true);
+    cs.reinitialize(conf, rmContext);
+    assertTrue(
+        "queue " + B2 + " should have inherited non-preemptability from parent",
+        queueB2.getPreemptionDisabled());
+
+    // When preemption is turned on for root.b.b2, it should be preemptable
+    // even though preemption is disabled on root.b (parent).
+    conf.setPreemptionDisabled(queueB2.getQueuePath(), false);
+    cs.reinitialize(conf, rmContext);
+    assertFalse("queue " + B2 + " should have been preemptable",
+        queueB2.getPreemptionDisabled());
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationRefresh() throws Exception {
+    // queue refresh should not allow changing the maximum allocation setting
+    // per queue to be smaller than previous setting
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+    assertEquals("max allocation in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("max allocation for A1",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        conf.getMaximumAllocationPerQueue(A1).getMemory());
+    assertEquals("max allocation",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        conf.getMaximumAllocation().getMemory());
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    assertEquals("queue max allocation", ((LeafQueue) queueA1)
+        .getMaximumAllocation().getMemory(), 8192);
+
+    setMaxAllocMb(conf, A1, 4096);
+
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+
+    setMaxAllocMb(conf, A1, 8192);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, A1,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception {
+    // verify we can't set the allocation per queue larger then cluster setting
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.init(conf);
+    cs.start();
+    // change max allocation for B3 queue to be larger then cluster max
+    setMaxAllocMb(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+
+    setMaxAllocMb(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception {
+    // queue refresh should allow max allocation per queue to go larger
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    setMaxAllocVcores(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 2);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+    assertEquals("max capability MB in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("max capability vcores in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("max allocation MB A1",
+        4096,
+        conf.getMaximumAllocationPerQueue(A1).getMemory());
+    assertEquals("max allocation vcores A1",
+        2,
+        conf.getMaximumAllocationPerQueue(A1).getVirtualCores());
+    assertEquals("cluster max allocation MB",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        conf.getMaximumAllocation().getMemory());
+    assertEquals("cluster max allocation vcores",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        conf.getMaximumAllocation().getVirtualCores());
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    assertEquals("queue max allocation", ((LeafQueue) queueA1)
+        .getMaximumAllocation().getMemory(), 4096);
+
+    setMaxAllocMb(conf, A1, 6144);
+    setMaxAllocVcores(conf, A1, 3);
+    cs.reinitialize(conf, null);
+    // conf will have changed but we shouldn't be able to change max allocation
+    // for the actual queue
+    assertEquals("max allocation MB A1", 6144,
+        conf.getMaximumAllocationPerQueue(A1).getMemory());
+    assertEquals("max allocation vcores A1", 3,
+        conf.getMaximumAllocationPerQueue(A1).getVirtualCores());
+    assertEquals("max allocation MB cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        conf.getMaximumAllocation().getMemory());
+    assertEquals("max allocation vcores cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        conf.getMaximumAllocation().getVirtualCores());
+    assertEquals("queue max allocation MB", 6144,
+        ((LeafQueue) queueA1).getMaximumAllocation().getMemory());
+    assertEquals("queue max allocation vcores", 3,
+        ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores());
+    assertEquals("max capability MB cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("cluster max capability vcores",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationCSError() throws Exception {
+    // Try to refresh the cluster level max allocation size to be smaller
+    // and it should error out
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf, 10240);
+    setMaxAllocVcores(conf, 10);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 4);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+    assertEquals("max allocation MB in CS", 10240,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("max allocation vcores in CS", 10,
+        cs.getMaximumResourceCapability().getVirtualCores());
+
+    setMaxAllocMb(conf, 6144);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+
+    setMaxAllocMb(conf, 10240);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, 8);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationCSLarger() throws Exception {
+    // Try to refresh the cluster level max allocation size to be larger
+    // and verify that if there is no setting per queue it uses the
+    // cluster level setting.
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf, 10240);
+    setMaxAllocVcores(conf, 10);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 4);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+    assertEquals("max allocation MB in CS", 10240,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("max allocation vcores in CS", 10,
+        cs.getMaximumResourceCapability().getVirtualCores());
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    CSQueue queueA2 = findQueue(queueA, A2);
+    CSQueue queueB2 = findQueue(queueB, B2);
+
+    assertEquals("queue A1 max allocation MB", 4096,
+        ((LeafQueue) queueA1).getMaximumAllocation().getMemory());
+    assertEquals("queue A1 max allocation vcores", 4,
+        ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores());
+    assertEquals("queue A2 max allocation MB", 10240,
+        ((LeafQueue) queueA2).getMaximumAllocation().getMemory());
+    assertEquals("queue A2 max allocation vcores", 10,
+        ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores());
+    assertEquals("queue B2 max allocation MB", 10240,
+        ((LeafQueue) queueB2).getMaximumAllocation().getMemory());
+    assertEquals("queue B2 max allocation vcores", 10,
+        ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores());
+
+    setMaxAllocMb(conf, 12288);
+    setMaxAllocVcores(conf, 12);
+    cs.reinitialize(conf, null);
+    // cluster level setting should change and any queues without
+    // per queue setting
+    assertEquals("max allocation MB in CS", 12288,
+        cs.getMaximumResourceCapability().getMemory());
+    assertEquals("max allocation vcores in CS", 12,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("queue A1 max MB allocation", 4096,
+        ((LeafQueue) queueA1).getMaximumAllocation().getMemory());
+    assertEquals("queue A1 max vcores allocation", 4,
+        ((LeafQueue) queueA1).getMaximumAllocation().getVirtualCores());
+    assertEquals("queue A2 max MB allocation", 12288,
+        ((LeafQueue) queueA2).getMaximumAllocation().getMemory());
+    assertEquals("queue A2 max vcores allocation", 12,
+        ((LeafQueue) queueA2).getMaximumAllocation().getVirtualCores());
+    assertEquals("queue B2 max MB allocation", 12288,
+        ((LeafQueue) queueB2).getMaximumAllocation().getMemory());
+    assertEquals("queue B2 max vcores allocation", 12,
+        ((LeafQueue) queueB2).getMaximumAllocation().getVirtualCores());
+  }
+
+  private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        maxAllocMb);
+  }
+
+  private void setMaxAllocMb(CapacitySchedulerConfiguration conf,
+      String queueName, int maxAllocMb) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
+    conf.setInt(propName, maxAllocMb);
+  }
+
+  private void setMaxAllocVcores(Configuration conf, int maxAllocVcores) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        maxAllocVcores);
+  }
+
+  private void setMaxAllocVcores(CapacitySchedulerConfiguration conf,
+      String queueName, int maxAllocVcores) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
+    conf.setInt(propName, maxAllocVcores);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index 73d8a55ac004e..ce3382fdb2307 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -217,7 +218,7 @@ public void testMoveAppToPlanQueue() throws Exception {
     assertEquals(1, appsInRoot.size());
 
     // create the default reservation queue
-    String defQName = "a" + PlanQueue.DEFAULT_QUEUE_SUFFIX;
+    String defQName = "a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
     ReservationQueue defQ =
         new ReservationQueue(scheduler, defQName,
             (PlanQueue) scheduler.getQueue("a"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
new file mode 100644
index 0000000000000..923c0a34990ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class TestCapacitySchedulerNodeLabelUpdate {
+  private final int GB = 1024;
+
+  private YarnConfiguration conf;
+  
+  RMNodeLabelsManager mgr;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+      ResourceScheduler.class);
+    mgr = new NullRMNodeLabelsManager();
+    mgr.init(conf);
+  }
+  
+  private Configuration getConfigurationWithQueueLabels(Configuration config) {
+    CapacitySchedulerConfiguration conf =
+        new CapacitySchedulerConfiguration(config);
+    
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    conf.setCapacity(A, 100);
+    conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
+    conf.setCapacityByLabel(A, "x", 100);
+    conf.setCapacityByLabel(A, "y", 100);
+    conf.setCapacityByLabel(A, "z", 100);
+    
+    return conf;
+  }
+  
+  private Set<String> toSet(String... elements) {
+    Set<String> set = Sets.newHashSet(elements);
+    return set;
+  }
+  
+  private void checkUsedResource(MockRM rm, String queueName, int memory) {
+    checkUsedResource(rm, queueName, memory, RMNodeLabelsManager.NO_LABEL);
+  }
+  
+  private void checkUsedResource(MockRM rm, String queueName, int memory,
+      String label) {
+    CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+    CSQueue queue = scheduler.getQueue(queueName);
+    Assert.assertEquals(memory, queue.getUsedResourceByLabel(label).getMemory());
+  }
+
+  @Test (timeout = 30000)
+  public void testNodeUpdate() throws Exception {
+    // set node -> label
+    mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
+    
+    // set mapping:
+    // h1 -> x
+    // h2 -> y
+    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+    // inject node label manager
+    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+
+    rm.getRMContext().setNodeLabelManager(mgr);
+    rm.start();
+    MockNM nm1 = rm.registerNode("h1:1234", 8000);
+    MockNM nm2 = rm.registerNode("h2:1234", 8000);
+    MockNM nm3 = rm.registerNode("h3:1234", 8000);
+    
+    ContainerId containerId;
+
+    // launch an app to queue a1 (label = x), and check all container will
+    // be allocated in h1
+    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+
+    // request a container.
+    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+    containerId =
+        ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+    Assert.assertTrue(rm.waitForState(nm1, containerId,
+        RMContainerState.ALLOCATED, 10 * 1000));
+    
+    // check used resource:
+    // queue-a used x=1G, ""=1G
+    checkUsedResource(rm, "a", 1024, "x");
+    checkUsedResource(rm, "a", 1024);
+    
+    // change h1's label to z, container should be killed
+    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
+        toSet("z")));
+    Assert.assertTrue(rm.waitForState(nm1, containerId,
+        RMContainerState.KILLED, 10 * 1000));
+    
+    // check used resource:
+    // queue-a used x=0G, ""=1G ("" not changed)
+    checkUsedResource(rm, "a", 0, "x");
+    checkUsedResource(rm, "a", 1024);
+    
+    // request a container with label = y
+    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
+    containerId =
+        ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
+    Assert.assertTrue(rm.waitForState(nm2, containerId,
+        RMContainerState.ALLOCATED, 10 * 1000));
+    
+    // check used resource:
+    // queue-a used y=1G, ""=1G
+    checkUsedResource(rm, "a", 1024, "y");
+    checkUsedResource(rm, "a", 1024);
+    
+    // change h2's label to no label, container should be killed
+    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h2", 0),
+        CommonNodeLabelsManager.EMPTY_STRING_SET));
+    Assert.assertTrue(rm.waitForState(nm1, containerId,
+        RMContainerState.KILLED, 10 * 1000));
+    
+    // check used resource:
+    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+    checkUsedResource(rm, "a", 0, "x");
+    checkUsedResource(rm, "a", 0, "y");
+    checkUsedResource(rm, "a", 1024);
+    
+    containerId =
+        ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+    
+    // change h3's label to z, AM container should be killed
+    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h3", 0),
+        toSet("z")));
+    Assert.assertTrue(rm.waitForState(nm1, containerId,
+        RMContainerState.KILLED, 10 * 1000));
+    
+    // check used resource:
+    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+    checkUsedResource(rm, "a", 0, "x");
+    checkUsedResource(rm, "a", 0, "y");
+    checkUsedResource(rm, "a", 0);
+
+    rm.close();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 9a29bff970aec..169517dd37246 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -45,7 +45,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.TestFifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -81,12 +81,13 @@ public void setUp() throws Exception {
     conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
       ResourceScheduler.class);
-    mgr = new MemoryRMNodeLabelsManager();
+    mgr = new NullRMNodeLabelsManager();
     mgr.init(conf);
   }
 
   @Test(timeout = 3000000)
   public void testExcessReservationThanNodeManagerCapacity() throws Exception {
+    @SuppressWarnings("resource")
     MockRM rm = new MockRM(conf);
     rm.start();
 
@@ -393,6 +394,7 @@ private void checkTaskContainersHost(ApplicationAttemptId attemptId,
     }
   }
   
+  @SuppressWarnings("unchecked")
   private <E> Set<E> toSet(E... elements) {
     Set<E> set = Sets.newHashSet(elements);
     return set;
@@ -449,9 +451,9 @@ private Configuration getComplexConfigurationWithQueueLabels(
     return conf;
   }
   
-  @Test(timeout = 300000)
+  @Test (timeout = 300000)
   public void testContainerAllocationWithSingleUserLimits() throws Exception {
-    final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
+    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
     mgr.init(conf);
 
     // set node -> label
@@ -470,7 +472,7 @@ public RMNodeLabelsManager createNodeLabelManager() {
     rm1.getRMContext().setNodeLabelManager(mgr);
     rm1.start();
     MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x
-    MockNM nm2 = rm1.registerNode("h2:1234", 8000); // label = y
+    rm1.registerNode("h2:1234", 8000); // label = y
     MockNM nm3 = rm1.registerNode("h3:1234", 8000); // label = <empty>
 
     // launch an app to queue a1 (label = x), and check all container will
@@ -518,9 +520,9 @@ public void testContainerAllocateWithComplexLabels() throws Exception {
      *                           
      * Node structure:
      * h1 : x
-     * h2 : x, y
+     * h2 : y
      * h3 : y
-     * h4 : y, z
+     * h4 : z
      * h5 : NO
      * 
      * Total resource:
@@ -540,9 +542,9 @@ public void testContainerAllocateWithComplexLabels() throws Exception {
     // set node -> label
     mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
     mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
-        toSet("x"), NodeId.newInstance("h2", 0), toSet("x", "y"),
+        toSet("x"), NodeId.newInstance("h2", 0), toSet("y"),
         NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0),
-        toSet("y", "z"), NodeId.newInstance("h5", 0),
+        toSet("z"), NodeId.newInstance("h5", 0),
         RMNodeLabelsManager.EMPTY_STRING_SET));
 
     // inject node label manager
@@ -568,12 +570,10 @@ public RMNodeLabelsManager createNodeLabelManager() {
     RMApp app1 = rm1.submitApp(1024, "app", "user", null, "a1");
     MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
 
-    // request a container (label = x && y). can only allocate on nm2 
-    am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "x && y");
+    // request a container (label = y). can be allocated on nm2 
+    am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y");
     containerId =
-        ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
-    Assert.assertFalse(rm1.waitForState(nm1, containerId,
-        RMContainerState.ALLOCATED, 10 * 1000));
+        ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L);
     Assert.assertTrue(rm1.waitForState(nm2, containerId,
         RMContainerState.ALLOCATED, 10 * 1000));
     checkTaskContainersHost(am1.getApplicationAttemptId(), containerId, rm1,
@@ -609,12 +609,10 @@ public RMNodeLabelsManager createNodeLabelManager() {
     checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
         "h3");
     
-    // try to allocate container (request label = y && z) on nm3 (label = y) and
-    // nm4 (label = y,z). Will sucessfully allocate on nm4 only.
-    am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y && z");
-    containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3);
-    Assert.assertFalse(rm1.waitForState(nm3, containerId,
-        RMContainerState.ALLOCATED, 10 * 1000));
+    // try to allocate container (request label = z) on nm4 (label = y,z). 
+    // Will successfully allocate on nm4 only.
+    am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "z");
+    containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3L);
     Assert.assertTrue(rm1.waitForState(nm4, containerId,
         RMContainerState.ALLOCATED, 10 * 1000));
     checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 642363e6bfe65..ead5719f376f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -37,11 +37,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.ConcurrentModificationException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -61,6 +63,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -98,6 +101,7 @@ public class TestLeafQueue {
 
   RMContext rmContext;
   RMContext spyRMContext;
+  ResourceRequest amResourceRequest;
   CapacityScheduler cs;
   CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
@@ -121,6 +125,10 @@ public void setUp() throws Exception {
         spy(new ConcurrentHashMap<ApplicationId, RMApp>());
     RMApp rmApp = mock(RMApp.class);
     when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
+    amResourceRequest = mock(ResourceRequest.class);
+    when(amResourceRequest.getCapability()).thenReturn(
+      Resources.createResource(0, 0));
+    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
     Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
     when(spyRMContext.getRMApps()).thenReturn(spyApps);
     
@@ -262,26 +270,37 @@ public Container answer(InvocationOnMock invocation)
   
   @Test
   public void testInitializeQueue() throws Exception {
-	  final float epsilon = 1e-5f;
-	  //can add more sturdy test with 3-layer queues 
-	  //once MAPREDUCE:3410 is resolved
-	  LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
-	  assertEquals(0.085, a.getCapacity(), epsilon);
-	  assertEquals(0.085, a.getAbsoluteCapacity(), epsilon);
-	  assertEquals(0.2, a.getMaximumCapacity(), epsilon);
-	  assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+    final float epsilon = 1e-5f;
+    //can add more sturdy test with 3-layer queues 
+    //once MAPREDUCE:3410 is resolved
+    LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    assertEquals(0.085, a.getCapacity(), epsilon);
+    assertEquals(0.085, a.getAbsoluteCapacity(), epsilon);
+    assertEquals(0.2, a.getMaximumCapacity(), epsilon);
+    assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+    
+    LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+    assertEquals(0.80, b.getCapacity(), epsilon);
+    assertEquals(0.80, b.getAbsoluteCapacity(), epsilon);
+    assertEquals(0.99, b.getMaximumCapacity(), epsilon);
+    assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+    
+    ParentQueue c = (ParentQueue)queues.get(C);
+    assertEquals(0.015, c.getCapacity(), epsilon);
+    assertEquals(0.015, c.getAbsoluteCapacity(), epsilon);
+    assertEquals(0.1, c.getMaximumCapacity(), epsilon);
+    assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
+
+	  //Verify the value for getAMResourceLimit for queues with < .1 maxcap
+	  Resource clusterResource = Resource.newInstance(50 * GB, 50);
 	  
-	  LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
-	  assertEquals(0.80, b.getCapacity(), epsilon);
-	  assertEquals(0.80, b.getAbsoluteCapacity(), epsilon);
-	  assertEquals(0.99, b.getMaximumCapacity(), epsilon);
-	  assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
-
-	  ParentQueue c = (ParentQueue)queues.get(C);
-	  assertEquals(0.015, c.getCapacity(), epsilon);
-	  assertEquals(0.015, c.getAbsoluteCapacity(), epsilon);
-	  assertEquals(0.1, c.getMaximumCapacity(), epsilon);
-	  assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
+	  a.updateClusterResource(clusterResource);
+	  assertEquals(Resource.newInstance(1 * GB, 1), 
+	    a.getAMResourceLimit());
+    
+	  b.updateClusterResource(clusterResource);
+	  assertEquals(Resource.newInstance(5 * GB, 1), 
+	    b.getAMResourceLimit());
   }
  
   @Test
@@ -676,7 +695,7 @@ public void testComputeUserLimitAndSetHeadroom(){
               TestUtils.getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 =
         new FiCaSchedulerApp(appAttemptId_0, user_0, qb,
-            qb.getActiveUsersManager(), rmContext);
+            qb.getActiveUsersManager(), spyRMContext);
     qb.submitApplicationAttempt(app_0, user_0);
     Priority u0Priority = TestUtils.createMockPriority(1);
     app_0.updateResourceRequests(Collections.singletonList(
@@ -699,7 +718,7 @@ public void testComputeUserLimitAndSetHeadroom(){
         TestUtils.getMockApplicationAttemptId(2, 0);
     FiCaSchedulerApp app_2 =
         new FiCaSchedulerApp(appAttemptId_2, user_1, qb,
-            qb.getActiveUsersManager(), rmContext);
+            qb.getActiveUsersManager(), spyRMContext);
     Priority u1Priority = TestUtils.createMockPriority(2);
     app_2.updateResourceRequests(Collections.singletonList(
         TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
@@ -733,12 +752,12 @@ public void testComputeUserLimitAndSetHeadroom(){
         TestUtils.getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 =
         new FiCaSchedulerApp(appAttemptId_1, user_0, qb,
-            qb.getActiveUsersManager(), rmContext);
+            qb.getActiveUsersManager(), spyRMContext);
     final ApplicationAttemptId appAttemptId_3 =
         TestUtils.getMockApplicationAttemptId(3, 0);
     FiCaSchedulerApp app_3 =
         new FiCaSchedulerApp(appAttemptId_3, user_1, qb,
-            qb.getActiveUsersManager(), rmContext);
+            qb.getActiveUsersManager(), spyRMContext);
     app_1.updateResourceRequests(Collections.singletonList(
         TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
             u0Priority, recordFactory)));
@@ -761,7 +780,7 @@ public void testComputeUserLimitAndSetHeadroom(){
               TestUtils.getMockApplicationAttemptId(4, 0);
     FiCaSchedulerApp app_4 =
               new FiCaSchedulerApp(appAttemptId_4, user_0, qb,
-                      qb.getActiveUsersManager(), rmContext);
+                      qb.getActiveUsersManager(), spyRMContext);
     qb.submitApplicationAttempt(app_4, user_0);
     app_4.updateResourceRequests(Collections.singletonList(
               TestUtils.createResourceRequest(ResourceRequest.ANY, 6*GB, 1, true,
@@ -977,7 +996,6 @@ public void testHeadroomWithMaxCap() throws Exception {
     assertEquals(0*GB, app_1.getHeadroom().getMemory());
     
     // Check headroom for app_2 
-    LOG.info("here");
     app_1.updateResourceRequests(Collections.singletonList(     // unset
         TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true,
             priority, recordFactory)));
@@ -1901,6 +1919,9 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception {
 
     // Users
     final String user_e = "user_e";
+    
+    when(amResourceRequest.getCapability()).thenReturn(
+      Resources.createResource(1 * GB, 0));
 
     // Submit applications
     final ApplicationAttemptId appAttemptId_0 =
@@ -1939,7 +1960,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception {
             newQueues, queues,
             TestUtils.spyHook);
     queues = newQueues;
-    root.reinitialize(newRoot, cs.getClusterResource());
+    root.reinitialize(newRoot, csContext.getClusterResource());
 
     // after reinitialization
     assertEquals(3, e.activeApplications.size());
@@ -1979,6 +2000,9 @@ public void testActivateApplicationByUpdatingClusterResource()
 
     // Users
     final String user_e = "user_e";
+    
+    when(amResourceRequest.getCapability()).thenReturn(
+      Resources.createResource(1 * GB, 0));
 
     // Submit applications
     final ApplicationAttemptId appAttemptId_0 =
@@ -2288,20 +2312,20 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
     csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
     LeafQueue a = new LeafQueue(csContext, A, root, null);
     assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
-    assertEquals(160, a.getMaximumActiveApplications());
+    assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
     
     csConf.setFloat(CapacitySchedulerConfiguration.
         MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
     LeafQueue newA = new LeafQueue(csContext, A, root, null);
     a.reinitialize(newA, clusterResource);
     assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
-    assertEquals(320, a.getMaximumActiveApplications());
+    assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));
 
     Resource newClusterResource = Resources.createResource(100 * 20 * GB,
         100 * 32);
     a.updateClusterResource(newClusterResource);
     //  100 * 20 * 0.2 = 400
-    assertEquals(400, a.getMaximumActiveApplications());
+    assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
   }
   
   @Test
@@ -2353,6 +2377,89 @@ public void testAllocateContainerOnNodeWithoutOffSwitchSpecified()
     }
   }
 
+  @Test
+  public void testConcurrentAccess() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    MockRM rm = new MockRM();
+    rm.init(conf);
+    rm.start();
+
+    final String queue = "default";
+    final String user = "user";
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    final LeafQueue defaultQueue = (LeafQueue) cs.getQueue(queue);
+
+    final List<FiCaSchedulerApp> listOfApps =
+        createListOfApps(10000, user, defaultQueue);
+
+    final CyclicBarrier cb = new CyclicBarrier(2);
+    final List<ConcurrentModificationException> conException =
+        new ArrayList<ConcurrentModificationException>();
+
+    Thread submitAndRemove = new Thread(new Runnable() {
+
+      @Override
+      public void run() {
+
+        for (FiCaSchedulerApp fiCaSchedulerApp : listOfApps) {
+          defaultQueue.submitApplicationAttempt(fiCaSchedulerApp, user);
+        }
+        try {
+          cb.await();
+        } catch (Exception e) {
+          // Ignore
+        }
+        for (FiCaSchedulerApp fiCaSchedulerApp : listOfApps) {
+          defaultQueue.finishApplicationAttempt(fiCaSchedulerApp, queue);
+        }
+      }
+    }, "SubmitAndRemoveApplicationAttempt Thread");
+
+    Thread getAppsInQueue = new Thread(new Runnable() {
+      List<ApplicationAttemptId> apps = new ArrayList<ApplicationAttemptId>();
+
+      @Override
+      public void run() {
+        try {
+          try {
+            cb.await();
+          } catch (Exception e) {
+            // Ignore
+          }
+          defaultQueue.collectSchedulerApplications(apps);
+        } catch (ConcurrentModificationException e) {
+          conException.add(e);
+        }
+      }
+
+    }, "GetAppsInQueue Thread");
+
+    submitAndRemove.start();
+    getAppsInQueue.start();
+
+    submitAndRemove.join();
+    getAppsInQueue.join();
+
+    assertTrue("ConcurrentModificationException is thrown",
+        conException.isEmpty());
+    rm.stop();
+
+  }
+
+  private List<FiCaSchedulerApp> createListOfApps(int noOfApps, String user,
+      LeafQueue defaultQueue) {
+    List<FiCaSchedulerApp> appsLists = new ArrayList<FiCaSchedulerApp>();
+    for (int i = 0; i < noOfApps; i++) {
+      ApplicationAttemptId appAttemptId_0 =
+          TestUtils.getMockApplicationAttemptId(i, 0);
+      FiCaSchedulerApp app_0 =
+          new FiCaSchedulerApp(appAttemptId_0, user, defaultQueue,
+              mock(ActiveUsersManager.class), spyRMContext);
+      appsLists.add(app_0);
+    }
+    return appsLists;
+  }
+
   private CapacitySchedulerContext mockCSContext(
       CapacitySchedulerConfiguration csConf, Resource clusterResource) {
     CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
new file mode 100644
index 0000000000000..6d2a42180f342
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestQueueCapacities {
+  private static final Log LOG = LogFactory.getLog(TestQueueCapacities.class);
+  private String suffix;
+
+  @Parameterized.Parameters
+  public static Collection<String[]> getParameters() {
+    return Arrays.asList(new String[][] { 
+        { "Capacity" },
+        { "AbsoluteCapacity" }, 
+        { "UsedCapacity" }, 
+        { "AbsoluteUsedCapacity" },
+        { "MaximumCapacity" }, 
+        { "AbsoluteMaximumCapacity" } });
+  }
+
+  public TestQueueCapacities(String suffix) {
+    this.suffix = suffix;
+  }
+
+  private static float get(QueueCapacities obj, String suffix,
+      String label) throws Exception {
+    return executeByName(obj, "get" + suffix, label, -1f);
+  }
+
+  private static void set(QueueCapacities obj, String suffix,
+      String label, float value) throws Exception {
+    executeByName(obj, "set" + suffix, label, value);
+  }
+
+  // Use reflection to avoid too much avoid code
+  private static float executeByName(QueueCapacities obj, String methodName,
+      String label, float value) throws Exception {
+    // We have 4 kinds of method
+    // 1. getXXX() : float
+    // 2. getXXX(label) : float
+    // 3. setXXX(float) : void
+    // 4. setXXX(label, float) : void
+    if (methodName.startsWith("get")) {
+      float result;
+      if (label == null) {
+        // 1.
+        Method method = QueueCapacities.class.getDeclaredMethod(methodName);
+        result = (float) method.invoke(obj);
+      } else {
+        // 2.
+        Method method =
+            QueueCapacities.class.getDeclaredMethod(methodName, String.class);
+        result = (float) method.invoke(obj, label);
+      }
+      return result;
+    } else {
+      if (label == null) {
+        // 3.
+        Method method =
+            QueueCapacities.class.getDeclaredMethod(methodName, Float.TYPE);
+        method.invoke(obj, value);
+      } else {
+        // 4.
+        Method method =
+            QueueCapacities.class.getDeclaredMethod(methodName, String.class,
+                Float.TYPE);
+        method.invoke(obj, label, value);
+      }
+      return -1f;
+    }
+  }
+
+  private void internalTestModifyAndRead(String label) throws Exception {
+    QueueCapacities qc = new QueueCapacities();
+
+    // First get returns 0 always
+    Assert.assertEquals(0f, get(qc, suffix, label), 1e-8);
+
+    // Set to 1, and check
+    set(qc, suffix, label, 1f);
+    Assert.assertEquals(1f, get(qc, suffix, label), 1e-8);
+
+    // Set to 2, and check
+    set(qc, suffix, label, 2f);
+    Assert.assertEquals(2f, get(qc, suffix, label), 1e-8);
+  }
+
+  void check(int mem, int cpu, Resource res) {
+    Assert.assertEquals(mem, res.getMemory());
+    Assert.assertEquals(cpu, res.getVirtualCores());
+  }
+
+  @Test
+  public void testModifyAndRead() throws Exception {
+    LOG.info("Test - " + suffix);
+    internalTestModifyAndRead(null);
+    internalTestModifyAndRead("label");
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 5a9fbe15031f6..f821e64d3c918 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@@ -47,7 +47,7 @@ public class TestQueueParsing {
   
   @Before
   public void setup() {
-    nodeLabelManager = new MemoryRMNodeLabelsManager();
+    nodeLabelManager = new NullRMNodeLabelsManager();
     nodeLabelManager.init(new YarnConfiguration());
     nodeLabelManager.start();
   }
@@ -566,7 +566,7 @@ public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     
@@ -594,7 +594,7 @@ public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     
@@ -622,7 +622,7 @@ public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     
@@ -649,7 +649,7 @@ public void testQueueParsingWhenLabelsNotExist() throws IOException {
             new NMTokenSecretManagerInRM(csConf),
             new ClientToAMTokenSecretManagerInRM(), null);
     
-    RMNodeLabelsManager nodeLabelsManager = new MemoryRMNodeLabelsManager();
+    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
     nodeLabelsManager.init(conf);
     nodeLabelsManager.start();
     
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 2a49545a407df..985609e008979 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -77,6 +77,7 @@ public class TestReservations {
       .getRecordFactory(null);
 
   RMContext rmContext;
+  RMContext spyRMContext;
   CapacityScheduler cs;
   // CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
@@ -132,7 +133,10 @@ private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
     root = CapacityScheduler.parseQueue(csContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
 
-    cs.setRMContext(rmContext);
+    spyRMContext = spy(rmContext);
+    when(spyRMContext.getScheduler()).thenReturn(cs);
+    
+    cs.setRMContext(spyRMContext);
     cs.init(csConf);
     cs.start();
   }
@@ -212,14 +216,14 @@ public void testReservation() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
@@ -361,14 +365,14 @@ public void testReservationNoContinueLook() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
@@ -506,14 +510,14 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
@@ -618,7 +622,7 @@ public void testGetAppToUnreserve() throws Exception {
         .getMockApplicationAttemptId(0, 0);
     LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     String host_0 = "host_0";
     FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
@@ -685,7 +689,7 @@ public void testFindNodeToUnreserve() throws Exception {
         .getMockApplicationAttemptId(0, 0);
     LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     String host_1 = "host_1";
     FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
@@ -742,14 +746,14 @@ public void testAssignToQueue() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
@@ -916,14 +920,14 @@ public void testAssignToUser() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
@@ -1042,14 +1046,14 @@ public void testReservationsNoneAvailable() throws Exception {
     final ApplicationAttemptId appAttemptId_0 = TestUtils
         .getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
 
     a.submitApplicationAttempt(app_0, user_0); 
 
     final ApplicationAttemptId appAttemptId_1 = TestUtils
         .getMockApplicationAttemptId(1, 0);
     FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-        mock(ActiveUsersManager.class), rmContext);
+        mock(ActiveUsersManager.class), spyRMContext);
     a.submitApplicationAttempt(app_1, user_0); 
 
     // Setup some nodes
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 7b6aaf3207070..8656175a7f483 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -60,7 +60,7 @@ public void tick(int seconds) {
     }
   }
 
-  protected final static String TEST_DIR =
+  public final static String TEST_DIR =
       new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
 
   private static RecordFactory
@@ -74,7 +74,7 @@ public void tick(int seconds) {
   protected ResourceManager resourceManager;
 
   // Helper methods
-  protected Configuration createConfiguration() {
+  public Configuration createConfiguration() {
     Configuration conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
         ResourceScheduler.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 9a66a940ca4e6..3c166a5edcb59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
@@ -550,6 +551,86 @@ public void testQueueNameContainingPeriods() throws Exception {
     allocLoader.reloadAllocations();
   }
   
+
+  @Test
+  public void testReservableQueue() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"reservable\">");
+    out.println("<reservation>");
+    out.println("</reservation>");
+    out.println("</queue>");
+    out.println("<queue name=\"other\">");
+    out.println("</queue>");
+    out.println("<reservation-agent>DummyAgentName</reservation-agent>");
+    out.println("<reservation-policy>AnyAdmissionPolicy</reservation-policy>");
+    out.println("</allocations>");
+    out.close();
+
+    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+    allocLoader.init(conf);
+    ReloadListener confHolder = new ReloadListener();
+    allocLoader.setReloadListener(confHolder);
+    allocLoader.reloadAllocations();
+
+    AllocationConfiguration allocConf = confHolder.allocConf;
+    String reservableQueueName = "root.reservable";
+    String nonreservableQueueName = "root.other";
+    assertFalse(allocConf.isReservable(nonreservableQueueName));
+    assertTrue(allocConf.isReservable(reservableQueueName));
+
+    assertTrue(allocConf.getMoveOnExpiry(reservableQueueName));
+    assertEquals(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW,
+        allocConf.getReservationWindow(reservableQueueName));
+    assertEquals(100, allocConf.getInstantaneousMaxCapacity
+            (reservableQueueName),
+        0.0001);
+    assertEquals(
+        "DummyAgentName",
+        allocConf.getReservationAgent(reservableQueueName));
+    assertEquals(100, allocConf.getAverageCapacity(reservableQueueName), 0.001);
+    assertFalse(allocConf.getShowReservationAsQueues(reservableQueueName));
+    assertEquals("AnyAdmissionPolicy",
+        allocConf.getReservationAdmissionPolicy(reservableQueueName));
+    assertEquals(ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_PLANNER_NAME,
+        allocConf.getReplanner(reservableQueueName));
+    assertEquals(ReservationSchedulerConfiguration
+        .DEFAULT_RESERVATION_ENFORCEMENT_WINDOW,
+        allocConf.getEnforcementWindow(reservableQueueName));
+  }
+
+  /**
+   * Verify that you can't have dynamic user queue and reservable queue on
+   * the same queue
+   */
+  @Test (expected = AllocationConfigurationException.class)
+  public void testReservableCannotBeCombinedWithDynamicUserQueue()
+      throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"notboth\" type=\"parent\" >");
+    out.println("<reservation>");
+    out.println("</reservation>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+    allocLoader.init(conf);
+    ReloadListener confHolder = new ReloadListener();
+    allocLoader.setReloadListener(confHolder);
+    allocLoader.reloadAllocations();
+  }
+
   private class ReloadListener implements AllocationFileLoaderService.Listener {
     public AllocationConfiguration allocConf;
     
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 97736bedd0427..385ea0be76b15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -28,12 +28,22 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
 import java.util.Collection;
-
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -222,4 +232,85 @@ public void testIsStarvedForFairShare() throws Exception {
     assertFalse(queueB1.isStarvedForFairShare());
     assertFalse(queueB2.isStarvedForFairShare());
   }
+
+  @Test
+  public void testConcurrentAccess() {
+    conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
+    resourceManager = new MockRM(conf);
+    resourceManager.start();
+    scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+
+    String queueName = "root.queue1";
+    final FSLeafQueue schedulable = scheduler.getQueueManager().
+      getLeafQueue(queueName, true);
+    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
+    RMContext rmContext = resourceManager.getRMContext();
+    final FSAppAttempt app =
+        new FSAppAttempt(scheduler, applicationAttemptId, "user1",
+            schedulable, null, rmContext);
+
+    // this needs to be in sync with the number of runnables declared below
+    int testThreads = 2;
+    List<Runnable> runnables = new ArrayList<Runnable>();
+
+    // add applications to modify the list
+    runnables.add(new Runnable() {
+      @Override
+      public void run() {
+        for (int i=0; i < 500; i++) {
+          schedulable.addAppSchedulable(app);
+        }
+      }
+    });
+
+    // iterate over the list a couple of times in a different thread
+    runnables.add(new Runnable() {
+      @Override
+      public void run() {
+        for (int i=0; i < 500; i++) {
+          schedulable.getResourceUsage();
+        }
+      }
+    });
+
+    final List<Throwable> exceptions = Collections.synchronizedList(
+        new ArrayList<Throwable>());
+    final ExecutorService threadPool = Executors.newFixedThreadPool(
+        testThreads);
+
+    try {
+      final CountDownLatch allExecutorThreadsReady =
+          new CountDownLatch(testThreads);
+      final CountDownLatch startBlocker = new CountDownLatch(1);
+      final CountDownLatch allDone = new CountDownLatch(testThreads);
+      for (final Runnable submittedTestRunnable : runnables) {
+        threadPool.submit(new Runnable() {
+          public void run() {
+            allExecutorThreadsReady.countDown();
+            try {
+              startBlocker.await();
+              submittedTestRunnable.run();
+            } catch (final Throwable e) {
+              exceptions.add(e);
+            } finally {
+              allDone.countDown();
+            }
+          }
+        });
+      }
+      // wait until all threads are ready
+      allExecutorThreadsReady.await();
+      // start all test runners
+      startBlocker.countDown();
+      int testTimeout = 2;
+      assertTrue("Timeout waiting for more than " + testTimeout + " seconds",
+          allDone.await(testTimeout, TimeUnit.SECONDS));
+    } catch (InterruptedException ie) {
+      exceptions.add(ie);
+    } finally {
+      threadPool.shutdownNow();
+    }
+    assertTrue("Test failed with exception(s)" + exceptions,
+        exceptions.isEmpty());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 67cea370fe819..c29dbfc149a34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -784,19 +784,75 @@ public void testSimpleContainerReservation() throws Exception {
 
   }
 
-  @Test (timeout = 5000)
-  public void testContainerReservationNotExceedingQueueMax() throws Exception {
+  @Test (timeout = 500000)
+  public void testContainerReservationAttemptExceedingQueueMax()
+      throws Exception {
     conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
     PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
     out.println("<?xml version=\"1.0\"?>");
     out.println("<allocations>");
     out.println("<queue name=\"root\">");
     out.println("<queue name=\"queue1\">");
-    out.println("<minResources>1024mb,5vcores</minResources>");
+    out.println("<maxResources>2048mb,5vcores</maxResources>");
+    out.println("</queue>");
+    out.println("<queue name=\"queue2\">");
     out.println("<maxResources>2048mb,10vcores</maxResources>");
     out.println("</queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    // Add a node
+    RMNode node1 =
+        MockNodes
+            .newNodeInfo(1, Resources.createResource(3072, 5), 1, "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+
+    // Queue 1 requests full capacity of the queue
+    createSchedulingRequest(2048, "queue1", "user1", 1);
+    scheduler.update();
+    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+    scheduler.handle(updateEvent);
+
+    // Make sure queue 1 is allocated app capacity
+    assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+        getResourceUsage().getMemory());
+
+    // Now queue 2 requests likewise
+    createSchedulingRequest(1024, "queue2", "user2", 1);
+    scheduler.update();
+    scheduler.handle(updateEvent);
+
+    // Make sure queue 2 is allocated app capacity
+    assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+        getResourceUsage().getMemory());
+
+    ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
+    scheduler.update();
+    scheduler.handle(updateEvent);
+
+    // Ensure the reservation does not get created as allocated memory of
+    // queue1 exceeds max
+    assertEquals(0, scheduler.getSchedulerApp(attId1).
+        getCurrentReservation().getMemory());
+  }
+
+    @Test (timeout = 500000)
+  public void testContainerReservationNotExceedingQueueMax() throws Exception {
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"root\">");
+    out.println("<queue name=\"queue1\">");
+    out.println("<maxResources>3072mb,10vcores</maxResources>");
+    out.println("</queue>");
     out.println("<queue name=\"queue2\">");
-    out.println("<minResources>1024mb,5vcores</minResources>");
     out.println("<maxResources>2048mb,10vcores</maxResources>");
     out.println("</queue>");
     out.println("</queue>");
@@ -806,7 +862,7 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception {
     scheduler.init(conf);
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
-    
+
     // Add a node
     RMNode node1 =
         MockNodes
@@ -825,7 +881,7 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception {
         getResourceUsage().getMemory());
 
     // Now queue 2 requests likewise
-    ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user2", 1);
+    createSchedulingRequest(1024, "queue2", "user2", 1);
     scheduler.update();
     scheduler.handle(updateEvent);
 
@@ -841,18 +897,34 @@ public void testContainerReservationNotExceedingQueueMax() throws Exception {
     assertEquals(1024, scheduler.getSchedulerApp(attId1)
         .getCurrentReservation().getMemory());
 
-    // Now remove app of queue2
-    AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
-        attId, RMAppAttemptState.FINISHED, false);
-    scheduler.update();
-    scheduler.handle(appRemovedEvent1);
+    // Exercise checks that reservation fits
+    scheduler.handle(updateEvent);
+
+    // Ensure the reservation still exists as allocated memory of queue1 doesn't
+    // exceed max
+    assertEquals(1024, scheduler.getSchedulerApp(attId1).
+        getCurrentReservation().getMemory());
+
+    // Now reduce max Resources of queue1 down to 2048
+    out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"root\">");
+    out.println("<queue name=\"queue1\">");
+    out.println("<maxResources>2048mb,10vcores</maxResources>");
+    out.println("</queue>");
+    out.println("<queue name=\"queue2\">");
+    out.println("<maxResources>2048mb,10vcores</maxResources>");
+    out.println("</queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
 
-    // Queue should have no apps
-    assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
-        getResourceUsage().getMemory());
-    
     createSchedulingRequest(1024, "queue2", "user2", 1);
     scheduler.handle(updateEvent);
+
     // Make sure allocated memory of queue1 doesn't exceed its maximum
     assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
         getResourceUsage().getMemory());
@@ -872,9 +944,9 @@ public void testUserAsDefaultQueue() throws Exception {
     ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
     createApplicationWithAMResource(appAttemptId, "default", "user1", null);
     assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
     assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
     assertEquals("root.user1", resourceManager.getRMContext().getRMApps()
         .get(appAttemptId.getApplicationId()).getQueue());
   }
@@ -888,11 +960,11 @@ public void testNotUserAsDefaultQueue() throws Exception {
     ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
     createApplicationWithAMResource(appAttemptId, "default", "user2", null);
     assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
     assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
     assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
   }
 
   @Test
@@ -1370,7 +1442,7 @@ public void testAppAdditionAndRemoval() throws Exception {
 
     // That queue should have one app
     assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
 
     AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
         createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
@@ -1380,7 +1452,7 @@ public void testAppAdditionAndRemoval() throws Exception {
 
     // Queue should have no apps
     assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
-        .getRunnableAppSchedulables().size());
+        .getNumRunnableApps());
   }
 
   @Test
@@ -2124,7 +2196,7 @@ public void testPreemptionVariablesForQueueCreatedRuntime() throws Exception {
     // The user1 queue should inherit the configurations from the root queue
     FSLeafQueue userQueue =
         scheduler.getQueueManager().getLeafQueue("user1", true);
-    assertEquals(1, userQueue.getRunnableAppSchedulables().size());
+    assertEquals(1, userQueue.getNumRunnableApps());
     assertEquals(10000, userQueue.getMinSharePreemptionTimeout());
     assertEquals(15000, userQueue.getFairSharePreemptionTimeout());
     assertEquals(.6f, userQueue.getFairSharePreemptionThreshold(), 0.001);
@@ -2257,10 +2329,9 @@ public void testReservationWhileMultiplePriorities() throws IOException {
     scheduler.handle(updateEvent);
     
     assertEquals(1, app.getLiveContainers().size());
-    // Reserved container should will be at higher priority,
-    // since old reservation cannot be satisfied
+    // Reserved container should still be at lower priority
     for (RMContainer container : app.getReservedContainers()) {
-      assertEquals(1, container.getReservedPriority().getPriority());
+      assertEquals(2, container.getReservedPriority().getPriority());
     }
     
     // Complete container
@@ -2273,11 +2344,12 @@ public void testReservationWhileMultiplePriorities() throws IOException {
     scheduler.update();
     scheduler.handle(updateEvent);
     
-    // Reserved container (at higher priority) should be run
+    // Reserved container (at lower priority) should be run
     Collection<RMContainer> liveContainers = app.getLiveContainers();
     assertEquals(1, liveContainers.size());
     for (RMContainer liveContainer : liveContainers) {
-      Assert.assertEquals(1, liveContainer.getContainer().getPriority().getPriority());
+      Assert.assertEquals(2, liveContainer.getContainer().getPriority()
+          .getPriority());
     }
     assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
     assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
@@ -3023,21 +3095,15 @@ public void testHostPortNodeName() throws Exception {
   private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) {
     FSAppAttempt app = scheduler.getSchedulerApp(attId);
     FSLeafQueue queue = app.getQueue();
-    Collection<FSAppAttempt> runnableApps =
-        queue.getRunnableAppSchedulables();
-    Collection<FSAppAttempt> nonRunnableApps =
-        queue.getNonRunnableAppSchedulables();
-    assertEquals(runnable, runnableApps.contains(app));
-    assertEquals(!runnable, nonRunnableApps.contains(app));
+    assertEquals(runnable, queue.isRunnableApp(app));
+    assertEquals(!runnable, queue.isNonRunnableApp(app));
   }
   
   private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue,
       int numNonRunnableInQueue) {
     FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(queueName, false);
-    assertEquals(numRunnableInQueue,
-        queue.getRunnableAppSchedulables().size());
-    assertEquals(numNonRunnableInQueue,
-        queue.getNonRunnableAppSchedulables().size());
+    assertEquals(numRunnableInQueue, queue.getNumRunnableApps());
+    assertEquals(numNonRunnableInQueue, queue.getNumNonRunnableApps());
   }
   
   @Test
@@ -3584,23 +3650,23 @@ public void testDontAllowUndeclaredPools() throws Exception{
     
     // Should get put into jerry
     createSchedulingRequest(1024, "jerry", "someuser");
-    assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
+    assertEquals(1, jerryQueue.getNumRunnableApps());
 
     // Should get forced into default
     createSchedulingRequest(1024, "newqueue", "someuser");
-    assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
-    assertEquals(1, defaultQueue.getRunnableAppSchedulables().size());
+    assertEquals(1, jerryQueue.getNumRunnableApps());
+    assertEquals(1, defaultQueue.getNumRunnableApps());
     
     // Would get put into someuser because of user-as-default-queue, but should
     // be forced into default
     createSchedulingRequest(1024, "default", "someuser");
-    assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
-    assertEquals(2, defaultQueue.getRunnableAppSchedulables().size());
+    assertEquals(1, jerryQueue.getNumRunnableApps());
+    assertEquals(2, defaultQueue.getNumRunnableApps());
     
     // Should get put into jerry because of user-as-default-queue
     createSchedulingRequest(1024, "default", "jerry");
-    assertEquals(2, jerryQueue.getRunnableAppSchedulables().size());
-    assertEquals(2, defaultQueue.getRunnableAppSchedulables().size());
+    assertEquals(2, jerryQueue.getNumRunnableApps());
+    assertEquals(2, defaultQueue.getNumRunnableApps());
   }
 
   @Test
@@ -3843,8 +3909,8 @@ public void testMoveRunnableApp() throws Exception {
     scheduler.moveApplication(appId, "queue2");
     FSAppAttempt app = scheduler.getSchedulerApp(appAttId);
     assertSame(targetQueue, app.getQueue());
-    assertFalse(oldQueue.getRunnableAppSchedulables().contains(app));
-    assertTrue(targetQueue.getRunnableAppSchedulables().contains(app));
+    assertFalse(oldQueue.isRunnableApp(app));
+    assertTrue(targetQueue.isRunnableApp(app));
     assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
     assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
     assertEquals(0, oldQueue.getNumRunnableApps());
@@ -3893,12 +3959,12 @@ public void testMoveMakesAppRunnable() throws Exception {
         createSchedulingRequest(1024, 1, "queue1", "user1", 3);
     
     FSAppAttempt app = scheduler.getSchedulerApp(appAttId);
-    assertTrue(oldQueue.getNonRunnableAppSchedulables().contains(app));
+    assertTrue(oldQueue.isNonRunnableApp(app));
     
     scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
-    assertFalse(oldQueue.getNonRunnableAppSchedulables().contains(app));
-    assertFalse(targetQueue.getNonRunnableAppSchedulables().contains(app));
-    assertTrue(targetQueue.getRunnableAppSchedulables().contains(app));
+    assertFalse(oldQueue.isNonRunnableApp(app));
+    assertFalse(targetQueue.isNonRunnableApp(app));
+    assertTrue(targetQueue.isRunnableApp(app));
     assertEquals(1, targetQueue.getNumRunnableApps());
     assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 903c7af9cb1b5..458b06dd60a03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -58,8 +58,7 @@ public void resetLastPreemptResources() {
     }
   }
 
-  @Override
-  protected Configuration createConfiguration() {
+  public Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class,
         ResourceScheduler.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
index 34c33b4bd9506..ac5748f8c2dd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
@@ -97,13 +97,13 @@ public void testRemoveDoesNotEnableAnyApp() {
     FSAppAttempt app1 = addApp(leaf1, "user");
     addApp(leaf2, "user");
     addApp(leaf2, "user");
-    assertEquals(1, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(1, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
     removeApp(app1);
-    assertEquals(0, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(0, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
   }
   
   @Test
@@ -114,13 +114,13 @@ public void testRemoveEnablesAppOnCousinQueue() {
     FSAppAttempt app1 = addApp(leaf1, "user");
     addApp(leaf2, "user");
     addApp(leaf2, "user");
-    assertEquals(1, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(1, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
     removeApp(app1);
-    assertEquals(0, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(2, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(0, leaf1.getNumRunnableApps());
+    assertEquals(2, leaf2.getNumRunnableApps());
+    assertEquals(0, leaf2.getNumNonRunnableApps());
   }
   
   @Test
@@ -133,14 +133,14 @@ public void testRemoveEnablesOneByQueueOneByUser() {
     addApp(leaf1, "user2");
     addApp(leaf1, "user3");
     addApp(leaf2, "user1");
-    assertEquals(2, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf1.getNonRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(2, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf1.getNumNonRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
     removeApp(app1);
-    assertEquals(2, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(0, leaf1.getNonRunnableAppSchedulables().size());
-    assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(2, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(0, leaf1.getNumNonRunnableApps());
+    assertEquals(0, leaf2.getNumNonRunnableApps());
   }
   
   @Test
@@ -153,14 +153,14 @@ public void testRemoveEnablingOrderedByStartTime() {
     addApp(leaf2, "user");
     clock.tick(20);
     addApp(leaf1, "user");
-    assertEquals(1, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf1.getNonRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(1, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(1, leaf1.getNumNonRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
     removeApp(app1);
-    assertEquals(0, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(2, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(0, leaf1.getNumRunnableApps());
+    assertEquals(2, leaf2.getNumRunnableApps());
+    assertEquals(0, leaf2.getNumNonRunnableApps());
   }
   
   @Test
@@ -172,13 +172,13 @@ public void testMultipleAppsWaitingOnCousinQueue() {
     addApp(leaf2, "user");
     addApp(leaf2, "user");
     addApp(leaf2, "user");
-    assertEquals(1, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(2, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(1, leaf1.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumRunnableApps());
+    assertEquals(2, leaf2.getNumNonRunnableApps());
     removeApp(app1);
-    assertEquals(0, leaf1.getRunnableAppSchedulables().size());
-    assertEquals(2, leaf2.getRunnableAppSchedulables().size());
-    assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+    assertEquals(0, leaf1.getNumRunnableApps());
+    assertEquals(2, leaf2.getNumRunnableApps());
+    assertEquals(1, leaf2.getNumNonRunnableApps());
   }
   
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index b4c4c1017dd80..3918bf70ba3ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -143,13 +143,14 @@ public void testFifoSchedulerCapacityWhenNoNMs() {
   @Test(timeout=5000)
   public void testAppAttemptMetrics() throws Exception {
     AsyncDispatcher dispatcher = new InlineDispatcher();
+    
+    FifoScheduler scheduler = new FifoScheduler();
     RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
     RMContext rmContext = new RMContextImpl(dispatcher, null,
-        null, null, null, null, null, null, null, writer);
+        null, null, null, null, null, null, null, writer, scheduler);
     ((RMContextImpl) rmContext).setSystemMetricsPublisher(
         mock(SystemMetricsPublisher.class));
 
-    FifoScheduler scheduler = new FifoScheduler();
     Configuration conf = new Configuration();
     scheduler.setRMContext(rmContext);
     scheduler.init(conf);
@@ -189,12 +190,14 @@ public void testNodeLocalAssignment() throws Exception {
         new NMTokenSecretManagerInRM(conf);
     nmTokenSecretManager.rollMasterKey();
     RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
+    
+    FifoScheduler scheduler = new FifoScheduler();
     RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
-        null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
+        null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
+        scheduler);
     ((RMContextImpl) rmContext).setSystemMetricsPublisher(
         mock(SystemMetricsPublisher.class));
 
-    FifoScheduler scheduler = new FifoScheduler();
     scheduler.setRMContext(rmContext);
     scheduler.init(conf);
     scheduler.start();
@@ -260,17 +263,19 @@ public void testUpdateResourceOnNode() throws Exception {
         new NMTokenSecretManagerInRM(conf);
     nmTokenSecretManager.rollMasterKey();
     RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
-    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
-        null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
-    ((RMContextImpl) rmContext).setSystemMetricsPublisher(
-        mock(SystemMetricsPublisher.class));
-
+    
     FifoScheduler scheduler = new FifoScheduler(){
       @SuppressWarnings("unused")
       public Map<NodeId, FiCaSchedulerNode> getNodes(){
         return nodes;
       }
     };
+    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
+        null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
+        scheduler);
+    ((RMContextImpl) rmContext).setSystemMetricsPublisher(
+        mock(SystemMetricsPublisher.class));
+
     scheduler.setRMContext(rmContext);
     scheduler.init(conf);
     scheduler.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 7275089c13b65..5d31404b4cca6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -79,6 +79,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
 import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -86,6 +87,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.junit.After;
@@ -116,11 +118,12 @@ public static class Renewer extends TokenRenewer {
     private static int counter = 0;
     private static Token<?> lastRenewed = null;
     private static Token<?> tokenToRenewIn2Sec = null;
-
+    private static boolean cancelled = false; 
     private static void reset() {
       counter = 0;
       lastRenewed = null;
       tokenToRenewIn2Sec = null;
+
     }
 
     @Override
@@ -136,7 +139,8 @@ public boolean isManaged(Token<?> token) throws IOException {
     @Override
     public long renew(Token<?> t, Configuration conf) throws IOException {
       if ( !(t instanceof MyToken)) {
-        return DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT;
+        // renew in 3 seconds
+        return System.currentTimeMillis() + 3000;
       }
       MyToken token = (MyToken)t;
       if(token.isCanceled()) {
@@ -158,9 +162,12 @@ public long renew(Token<?> t, Configuration conf) throws IOException {
 
     @Override
     public void cancel(Token<?> t, Configuration conf) {
-      MyToken token = (MyToken)t;
-      LOG.info("Cancel token " + token);
-      token.cancelToken();
+      cancelled = true;
+      if (t instanceof MyToken) {
+        MyToken token = (MyToken) t;
+        LOG.info("Cancel token " + token);
+        token.cancelToken();
+      }
    }
     
   }
@@ -921,6 +928,7 @@ public Boolean get() {
   // YARN will get the token for the app submitted without the delegation token.
   @Test
   public void testAppSubmissionWithoutDelegationToken() throws Exception {
+    conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
     // create token2
     Text userText2 = new Text("user2");
     DelegationTokenIdentifier dtId2 =
@@ -970,4 +978,48 @@ public Boolean get() {
     appCredentials.readTokenStorageStream(buf);
     Assert.assertTrue(appCredentials.getAllTokens().contains(token2));
   }
+
+  // Test submitting an application with the token obtained by a previously
+  // submitted application.
+  @Test (timeout = 30000)
+  public void testAppSubmissionWithPreviousToken() throws Exception{
+    MockRM rm = new TestSecurityMockRM(conf, null);
+    rm.start();
+    final MockNM nm1 =
+        new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
+    nm1.registerNode();
+
+    // create Token1:
+    Text userText1 = new Text("user");
+    DelegationTokenIdentifier dtId1 =
+        new DelegationTokenIdentifier(userText1, new Text("renewer1"),
+          userText1);
+    final Token<DelegationTokenIdentifier> token1 =
+        new Token<DelegationTokenIdentifier>(dtId1.getBytes(),
+          "password1".getBytes(), dtId1.getKind(), new Text("service1"));
+
+    Credentials credentials = new Credentials();
+    credentials.addToken(userText1, token1);
+
+    // submit app1 with a token, set cancelTokenWhenComplete to false;
+    RMApp app1 =
+        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+          null, true, false, false, null, 0, null, false);
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+    rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
+
+    // submit app2 with the same token, set cancelTokenWhenComplete to true;
+    RMApp app2 =
+        rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+          null, true, false, false, null, 0, null, true);
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
+    rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
+    MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
+    Assert.assertTrue(rm.getRMContext().getDelegationTokenRenewer()
+      .getAllTokens().containsKey(token1));
+
+    MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);
+    // app2 completes, app1 is still running, check the token is not cancelled
+    Assert.assertFalse(Renewer.cancelled);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index bb38079ccea50..62713cfc7c183 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -106,4 +106,49 @@ public void testNodesBlockRenderForLostNodes() {
             * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
         "<td");
   }
+  
+  @Test
+  public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
+    NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
+    nodesBlock.set("node.label", "x");
+    nodesBlock.render();
+    PrintWriter writer = injector.getInstance(PrintWriter.class);
+    WebAppTests.flushOutput(injector);
+
+    Mockito.verify(
+        writer,
+        Mockito.times(numberOfRacks
+            * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
+        "<td");
+  }
+  
+  @Test
+  public void testNodesBlockRenderForNodeLabelFilterWithEmptyLabel() {
+    NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
+    nodesBlock.set("node.label", "");
+    nodesBlock.render();
+    PrintWriter writer = injector.getInstance(PrintWriter.class);
+    WebAppTests.flushOutput(injector);
+
+    Mockito.verify(
+        writer,
+        Mockito.times(numberOfRacks * (numberOfNodesPerRack - 1)
+            * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
+        "<td");
+  }
+  
+  @Test
+  public void testNodesBlockRenderForNodeLabelFilterWithAnyLabel() {
+    NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
+    nodesBlock.set("node.label", "*");
+    nodesBlock.render();
+    PrintWriter writer = injector.getInstance(PrintWriter.class);
+    WebAppTests.flushOutput(injector);
+
+    Mockito.verify(
+        writer,
+        Mockito.times(numberOfRacks * numberOfNodesPerRack
+            * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
+        "<td");
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index abc701db192fb..fb1e61de0547f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -39,7 +39,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -179,7 +179,7 @@ public ConcurrentMap<NodeId, RMNode> getRMNodes() {
          return nodesMap;
        }
      }; 
-    rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
+    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
     return rmContext;
   }
 
@@ -211,7 +211,7 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException {
         null, new RMContainerTokenSecretManager(conf),
         new NMTokenSecretManagerInRM(conf),
         new ClientToAMTokenSecretManagerInRM(), null);
-    rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
+    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
     cs.setRMContext(rmContext);
     cs.init(conf);
     return cs;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 9f091d2398f65..298246ca301e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -21,9 +21,18 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.io.StringReader;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -31,16 +40,24 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
 import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -61,10 +78,9 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServices extends JerseyTest {
+public class TestRMWebServices extends JerseyTestBase {
 
   private static MockRM rm;
 
@@ -586,4 +602,43 @@ public void verifyClusterSchedulerFifoGeneric(String type, String state,
 
   }
 
+  // Test the scenario where the RM removes an app just as we try to
+  // look at it in the apps list
+  @Test
+  public void testAppsRace() throws Exception {
+    // mock up an RM that returns app reports for apps that don't exist
+    // in the RMApps list
+    ApplicationId appId = ApplicationId.newInstance(1, 1);
+    ApplicationReport mockReport = mock(ApplicationReport.class);
+    when(mockReport.getApplicationId()).thenReturn(appId);
+    GetApplicationsResponse mockAppsResponse =
+        mock(GetApplicationsResponse.class);
+    when(mockAppsResponse.getApplicationList())
+      .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
+    ClientRMService mockClientSvc = mock(ClientRMService.class);
+    when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
+        anyBoolean())).thenReturn(mockAppsResponse);
+    ResourceManager mockRM = mock(ResourceManager.class);
+    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+        null, null, null, null, null);
+    when(mockRM.getRMContext()).thenReturn(rmContext);
+    when(mockRM.getClientRMService()).thenReturn(mockClientSvc);
+
+    RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
+        mock(HttpServletResponse.class));
+
+    final Set<String> emptySet =
+        Collections.unmodifiableSet(Collections.<String>emptySet());
+
+    // verify we don't get any apps when querying
+    HttpServletRequest mockHsr = mock(HttpServletRequest.class);
+    AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
+        null, null, null, null, null, null, null, emptySet, emptySet);
+    assertTrue(appsInfo.getApps().isEmpty());
+
+    // verify we don't get an NPE when specifying a final status query
+    appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
+        null, null, null, null, null, null, null, emptySet, emptySet);
+    assertTrue(appsInfo.getApps().isEmpty());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 23ea22e37fb27..705fd316af1e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -67,10 +68,9 @@
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.core.util.MultivaluedMapImpl;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServicesApps extends JerseyTest {
+public class TestRMWebServicesApps extends JerseyTestBase {
 
   private static MockRM rm;
   
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index df23e85ab1b2b..8e5e6015b8e67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -43,7 +43,6 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
@@ -72,12 +71,15 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -102,11 +104,10 @@
 import com.sun.jersey.api.json.JSONJAXBContext;
 import com.sun.jersey.api.json.JSONMarshaller;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
 @RunWith(Parameterized.class)
-public class TestRMWebServicesAppsModification extends JerseyTest {
+public class TestRMWebServicesAppsModification extends JerseyTestBase {
   private static MockRM rm;
 
   private static final int CONTAINER_MB = 1024;
@@ -199,6 +200,9 @@ public void configureScheduler() {
         out.println("  <queue name=\"default\">");
         out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
         out.println("  </queue>");
+        out.println("  <queue name=\"test\">");
+        out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
+        out.println("  </queue>");
         out.println("</queue>");
         out.println("</allocations>");
         out.close();
@@ -358,7 +362,7 @@ public void testSingleAppKill() throws Exception {
             new AppState(YarnApplicationState.KILLED.toString());
 
         Object entity;
-        if (contentType == MediaType.APPLICATION_JSON_TYPE) {
+        if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
           entity = appStateToJSON(targetState);
         } else {
           entity = targetState;
@@ -439,7 +443,7 @@ public void testSingleAppKillInvalidState() throws Exception {
           ClientResponse response;
           AppState targetState = new AppState(targetStateString);
           Object entity;
-          if (contentType == MediaType.APPLICATION_JSON_TYPE) {
+          if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
             entity = appStateToJSON(targetState);
           } else {
             entity = targetState;
@@ -555,7 +559,6 @@ public void testSingleAppKillUnauthorized() throws Exception {
       validateResponseStatus(response, Status.FORBIDDEN);
     }
     rm.stop();
-
   }
 
   @Test
@@ -736,20 +739,19 @@ public void testAppSubmit(String acceptMedia, String contentMedia)
     String appType = "test-type";
     String urlPath = "apps";
     String appId = testGetNewApplication(acceptMedia);
-    List<String> commands = new ArrayList<String>();
+    List<String> commands = new ArrayList<>();
     commands.add("/bin/sleep 5");
-    HashMap<String, String> environment = new HashMap<String, String>();
+    HashMap<String, String> environment = new HashMap<>();
     environment.put("APP_VAR", "ENV_SETTING");
-    HashMap<ApplicationAccessType, String> acls =
-        new HashMap<ApplicationAccessType, String>();
+    HashMap<ApplicationAccessType, String> acls = new HashMap<>();
     acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2");
     acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4");
-    Set<String> tags = new HashSet<String>();
+    Set<String> tags = new HashSet<>();
     tags.add("tag1");
     tags.add("tag 2");
     CredentialsInfo credentials = new CredentialsInfo();
-    HashMap<String, String> tokens = new HashMap<String, String>();
-    HashMap<String, String> secrets = new HashMap<String, String>();
+    HashMap<String, String> tokens = new HashMap<>();
+    HashMap<String, String> secrets = new HashMap<>();
     secrets.put("secret1", Base64.encodeBase64String(
         "mysecret".getBytes("UTF8")));
     credentials.setSecrets(secrets);
@@ -761,8 +763,7 @@ public void testAppSubmit(String acceptMedia, String contentMedia)
     appInfo.setMaxAppAttempts(2);
     appInfo.setQueue(queueName);
     appInfo.setApplicationType(appType);
-    HashMap<String, LocalResourceInfo> lr =
-        new HashMap<String, LocalResourceInfo>();
+    HashMap<String, LocalResourceInfo> lr =  new HashMap<>();
     LocalResourceInfo y = new LocalResourceInfo();
     y.setUrl(new URI("http://www.test.com/file.txt"));
     y.setSize(100);
@@ -911,8 +912,7 @@ public void testAppSubmitBadJsonAndXML() throws Exception {
     appInfo.setMaxAppAttempts(2);
     appInfo.setQueue("testqueue");
     appInfo.setApplicationType("test-type");
-    HashMap<String, LocalResourceInfo> lr =
-        new HashMap<String, LocalResourceInfo>();
+    HashMap<String, LocalResourceInfo> lr = new HashMap<>();
     LocalResourceInfo y = new LocalResourceInfo();
     y.setUrl(new URI("http://www.test.com/file.txt"));
     y.setSize(100);
@@ -939,4 +939,150 @@ public void testAppSubmitBadJsonAndXML() throws Exception {
     rm.stop();
   }
 
+  @Test
+  public void testGetAppQueue() throws Exception {
+    client().addFilter(new LoggingFilter(System.out));
+    boolean isCapacityScheduler =
+        rm.getResourceScheduler() instanceof CapacityScheduler;
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    String[] contentTypes =
+        { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+    for (String contentType : contentTypes) {
+      RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
+      amNodeManager.nodeHeartbeat(true);
+      ClientResponse response =
+          this
+            .constructWebResource("apps", app.getApplicationId().toString(),
+              "queue").accept(contentType).get(ClientResponse.class);
+      assertEquals(Status.OK, response.getClientResponseStatus());
+      String expectedQueue = "default";
+      if(!isCapacityScheduler) {
+        expectedQueue = "root." + webserviceUserName;
+      }
+      if (contentType.equals(MediaType.APPLICATION_JSON)) {
+        verifyAppQueueJson(response, expectedQueue);
+      } else {
+        verifyAppQueueXML(response, expectedQueue);
+      }
+    }
+    rm.stop();
+  }
+
+  @Test(timeout = 90000)
+  public void testAppMove() throws Exception {
+
+    client().addFilter(new LoggingFilter(System.out));
+
+    boolean isCapacityScheduler =
+        rm.getResourceScheduler() instanceof CapacityScheduler;
+
+    // default root queue allows anyone to have admin acl
+    CapacitySchedulerConfiguration csconf =
+        new CapacitySchedulerConfiguration();
+    String[] queues = { "default", "test" };
+    csconf.setQueues("root", queues);
+    csconf.setCapacity("root.default", 50.0f);
+    csconf.setCapacity("root.test", 50.0f);
+    csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
+    csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
+    csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser");
+    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
+
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    String[] mediaTypes =
+        { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+    MediaType[] contentTypes =
+        { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
+    for (String mediaType : mediaTypes) {
+      for (MediaType contentType : contentTypes) {
+        RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
+        amNodeManager.nodeHeartbeat(true);
+        AppQueue targetQueue = new AppQueue("test");
+        Object entity;
+        if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
+          entity = appQueueToJSON(targetQueue);
+        } else {
+          entity = targetQueue;
+        }
+        ClientResponse response =
+            this
+              .constructWebResource("apps", app.getApplicationId().toString(),
+                "queue").entity(entity, contentType).accept(mediaType)
+              .put(ClientResponse.class);
+
+        if (!isAuthenticationEnabled()) {
+          assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
+          continue;
+        }
+        assertEquals(Status.OK, response.getClientResponseStatus());
+        String expectedQueue = "test";
+        if(!isCapacityScheduler) {
+          expectedQueue = "root.test";
+        }
+        if (mediaType.equals(MediaType.APPLICATION_JSON)) {
+          verifyAppQueueJson(response, expectedQueue);
+        } else {
+          verifyAppQueueXML(response, expectedQueue);
+        }
+        Assert.assertEquals(expectedQueue, app.getQueue());
+
+        // check unauthorized
+        app = rm.submitApp(CONTAINER_MB, "", "someuser");
+        amNodeManager.nodeHeartbeat(true);
+        response =
+            this
+              .constructWebResource("apps", app.getApplicationId().toString(),
+                "queue").entity(entity, contentType).accept(mediaType)
+              .put(ClientResponse.class);
+        assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+        if(isCapacityScheduler) {
+          Assert.assertEquals("default", app.getQueue());
+        }
+        else {
+          Assert.assertEquals("root.someuser", app.getQueue());
+        }
+
+      }
+    }
+    rm.stop();
+  }
+
+  protected static String appQueueToJSON(AppQueue targetQueue) throws Exception {
+    StringWriter sw = new StringWriter();
+    JSONJAXBContext ctx = new JSONJAXBContext(AppQueue.class);
+    JSONMarshaller jm = ctx.createJSONMarshaller();
+    jm.marshallToJSON(targetQueue, sw);
+    return sw.toString();
+  }
+
+  protected static void
+      verifyAppQueueJson(ClientResponse response, String queue)
+          throws JSONException {
+
+    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    String responseQueue = json.getString("queue");
+    assertEquals(queue, responseQueue);
+  }
+
+  protected static void
+      verifyAppQueueXML(ClientResponse response, String queue)
+          throws ParserConfigurationException, IOException, SAXException {
+    assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+    String xml = response.getEntity(String.class);
+    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilder db = dbf.newDocumentBuilder();
+    InputSource is = new InputSource();
+    is.setCharacterStream(new StringReader(xml));
+    Document dom = db.parse(is);
+    NodeList nodes = dom.getElementsByTagName("appqueue");
+    assertEquals("incorrect number of elements", 1, nodes.getLength());
+    Element element = (Element) nodes.item(0);
+    String responseQueue = WebServicesTestUtils.getXmlString(element, "queue");
+    assertEquals(queue, responseQueue);
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 87bacc6eca3fb..94040b5db22d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -55,10 +56,9 @@
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServicesCapacitySched extends JerseyTest {
+public class TestRMWebServicesCapacitySched extends JerseyTestBase {
 
   private static MockRM rm;
   private CapacitySchedulerConfiguration csConf;
@@ -82,8 +82,6 @@ private class LeafQueueInfo extends QueueInfo {
     int numContainers;
     int maxApplications;
     int maxApplicationsPerUser;
-    int maxActiveApplications;
-    int maxActiveApplicationsPerUser;
     int userLimit;
     float userLimitFactor;
   }
@@ -303,10 +301,6 @@ public void verifySubQueueXML(Element qElem, String q,
           WebServicesTestUtils.getXmlInt(qElem, "maxApplications");
       lqi.maxApplicationsPerUser =
           WebServicesTestUtils.getXmlInt(qElem, "maxApplicationsPerUser");
-      lqi.maxActiveApplications =
-          WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplications");
-      lqi.maxActiveApplicationsPerUser =
-          WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplicationsPerUser");
       lqi.userLimit = WebServicesTestUtils.getXmlInt(qElem, "userLimit");
       lqi.userLimitFactor =
           WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor");
@@ -353,7 +347,7 @@ private void verifySubQueue(JSONObject info, String q,
     int numExpectedElements = 13;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 23;
+      numExpectedElements = 24;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());
@@ -386,8 +380,6 @@ private void verifySubQueue(JSONObject info, String q,
       lqi.numContainers = info.getInt("numContainers");
       lqi.maxApplications = info.getInt("maxApplications");
       lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser");
-      lqi.maxActiveApplications = info.getInt("maxActiveApplications");
-      lqi.maxActiveApplicationsPerUser = info.getInt("maxActiveApplicationsPerUser");
       lqi.userLimit = info.getInt("userLimit");
       lqi.userLimitFactor = (float) info.getDouble("userLimitFactor");
       verifyLeafQueueGeneric(q, lqi);
@@ -449,10 +441,6 @@ private void verifyLeafQueueGeneric(String q, LeafQueueInfo info)
         (float)expectedMaxAppsPerUser,
         (float)info.maxApplicationsPerUser, info.userLimitFactor);
 
-    assertTrue("maxActiveApplications doesn't match",
-        info.maxActiveApplications > 0);
-    assertTrue("maxActiveApplicationsPerUser doesn't match",
-        info.maxActiveApplicationsPerUser > 0);
     assertEquals("userLimit doesn't match", csConf.getUserLimit(q),
         info.userLimit);
     assertEquals("userLimitFactor doesn't match",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
index c5c048fa65ed4..dab83433a45b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
@@ -53,6 +53,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -87,7 +88,7 @@
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
 @RunWith(Parameterized.class)
-public class TestRMWebServicesDelegationTokens extends JerseyTest {
+public class TestRMWebServicesDelegationTokens extends JerseyTestBase {
 
   private static File testRootDir;
   private static File httpSpnegoKeytabFile = new File(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
index 9de3f76f12c94..21ca6a724ac72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Test;
@@ -39,10 +40,9 @@
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServicesFairScheduler extends JerseyTest {
+public class TestRMWebServicesFairScheduler extends JerseyTestBase {
   private static MockRM rm;
   private YarnConfiguration conf;
   
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index df5aecbba71e6..0cc576cb12ef7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -36,6 +36,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -51,10 +52,9 @@
 import com.sun.jersey.api.json.JSONMarshaller;
 import com.sun.jersey.api.json.JSONUnmarshaller;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServicesNodeLabels extends JerseyTest {
+public class TestRMWebServicesNodeLabels extends JerseyTestBase {
 
   private static final Log LOG = LogFactory
       .getLog(TestRMWebServicesNodeLabels.class);
@@ -155,7 +155,7 @@ public void testNodeLabels() throws JSONException, Exception {
             .path("replace-labels")
             .queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+            .entity("{\"nodeLabels\": [\"a\"]}",
               MediaType.APPLICATION_JSON)
             .post(ClientResponse.class);
     LOG.info("posted node nodelabel");
@@ -168,8 +168,8 @@ public void testNodeLabels() throws JSONException, Exception {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     json = response.getEntity(JSONObject.class);
-    jarr = json.getJSONArray("nodeLabels");
-    assertEquals(2, jarr.length());
+    assertEquals("a", json.getString("nodeLabels"));
+
     
     // Replace
     response =
@@ -178,9 +178,10 @@ public void testNodeLabels() throws JSONException, Exception {
             .path("replace-labels")
             .queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+            .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
             .post(ClientResponse.class);
     LOG.info("posted node nodelabel");
+
     // Verify
     response =
         r.path("ws").path("v1").path("cluster")
@@ -189,13 +190,12 @@ public void testNodeLabels() throws JSONException, Exception {
             .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     json = response.getEntity(JSONObject.class);
-    assertEquals("a", json.getString("nodeLabels"));
+    assertEquals("b", json.getString("nodeLabels"));
             
     // Replace labels using node-to-labels
     NodeToLabelsInfo ntli = new NodeToLabelsInfo();
     NodeLabelsInfo nli = new NodeLabelsInfo();
     nli.getNodeLabels().add("a");
-    nli.getNodeLabels().add("b");
     ntli.getNodeToLabels().put("nid:0", nli);
     response =
         r.path("ws").path("v1").path("cluster")
@@ -214,9 +214,8 @@ public void testNodeLabels() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
     ntli = response.getEntity(NodeToLabelsInfo.class);
     nli = ntli.getNodeToLabels().get("nid:0");
-    assertEquals(2, nli.getNodeLabels().size());
+    assertEquals(1, nli.getNodeLabels().size());
     assertTrue(nli.getNodeLabels().contains("a"));
-    assertTrue(nli.getNodeLabels().contains("b"));
     
     // Remove all
     response =
@@ -267,7 +266,7 @@ public void testNodeLabels() throws JSONException, Exception {
             .path("replace-labels")
             .queryParam("user.name", notUserName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+            .entity("{\"nodeLabels\": [\"b\"]}",
               MediaType.APPLICATION_JSON)
             .post(ClientResponse.class);
     // Verify
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index e685f221ad2e5..f507e1789f714 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -43,6 +43,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -64,10 +65,9 @@
 import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
-public class TestRMWebServicesNodes extends JerseyTest {
+public class TestRMWebServicesNodes extends JerseyTestBase {
 
   private static MockRM rm;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java
index 593be4a340267..6748387465f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/CleanerService.java
@@ -91,7 +91,7 @@ protected void serviceStart() throws Exception {
           "It appears there is another CleanerService running in the cluster");
     }
 
-    this.metrics = CleanerMetrics.initSingleton(conf);
+    this.metrics = CleanerMetrics.getInstance();
 
     // Start dependent services (i.e. AppChecker)
     super.serviceStart();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java
index bd13573db3be3..1dcca6c96e9d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/ClientProtocolService.java
@@ -94,7 +94,7 @@ InetSocketAddress getBindAddress(Configuration conf) {
   @Override
   protected void serviceStart() throws Exception {
     Configuration conf = getConfig();
-    this.metrics = ClientSCMMetrics.initSingleton(conf);
+    this.metrics = ClientSCMMetrics.getInstance();
 
     YarnRPC rpc = YarnRPC.create(conf);
     this.server =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
index 5c33b2b8f954c..331e29ee32b0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheManager.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
+import org.apache.hadoop.yarn.server.sharedcachemanager.webapp.SCMWebServer;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -77,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
     SCMAdminProtocolService saps = createSCMAdminProtocolService(cs);
     addService(saps);
 
+    SCMWebServer webUI = createSCMWebServer(this);
+    addService(webUI);
+
     // init metrics
     DefaultMetricsSystem.initialize("SharedCacheManager");
     JvmMetrics.initSingleton("SharedCacheManager", null);
@@ -121,6 +125,10 @@ private SCMAdminProtocolService createSCMAdminProtocolService(
     return new SCMAdminProtocolService(cleanerService);
   }
 
+  private SCMWebServer createSCMWebServer(SharedCacheManager scm) {
+    return new SCMWebServer(scm);
+  }
+
   @Override
   protected void serviceStop() throws Exception {
 
@@ -152,24 +160,4 @@ public static void main(String[] args) {
       System.exit(-1);
     }
   }
-
-  @Private
-  @SuppressWarnings("unchecked")
-  public static AppChecker createAppCheckerService(Configuration conf) {
-    Class<? extends AppChecker> defaultCheckerClass;
-    try {
-      defaultCheckerClass =
-          (Class<? extends AppChecker>) Class
-              .forName(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
-    } catch (Exception e) {
-      throw new YarnRuntimeException("Invalid default scm app checker class"
-          + YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS, e);
-    }
-
-    AppChecker checker =
-        ReflectionUtils.newInstance(conf.getClass(
-            YarnConfiguration.SCM_APP_CHECKER_CLASS, defaultCheckerClass,
-            AppChecker.class), conf);
-    return checker;
-  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheUploaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheUploaderService.java
index f9494386b02b3..dd87b679cb4f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheUploaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SharedCacheUploaderService.java
@@ -72,7 +72,7 @@ InetSocketAddress getBindAddress(Configuration conf) {
   @Override
   protected void serviceStart() throws Exception {
     Configuration conf = getConfig();
-    this.metrics = SharedCacheUploaderMetrics.initSingleton(conf);
+    this.metrics = SharedCacheUploaderMetrics.getInstance();
 
     YarnRPC rpc = YarnRPC.create(conf);
     this.server =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java
index 5c8ea3d493a55..b86a469f823f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/CleanerMetrics.java
@@ -21,7 +21,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.MetricsSource;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
@@ -43,31 +42,10 @@
 public class CleanerMetrics {
   public static final Log LOG = LogFactory.getLog(CleanerMetrics.class);
   private final MetricsRegistry registry = new MetricsRegistry("cleaner");
-
-  enum Singleton {
-    INSTANCE;
-
-    CleanerMetrics impl;
-
-    synchronized CleanerMetrics init(Configuration conf) {
-      if (impl == null) {
-        impl = create(conf);
-      }
-      return impl;
-    }
-  }
-
-  public static CleanerMetrics initSingleton(Configuration conf) {
-    return Singleton.INSTANCE.init(conf);
-  }
-
+  private final static CleanerMetrics INSTANCE = create();
+  
   public static CleanerMetrics getInstance() {
-    CleanerMetrics topMetrics = Singleton.INSTANCE.impl;
-    if (topMetrics == null)
-      throw new IllegalStateException(
-          "The CleanerMetics singlton instance is not initialized."
-              + " Have you called init first?");
-    return topMetrics;
+    return INSTANCE;
   }
 
   @Metric("number of deleted files over all runs")
@@ -120,7 +98,7 @@ private CleanerMetrics() {
    */
   MetricsSource metricSource;
 
-  static CleanerMetrics create(Configuration conf) {
+  static CleanerMetrics create() {
     MetricsSystem ms = DefaultMetricsSystem.instance();
 
     CleanerMetrics metricObject = new CleanerMetrics();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java
index 3ae88e0e4bcac..fe960c6e6f7ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/ClientSCMMetrics.java
@@ -21,7 +21,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -40,37 +39,15 @@ public class ClientSCMMetrics {
 
   private static final Log LOG = LogFactory.getLog(ClientSCMMetrics.class);
   final MetricsRegistry registry;
+  private final static ClientSCMMetrics INSTANCE = create();
 
-  ClientSCMMetrics() {
+  private ClientSCMMetrics() {
     registry = new MetricsRegistry("clientRequests");
     LOG.debug("Initialized " + registry);
   }
-
-  enum Singleton {
-    INSTANCE;
-
-    ClientSCMMetrics impl;
-
-    synchronized ClientSCMMetrics init(Configuration conf) {
-      if (impl == null) {
-        impl = create();
-      }
-      return impl;
-    }
-  }
-
-  public static ClientSCMMetrics initSingleton(Configuration conf) {
-    return Singleton.INSTANCE.init(conf);
-  }
-
+  
   public static ClientSCMMetrics getInstance() {
-    ClientSCMMetrics topMetrics = Singleton.INSTANCE.impl;
-    if (topMetrics == null) {
-      throw new IllegalStateException(
-          "The ClientSCMMetrics singleton instance is not initialized."
-          + " Have you called init first?");
-    }
-    return topMetrics;
+    return INSTANCE;
   }
 
   static ClientSCMMetrics create() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/SharedCacheUploaderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/SharedCacheUploaderMetrics.java
index 6fd816f29a61b..7fff13a6aec83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/SharedCacheUploaderMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/SharedCacheUploaderMetrics.java
@@ -21,7 +21,6 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -41,37 +40,15 @@ public class SharedCacheUploaderMetrics {
   static final Log LOG =
       LogFactory.getLog(SharedCacheUploaderMetrics.class);
   final MetricsRegistry registry;
+  private final static SharedCacheUploaderMetrics INSTANCE = create();
 
-  SharedCacheUploaderMetrics() {
+  private SharedCacheUploaderMetrics() {
     registry = new MetricsRegistry("SharedCacheUploaderRequests");
     LOG.debug("Initialized "+ registry);
   }
 
-  enum Singleton {
-    INSTANCE;
-
-    SharedCacheUploaderMetrics impl;
-
-    synchronized SharedCacheUploaderMetrics init(Configuration conf) {
-      if (impl == null) {
-        impl = create();
-      }
-      return impl;
-    }
-  }
-
-  public static SharedCacheUploaderMetrics
-      initSingleton(Configuration conf) {
-    return Singleton.INSTANCE.init(conf);
-  }
-
   public static SharedCacheUploaderMetrics getInstance() {
-    SharedCacheUploaderMetrics topMetrics = Singleton.INSTANCE.impl;
-    if (topMetrics == null)
-      throw new IllegalStateException(
-          "The SharedCacheUploaderMetrics singleton instance is not"
-          + "initialized. Have you called init first?");
-    return topMetrics;
+    return INSTANCE;
   }
 
   static SharedCacheUploaderMetrics create() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java
index b8fe14f2f5519..d2efb6a14d8e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/InMemorySCMStore.java
@@ -86,6 +86,11 @@ public class InMemorySCMStore extends SCMStore {
   private int initialDelayMin;
   private int checkPeriodMin;
 
+  public InMemorySCMStore() {
+    super(InMemorySCMStore.class.getName());
+  }
+
+  @VisibleForTesting
   public InMemorySCMStore(AppChecker appChecker) {
     super(InMemorySCMStore.class.getName(), appChecker);
   }
@@ -137,18 +142,22 @@ protected void serviceStart() throws Exception {
 
   @Override
   protected void serviceStop() throws Exception {
-    LOG.info("Shutting down the background thread.");
-    scheduler.shutdownNow();
-    try {
-      if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
-        LOG.warn("Gave up waiting for the app check task to shutdown.");
+    LOG.info("Stopping the " + InMemorySCMStore.class.getSimpleName()
+        + " service.");
+    if (scheduler != null) {
+      LOG.info("Shutting down the background thread.");
+      scheduler.shutdownNow();
+      try {
+        if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
+          LOG.warn("Gave up waiting for the app check task to shutdown.");
+        }
+      } catch (InterruptedException e) {
+        LOG.warn(
+            "The InMemorySCMStore was interrupted while shutting down the "
+                + "app check task.", e);
       }
-    } catch (InterruptedException e) {
-      LOG.warn("The InMemorySCMStore was interrupted while shutting down the "
-          + "app check task.", e);
+      LOG.info("The background thread stopped.");
     }
-    LOG.info("The background thread stopped.");
-
     super.serviceStop();
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java
index 6be00b9e1a3df..9eae556cca8b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStore.java
@@ -24,11 +24,17 @@
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.sharedcachemanager.AppChecker;
 
+import com.google.common.annotations.VisibleForTesting;
+
 
 /**
  * An abstract class for the data store used by the shared cache manager
@@ -39,13 +45,27 @@
 @Evolving
 public abstract class SCMStore extends CompositeService {
 
-  protected final AppChecker appChecker;
+  protected AppChecker appChecker;
+
+  protected SCMStore(String name) {
+    super(name);
+  }
 
-  protected SCMStore(String name, AppChecker appChecker) {
+  @VisibleForTesting
+  SCMStore(String name, AppChecker appChecker) {
     super(name);
     this.appChecker = appChecker;
   }
 
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+    if (this.appChecker == null) {
+      this.appChecker = createAppCheckerService(conf);
+    }
+    addService(appChecker);
+    super.serviceInit(conf);
+  }
+
   /**
    * Add a resource to the shared cache and it's associated filename. The
    * resource is identified by a unique key. If the key already exists no action
@@ -164,4 +184,30 @@ public void cleanResourceReferences(String key) throws YarnException {
   @Private
   public abstract boolean isResourceEvictable(String key, FileStatus file);
 
+  /**
+   * Create an instance of the AppChecker service via reflection based on the
+   * {@link YarnConfiguration#SCM_APP_CHECKER_CLASS} parameter.
+   * 
+   * @param conf
+   * @return an instance of the AppChecker class
+   */
+  @Private
+  @SuppressWarnings("unchecked")
+  public static AppChecker createAppCheckerService(Configuration conf) {
+    Class<? extends AppChecker> defaultCheckerClass;
+    try {
+      defaultCheckerClass =
+          (Class<? extends AppChecker>) Class
+              .forName(YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS);
+    } catch (Exception e) {
+      throw new YarnRuntimeException("Invalid default scm app checker class"
+          + YarnConfiguration.DEFAULT_SCM_APP_CHECKER_CLASS, e);
+    }
+
+    AppChecker checker =
+        ReflectionUtils.newInstance(conf.getClass(
+            YarnConfiguration.SCM_APP_CHECKER_CLASS, defaultCheckerClass,
+            AppChecker.class), conf);
+    return checker;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java
new file mode 100644
index 0000000000000..f597d14bf4e08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMController.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.webapp.Controller;
+
+/**
+ * The controller class for the shared cache manager web app.
+ */
+@Private
+@Unstable
+public class SCMController extends Controller {
+  @Override
+  public void index() {
+    setTitle("Shared Cache Manager");
+  }
+
+  /**
+   * It is referenced in SCMWebServer.SCMWebApp.setup()
+   */
+  @SuppressWarnings("unused")
+  public void overview() {
+    render(SCMOverviewPage.class);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java
new file mode 100644
index 0000000000000..24aa1452a675f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMMetricsInfo.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
+
+/**
+ * This class is used to summarize useful shared cache manager metrics for the
+ * webUI display.
+ */
+@XmlRootElement(name = "SCMMetrics")
+@XmlAccessorType(XmlAccessType.FIELD)
+@Private
+@Unstable
+public class SCMMetricsInfo {
+  protected long totalDeletedFiles;
+  protected long totalProcessedFiles;
+  protected long cacheHits;
+  protected long cacheMisses;
+  protected long cacheReleases;
+  protected long acceptedUploads;
+  protected long rejectedUploads;
+
+  public SCMMetricsInfo() {
+  }
+  
+  public SCMMetricsInfo(CleanerMetrics cleanerMetrics,
+      ClientSCMMetrics clientSCMMetrics,
+      SharedCacheUploaderMetrics scmUploaderMetrics) {
+    totalDeletedFiles = cleanerMetrics.getTotalDeletedFiles();
+    totalProcessedFiles = cleanerMetrics.getTotalProcessedFiles();
+    cacheHits = clientSCMMetrics.getCacheHits();
+    cacheMisses = clientSCMMetrics.getCacheMisses();
+    cacheReleases = clientSCMMetrics.getCacheReleases();
+    acceptedUploads = scmUploaderMetrics.getAcceptedUploads();
+    rejectedUploads = scmUploaderMetrics.getRejectUploads();
+  }
+
+  public long getTotalDeletedFiles() { return totalDeletedFiles; }
+  public long getTotalProcessedFiles() { return totalProcessedFiles; }
+  public long getCacheHits() { return cacheHits; }
+  public long getCacheMisses() { return cacheMisses; }
+  public long getCacheReleases() { return cacheReleases; }
+  public long getAcceptedUploads() { return acceptedUploads; }
+  public long getRejectUploads() { return rejectedUploads; }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java
new file mode 100644
index 0000000000000..27944d3919244
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMOverviewPage.java
@@ -0,0 +1,95 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.ClientSCMMetrics;
+import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.SharedCacheUploaderMetrics;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+import com.google.inject.Inject;
+
+/**
+ * This class is to render the shared cache manager web ui overview page.
+ */
+@Private
+@Unstable
+public class SCMOverviewPage extends TwoColumnLayout {
+
+  @Override protected void preHead(Page.HTML<_> html) {
+    set(ACCORDION_ID, "nav");
+    set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+  }
+
+  @Override protected Class<? extends SubView> content() {
+    return SCMOverviewBlock.class;
+  }
+
+  @Override
+  protected Class<? extends SubView> nav() {
+    return SCMOverviewNavBlock.class;
+  }
+
+  static private class SCMOverviewNavBlock extends HtmlBlock {
+    @Override
+    protected void render(Block html) {
+      html.div("#nav").h3("Tools").ul().li().a("/conf", "Configuration")._()
+          .li().a("/stacks", "Thread dump")._().li().a("/logs", "Logs")._()
+          .li().a("/metrics", "Metrics")._()._()._();
+    }
+  }
+
+  static private class SCMOverviewBlock extends HtmlBlock {
+    final SharedCacheManager scm;
+
+    @Inject
+    SCMOverviewBlock(SharedCacheManager scm, ViewContext ctx) {
+      super(ctx);
+      this.scm = scm;
+    }
+
+    @Override
+    protected void render(Block html) {
+      SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
+          CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
+              SharedCacheUploaderMetrics.getInstance());
+      info("Shared Cache Manager overview").
+          _("Started on:", Times.format(scm.getStartTime())).
+          _("Cache hits: ", metricsInfo.getCacheHits()).
+          _("Cache misses: ", metricsInfo.getCacheMisses()).
+          _("Cache releases: ", metricsInfo.getCacheReleases()).
+          _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
+          _("Rejected uploads: ", metricsInfo.getRejectUploads()).
+          _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
+          _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
+      html._(InfoBlock.class);
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java
new file mode 100644
index 0000000000000..b81ed29f1c793
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/webapp/SCMWebServer.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.webapp;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+
+/**
+ * A very simple web interface for the metrics reported by
+ * {@link org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager}
+ * TODO: Security for web ui (See YARN-2774)
+ */
+@Private
+@Unstable
+public class SCMWebServer extends AbstractService {
+  private static final Log LOG = LogFactory.getLog(SCMWebServer.class);
+
+  private final SharedCacheManager scm;
+  private WebApp webApp;
+  private String bindAddress;
+
+  public SCMWebServer(SharedCacheManager scm) {
+    super(SCMWebServer.class.getName());
+    this.scm = scm;
+  }
+
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+    this.bindAddress = getBindAddress(conf);
+    super.serviceInit(conf);
+  }
+
+  private String getBindAddress(Configuration conf) {
+    return conf.get(YarnConfiguration.SCM_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_SCM_WEBAPP_ADDRESS);
+  }
+
+  @Override
+  protected void serviceStart() throws Exception {
+    SCMWebApp scmWebApp = new SCMWebApp(scm);
+    this.webApp = WebApps.$for("sharedcache").at(bindAddress).start(scmWebApp);
+    LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress);
+  }
+
+  @Override
+  protected void serviceStop() throws Exception {
+    if (this.webApp != null) {
+      this.webApp.stop();
+    }
+  }
+
+  private class SCMWebApp extends WebApp {
+    private final SharedCacheManager scm;
+
+    public SCMWebApp(SharedCacheManager scm) {
+      this.scm = scm;
+    }
+
+    @Override
+    public void setup() {
+      if (scm != null) {
+        bind(SharedCacheManager.class).toInstance(scm);
+      }
+      route("/", SCMController.class, "overview");
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/DummyAppChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/DummyAppChecker.java
new file mode 100644
index 0000000000000..2b7c72cf28350
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/DummyAppChecker.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * A dummy app checker class for testing only.
+ */
+public class DummyAppChecker extends AppChecker {
+  @Override
+  @Private
+  public boolean isApplicationActive(ApplicationId id) throws YarnException {
+    return false;
+  }
+
+  @Override
+  @Private
+  public Collection<ApplicationId> getActiveApplications() throws YarnException {
+    return new ArrayList<ApplicationId>();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
index 68f98510679f6..ca4bdce7cf19d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
@@ -21,7 +21,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 
 import java.io.File;
 import java.io.IOException;
@@ -84,7 +84,7 @@ public void startUp() {
     conf.set(YarnConfiguration.SCM_STORE_CLASS,
         InMemorySCMStore.class.getName());
     conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
-    AppChecker appChecker = mock(AppChecker.class);
+    AppChecker appChecker = spy(new DummyAppChecker());
     store = new InMemorySCMStore(appChecker);
     store.init(conf);
     store.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
index 1cb0663c4efbb..048523e8457d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
@@ -22,7 +22,7 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 
 import java.io.File;
 import java.io.IOException;
@@ -82,7 +82,7 @@ public void startUp() {
     conf.set(YarnConfiguration.SCM_STORE_CLASS,
         InMemorySCMStore.class.getName());
     conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
-    AppChecker appChecker = mock(AppChecker.class);
+    AppChecker appChecker = spy(new DummyAppChecker());
     store = new InMemorySCMStore(appChecker);
     store.init(conf);
     store.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
index 26ab17927a6d0..cca3f4286b321 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
@@ -30,7 +30,6 @@ public class TestCleanerMetrics {
 
   @Before
   public void init() {
-    CleanerMetrics.initSingleton(conf);
     cleanerMetrics = CleanerMetrics.getInstance();
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java
new file mode 100644
index 0000000000000..4e960b29c779d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.sharedcachemanager.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.Test;
+
+/**
+ * All test classes that test an SCMStore implementation must extend this class.
+ */
+public abstract class SCMStoreBaseTest {
+
+  /**
+   * Get the SCMStore implementation class associated with this test class.
+   */
+  abstract Class<? extends SCMStore> getStoreClass();
+
+  @Test
+  public void TestZeroArgConstructor() throws Exception {
+    // Test that the SCMStore implementation class is compatible with
+    // ReflectionUtils#newInstance
+    ReflectionUtils.newInstance(getStoreClass(), new Configuration());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
index 831ef6eb8a793..f934dbfda41ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
@@ -42,22 +42,26 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.sharedcachemanager.AppChecker;
+import org.apache.hadoop.yarn.server.sharedcachemanager.DummyAppChecker;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-public class TestInMemorySCMStore {
+public class TestInMemorySCMStore extends SCMStoreBaseTest {
 
   private InMemorySCMStore store;
   private AppChecker checker;
 
+  @Override
+  Class<? extends SCMStore> getStoreClass() {
+    return InMemorySCMStore.class;
+  }
+
   @Before
   public void setup() {
     this.checker = spy(new DummyAppChecker());
@@ -310,23 +314,4 @@ public void testEvictableWithInitialApps() throws Exception {
   private ApplicationId createAppId(int id, long timestamp) {
     return ApplicationId.newInstance(timestamp, id);
   }
-
-  class DummyAppChecker extends AppChecker {
-
-    @Override
-    @Private
-    public boolean isApplicationActive(ApplicationId id) throws YarnException {
-      // stub
-      return false;
-    }
-
-    @Override
-    @Private
-    public Collection<ApplicationId> getActiveApplications()
-        throws YarnException {
-      // stub
-      return null;
-    }
-
-  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 2eb5034eab17a..365e0bb66b143 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,6 +57,7 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -70,6 +71,8 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
+import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -119,6 +122,7 @@ public class MiniYARNCluster extends CompositeService {
   private int numLocalDirs;
   // Number of nm-log-dirs per nodemanager
   private int numLogDirs;
+  private boolean enableAHS;
 
   /**
    * @param testName name of the test
@@ -126,13 +130,15 @@ public class MiniYARNCluster extends CompositeService {
    * @param numNodeManagers the number of node managers in the cluster
    * @param numLocalDirs the number of nm-local-dirs per nodemanager
    * @param numLogDirs the number of nm-log-dirs per nodemanager
+   * @param enableAHS enable ApplicationHistoryServer or not
    */
   public MiniYARNCluster(
       String testName, int numResourceManagers, int numNodeManagers,
-      int numLocalDirs, int numLogDirs) {
+      int numLocalDirs, int numLogDirs, boolean enableAHS) {
     super(testName.replace("$", ""));
     this.numLocalDirs = numLocalDirs;
     this.numLogDirs = numLogDirs;
+    this.enableAHS = enableAHS;
     String testSubDir = testName.replace("$", "");
     File targetWorkDir = new File("target", testSubDir);
     try {
@@ -182,6 +188,20 @@ public MiniYARNCluster(
     nodeManagers = new NodeManager[numNodeManagers];
   }
 
+  /**
+   * @param testName name of the test
+   * @param numResourceManagers the number of resource managers in the cluster
+   * @param numNodeManagers the number of node managers in the cluster
+   * @param numLocalDirs the number of nm-local-dirs per nodemanager
+   * @param numLogDirs the number of nm-log-dirs per nodemanager
+   */
+  public MiniYARNCluster(
+      String testName, int numResourceManagers, int numNodeManagers,
+      int numLocalDirs, int numLogDirs) {
+    this(testName, numResourceManagers, numNodeManagers, numLocalDirs,
+        numLogDirs, false);
+  }
+
   /**
    * @param testName name of the test
    * @param numNodeManagers the number of node managers in the cluster
@@ -242,8 +262,8 @@ public void serviceInit(Configuration conf) throws Exception {
       addService(new NodeManagerWrapper(index));
     }
 
-    if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
-        addService(new ApplicationHistoryServerWrapper());
+    if (enableAHS) {
+      addService(new ApplicationHistoryServerWrapper());
     }
     
     super.serviceInit(
@@ -488,8 +508,10 @@ protected synchronized void serviceInit(Configuration conf)
       String logDirsString = prepareDirs("log", numLogDirs);
       config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
 
-      // By default AM + 2 containers
-      config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
+      config.setInt(YarnConfiguration.NM_PMEM_MB, config.getInt(
+          YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB,
+          YarnConfiguration.DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB));
+
       config.set(YarnConfiguration.NM_ADDRESS,
           MiniYARNCluster.getHostname() + ":0");
       config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
@@ -664,6 +686,8 @@ protected synchronized void serviceInit(Configuration conf)
           MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
       conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
           MemoryTimelineStore.class, TimelineStore.class);
+      conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
+          MemoryTimelineStateStore.class, TimelineStateStore.class);
       appHistoryServer.init(conf);
       super.serviceInit(conf);
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 31457a548da23..408fda76977db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -22,6 +22,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.io.ObjectInputStream;
 import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -68,9 +69,9 @@ public class WebAppProxyServlet extends HttpServlet {
   
   public static final String PROXY_USER_COOKIE_NAME = "proxy-user";
 
-  private final List<TrackingUriPlugin> trackingUriPlugins;
+  private transient List<TrackingUriPlugin> trackingUriPlugins;
   private final String rmAppPageUrlBase;
-  private final transient YarnConfiguration conf;
+  private transient YarnConfiguration conf;
 
   private static class _ implements Hamlet._ {
     //Empty
@@ -350,4 +351,13 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
       throw new IOException(e);
     }
   }
+
+  private void readObject(ObjectInputStream input)
+      throws IOException, ClassNotFoundException {
+    input.defaultReadObject();
+    conf = new YarnConfiguration();
+    this.trackingUriPlugins =
+        conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
+            TrackingUriPlugin.class);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
index bc3b9eaa5c897..8528c1ab92dbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
@@ -226,6 +226,18 @@ Hadoop MapReduce Next Generation - Capacity Scheduler
 | | ensures that a single user can never take more than the queue's configured | 
 | | capacity irrespective of how idle th cluster is. Value is specified as |
 | | a float.|
+*--------------------------------------+--------------------------------------+
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-allocation-mb>>> |   |
+| | The per queue maximum limit of memory to allocate to each container |
+| | request at the Resource Manager. This setting overrides the cluster |
+| | configuration <<<yarn.scheduler.maximum-allocation-mb>>>. This value |
+| | must be smaller than or equal to the cluster maximum. |
+*--------------------------------------+--------------------------------------+
+| <<<yarn.scheduler.capacity.<queue-path>.maximum-allocation-vcores>>> |   |
+| | The per queue maximum limit of virtual cores to allocate to each container |
+| | request at the Resource Manager. This setting overrides the cluster |
+| | configuration <<<yarn.scheduler.maximum-allocation-vcores>>>. This value |
+| | must be smaller than or equal to the cluster maximum. |
 *--------------------------------------+--------------------------------------+
 
     * Running and Pending Application Limits
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
index 13bf7f9d2d574..10de3e010e3db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
@@ -375,7 +375,7 @@ Allocation file format
 
   <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
 
-  <!—- Queue 'secondary_group_queueue' is a parent queue and may have
+  <!-- Queue 'secondary_group_queue' is a parent queue and may have
        user queues under it -->
   <queue name="secondary_group_queue" type="parent">
   <weight>3.0</weight>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerCgroups.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerCgroups.apt.vm
new file mode 100644
index 0000000000000..f228e3b94bfca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerCgroups.apt.vm
@@ -0,0 +1,77 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~   http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+  ---
+  Using CGroups with YARN
+  ---
+  ---
+  ${maven.build.timestamp}
+
+Using CGroups with YARN
+
+%{toc|section=1|fromDepth=0|toDepth=2}
+
+ CGroups is a mechanism for aggregating/partitioning sets of tasks, and all their future children, into hierarchical groups with specialized behaviour. CGroups is a Linux kernel feature and was merged into kernel version 2.6.24. From a YARN perspective, this allows containers to be limited in their resource usage. A good example of this is CPU usage. Without CGroups, it becomes hard to limit container CPU usage. Currently, CGroups is only used for limiting CPU usage.
+
+* CGroups configuration
+
+ The config variables related to using CGroups are the following:
+
+ The following settings are related to setting up CGroups. All of these need to be set in yarn-site.xml.
+
+  [[1]] yarn.nodemanager.container-executor.class
+
+    This should be set to "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor". CGroups is a Linux kernel feature and is exposed via the LinuxContainerExecutor.
+
+  [[2]] yarn.nodemanager.linux-container-executor.resources-handler.class
+
+    This should be set to "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler".Using the LinuxContainerExecutor doesn't force you to use CGroups. If you wish to use CGroups, the resource-handler-class must be set to CGroupsLCEResourceHandler.
+
+  [[3]] yarn.nodemanager.linux-container-executor.cgroups.hierarchy
+
+    The cgroups hierarchy under which to place YARN proccesses(cannot contain commas). If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that is, if cgroups have been pre-configured), then this cgroups hierarchy must already exist
+
+  [[4]] yarn.nodemanager.linux-container-executor.cgroups.mount
+
+    Whether the LCE should attempt to mount cgroups if not found - can be true or false
+
+  [[5]] yarn.nodemanager.linux-container-executor.cgroups.mount-path
+
+    Where the LCE should attempt to mount cgroups if not found. Common locations include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux distribution in use. This path must exist before the NodeManager is launched. Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and yarn.nodemanager.linux-container-executor.cgroups.mount is true. A point to note here is that the container-executor binary will try to mount the path specified + "/" + the subsystem. In our case, since we are trying to limit CPU the binary tries to mount the path specified + "/cpu" and that's the path it expects to exist.
+
+  [[6]] yarn.nodemanager.linux-container-executor.group
+
+    The Unix group of the NodeManager. It should match the setting in "container-executor.cfg". This configuration is required for validating the secure access of the container-executor binary.
+
+ The following settings are related to limiting resource usage of YARN containers
+
+  [[1]] yarn.nodemanager.resource.percentage-physical-cpu-limit
+
+    This setting lets you limit the cpu usage of all YARN containers. It sets a hard upper limit on the cumulative CPU usage of the containers. For example, if set to 60, the combined CPU usage of all YARN containers will not exceed 60%.
+
+  [[2]] yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage
+
+    CGroups allows cpu usage limits to be hard or soft. When this setting is true, containers cannot use more CPU usage than allocated even if spare CPU is available. This ensures that containers can only use CPU that they were allocated. When set to false, containers can use spare CPU if available. It should be noted that irrespective of whether set to true or false, at no time can the combined CPU usage of all containers exceed the value specified in "yarn.nodemanager.resource.percentage-physical-cpu-limit".
+
+* CGroups and security
+
+ CGroups itself has no requirements related to security. However, the LinuxContainerExecutor does have some requirements. If running in non-secure mode, by default, the LCE runs all jobs as user "nobody". This user can be changed by setting "yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user" to the desired user. However, it can also be configured to run jobs as the user submitting the job. In that case "yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users" should be set to false.
+
+*-----------+-----------+---------------------------+
+|| yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user || yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users || User running jobs |
+*-----------+-----------+---------------------------+
+| (default) | (default) | nobody                    |
+*-----------+-----------+---------------------------+
+| yarn      | (default) | yarn                      |
+*-----------+-----------+---------------------------+
+| yarn      | false     | (User submitting the job) |
+*-----------+-----------+---------------------------+
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
index 8cfdd79b00e07..0346cda8f1ecd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
@@ -208,6 +208,7 @@ ResourceManager High Availability
 +---+
 
    If automatic failover is enabled, you can not use manual transition command.
+   Though you can override this by --forcemanual flag, you need caution.
 
 +---+
  $ yarn rmadmin -transitionToStandby rm1
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
index 2f73082a36c42..69728fb6c6517 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
@@ -2595,7 +2595,7 @@ Server: Jetty(6.1.26)
 
 +---+
 HTTP/1.1 202 Accepted
-Content-Type: application/json
+Content-Type: application/xml
 Content-Length: 794
 Location: http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003
 Server: Jetty(6.1.26)
@@ -2664,8 +2664,6 @@ Server: Jetty(6.1.26)
 
 +---+
 HTTP/1.1 403 Unauthorized
-Content-Type: application/json
-Transfer-Encoding: chunked
 Server: Jetty(6.1.26)
 +---+
 
@@ -2708,6 +2706,161 @@ Server: Jetty(6.1.26)
 +---+
 
 
+* Cluster Application Queue API
+
+  With the application queue API, you can query the queue of a submitted app as well move a running app to another queue using a PUT request specifying the target queue. To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to move the app. Currently you can only move the app if you're using the Capacity scheduler or the Fair scheduler.
+
+  Please note that in order to move an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response.
+
+  This feature is currently in the alpha stage and may change in the future.
+
+** URI
+
+-----
+  * http://<rm http address:port>/ws/v1/cluster/apps/{appid}/queue
+-----
+
+** HTTP Operations Supported
+
+------
+  * GET
+  * PUT
+------
+
+** Query Parameters Supported
+
+------
+  None
+------
+
+** Elements of <appqueue> object
+
+  When you make a request for the state of an app, the information returned has the following fields
+
+*---------------+--------------+-------------------------------+
+|| Item         || Data Type   || Description                  |
+*---------------+--------------+-------------------------------+
+| queue         | string       | The application queue         |
+*---------------+--------------+--------------------------------+
+
+
+** Response Examples
+
+  <<JSON responses>>
+
+  HTTP Request
+
+-----
+  GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/queue
+-----
+
+  Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+{
+  "queue":"default"
+}
++---+
+
+  HTTP Request
+
+-----
+  PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/queue
+----
+
+  Request Body:
+
++---+
+{
+  "queue":"test"
+}
++---+
+
+  Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+{
+  "queue":"test"
+}
++---+
+
+  <<XML responses>>
+
+  HTTP Request
+
+-----
+  GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/queue
+-----
+
+  Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 98
+Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appqueue>
+  <queue>default</queue>
+</appqueue>
++---+
+
+  HTTP Request
+
+-----
+  PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/queue
+----
+
+  Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appqueue>
+  <queue>test</queue>
+</appqueue>
++---+
+
+  Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 95
+Server: Jetty(6.1.26)
++---+
+
+  Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appqueue>
+  <queue>test</queue>
+</appqueue>
++---+
+
 * Cluster {Delegation Tokens API}
 
   The Delegation Tokens API can be used to create, renew and cancel YARN ResourceManager delegation tokens. All delegation token requests must be carried out on a Kerberos authenticated connection(using SPNEGO). Carrying out operations on a non-kerberos connection will result in a FORBIDDEN response. In case of renewing a token, only the renewer specified when creating the token can renew the token. Other users(including the owner) are forbidden from renewing tokens. It should be noted that when cancelling or renewing a token, the token to be cancelled or renewed is specified by setting a header.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
index 6c8203ce74394..7bb504e368ff4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
@@ -215,7 +215,7 @@ YARN Timeline Server
   Or users can start the Timeline server / history service as a daemon:
 
 +---+
-  $ yarn-daemon.sh start timelineserver
+  $ yarn --daemon start timelineserver
 +---+
 
 * Accessing generic-data via command-line
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
index cf41271af1f3c..6333d70163079 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
@@ -11,70 +11,49 @@
 ~~ limitations under the License. See accompanying LICENSE file.
 
   ---
-  Yarn Commands
+  YARN Commands
   ---
   ---
   ${maven.build.timestamp}
 
-Yarn Commands
+YARN Commands
 
 %{toc|section=1|fromDepth=0}
 
 * Overview
 
-  Yarn commands are invoked by the bin/yarn script. Running the yarn script
+  YARN commands are invoked by the bin/yarn script. Running the yarn script
   without any arguments prints the description for all commands.
 
-------
-Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
-------
+ Usage: <<<yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
 
-  Yarn has an option parsing framework that employs parsing generic options as
+  YARN has an option parsing framework that employs parsing generic options as
   well as running classes.
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
-*---------------+--------------+
-| --config confdir | Overwrites the default Configuration directory. Default
-|                  | is $\{HADOOP_PREFIX\}/conf.
-*---------------+--------------+
-| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
-|                     | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
-*---------------+--------------+
+*-------------------------+-------------+
+| SHELL_OPTIONS | The common set of shell options. These are documented on the {{{../../hadoop-project-dist/hadoop-common/CommandsManual.html#Shell Options}Commands Manual}} page.
+*-------------------------+----+
+| GENERIC_OPTIONS | The common set of options supported by multiple commands. See the Hadoop {{{../../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic Options}Commands Manual}} for more information.
+*------------------+---------------+
 | COMMAND COMMAND_OPTIONS | Various commands with their options are described
 |                         | in the following sections. The commands have been
 |                         | grouped into {{User Commands}} and
 |                         | {{Administration Commands}}.
-*---------------+--------------+
+*-------------------------+--------------+
 
 * {User Commands}
 
   Commands useful for users of a Hadoop cluster.
 
-** jar
-
-  Runs a jar file. Users can bundle their Yarn code in a jar file and execute
-  it using this command.
-
--------
-  Usage: yarn jar <jar> [mainClass] args...
--------
-
-** application
-
-  Prints application(s) report/kill application
+** <<<application>>>
 
--------
-  Usage: yarn application <options>
--------
+  Usage: <<<yarn application [options] >>>
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
 *---------------+--------------+
-| -list | Lists applications from the RM. Supports optional use of -appTypes
-|       | to filter applications based on application type, and -appStates to
-|       | filter applications based on application state.
-*---------------+--------------+
 | -appStates States | Works with -list to filter applications based on input
 |                   | comma-separated list of application states. The valid
 |                   | application state can be one of the following: \
@@ -84,40 +63,66 @@ Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
 | -appTypes Types | Works with -list to filter applications based on input
 |                 | comma-separated list of application types.
 *---------------+--------------+
-| -status  ApplicationId | Prints the status of the application.
+| -list | Lists applications from the RM. Supports optional use of -appTypes
+|       | to filter applications based on application type, and -appStates to
+|       | filter applications based on application state.
 *---------------+--------------+
 | -kill ApplicationId | Kills the application.
+*---------------+--------------+
+| -status  ApplicationId | Prints the status of the application.
 *---------------+--------------+
 
-** node
+  Prints application(s) report/kill application
 
-  Prints node report(s)
+** <<<applicationattempt>>>
 
--------
-  Usage: yarn node <options>
--------
+  Usage: <<<yarn applicationattempt [options] >>>
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
 *---------------+--------------+
-| -list | Lists all running nodes. Supports optional use of -states to filter
-|       | nodes based on node state, and -all to list all nodes.
+| -help | Help
 *---------------+--------------+
-| -states States | Works with -list to filter nodes based on input
-|                | comma-separated list of node states.
+| -list ApplicationId | Lists applications attempts from the RM
 *---------------+--------------+
-| -all | Works with -list to list all nodes.
+| -status  Application Attempt Id | Prints the status of the application attempt.
 *---------------+--------------+
-| -status NodeId | Prints the status report of the node.
+
+  prints applicationattempt(s) report
+
+** <<<classpath>>>
+
+  Usage: <<<yarn classpath>>>
+
+  Prints the class path needed to get the Hadoop jar and the required libraries
+
+
+** <<<container>>>
+
+  Usage: <<<yarn container [options] >>>
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description                   |
+*---------------+--------------+
+| -help | Help
+*---------------+--------------+
+| -list ApplicationId | Lists containers for the application attempt.
+*---------------+--------------+
+| -status  ContainerId | Prints the status of the container.
 *---------------+--------------+
 
-** logs
+  prints container(s) report
 
-  Dump the container logs
+** <<<jar>>>
 
--------
-  Usage: yarn logs -applicationId <application ID> <options>
--------
+  Usage: <<<yarn jar <jar> [mainClass] args... >>>
+
+  Runs a jar file. Users can bundle their YARN code in a jar file and execute
+  it using this command.
+
+** <<<logs>>>
+
+  Usage: <<<yarn logs -applicationId <application ID> [options] >>>
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
@@ -130,77 +135,131 @@ Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
 | -containerId ContainerId | ContainerId (must be specified if node address is
 |                          | specified)
 *---------------+--------------+
+| -help | Help
+*---------------+--------------+
 | -nodeAddress NodeAddress | NodeAddress in the format nodename:port (must be
 |                          | specified if container id is specified)
 *---------------+--------------+
 
-** classpath
+  Dump the container logs
 
-  Prints the class path needed to get the Hadoop jar and the required libraries
 
--------
-  Usage: yarn classpath
--------
+** <<<node>>>
+
+  Usage: <<<yarn node [options] >>>
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description                   |
+*---------------+--------------+
+| -all | Works with -list to list all nodes.
+*---------------+--------------+
+| -list | Lists all running nodes. Supports optional use of -states to filter
+|       | nodes based on node state, and -all to list all nodes.
+*---------------+--------------+
+| -states States | Works with -list to filter nodes based on input
+|                | comma-separated list of node states.
+*---------------+--------------+
+| -status NodeId | Prints the status report of the node.
+*---------------+--------------+
+
+  Prints node report(s)
+
+** <<<queue>>>
+
+  Usage: <<<yarn queue [options] >>>
 
-** version
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description                   |
+*---------------+--------------+
+| -help | Help
+*---------------+--------------+
+| -status  QueueName | Prints the status of the queue.
+*---------------+--------------+
 
-  Prints the version.
+  Prints queue information
 
--------
-  Usage: yarn version
--------
 
+** <<<version>>>
+
+  Usage: <<<yarn version>>>
+
+  Prints the Hadoop version.
 
 * {Administration Commands}
 
   Commands useful for administrators of a Hadoop cluster.
 
-** resourcemanager
+** <<<daemonlog>>>
 
-  Start the ResourceManager
+  Usage:
 
--------
-  Usage: yarn resourcemanager [-format-state-store]
--------
+---------------------------------
+   yarn daemonlog -getlevel <host:port> <name>
+   yarn daemonlog -setlevel <host:port> <name> <level>
+---------------------------------
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
 *---------------+--------------+
-| -format-state-store | Formats the RMStateStore. This will clear the
-|                     | RMStateStore and is useful if past applications are no
-|                     | longer needed. This should be run only when the
-|                     | ResourceManager is not running.
+| -getlevel \<host:port\> \<name\> | Prints the log level of the daemon running
+| | at \<host:port\>. This command internally connects to
+| | http://\<host:port\>/logLevel?log=\<name\>
 *---------------+--------------+
+| -setlevel \<host:port\> \<name\> \<level\> | Sets the log level of the daemon
+| | running at \<host:port\>. This command internally connects to
+| | http://\<host:port\>/logLevel?log=\<name\>
+*---------------+--------------+
+
+  Get/Set the log level for each daemon.
+
+
+** <<<nodemanager>>>
 
-** nodemanager
+  Usage: <<<yarn nodemanager>>>
 
   Start the NodeManager
 
--------
-  Usage: yarn nodemanager
--------
+** <<<proxyserver>>>
 
-** proxyserver
+  Usage: <<<yarn proxyserver>>>
 
   Start the web proxy server
 
--------
-  Usage: yarn proxyserver
--------
+** <<<resourcemanager>>>
 
-** rmadmin
+  Usage: <<<yarn resourcemanager [-format-state-store]>>>
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description                   |
+*---------------+--------------+
+| -format-state-store | Formats the RMStateStore. This will clear the
+|                     | RMStateStore and is useful if past applications are no
+|                     | longer needed. This should be run only when the
+|                     | ResourceManager is not running.
+*---------------+--------------+
+
+  Start the ResourceManager
 
-  Runs ResourceManager admin client
 
--------
-  Usage: yarn rmadmin [-refreshQueues] [-refreshNodes] [-refreshUserToGroupsMapping] 
-                      [-refreshSuperUserGroupsConfiguration] [-refreshAdminAcls] 
-                      [-refreshServiceAcl] [-getGroups [username]] [-help [cmd]]
-                      [-transitionToActive <serviceId>]
-                      [-transitionToStandby <serviceId>]
-                      [-getServiceState <serviceId>]
-                      [-checkHealth <serviceId>]
--------
+** <<<rmadmin>>>
+
+  Usage:
+
+----
+  yarn rmadmin [-refreshQueues]
+               [-refreshNodes]
+               [-refreshUserToGroupsMapping] 
+               [-refreshSuperUserGroupsConfiguration]
+               [-refreshAdminAcls] 
+               [-refreshServiceAcl]
+               [-getGroups [username]]
+               [-transitionToActive [--forceactive] [--forcemanual] <serviceId>]
+               [-transitionToStandby [--forcemanual] <serviceId>]
+               [-failover [--forcefence] [--forceactive] <serviceId1> <serviceId2>]
+               [-getServiceState <serviceId>]
+               [-checkHealth <serviceId>]
+               [-help [cmd]]
+----
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
@@ -224,41 +283,84 @@ Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
 *---------------+--------------+
 | -getGroups [username] | Get groups the specified user belongs to.
 *---------------+--------------+
-| -help [cmd] | Displays help for the given command or all commands if none is
-|             | specified.
-*---------------+--------------+
-| -transitionToActive \<serviceId\> | Transitions the service into Active
-|                                   | state.
-*---------------+--------------+
-| -transitionToStandby \<serviceId\> | Transitions the service into Standby
-|                                    | state.
+| -transitionToActive [--forceactive] [--forcemanual] \<serviceId\> |
+|               | Transitions the service into Active state.
+|               | Try to make the target active
+|               | without checking that there is no active node
+|               | if the --forceactive option is used.
+|               | This command can not be used if automatic failover is enabled.
+|               | Though you can override this by --forcemanual option,
+|               | you need caution.
+*---------------+--------------+
+| -transitionToStandby [--forcemanual] \<serviceId\> |
+|               | Transitions the service into Standby state.
+|               | This command can not be used if automatic failover is enabled.
+|               | Though you can override this by --forcemanual option,
+|               | you need caution.
+*---------------+--------------+
+| -failover [--forceactive] \<serviceId1\> \<serviceId2\> |
+|               | Initiate a failover from serviceId1 to serviceId2.
+|               | Try to failover to the target service even if it is not ready
+|               | if the --forceactive option is used.
+|               | This command can not be used if automatic failover is enabled.
 *---------------+--------------+
 | -getServiceState \<serviceId\> | Returns the state of the service.
 *---------------+--------------+
 | -checkHealth \<serviceId\> | Requests that the service perform a health
 |                            | check. The RMAdmin tool will exit with a
 |                            | non-zero exit code if the check fails.
+*---------------+--------------+
+| -help [cmd] | Displays help for the given command or all commands if none is
+|             | specified.
 *---------------+--------------+
 
-** daemonlog
 
-  Get/Set the log level for each daemon.
+  Runs ResourceManager admin client
 
--------
-  Usage: yarn daemonlog -getlevel <host:port> <name> 
-  Usage: yarn daemonlog -setlevel <host:port> <name> <level>
--------
+** scmadmin
+
+  Usage: <<<yarn scmadmin [options] >>>
 
 *---------------+--------------+
 || COMMAND_OPTIONS || Description                   |
 *---------------+--------------+
-| -getlevel \<host:port\> \<name\> | Prints the log level of the daemon running
-| | at \<host:port\>. This command internally connects to
-| | http://\<host:port\>/logLevel?log=\<name\>
+| -help | Help
 *---------------+--------------+
-| -setlevel \<host:port\> \<name\> \<level\> | Sets the log level of the daemon
-| | running at \<host:port\>. This command internally connects to
-| | http://\<host:port\>/logLevel?log=\<name\>
+| -runCleanerTask | Runs the cleaner task
 *---------------+--------------+
 
+  Runs Shared Cache Manager admin client
+
+
+** sharedcachemanager
+
+  Usage: <<<yarn sharedcachemanager>>>
+
+  Start the Shared Cache Manager
+
+** timelineserver
+
+  Usage: <<<yarn timelineserver>>>
+
+  Start the TimeLineServer
+
+
+* Files
+
+** <<etc/hadoop/hadoop-env.sh>>
+
+    This file stores the global settings used by all Hadoop shell commands.
+
+** <<etc/hadoop/yarn-env.sh>>
+
+    This file stores overrides used by all YARN shell commands.
+
+** <<etc/hadoop/hadoop-user-functions.sh>>
+
+    This file allows for advanced users to override some shell functionality.
+
+** <<~/.hadooprc>>
 
+    This stores the personal environment for an individual user.  It is
+    processed after the <<<hadoop-env.sh>>>, <<<hadoop-user-functions.sh>>>, and <<<yarn-env.sh>>> files
+    and can contain the same settings.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
index 79b8e1125ed25..43e5b028986d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
@@ -36,29 +36,47 @@ MapReduce NextGen aka YARN aka MRv2
   library and is tasked with negotiating resources from the ResourceManager and 
   working with the NodeManager(s) to execute and monitor the tasks.
 
-  More details are available in the {{{./YARN.html}YARN}}
-  document.
+  More details are available in the {{{./YARN.html}Architecture}} document.
 
-* Documentation
 
-  * {{{./YARN.html}NextGen MapReduce}}
-  
-  * {{{./WritingYarnApplications.html}Writing YARN Applications}}
+Documentation Index
 
-  * {{{./CapacityScheduler.html}Capacity Scheduler}}
+* YARN
 
+  * {{{./YARN.html}YARN Architecture}}
+ 
+  * {{{./CapacityScheduler.html}Capacity Scheduler}}
+ 
   * {{{./FairScheduler.html}Fair Scheduler}}
-
+ 
+  * {{{./ResourceManagerRestart.htaml}ResourceManager Restart}}
+ 
+  * {{{./ResourceManagerHA.html}ResourceManager HA}}
+ 
   * {{{./WebApplicationProxy.html}Web Application Proxy}}
-
+ 
   * {{{./TimelineServer.html}YARN Timeline Server}}
-  
-  * {{{./SecureContainer.html}YARN Secure Containers}}
+ 
+  * {{{./WritingYarnApplications.html}Writing YARN Applications}}
+ 
+  * {{{./YarnCommands.html}YARN Commands}}
+ 
+  * {{{hadoop-sls/SchedulerLoadSimulator.html}Scheduler Load Simulator}}
+ 
+  * {{{./NodeManagerRestart.html}NodeManager Restart}}
+ 
+  * {{{./DockerContainerExecutor.html}DockerContainerExecutor}}
+ 
+  * {{{./NodeManagerCGroups.html}Using CGroups}}
+ 
+  * {{{./SecureContainer.html}Secure Containers}}
+ 
+  * {{{./registry/index.html}Registry}}
 
-  * {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}}
+* YARN REST APIs
 
-  * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html}Backward Compatibility between Apache Hadoop 1.x and 2.x for MapReduce}}
+  * {{{./WebServicesIntro.html}Introduction}}
 
-  * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted Shuffle}}
+  * {{{./ResourceManagerRest.html}Resource Manager}}
 
-  * {{{./registry/index.html}Registry}}
+  * {{{./NodeManagerRest.html}Node Manager}}
diff --git a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn
new file mode 100644
index 0000000000000..4aa20b1fcbfbc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn
@@ -0,0 +1,62 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile yarn
+
+function _yarn_hadoop_classpath
+{
+  local i
+  #
+  # get all of the yarn jars+config in the path
+  #
+  # developers
+  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+    for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
+             yarn-server/yarn-server-nodemanager \
+             yarn-server/yarn-server-common \
+             yarn-server/yarn-server-resourcemanager; do
+      hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
+    done
+
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
+  fi
+
+  if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
+    hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
+  fi
+
+  hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
+  hadoop_add_classpath  "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
+}
+
+function _yarn_hadoop_finalize
+{
+  # Add YARN custom options to comamnd line in case someone actaully
+  # used these.
+  #
+  # Note that we are replacing ' ' with '\ ' so that when we exec
+  # stuff it works
+  #
+  local yld=$HADOOP_LOG_DIR
+  hadoop_translate_cygwin_path yld
+  hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
+  hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
+  local yhd=$HADOOP_YARN_HOME
+  hadoop_translate_cygwin_path yhd
+  hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
+  hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+}
diff --git a/pom.xml b/pom.xml
index 5cc30c24be1b3..3bad969ee828c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
 
   <licenses>
     <license>
-      <name>The Apache Software License, Version 2.0</name>
+      <name>Apache License, Version 2.0</name>
       <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
     </license>
   </licenses>
@@ -89,6 +89,20 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <!-- platform encoding override -->
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+    <!-- maven plugin versions -->
+    <maven-deploy-plugin.version>2.8.1</maven-deploy-plugin.version>
+    <maven-site-plugin.version>3.4</maven-site-plugin.version>
+    <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
+    <maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
+    <maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
+    <maven-enforcer-plugin.version>1.3.1</maven-enforcer-plugin.version>
+    <maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
+    <apache-rat-plugin.version>0.10</apache-rat-plugin.version>
+    <wagon-ssh.version>1.0</wagon-ssh.version>
+    <maven-clover2-plugin.version>3.3.0</maven-clover2-plugin.version>
+    <maven-bundle-plugin.version>2.5.0</maven-bundle-plugin.version>
+    <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
   </properties>
 
   <modules>
@@ -112,19 +126,19 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-dependency-plugin</artifactId>
-          <version>2.4</version>
+          <version>${maven-dependency-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-enforcer-plugin</artifactId>
-          <version>1.3.1</version>
+          <version>${maven-enforcer-plugin.version}</version>
           <configuration>
             <rules>
               <requireMavenVersion>
                 <version>[3.0.2,)</version>
               </requireMavenVersion>
               <requireJavaVersion>
-                <version>1.6</version>
+                <version>[1.7,)</version>
               </requireJavaVersion>
             </rules>
           </configuration>
@@ -132,37 +146,32 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-assembly-plugin</artifactId>
-          <version>2.3</version>
+          <version>${maven-assembly-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-deploy-plugin</artifactId>
-          <version>2.5</version>
+          <version>${maven-deploy-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.rat</groupId>
           <artifactId>apache-rat-plugin</artifactId>
-          <version>0.7</version>
+          <version>${apache-rat-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-antrun-plugin</artifactId>
-          <version>1.7</version>
+          <version>${maven-antrun-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-site-plugin</artifactId>
-          <version>3.3</version>
+          <version>${maven-site-plugin.version}</version>
           <dependencies>
             <dependency><!-- add support for ssh/scp -->
               <groupId>org.apache.maven.wagon</groupId>
               <artifactId>wagon-ssh</artifactId>
-              <version>1.0</version>
-            </dependency>
-            <dependency> <!-- add support for markdown -->
-              <groupId>org.apache.maven.doxia</groupId>
-              <artifactId>doxia-module-markdown</artifactId>
-              <version>1.3</version>
+              <version>${wagon-ssh.version}</version>
             </dependency>
           </dependencies>
         </plugin>
@@ -171,7 +180,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <plugin>
           <groupId>org.eclipse.m2e</groupId>
           <artifactId>lifecycle-mapping</artifactId>
-          <version>1.0.0</version>
+          <version>${lifecycle-mapping.version}</version>
           <configuration>
             <lifecycleMappingMetadata>
               <pluginExecutions>
@@ -276,12 +285,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
         <plugin>
           <groupId>com.atlassian.maven.plugins</groupId>
           <artifactId>maven-clover2-plugin</artifactId>
-          <version>3.0.5</version>
+          <version>${maven-clover2-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.felix</groupId>
           <artifactId>maven-bundle-plugin</artifactId>
-          <version>2.4.0</version>
+          <version>${maven-bundle-plugin.version}</version>
         </plugin>
       </plugins>
     </pluginManagement>
@@ -330,7 +339,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
       </plugin>
       <plugin>
         <artifactId>maven-site-plugin</artifactId>
-        <version>3.3</version>
         <executions>
           <execution>
             <id>attach-descriptor</id>
@@ -358,7 +366,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.8.1</version>
+        <version>${maven-javadoc-plugin.version}</version>
         <inherited>false</inherited>
         <reportSets>
           <reportSet>
@@ -373,12 +381,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
               <reportOutputDirectory>${project.build.directory}/site</reportOutputDirectory>
               <destDir>hadoop-project/api</destDir>
               <!-- Non-public APIs -->
-              <excludePackageNames>org.apache.hadoop.authentication*,org.apache.hadoop.hdfs*,org.apache.hadoop.mapreduce.v2.proto,org.apache.hadoop.yarn.proto,org.apache.hadoop.yarn.server*,org.apache.hadoop.yarn.webapp*</excludePackageNames>
+              <excludePackageNames>org.apache.hadoop.authentication*,org.apache.hadoop.mapreduce.v2.proto,org.apache.hadoop.yarn.proto,org.apache.hadoop.yarn.server*,org.apache.hadoop.yarn.webapp*</excludePackageNames>
               <groups>
                 <group>
                   <title>Common</title>
                   <packages>org.apache.hadoop*</packages>
                 </group>
+                <group>
+                  <title>HDFS</title>
+                  <packages>org.apache.hadoop.hdfs*</packages>
+                </group>
                 <group>
                   <title>MapReduce</title>
                   <packages>org.apache.hadoop.mapred*</packages>
@@ -417,7 +429,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>2.4</version>
+        <version>${maven-dependency-plugin.version}</version>
         <reportSets>
           <reportSet>
             <reports>