HDDS-1368. Cleanup old ReplicationManager code from SCM.

LICENSE.txt

@@ -1636,7 +1636,6 @@ derived.)
 
 The binary distribution of this product bundles these dependencies under the
 following license:
-JUnit 4.11
 Eclipse JDT Core 3.1.1
 --------------------------------------------------------------------------------
 (EPL v1.0)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -1840,7 +1840,7 @@ public long getTimeDuration(String name, long defaultValue,
       TimeUnit defaultUnit, TimeUnit returnUnit) {
     String vStr = get(name);
     if (null == vStr) {
-      return defaultValue;
+      return returnUnit.convert(defaultValue, defaultUnit);
     } else {
       return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java

@@ -51,6 +51,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.LambdaUtils;
 import org.apache.hadoop.util.Progressable;
@@ -456,8 +457,16 @@ public Path getInitialWorkingDirectory() {
    * @return current user's home directory.
    */
   public Path getHomeDirectory() {
-    return new Path("/user/"+System.getProperty("user.name")).makeQualified(
-                                                                getUri(), null);
+    String username;
+    try {
+      username = UserGroupInformation.getCurrentUser().getShortUserName();
+    } catch(IOException ex) {
+      LOG.warn("Unable to get user name. Fall back to system property " +
+          "user.name", ex);
+      username = System.getProperty("user.name");
+    }
+    return new Path("/user/" + username)
+        .makeQualified(getUri(), null);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -2241,8 +2241,16 @@ public LocatedFileStatus next() throws IOException {
    * The default implementation returns {@code "/user/$USER/"}.
    */
   public Path getHomeDirectory() {
+    String username;
+    try {
+      username = UserGroupInformation.getCurrentUser().getShortUserName();
+    } catch(IOException ex) {
+      LOGGER.warn("Unable to get user name. Fall back to system property " +
+          "user.name", ex);
+      username = System.getProperty("user.name");
+    }
     return this.makeQualified(
-        new Path(USER_HOME_PREFIX + "/" + System.getProperty("user.name")));
+        new Path(USER_HOME_PREFIX + "/" + username));
   }
 
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java

@@ -145,7 +145,14 @@ public synchronized void reset() {
     setDelegationTokenSeqNum(0);
     currentTokens.clear();
   }
-
+
+  /**
+   * Total count of active delegation tokens.
+   */
+  public long getCurrentTokensSize() {
+    return currentTokens.size();
+  }
+
   /** 
    * Add a previously used master key to cache (when NN restarts), 
    * should be called before activate(). 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -1401,6 +1401,10 @@ public void testEnumFromXml() throws IOException {
   @Test
   public void testTimeDuration() {
     Configuration conf = new Configuration(false);
+
+    assertEquals(7000L,
+        conf.getTimeDuration("test.time.a", 7L, SECONDS, MILLISECONDS));
+
     conf.setTimeDuration("test.time.a", 7L, SECONDS);
     assertEquals("7s", conf.get("test.time.a"));
     assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java

@@ -259,6 +259,29 @@ public void testGetUserWithOwnerAndReal() {
                  ugi.getRealUser().getAuthenticationMethod());
   }
 
+  @Test
+  public void testDelegationTokenCount() throws Exception {
+    final TestDelegationTokenSecretManager dtSecretManager =
+        new TestDelegationTokenSecretManager(24*60*60*1000,
+            3*1000, 1*1000, 3600000);
+    try {
+      dtSecretManager.startThreads();
+      Assert.assertEquals(dtSecretManager.getCurrentTokensSize(), 0);
+      final Token<TestDelegationTokenIdentifier> token1 =
+          generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+      Assert.assertEquals(dtSecretManager.getCurrentTokensSize(), 1);
+      final Token<TestDelegationTokenIdentifier> token2 =
+          generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+      Assert.assertEquals(dtSecretManager.getCurrentTokensSize(), 2);
+      dtSecretManager.cancelToken(token1, "JobTracker");
+      Assert.assertEquals(dtSecretManager.getCurrentTokensSize(), 1);
+      dtSecretManager.cancelToken(token2, "JobTracker");
+      Assert.assertEquals(dtSecretManager.getCurrentTokensSize(), 0);
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+
   @Test
   public void testDelegationTokenSecretManager() throws Exception {
     final TestDelegationTokenSecretManager dtSecretManager = 

hadoop-hdds/common/pom.xml

@@ -137,6 +137,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>opentracing-util</artifactId>
       <version>0.31.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.yaml</groupId>
+      <artifactId>snakeyaml</artifactId>
+      <version>1.16</version>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -368,6 +368,8 @@ public final class ScmConfigKeys {
       "hdds.scm.http.kerberos.keytab";
 
   // Network topology
+  public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_TYPE =
+          "ozone.scm.network.topology.schema.file.type";
   public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE =
       "ozone.scm.network.topology.schema.file";
   public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT =

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchema.java

@@ -19,6 +19,8 @@
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 
+import java.util.List;
+
 /**
  * Network topology schema to housekeeper relevant information.
  */
@@ -59,13 +61,15 @@ public static LayerType getType(String typeStr) {
   }
 
   // default cost
-  private final int cost;
+  private int cost;
   // layer Type, mandatory property
-  private final LayerType type;
+  private LayerType type;
   // default name, can be null or ""
-  private final String defaultName;
+  private String defaultName;
   // layer prefix, can be null or ""
-  private final String prefix;
+  private String prefix;
+  // sublayer
+  private List<NodeSchema> sublayer;
 
   /**
    * Builder for NodeSchema.
@@ -123,6 +127,14 @@ public NodeSchema(LayerType type, int cost, String prefix,
     this.defaultName = defaultName;
   }
 
+  /**
+   * Constructor. This constructor is only used when build NodeSchema from
+   * YAML file.
+   */
+  public NodeSchema() {
+    this.type = LayerType.INNER_NODE;
+  }
+
   public boolean matchPrefix(String name) {
     if (name == null || name.isEmpty() || prefix == null || prefix.isEmpty()) {
       return false;
@@ -134,15 +146,38 @@ public LayerType getType() {
     return this.type;
   }
 
+  public void setType(LayerType type) {
+    this.type = type;
+  }
+
   public String getPrefix() {
     return this.prefix;
   }
 
+  public void setPrefix(String prefix) {
+    this.prefix = prefix;
+  }
+
   public String getDefaultName() {
     return this.defaultName;
   }
 
+  public void setDefaultName(String name) {
+    this.defaultName = name;
+  }
+
   public int getCost() {
     return this.cost;
   }
+  public void setCost(int cost) {
+    this.cost = cost;
+  }
+
+  public void setSublayer(List<NodeSchema> sublayer) {
+    this.sublayer = sublayer;
+  }
+
+  public List<NodeSchema> getSublayer() {
+    return sublayer;
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java

@@ -30,13 +30,15 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import  org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
+import org.yaml.snakeyaml.Yaml;
 
 /**
  * A Network topology layer schema loading tool that loads user defined network
@@ -95,7 +97,7 @@ public List<NodeSchema> getSchemaList() {
    * @param schemaFilePath path of schema file
    * @return all valid node schemas defined in schema file
    */
-  public NodeSchemaLoadResult loadSchemaFromFile(String schemaFilePath)
+  public NodeSchemaLoadResult loadSchemaFromXml(String schemaFilePath)
       throws IllegalArgumentException {
     try {
       File schemaFile = new File(schemaFilePath);
@@ -165,6 +167,88 @@ private NodeSchemaLoadResult loadSchema(File schemaFile) throws
     return schemaList;
   }
 
+  /**
+   * Load user defined network layer schemas from a YAML configuration file.
+   * @param schemaFilePath path of schema file
+   * @return all valid node schemas defined in schema file
+   */
+  public NodeSchemaLoadResult loadSchemaFromYaml(String schemaFilePath)
+          throws IllegalArgumentException {
+    try {
+      File schemaFile = new File(schemaFilePath);
+      if (!schemaFile.exists()) {
+        String msg = "Network topology layer schema file " + schemaFilePath +
+                " is not found.";
+        LOG.warn(msg);
+        throw new IllegalArgumentException(msg);
+      }
+      return loadSchemaFromYaml(schemaFile);
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Fail to load network topology node"
+              + " schema file: " + schemaFilePath + " , error:"
+              + e.getMessage());
+    }
+  }
+
+  /**
+   * Load network topology layer schemas from a YAML configuration file.
+   * @param schemaFile schema file
+   * @return all valid node schemas defined in schema file
+   * @throws ParserConfigurationException ParserConfigurationException happen
+   * @throws IOException no such schema file
+   * @throws SAXException xml file has some invalid elements
+   * @throws IllegalArgumentException xml file content is logically invalid
+   */
+  private NodeSchemaLoadResult loadSchemaFromYaml(File schemaFile) {
+    LOG.info("Loading network topology layer schema file {}", schemaFile);
+    NodeSchemaLoadResult finalSchema;
+
+    try {
+      Yaml yaml = new Yaml();
+      NodeSchema nodeTree;
+
+      try (FileInputStream fileInputStream = new FileInputStream(schemaFile)) {
+        nodeTree = yaml.loadAs(fileInputStream, NodeSchema.class);
+      }
+      List<NodeSchema> schemaList = new ArrayList<>();
+      if (nodeTree.getType() != LayerType.ROOT) {
+        throw new IllegalArgumentException("First layer is not a ROOT node."
+                + " schema file: " + schemaFile.getAbsolutePath());
+      }
+      schemaList.add(nodeTree);
+      if (nodeTree.getSublayer() != null) {
+        nodeTree = nodeTree.getSublayer().get(0);
+      }
+
+      while (nodeTree != null) {
+        if (nodeTree.getType() == LayerType.LEAF_NODE
+                && nodeTree.getSublayer() != null) {
+          throw new IllegalArgumentException("Leaf node in the middle of path."
+                  + " schema file: " + schemaFile.getAbsolutePath());
+        }
+        if (nodeTree.getType() == LayerType.ROOT) {
+          throw new IllegalArgumentException("Multiple root nodes are defined."
+                  + " schema file: " + schemaFile.getAbsolutePath());
+        }
+        schemaList.add(nodeTree);
+        if (nodeTree.getSublayer() != null) {
+          nodeTree = nodeTree.getSublayer().get(0);
+        } else {
+          break;
+        }
+      }
+      finalSchema = new NodeSchemaLoadResult(schemaList, true);
+    } catch (RuntimeException e) {
+      throw  e;
+    } catch (Exception e) {
+      throw new IllegalArgumentException("Fail to load network topology node"
+              + " schema file: " + schemaFile.getAbsolutePath() + " , error:"
+              + e.getMessage());
+    }
+
+    return finalSchema;
+  }
+
   /**
    * Load layoutVersion from root element in the XML configuration file.
    * @param root root element

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java

@@ -59,13 +59,20 @@ public void init(Configuration conf) {
     /**
      * Load schemas from network topology schema configuration file
      */
+    String schemaFileType = conf.get(
+            ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_TYPE);
+
     String schemaFile = conf.get(
         ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE,
         ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT);
 
     NodeSchemaLoadResult result;
     try {
-      result = NodeSchemaLoader.getInstance().loadSchemaFromFile(schemaFile);
+      if (schemaFileType.toLowerCase().compareTo("yaml") == 0) {
+        result = NodeSchemaLoader.getInstance().loadSchemaFromYaml(schemaFile);
+      } else {
+        result = NodeSchemaLoader.getInstance().loadSchemaFromXml(schemaFile);
+      }
       allSchema = result.getSchemaList();
       enforcePrefix = result.isEnforePrefix();
       maxLevel = allSchema.size();

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java

@@ -190,7 +190,7 @@ private String useFirstLine(String message) {
   public ScmInfo getScmInfo() throws IOException {
     HddsProtos.GetScmInfoRequestProto request =
         HddsProtos.GetScmInfoRequestProto.getDefaultInstance();
-    HddsProtos.GetScmInfoRespsonseProto resp;
+    HddsProtos.GetScmInfoResponseProto resp;
     try {
       resp = rpcProxy.getScmInfo(NULL_RPC_CONTROLLER, request);
     } catch (ServiceException e) {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -357,7 +357,7 @@ public ScmInfo getScmInfo() throws IOException {
             .setTraceID(TracingUtil.exportCurrentSpan())
             .build();
     try {
-      HddsProtos.GetScmInfoRespsonseProto resp = rpcProxy.getScmInfo(
+      HddsProtos.GetScmInfoResponseProto resp = rpcProxy.getScmInfo(
           NULL_RPC_CONTROLLER, request);
       ScmInfo.Builder builder = new ScmInfo.Builder()
           .setClusterId(resp.getClusterId())

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java

@@ -133,7 +133,7 @@ public DeleteScmKeyBlocksResponseProto deleteScmKeyBlocks(
   }
 
   @Override
-  public HddsProtos.GetScmInfoRespsonseProto getScmInfo(
+  public HddsProtos.GetScmInfoResponseProto getScmInfo(
       RpcController controller, HddsProtos.GetScmInfoRequestProto req)
       throws ServiceException {
     ScmInfo scmInfo;
@@ -142,7 +142,7 @@ public HddsProtos.GetScmInfoRespsonseProto getScmInfo(
     } catch (IOException ex) {
       throw new ServiceException(ex);
     }
-    return HddsProtos.GetScmInfoRespsonseProto.newBuilder()
+    return HddsProtos.GetScmInfoResponseProto.newBuilder()
         .setClusterId(scmInfo.getClusterId())
         .setScmId(scmInfo.getScmId())
         .build();