From b5f8331b010b16603879884d9c6180d8c8bcace0 Mon Sep 17 00:00:00 2001
From: Li Cheng <timmycheng@tencent.com>
Date: Wed, 8 Jan 2020 22:24:29 +0800
Subject: [PATCH] HDDS-2756 Better logging and message for pipeline creation
 failure.

---
 .../scm/pipeline/PipelinePlacementPolicy.java | 30 +++++++++++--------
 .../hdds/scm/pipeline/SCMPipelineManager.java |  1 -
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index bc65d14d4a53..f4a13e17d1c3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -114,10 +114,10 @@ boolean meetCriteria(DatanodeDetails datanodeDetails, int nodesRequired) {
     boolean meet = (nodeManager.getPipelinesCount(datanodeDetails)
         - pipelineNumDeductable) < heavyNodeCriteria;
     if (!meet) {
-      LOG.info("Pipeline Placement: can't place more pipeline on heavy " +
-          "datanode： " + datanodeDetails.getUuid().toString() + " Heaviness: " +
-          nodeManager.getPipelinesCount(datanodeDetails) + " limit: " +
-          heavyNodeCriteria);
+      LOG.debug("Pipeline Placement: can't place more pipeline on heavy " +
+          "datanode： " + datanodeDetails.getUuid().toString() +
+          " Heaviness: " + nodeManager.getPipelinesCount(datanodeDetails) +
+          " limit: " + heavyNodeCriteria);
     }
     return meet;
   }
@@ -144,17 +144,19 @@ List<DatanodeDetails> filterViableNodes(
     int initialHealthyNodesCount = healthyNodes.size();
     String msg;
     if (initialHealthyNodesCount == 0) {
-      msg = "No healthy node found to allocate pipeline.";
+      msg = "No healthy nodes found to allocate pipeline.";
       LOG.error(msg);
       throw new SCMException(msg, SCMException.ResultCodes
           .FAILED_TO_FIND_HEALTHY_NODES);
     }
 
     if (initialHealthyNodesCount < nodesRequired) {
-      msg = String.format("Not enough healthy nodes to allocate pipeline. %d "
+      LOG.warn("Not enough healthy nodes to allocate pipeline. %d "
               + " datanodes required. Found %d",
           nodesRequired, initialHealthyNodesCount);
-      LOG.error(msg);
+      msg = String.format("Pipeline creation failed due to no sufficient" +
+              " healthy datanodes. Required %d. Found %d.",
+          nodesRequired, initialHealthyNodesCount);
       throw new SCMException(msg,
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
     }
@@ -166,13 +168,15 @@ List<DatanodeDetails> filterViableNodes(
         .collect(Collectors.toList());
 
     if (healthyList.size() < nodesRequired) {
-      msg = String.format("Unable to find enough nodes that meet " +
+      LOG.debug("Unable to find enough nodes that meet " +
               "the criteria that cannot engage in more than %d pipelines." +
               " Nodes required: %d Found: %d, healthy nodes count in " +
               "NodeManager: %d.",
           heavyNodeCriteria, nodesRequired, healthyList.size(),
           initialHealthyNodesCount);
-      LOG.error(msg);
+      msg = String.format("Pipeline creation failed due to not enough" +
+              " healthy datanodes after filter. Required %d. Found %d",
+          nodesRequired, initialHealthyNodesCount);
       throw new SCMException(msg,
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
     }
@@ -229,8 +233,8 @@ public List<DatanodeDetails> getResultSet(
     // First choose an anchor nodes randomly
     DatanodeDetails anchor = chooseNode(healthyNodes);
     if (anchor == null) {
-      LOG.error("Pipeline Placement: Unable to find the first healthy nodes " +
-              "that meet the criteria. Required nodes: {}, Found nodes: {}",
+      LOG.warn("Unable to find healthy nodes." +
+              " Required nodes: {}, Found nodes: {}",
           nodesRequired, results.size());
       throw new SCMException("Unable to find required number of nodes.",
           SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
@@ -245,7 +249,7 @@ public List<DatanodeDetails> getResultSet(
         healthyNodes, exclude,
         nodeManager.getClusterNetworkTopologyMap(), anchor);
     if (nodeOnDifferentRack == null) {
-      LOG.error("Pipeline Placement: Unable to find nodes on different racks " +
+      LOG.warn("Pipeline Placement: Unable to find nodes on different racks " +
               " that meet the criteria. Required nodes: {}, Found nodes: {}",
           nodesRequired, results.size());
       throw new SCMException("Unable to find required number of nodes.",
@@ -269,7 +273,7 @@ public List<DatanodeDetails> getResultSet(
     }
 
     if (results.size() < nodesRequired) {
-      LOG.error("Pipeline Placement: Unable to find the required number of " +
+      LOG.warn("Unable to find the required number of " +
               "healthy nodes that  meet the criteria. Required nodes: {}, " +
               "Found nodes: {}", nodesRequired, results.size());
       throw new SCMException("Unable to find required number of nodes.",
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 58fce89f25af..67856afea714 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -192,7 +192,6 @@ public synchronized Pipeline createPipeline(ReplicationType type,
       return pipeline;
     } catch (IOException ex) {
       metrics.incNumPipelineCreationFailed();
-      LOG.error("Pipeline creation failed.", ex);
       throw ex;
     } finally {
       lock.writeLock().unlock();