From 81cf29770d980c10eaca56aed1d929d0e3b5cec2 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 20 Sep 2019 11:42:21 -0700 Subject: [PATCH 01/78] HDDS-1982. Extend SCMNodeManager to support decommission and maintenance states. Contributed by Stephen O'Donnell. --- hadoop-hdds/common/src/main/proto/hdds.proto | 10 +- .../scm/block/SCMBlockDeletingService.java | 7 +- .../placement/algorithms/SCMCommonPolicy.java | 4 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 50 ++- .../hadoop/hdds/scm/node/NodeManager.java | 31 +- .../hdds/scm/node/NodeStateManager.java | 291 +++++++++--------- .../hadoop/hdds/scm/node/NodeStatus.java | 93 ++++++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 59 +++- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 - .../hdds/scm/node/states/NodeStateMap.java | 241 +++++++++++---- .../scm/pipeline/RatisPipelineProvider.java | 4 +- .../scm/pipeline/SimplePipelineProvider.java | 4 +- .../scm/server/SCMClientProtocolServer.java | 4 +- .../scm/server/StorageContainerManager.java | 3 +- .../hdds/scm/block/TestBlockManager.java | 3 +- .../hdds/scm/container/MockNodeManager.java | 37 ++- .../container/TestContainerReportHandler.java | 12 +- .../TestContainerPlacementFactory.java | 4 +- .../TestSCMContainerPlacementCapacity.java | 4 +- .../TestSCMContainerPlacementRackAware.java | 4 +- .../TestSCMContainerPlacementRandom.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hdds/scm/node/TestNodeStateManager.java | 223 ++++++++++++++ .../hdds/scm/node/TestSCMNodeManager.java | 83 ++--- .../scm/node/states/TestNodeStateMap.java | 140 +++++++++ .../placement/TestContainerPlacement.java | 6 +- .../testutils/ReplicationNodeManagerMock.java | 36 ++- .../hdds/scm/cli/TopologySubcommand.java | 4 - .../ozone/TestStorageContainerManager.java | 5 +- .../ozone/scm/node/TestSCMNodeMetrics.java | 4 - 30 files changed, 1063 insertions(+), 317 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index d2bb355ff8a4..294f2b769512 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -100,8 +100,14 @@ enum NodeState { HEALTHY = 1; STALE = 2; DEAD = 3; - DECOMMISSIONING = 4; - DECOMMISSIONED = 5; +} + +enum NodeOperationalState { + IN_SERVICE = 1; + DECOMMISSIONING = 2; + DECOMMISSIONED = 3; + ENTERING_MAINTENANCE = 4; + IN_MAINTENANCE = 5; } enum QueryScope { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java index ad7762482ce4..b5e5d16b3f4b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand; @@ -137,7 +137,10 @@ public EmptyTaskResult call() throws Exception { // to delete blocks. LOG.debug("Running DeletedBlockTransactionScanner"); DatanodeDeletedBlockTransactions transactions = null; - List datanodes = nodeManager.getNodes(NodeState.HEALTHY); + // TODO - DECOMM - should we be deleting blocks from decom nodes + // and what about entering maintenance. + List datanodes = + nodeManager.getNodes(NodeStatus.inServiceHealthy()); Map transactionMap = null; if (datanodes != null) { transactions = new DatanodeDeletedBlockTransactions(containerManager, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java index 77cdd83f7938..63cd8e1f4298 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +108,7 @@ public List chooseDatanodes( List excludedNodes, List favoredNodes, int nodesRequired, final long sizeRequired) throws SCMException { List healthyNodes = - nodeManager.getNodes(HddsProtos.NodeState.HEALTHY); + nodeManager.getNodes(NodeStatus.inServiceHealthy()); if (excludedNodes != null) { healthyNodes.removeAll(excludedNodes); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java index d06ea2a3b3f3..66e1ca47fade 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.node; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.StorageReportProto; @@ -41,25 +42,39 @@ public class DatanodeInfo extends DatanodeDetails { private List storageReports; + private NodeStatus nodeStatus; + /** * Constructs DatanodeInfo from DatanodeDetails. * * @param datanodeDetails Details about the datanode */ - public DatanodeInfo(DatanodeDetails datanodeDetails) { + public DatanodeInfo(DatanodeDetails datanodeDetails, NodeStatus nodeStatus) { super(datanodeDetails); this.lock = new ReentrantReadWriteLock(); this.lastHeartbeatTime = Time.monotonicNow(); this.storageReports = Collections.emptyList(); + this.nodeStatus = nodeStatus; } /** * Updates the last heartbeat time with current time. */ public void updateLastHeartbeatTime() { + updateLastHeartbeatTime(Time.monotonicNow()); + } + + /** + * Sets the last heartbeat time to a given value. Intended to be used + * only for tests. + * + * @param milliSecondsSinceEpoch - ms since Epoch to set as the heartbeat time + */ + @VisibleForTesting + public void updateLastHeartbeatTime(long milliSecondsSinceEpoch) { try { lock.writeLock().lock(); - lastHeartbeatTime = Time.monotonicNow(); + lastHeartbeatTime = milliSecondsSinceEpoch; } finally { lock.writeLock().unlock(); } @@ -108,6 +123,37 @@ public List getStorageReports() { } } + /** + * Return the current NodeStatus for the datanode. + * + * @return NodeStatus - the current nodeStatus + */ + public NodeStatus getNodeStatus() { + try { + lock.readLock().lock(); + return nodeStatus; + } finally { + lock.readLock().unlock(); + } + } + + /** + * Update the NodeStatus for this datanode. When using this method + * be ware of the potential for lost updates if two threads read the + * current status, update one field and then write it back without + * locking enforced outside of this class. + * + * @param newNodeStatus - the new NodeStatus object + */ + public void setNodeStatus(NodeStatus newNodeStatus) { + try { + lock.writeLock().lock(); + this.nodeStatus = newNodeStatus; + } finally { + lock.writeLock().unlock(); + } + } + /** * Returns the last updated time of datanode info. * @return the last updated time of datanode info. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index d8890fb0b920..afff7a3c13ae 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; @@ -62,19 +63,39 @@ public interface NodeManager extends StorageContainerNodeProtocol, EventHandler, NodeManagerMXBean, Closeable { + /** + * Gets all Live Datanodes that are currently communicating with SCM. + * @param nodeStatus - Status of the node to return + * @return List of Datanodes that are Heartbeating SCM. + */ + List getNodes(NodeStatus nodeStatus); + /** * Gets all Live Datanodes that is currently communicating with SCM. - * @param nodeState - State of the node + * @param opState - The operational state of the node + * @param health - The health of the node * @return List of Datanodes that are Heartbeating SCM. */ - List getNodes(NodeState nodeState); + List getNodes( + NodeOperationalState opState, NodeState health); + + /** + * Returns the Number of Datanodes that are communicating with SCM with the + * given status. + * @param nodeStatus - State of the node + * @return int -- count + */ + int getNodeCount(NodeStatus nodeStatus); /** - * Returns the Number of Datanodes that are communicating with SCM. - * @param nodeState - State of the node + * Returns the Number of Datanodes that are communicating with SCM in the + * given state. + * @param opState - The operational state of the node + * @param health - The health of the node * @return int -- count */ - int getNodeCount(NodeState nodeState); + int getNodeCount( + NodeOperationalState opState, NodeState health); /** * Get all datanodes known to SCM. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index 954cb0e8ea46..1e1a50cd3db0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -23,7 +23,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.scm.HddsServerUtil; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -75,7 +77,12 @@ public class NodeStateManager implements Runnable, Closeable { * Node's life cycle events. */ private enum NodeLifeCycleEvent { - TIMEOUT, RESTORE, RESURRECT, DECOMMISSION, DECOMMISSIONED + TIMEOUT, RESTORE, RESURRECT + } + + private enum NodeOperationStateEvent { + START_DECOMMISSION, COMPLETE_DECOMMISSION, START_MAINTENANCE, + ENTER_MAINTENANCE, RETURN_TO_SERVICE } private static final Logger LOG = LoggerFactory @@ -84,7 +91,12 @@ private enum NodeLifeCycleEvent { /** * StateMachine for node lifecycle. */ - private final StateMachine stateMachine; + private final StateMachine nodeHealthSM; + /** + * StateMachine for node operational state. + */ + private final StateMachine nodeOpStateSM; /** * This is the map which maintains the current state of all datanodes. */ @@ -100,7 +112,7 @@ private enum NodeLifeCycleEvent { /** * Maps the event to be triggered when a node state us updated. */ - private final Map> state2EventMap; + private final Map> state2EventMap; /** * ExecutorService used for scheduling heartbeat processing thread. */ @@ -150,9 +162,11 @@ public NodeStateManager(Configuration conf, EventPublisher eventPublisher) { this.state2EventMap = new HashMap<>(); initialiseState2EventMap(); Set finalStates = new HashSet<>(); - finalStates.add(NodeState.DECOMMISSIONED); - this.stateMachine = new StateMachine<>(NodeState.HEALTHY, finalStates); - initializeStateMachine(); + Set opStateFinalStates = new HashSet<>(); + this.nodeHealthSM = new StateMachine<>(NodeState.HEALTHY, finalStates); + this.nodeOpStateSM = new StateMachine<>( + NodeOperationalState.IN_SERVICE, opStateFinalStates); + initializeStateMachines(); heartbeatCheckerIntervalMs = HddsServerUtil .getScmheartbeatCheckerInterval(conf); staleNodeIntervalMs = HddsServerUtil.getStaleNodeInterval(conf); @@ -176,10 +190,12 @@ public NodeStateManager(Configuration conf, EventPublisher eventPublisher) { * Populates state2event map. */ private void initialiseState2EventMap() { - state2EventMap.put(NodeState.STALE, SCMEvents.STALE_NODE); - state2EventMap.put(NodeState.DEAD, SCMEvents.DEAD_NODE); - state2EventMap - .put(NodeState.HEALTHY, SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE); + state2EventMap.put(NodeStatus.inServiceStale(), SCMEvents.STALE_NODE); + state2EventMap.put(NodeStatus.inServiceDead(), SCMEvents.DEAD_NODE); + state2EventMap.put(NodeStatus.inServiceHealthy(), + SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE); + // TODO - add whatever events are needed for decomm / maint to stale, dead, + // healthy } /* @@ -198,18 +214,6 @@ private void initialiseState2EventMap() { * State: DEAD -------------------> HEALTHY * Event: RESURRECT * - * State: HEALTHY -------------------> DECOMMISSIONING - * Event: DECOMMISSION - * - * State: STALE -------------------> DECOMMISSIONING - * Event: DECOMMISSION - * - * State: DEAD -------------------> DECOMMISSIONING - * Event: DECOMMISSION - * - * State: DECOMMISSIONING -------------------> DECOMMISSIONED - * Event: DECOMMISSIONED - * * Node State Flow * * +--------------------------------------------------------+ @@ -219,47 +223,51 @@ private void initialiseState2EventMap() { * | | | | * V V | | * [HEALTHY]------------------->[STALE]------------------->[DEAD] - * | (TIMEOUT) | (TIMEOUT) | - * | | | - * | | | - * | | | - * | | | - * | (DECOMMISSION) | (DECOMMISSION) | (DECOMMISSION) - * | V | - * +------------------->[DECOMMISSIONING]<----------------+ - * | - * | (DECOMMISSIONED) - * | - * V - * [DECOMMISSIONED] * */ /** * Initializes the lifecycle of node state machine. */ - private void initializeStateMachine() { - stateMachine.addTransition( + private void initializeStateMachines() { + nodeHealthSM.addTransition( NodeState.HEALTHY, NodeState.STALE, NodeLifeCycleEvent.TIMEOUT); - stateMachine.addTransition( + nodeHealthSM.addTransition( NodeState.STALE, NodeState.DEAD, NodeLifeCycleEvent.TIMEOUT); - stateMachine.addTransition( + nodeHealthSM.addTransition( NodeState.STALE, NodeState.HEALTHY, NodeLifeCycleEvent.RESTORE); - stateMachine.addTransition( + nodeHealthSM.addTransition( NodeState.DEAD, NodeState.HEALTHY, NodeLifeCycleEvent.RESURRECT); - stateMachine.addTransition( - NodeState.HEALTHY, NodeState.DECOMMISSIONING, - NodeLifeCycleEvent.DECOMMISSION); - stateMachine.addTransition( - NodeState.STALE, NodeState.DECOMMISSIONING, - NodeLifeCycleEvent.DECOMMISSION); - stateMachine.addTransition( - NodeState.DEAD, NodeState.DECOMMISSIONING, - NodeLifeCycleEvent.DECOMMISSION); - stateMachine.addTransition( - NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONED, - NodeLifeCycleEvent.DECOMMISSIONED); + nodeOpStateSM.addTransition( + NodeOperationalState.IN_SERVICE, NodeOperationalState.DECOMMISSIONING, + NodeOperationStateEvent.START_DECOMMISSION); + nodeOpStateSM.addTransition( + NodeOperationalState.DECOMMISSIONING, NodeOperationalState.IN_SERVICE, + NodeOperationStateEvent.RETURN_TO_SERVICE); + nodeOpStateSM.addTransition( + NodeOperationalState.DECOMMISSIONING, + NodeOperationalState.DECOMMISSIONED, + NodeOperationStateEvent.COMPLETE_DECOMMISSION); + nodeOpStateSM.addTransition( + NodeOperationalState.DECOMMISSIONED, NodeOperationalState.IN_SERVICE, + NodeOperationStateEvent.RETURN_TO_SERVICE); + + nodeOpStateSM.addTransition( + NodeOperationalState.IN_SERVICE, + NodeOperationalState.ENTERING_MAINTENANCE, + NodeOperationStateEvent.START_MAINTENANCE); + nodeOpStateSM.addTransition( + NodeOperationalState.ENTERING_MAINTENANCE, + NodeOperationalState.IN_SERVICE, + NodeOperationStateEvent.RETURN_TO_SERVICE); + nodeOpStateSM.addTransition( + NodeOperationalState.ENTERING_MAINTENANCE, + NodeOperationalState.IN_MAINTENANCE, + NodeOperationStateEvent.ENTER_MAINTENANCE); + nodeOpStateSM.addTransition( + NodeOperationalState.IN_MAINTENANCE, NodeOperationalState.IN_SERVICE, + NodeOperationStateEvent.RETURN_TO_SERVICE); } /** @@ -271,7 +279,8 @@ private void initializeStateMachine() { */ public void addNode(DatanodeDetails datanodeDetails) throws NodeAlreadyExistsException { - nodeStateMap.addNode(datanodeDetails, stateMachine.getInitialState()); + nodeStateMap.addNode(datanodeDetails, new NodeStatus( + nodeOpStateSM.getInitialState(), nodeHealthSM.getInitialState())); eventPublisher.fireEvent(SCMEvents.NEW_NODE, datanodeDetails); } @@ -317,59 +326,63 @@ public void updateLastHeartbeatTime(DatanodeDetails datanodeDetails) * * @throws NodeNotFoundException if the node is not present */ - public NodeState getNodeState(DatanodeDetails datanodeDetails) + public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) throws NodeNotFoundException { - return nodeStateMap.getNodeState(datanodeDetails.getUuid()); + return nodeStateMap.getNodeStatus(datanodeDetails.getUuid()); } /** - * Returns all the node which are in healthy state. + * Returns all the node which are in healthy state, ignoring the operational + * state. * * @return list of healthy nodes */ public List getHealthyNodes() { - return getNodes(NodeState.HEALTHY); + return getNodes(null, NodeState.HEALTHY); } /** - * Returns all the node which are in stale state. + * Returns all the node which are in stale state, ignoring the operational + * state. * * @return list of stale nodes */ public List getStaleNodes() { - return getNodes(NodeState.STALE); + return getNodes(null, NodeState.STALE); } /** - * Returns all the node which are in dead state. + * Returns all the node which are in dead state, ignoring the operational + * state. * * @return list of dead nodes */ public List getDeadNodes() { - return getNodes(NodeState.DEAD); + return getNodes(null, NodeState.DEAD); } /** - * Returns all the node which are in the specified state. + * Returns all the nodes with the specified status. * - * @param state NodeState + * @param status NodeStatus * * @return list of nodes */ - public List getNodes(NodeState state) { - List nodes = new ArrayList<>(); - nodeStateMap.getNodes(state).forEach( - uuid -> { - try { - nodes.add(nodeStateMap.getNodeInfo(uuid)); - } catch (NodeNotFoundException e) { - // This should not happen unless someone else other than - // NodeStateManager is directly modifying NodeStateMap and removed - // the node entry after we got the list of UUIDs. - LOG.error("Inconsistent NodeStateMap! " + nodeStateMap); - } - }); - return nodes; + public List getNodes(NodeStatus status) { + return nodeStateMap.getDatanodeInfos(status); + } + + /** + * Returns all the nodes with the specified operationalState and health. + * + * @param opState The operationalState of the node + * @param health The node health + * + * @return list of nodes matching the passed states + */ + public List getNodes( + NodeOperationalState opState, NodeState health) { + return nodeStateMap.getDatanodeInfos(opState, health); } /** @@ -378,19 +391,7 @@ public List getNodes(NodeState state) { * @return all the managed nodes */ public List getAllNodes() { - List nodes = new ArrayList<>(); - nodeStateMap.getAllNodes().forEach( - uuid -> { - try { - nodes.add(nodeStateMap.getNodeInfo(uuid)); - } catch (NodeNotFoundException e) { - // This should not happen unless someone else other than - // NodeStateManager is directly modifying NodeStateMap and removed - // the node entry after we got the list of UUIDs. - LOG.error("Inconsistent NodeStateMap! " + nodeStateMap); - } - }); - return nodes; + return nodeStateMap.getAllDatanodeInfos(); } /** @@ -403,41 +404,53 @@ public Set getPipelineByDnID(UUID dnId) { } /** - * Returns the count of healthy nodes. + * Returns the count of healthy nodes, ignoring operational state. * * @return healthy node count */ public int getHealthyNodeCount() { - return getNodeCount(NodeState.HEALTHY); + return getHealthyNodes().size(); } /** - * Returns the count of stale nodes. + * Returns the count of stale nodes, ignoring operational state. * * @return stale node count */ public int getStaleNodeCount() { - return getNodeCount(NodeState.STALE); + return getStaleNodes().size(); } /** - * Returns the count of dead nodes. + * Returns the count of dead nodes, ignoring operational state. * * @return dead node count */ public int getDeadNodeCount() { - return getNodeCount(NodeState.DEAD); + return getDeadNodes().size(); } /** - * Returns the count of nodes in specified state. + * Returns the count of nodes in specified status. * - * @param state NodeState + * @param status NodeState * * @return node count */ - public int getNodeCount(NodeState state) { - return nodeStateMap.getNodeCount(state); + public int getNodeCount(NodeStatus status) { + return nodeStateMap.getNodeCount(status); + } + + /** + * Returns the count of nodes in the specified states. + * + * @param opState The operational state of the node + * @param health The health of the node + * + * @return node count + */ + public int getNodeCount(NodeOperationalState opState, NodeState health) { + return nodeStateMap.getNodeCount(opState, health); } /** @@ -536,7 +549,8 @@ public void run() { scheduleNextHealthCheck(); } - private void checkNodesHealth() { + @VisibleForTesting + public void checkNodesHealth() { /* * @@ -578,39 +592,32 @@ private void checkNodesHealth() { Predicate deadNodeCondition = (lastHbTime) -> lastHbTime < staleNodeDeadline; try { - for (NodeState state : NodeState.values()) { - List nodes = nodeStateMap.getNodes(state); - for (UUID id : nodes) { - DatanodeInfo node = nodeStateMap.getNodeInfo(id); - switch (state) { - case HEALTHY: - // Move the node to STALE if the last heartbeat time is less than - // configured stale-node interval. - updateNodeState(node, staleNodeCondition, state, - NodeLifeCycleEvent.TIMEOUT); - break; - case STALE: - // Move the node to DEAD if the last heartbeat time is less than - // configured dead-node interval. - updateNodeState(node, deadNodeCondition, state, - NodeLifeCycleEvent.TIMEOUT); - // Restore the node if we have received heartbeat before configured - // stale-node interval. - updateNodeState(node, healthyNodeCondition, state, - NodeLifeCycleEvent.RESTORE); - break; - case DEAD: - // Resurrect the node if we have received heartbeat before - // configured stale-node interval. - updateNodeState(node, healthyNodeCondition, state, - NodeLifeCycleEvent.RESURRECT); - break; - // We don't do anything for DECOMMISSIONING and DECOMMISSIONED in - // heartbeat processing. - case DECOMMISSIONING: - case DECOMMISSIONED: - default: - } + for(DatanodeInfo node : nodeStateMap.getAllDatanodeInfos()) { + NodeStatus status = nodeStateMap.getNodeStatus(node.getUuid()); + switch (status.getHealth()) { + case HEALTHY: + // Move the node to STALE if the last heartbeat time is less than + // configured stale-node interval. + updateNodeState(node, staleNodeCondition, status, + NodeLifeCycleEvent.TIMEOUT); + break; + case STALE: + // Move the node to DEAD if the last heartbeat time is less than + // configured dead-node interval. + updateNodeState(node, deadNodeCondition, status, + NodeLifeCycleEvent.TIMEOUT); + // Restore the node if we have received heartbeat before configured + // stale-node interval. + updateNodeState(node, healthyNodeCondition, status, + NodeLifeCycleEvent.RESTORE); + break; + case DEAD: + // Resurrect the node if we have received heartbeat before + // configured stale-node interval. + updateNodeState(node, healthyNodeCondition, status, + NodeLifeCycleEvent.RESURRECT); + break; + default: } } } catch (NodeNotFoundException e) { @@ -669,27 +676,29 @@ private boolean shouldSkipCheck() { * * @param node DatanodeInfo * @param condition condition to check - * @param state current state of node + * @param status current status of node * @param lifeCycleEvent NodeLifeCycleEvent to be applied if condition * matches * * @throws NodeNotFoundException if the node is not present */ private void updateNodeState(DatanodeInfo node, Predicate condition, - NodeState state, NodeLifeCycleEvent lifeCycleEvent) + NodeStatus status, NodeLifeCycleEvent lifeCycleEvent) throws NodeNotFoundException { try { if (condition.test(node.getLastHeartbeatTime())) { - NodeState newState = stateMachine.getNextState(state, lifeCycleEvent); - nodeStateMap.updateNodeState(node.getUuid(), state, newState); - if (state2EventMap.containsKey(newState)) { - eventPublisher.fireEvent(state2EventMap.get(newState), node); + NodeState newHealthState = nodeHealthSM. + getNextState(status.getHealth(), lifeCycleEvent); + NodeStatus newStatus = + nodeStateMap.updateNodeHealthState(node.getUuid(), newHealthState); + if (state2EventMap.containsKey(newStatus)) { + eventPublisher.fireEvent(state2EventMap.get(newStatus), node); } } } catch (InvalidStateTransitionException e) { LOG.warn("Invalid state transition of node {}." + " Current state: {}, life cycle event: {}", - node, state, lifeCycleEvent); + node, status.getHealth(), lifeCycleEvent); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java new file mode 100644 index 000000000000..0776c2894e15 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + +import java.util.Objects; + +/** + * This class is used to capture the current status of a datanode. This + * includes its health (healthy, stale or dead) and its operation status ( + * in_service, decommissioned and maintenance mode. + */ +public class NodeStatus { + + private HddsProtos.NodeOperationalState operationalState; + private HddsProtos.NodeState health; + + public NodeStatus(HddsProtos.NodeOperationalState operationalState, + HddsProtos.NodeState health) { + this.operationalState = operationalState; + this.health = health; + } + + public static NodeStatus inServiceHealthy() { + return new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE, + HddsProtos.NodeState.HEALTHY); + } + + public static NodeStatus inServiceStale() { + return new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE, + HddsProtos.NodeState.STALE); + } + + public static NodeStatus inServiceDead() { + return new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE, + HddsProtos.NodeState.DEAD); + } + + public HddsProtos.NodeState getHealth() { + return health; + } + + public HddsProtos.NodeOperationalState getOperationalState() { + return operationalState; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + NodeStatus other = (NodeStatus) obj; + if (this.operationalState == other.operationalState && + this.health == other.health) { + return true; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(health, operationalState); + } + + @Override + public String toString() { + return "OperationalState: "+operationalState+" Health: "+health; + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index d3df858e6e6e..c277ea9be0f4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; @@ -150,8 +151,28 @@ private void unregisterMXBean() { * @return List of Datanodes that are known to SCM in the requested state. */ @Override - public List getNodes(NodeState nodestate) { - return nodeStateManager.getNodes(nodestate).stream() + public List getNodes(NodeStatus nodeStatus) { + return nodeStateManager.getNodes(nodeStatus) + .stream() + .map(node -> (DatanodeDetails)node).collect(Collectors.toList()); + } + + /** + * Returns all datanode that are in the given states. Passing null for one of + * of the states acts like a wildcard for that state. This function works by + * taking a snapshot of the current collection and then returning the list + * from that collection. This means that real map might have changed by the + * time we return this list. + * + * @param opState The operational state of the node + * @param health The health of the node + * @return List of Datanodes that are known to SCM in the requested states. + */ + @Override + public List getNodes( + NodeOperationalState opState, NodeState health) { + return nodeStateManager.getNodes(opState, health) + .stream() .map(node -> (DatanodeDetails)node).collect(Collectors.toList()); } @@ -172,8 +193,21 @@ public List getAllNodes() { * @return count */ @Override - public int getNodeCount(NodeState nodestate) { - return nodeStateManager.getNodeCount(nodestate); + public int getNodeCount(NodeStatus nodeStatus) { + return nodeStateManager.getNodeCount(nodeStatus); + } + + /** + * Returns the Number of Datanodes by State they are in. Passing null for + * either of the states acts like a wildcard for that state. + * + * @parem nodeOpState - The Operational State of the node + * @param health - The health of the node + * @return count + */ + @Override + public int getNodeCount(NodeOperationalState nodeOpState, NodeState health) { + return nodeStateManager.getNodeCount(nodeOpState, health); } /** @@ -185,7 +219,7 @@ public int getNodeCount(NodeState nodestate) { @Override public NodeState getNodeState(DatanodeDetails datanodeDetails) { try { - return nodeStateManager.getNodeState(datanodeDetails); + return nodeStateManager.getNodeStatus(datanodeDetails).getHealth(); } catch (NodeNotFoundException e) { // TODO: should we throw NodeNotFoundException? return null; @@ -365,9 +399,9 @@ public Map getNodeStats() { final Map nodeStats = new HashMap<>(); final List healthyNodes = nodeStateManager - .getNodes(NodeState.HEALTHY); + .getHealthyNodes(); final List staleNodes = nodeStateManager - .getNodes(NodeState.STALE); + .getStaleNodes(); final List datanodes = new ArrayList<>(healthyNodes); datanodes.addAll(staleNodes); @@ -417,9 +451,12 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) { @Override public Map getNodeCount() { + // TODO - This does not consider decom, maint etc. Map nodeCountMap = new HashMap(); for(NodeState state : NodeState.values()) { - nodeCountMap.put(state.toString(), getNodeCount(state)); + // TODO - this iterate the node list once per state and needs + // fixed to only perform one pass. + nodeCountMap.put(state.toString(), getNodeCount(null, state)); } return nodeCountMap; } @@ -436,10 +473,8 @@ public Map getNodeInfo() { long ssdUsed = 0L; long ssdRemaining = 0L; - List healthyNodes = nodeStateManager - .getNodes(NodeState.HEALTHY); - List staleNodes = nodeStateManager - .getNodes(NodeState.STALE); + List healthyNodes = nodeStateManager.getHealthyNodes(); + List staleNodes = nodeStateManager.getStaleNodes(); List datanodes = new ArrayList<>(healthyNodes); datanodes.addAll(staleNodes); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index 1596523bbcd1..fd9c9c126654 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hdds.scm.node; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONED; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONING; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; @@ -132,12 +130,6 @@ public void getMetrics(MetricsCollector collector, boolean all) { .addGauge(Interns.info("DeadNodes", "Number of dead datanodes"), nodeCount.get(DEAD.toString())) - .addGauge(Interns.info("DecommissioningNodes", - "Number of decommissioning datanodes"), - nodeCount.get(DECOMMISSIONING.toString())) - .addGauge(Interns.info("DecommissionedNodes", - "Number of decommissioned datanodes"), - nodeCount.get(DECOMMISSIONED.toString())) .addGauge(Interns.info("DiskCapacity", "Total disk capacity"), nodeInfo.get("DISKCapacity")) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java index 0c1ab2c3838a..6565e8117c73 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java @@ -20,13 +20,16 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.node.DatanodeInfo; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; /** * Maintains the state of datanodes in SCM. This class should only be used by @@ -35,15 +38,10 @@ * this class. */ public class NodeStateMap { - /** * Node id to node info map. */ private final ConcurrentHashMap nodeMap; - /** - * Represents the current state of node. - */ - private final ConcurrentHashMap> stateMap; /** * Node to set of containers on the node. */ @@ -57,29 +55,18 @@ public class NodeStateMap { public NodeStateMap() { lock = new ReentrantReadWriteLock(); nodeMap = new ConcurrentHashMap<>(); - stateMap = new ConcurrentHashMap<>(); nodeToContainer = new ConcurrentHashMap<>(); - initStateMap(); - } - - /** - * Initializes the state map with available states. - */ - private void initStateMap() { - for (NodeState state : NodeState.values()) { - stateMap.put(state, new HashSet<>()); - } } /** * Adds a node to NodeStateMap. * * @param datanodeDetails DatanodeDetails - * @param nodeState initial NodeState + * @param nodeStatus initial NodeStatus * * @throws NodeAlreadyExistsException if the node already exist */ - public void addNode(DatanodeDetails datanodeDetails, NodeState nodeState) + public void addNode(DatanodeDetails datanodeDetails, NodeStatus nodeStatus) throws NodeAlreadyExistsException { lock.writeLock().lock(); try { @@ -87,34 +74,54 @@ public void addNode(DatanodeDetails datanodeDetails, NodeState nodeState) if (nodeMap.containsKey(id)) { throw new NodeAlreadyExistsException("Node UUID: " + id); } - nodeMap.put(id, new DatanodeInfo(datanodeDetails)); + nodeMap.put(id, new DatanodeInfo(datanodeDetails, nodeStatus)); nodeToContainer.put(id, Collections.emptySet()); - stateMap.get(nodeState).add(id); } finally { lock.writeLock().unlock(); } } /** - * Updates the node state. + * Updates the node health state. * * @param nodeId Node Id - * @param currentState current state - * @param newState new state + * @param newHealth new health state * * @throws NodeNotFoundException if the node is not present */ - public void updateNodeState(UUID nodeId, NodeState currentState, - NodeState newState)throws NodeNotFoundException { - lock.writeLock().lock(); + public NodeStatus updateNodeHealthState(UUID nodeId, NodeState newHealth) + throws NodeNotFoundException { try { - checkIfNodeExist(nodeId); - if (stateMap.get(currentState).remove(nodeId)) { - stateMap.get(newState).add(nodeId); - } else { - throw new NodeNotFoundException("Node UUID: " + nodeId + - ", not found in state: " + currentState); - } + lock.writeLock().lock(); + DatanodeInfo dn = getNodeInfo(nodeId); + NodeStatus oldStatus = dn.getNodeStatus(); + NodeStatus newStatus = new NodeStatus( + oldStatus.getOperationalState(), newHealth); + dn.setNodeStatus(newStatus); + return newStatus; + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Updates the node operational state. + * + * @param nodeId Node Id + * @param newOpState new operational state + * + * @throws NodeNotFoundException if the node is not present + */ + public NodeStatus updateNodeOperationalState(UUID nodeId, + NodeOperationalState newOpState) throws NodeNotFoundException { + try { + lock.writeLock().lock(); + DatanodeInfo dn = getNodeInfo(nodeId); + NodeStatus oldStatus = dn.getNodeStatus(); + NodeStatus newStatus = new NodeStatus( + newOpState, oldStatus.getHealth()); + dn.setNodeStatus(newStatus); + return newStatus; } finally { lock.writeLock().unlock(); } @@ -139,21 +146,38 @@ public DatanodeInfo getNodeInfo(UUID uuid) throws NodeNotFoundException { } } - /** * Returns the list of node ids which are in the specified state. * - * @param state NodeState + * @param status NodeStatus * * @return list of node ids */ - public List getNodes(NodeState state) { - lock.readLock().lock(); - try { - return new ArrayList<>(stateMap.get(state)); - } finally { - lock.readLock().unlock(); + public List getNodes(NodeStatus status) { + ArrayList nodes = new ArrayList<>(); + for (DatanodeInfo dn : filterNodes(status)) { + nodes.add(dn.getUuid()); + } + return nodes; + } + + /** + * Returns the list of node ids which match the desired operational state + * and health. Passing a null for either value is equivalent to a wild card. + * + * Therefore, passing opState = null, health=stale will return all stale nodes + * regardless of their operational state. + * + * @param opState + * @param health + * @return The list of nodes matching the given states + */ + public List getNodes(NodeOperationalState opState, NodeState health) { + ArrayList nodes = new ArrayList<>(); + for (DatanodeInfo dn : filterNodes(opState, health)) { + nodes.add(dn.getUuid()); } + return nodes; } /** @@ -162,8 +186,8 @@ public List getNodes(NodeState state) { * @return list of all the node ids */ public List getAllNodes() { - lock.readLock().lock(); try { + lock.readLock().lock(); return new ArrayList<>(nodeMap.keySet()); } finally { lock.readLock().unlock(); @@ -171,21 +195,71 @@ public List getAllNodes() { } /** - * Returns the count of nodes in the specified state. - * - * @param state NodeState + * Returns the list of all the nodes as DatanodeInfo objects. * - * @return Number of nodes in the specified state + * @return list of all the node ids */ - public int getNodeCount(NodeState state) { - lock.readLock().lock(); + public List getAllDatanodeInfos() { try { - return stateMap.get(state).size(); + lock.readLock().lock(); + return new ArrayList<>(nodeMap.values()); } finally { lock.readLock().unlock(); } } + /** + * Returns a list of the nodes as DatanodeInfo objects matching the passed + * status. + * + * @param status - The status of the nodes to return + * @return List of DatanodeInfo for the matching nodes + */ + public List getDatanodeInfos(NodeStatus status) { + return filterNodes(status); + } + + /** + * Returns a list of the nodes as DatanodeInfo objects matching the passed + * states. Passing null for either of the state values acts as a wildcard + * for that state. + * + * @param opState - The node operational state + * @param health - The node health + * @return List of DatanodeInfo for the matching nodes + */ + public List getDatanodeInfos( + NodeOperationalState opState, NodeState health) { + return filterNodes(opState, health); + } + + /** + * Returns the count of nodes in the specified state. + * + * @param state NodeStatus + * + * @return Number of nodes in the specified state + */ + public int getNodeCount(NodeStatus state) { + return getNodes(state).size(); + } + + /** + * Returns the count of node ids which match the desired operational state + * and health. Passing a null for either value is equivalent to a wild card. + * + * Therefore, passing opState=null, health=stale will count all stale nodes + * regardless of their operational state. + * + * @param opState + * @param health + * + * @return Number of nodes in the specified state + */ + public int getNodeCount(NodeOperationalState opState, NodeState health) { + return getNodes(opState, health).size(); + } + /** * Returns the total node count. * @@ -209,17 +283,15 @@ public int getTotalNodeCount() { * * @throws NodeNotFoundException if the node is not found */ - public NodeState getNodeState(UUID uuid) throws NodeNotFoundException { + public NodeStatus getNodeStatus(UUID uuid) throws NodeNotFoundException { lock.readLock().lock(); try { - checkIfNodeExist(uuid); - for (Map.Entry> entry : stateMap.entrySet()) { - if (entry.getValue().contains(uuid)) { - return entry.getKey(); - } + DatanodeInfo dn = nodeMap.get(uuid); + if (dn == null) { + throw new NodeNotFoundException("Node not found in node map." + + " UUID: " + uuid); } - throw new NodeNotFoundException("Node not found in node state map." + - " UUID: " + uuid); + return dn.getNodeStatus(); } finally { lock.readLock().unlock(); } @@ -289,12 +361,13 @@ public void removeContainer(UUID uuid, ContainerID containerID) throws */ @Override public String toString() { + // TODO - fix this method to include the commented out values StringBuilder builder = new StringBuilder(); builder.append("Total number of nodes: ").append(getTotalNodeCount()); - for (NodeState state : NodeState.values()) { - builder.append("Number of nodes in ").append(state).append(" state: ") - .append(getNodeCount(state)); - } + // for (NodeState state : NodeState.values()) { + // builder.append("Number of nodes in ").append(state).append(" state: ") + // .append(getNodeCount(state)); + // } return builder.toString(); } @@ -309,4 +382,50 @@ private void checkIfNodeExist(UUID uuid) throws NodeNotFoundException { throw new NodeNotFoundException("Node UUID: " + uuid); } } + + /** + * Create a list of datanodeInfo for all nodes matching the passed states. + * Passing null for one of the states acts like a wildcard for that state. + * + * @param opState + * @param health + * @return List of DatanodeInfo objects matching the passed state + */ + private List filterNodes( + NodeOperationalState opState, NodeState health) { + if (opState != null && health != null) { + return filterNodes(new NodeStatus(opState, health)); + } + if (opState == null && health == null) { + return getAllDatanodeInfos(); + } + try { + lock.readLock().lock(); + return nodeMap.values().stream() + .filter(n -> opState == null + || n.getNodeStatus().getOperationalState() == opState) + .filter(n -> health == null + || n.getNodeStatus().getHealth() == health) + .collect(Collectors.toList()); + } finally { + lock.readLock().unlock(); + } + } + + /** + * Create a list of datanodeInfo for all nodes matching the passsed status. + * + * @param status + * @return List of DatanodeInfo objects matching the passed state + */ + private List filterNodes(NodeStatus status) { + try { + lock.readLock().lock(); + return nodeMap.values().stream() + .filter(n -> n.getNodeStatus().equals(status)) + .collect(Collectors.toList()); + } finally { + lock.readLock().unlock(); + } + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index a5e3d37eb92d..1cebef629ba6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@ -22,12 +22,12 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.hdds.ratis.RatisHelper; @@ -141,7 +141,7 @@ public Pipeline create(ReplicationFactor factor) throws IOException { // Get list of healthy nodes List dns = - nodeManager.getNodes(NodeState.HEALTHY) + nodeManager.getNodes(NodeStatus.inServiceHealthy()) .parallelStream() .filter(dn -> !dnsUsed.contains(dn)) .limit(factor.getNumber()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java index ab98dfa3ed7b..dad26b5842fe 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java @@ -21,8 +21,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState; import java.io.IOException; @@ -43,7 +43,7 @@ public SimplePipelineProvider(NodeManager nodeManager) { @Override public Pipeline create(ReplicationFactor factor) throws IOException { List dns = - nodeManager.getNodes(NodeState.HEALTHY); + nodeManager.getNodes(NodeStatus.inServiceHealthy()); if (dns.size() < factor.getNumber()) { String e = String .format("Cannot create pipeline of factor %d using %d nodes.", diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 7d9cb3e24646..9e9d2fe3afab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -548,7 +548,9 @@ public boolean getSafeModeStatus() { */ private Set queryNodeState(HddsProtos.NodeState nodeState) { Set returnSet = new TreeSet<>(); - List tmp = scm.getScmNodeManager().getNodes(nodeState); + // TODO - decomm states needed + List tmp = scm.getScmNodeManager() + .getNodes(null, nodeState); if ((tmp != null) && (tmp.size() > 0)) { returnSet.addAll(tmp); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 4ecab375c1bb..702102b5ba16 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -923,7 +923,8 @@ public void join() { * @return int -- count */ public int getNodeCount(NodeState nodestate) { - return scmNodeManager.getNodeCount(nodestate); + // TODO - decomm - this probably needs to accept opState and health + return scmNodeManager.getNodeCount(null, nodestate); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index e5c4766697d5..bc1e77113a1a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus; import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -264,7 +265,7 @@ public void testMultipleBlockAllocationWithClosedContainer() // create pipelines for (int i = 0; - i < nodeManager.getNodes(HddsProtos.NodeState.HEALTHY).size(); i++) { + i < nodeManager.getNodes(NodeStatus.inServiceHealthy()).size(); i++) { pipelineManager.createPipeline(type, factor); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index b7a9813483d1..e27a451ef123 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.scm.net.NetConstants; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.Node; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; @@ -147,14 +148,28 @@ public void setSafemode(boolean safemode) { this.safemode = safemode; } + /** * Gets all Live Datanodes that is currently communicating with SCM. * - * @param nodestate - State of the node + * @param status The status of the node + * @return List of Datanodes that are Heartbeating SCM. + */ + @Override + public List getNodes(NodeStatus status) { + return getNodes(status.getOperationalState(), status.getHealth()); + } + + /** + * Gets all Live Datanodes that is currently communicating with SCM. + * + * @param opState - The operational State of the node + * @param nodestate - The health of the node * @return List of Datanodes that are Heartbeating SCM. */ @Override - public List getNodes(HddsProtos.NodeState nodestate) { + public List getNodes( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState nodestate) { if (nodestate == HEALTHY) { return healthyNodes; } @@ -170,6 +185,17 @@ public List getNodes(HddsProtos.NodeState nodestate) { return null; } + /** + * Returns the Number of Datanodes that are communicating with SCM. + * + * @param status - Status of the node + * @return int -- count + */ + @Override + public int getNodeCount(NodeStatus status) { + return getNodeCount(status.getOperationalState(), status.getHealth()); + } + /** * Returns the Number of Datanodes that are communicating with SCM. * @@ -177,8 +203,9 @@ public List getNodes(HddsProtos.NodeState nodestate) { * @return int -- count */ @Override - public int getNodeCount(HddsProtos.NodeState nodestate) { - List nodes = getNodes(nodestate); + public int getNodeCount( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState nodestate) { + List nodes = getNodes(opState, nodestate); if (nodes != null) { return nodes.size(); } @@ -419,7 +446,7 @@ public Boolean isNodeRegistered( public Map getNodeCount() { Map nodeCountMap = new HashMap(); for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) { - nodeCountMap.put(state.toString(), getNodeCount(state)); + nodeCountMap.put(state.toString(), getNodeCount(null, state)); } return nodeCountMap; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java index 41585bc8f7d2..2a93e3bc97c0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; @@ -28,6 +27,7 @@ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server .SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode; @@ -114,7 +114,7 @@ public void testUnderReplicatedContainer() final ContainerReportHandler reportHandler = new ContainerReportHandler( nodeManager, containerManager); final Iterator nodeIterator = nodeManager.getNodes( - NodeState.HEALTHY).iterator(); + NodeStatus.inServiceHealthy()).iterator(); final DatanodeDetails datanodeOne = nodeIterator.next(); final DatanodeDetails datanodeTwo = nodeIterator.next(); final DatanodeDetails datanodeThree = nodeIterator.next(); @@ -183,7 +183,7 @@ public void testOverReplicatedContainer() throws NodeNotFoundException, nodeManager, containerManager); final Iterator nodeIterator = nodeManager.getNodes( - NodeState.HEALTHY).iterator(); + NodeStatus.inServiceHealthy()).iterator(); final DatanodeDetails datanodeOne = nodeIterator.next(); final DatanodeDetails datanodeTwo = nodeIterator.next(); final DatanodeDetails datanodeThree = nodeIterator.next(); @@ -262,7 +262,7 @@ public void testClosingToClosed() throws NodeNotFoundException, IOException { nodeManager, containerManager); final Iterator nodeIterator = nodeManager.getNodes( - NodeState.HEALTHY).iterator(); + NodeStatus.inServiceHealthy()).iterator(); final DatanodeDetails datanodeOne = nodeIterator.next(); final DatanodeDetails datanodeTwo = nodeIterator.next(); final DatanodeDetails datanodeThree = nodeIterator.next(); @@ -341,7 +341,7 @@ public void testClosingToQuasiClosed() nodeManager, containerManager); final Iterator nodeIterator = nodeManager.getNodes( - NodeState.HEALTHY).iterator(); + NodeStatus.inServiceHealthy()).iterator(); final DatanodeDetails datanodeOne = nodeIterator.next(); final DatanodeDetails datanodeTwo = nodeIterator.next(); final DatanodeDetails datanodeThree = nodeIterator.next(); @@ -418,7 +418,7 @@ public void testQuasiClosedToClosed() final ContainerReportHandler reportHandler = new ContainerReportHandler( nodeManager, containerManager); final Iterator nodeIterator = nodeManager.getNodes( - NodeState.HEALTHY).iterator(); + NodeStatus.inServiceHealthy()).iterator(); final DatanodeDetails datanodeOne = nodeIterator.next(); final DatanodeDetails datanodeTwo = nodeIterator.next(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index 18c4a64a0404..54c4080c4afc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -19,7 +19,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; @@ -29,6 +28,7 @@ import org.apache.hadoop.hdds.scm.net.NodeSchema; import org.apache.hadoop.hdds.scm.net.NodeSchemaManager; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -88,7 +88,7 @@ public void testRackAwarePolicy() throws IOException { // create mock node manager nodeManager = Mockito.mock(NodeManager.class); - when(nodeManager.getNodes(NodeState.HEALTHY)) + when(nodeManager.getNodes(NodeStatus.inServiceHealthy())) .thenReturn(new ArrayList<>(datanodes)); when(nodeManager.getNodeStat(anyObject())) .thenReturn(new SCMNodeMetric(storageCapacity, 0L, 100L)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java index 00ec3988e8af..fcb4f447874b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java @@ -24,12 +24,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.junit.Assert; import org.junit.Test; import static org.mockito.Matchers.anyObject; @@ -51,7 +51,7 @@ public void chooseDatanodes() throws SCMException { } NodeManager mockNodeManager = Mockito.mock(NodeManager.class); - when(mockNodeManager.getNodes(NodeState.HEALTHY)) + when(mockNodeManager.getNodes(NodeStatus.inServiceHealthy())) .thenReturn(new ArrayList<>(datanodes)); when(mockNodeManager.getNodeStat(anyObject())) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index 2d8b81633e75..03dd82989d24 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.exceptions.SCMException; @@ -30,6 +29,7 @@ import org.apache.hadoop.hdds.scm.net.NodeSchema; import org.apache.hadoop.hdds.scm.net.NodeSchemaManager; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -102,7 +102,7 @@ public void setup() { // create mock node manager nodeManager = Mockito.mock(NodeManager.class); - when(nodeManager.getNodes(NodeState.HEALTHY)) + when(nodeManager.getNodes(NodeStatus.inServiceHealthy())) .thenReturn(new ArrayList<>(datanodes)); when(nodeManager.getNodeStat(anyObject())) .thenReturn(new SCMNodeMetric(STORAGE_CAPACITY, 0L, 100L)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java index 43e3a8d13465..5edb25f25c70 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java @@ -22,12 +22,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.junit.Assert; import org.junit.Test; import static org.mockito.Matchers.anyObject; @@ -50,7 +50,7 @@ public void chooseDatanodes() throws SCMException { } NodeManager mockNodeManager = Mockito.mock(NodeManager.class); - when(mockNodeManager.getNodes(NodeState.HEALTHY)) + when(mockNodeManager.getNodes(NodeStatus.inServiceHealthy())) .thenReturn(new ArrayList<>(datanodes)); when(mockNodeManager.getNodeStat(anyObject())) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 26ffd8d1d34d..af9d5e51f467 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -150,7 +150,7 @@ public void testContainerPlacementCapacity() throws IOException, //TODO: wait for heartbeat to be processed Thread.sleep(4 * 1000); - assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); + assertEquals(nodeCount, nodeManager.getNodeCount(null, HEALTHY)); assertEquals(capacity * nodeCount, (long) nodeManager.getStats().getCapacity().get()); assertEquals(used * nodeCount, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java new file mode 100644 index 000000000000..bc28a438892a --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.scm.HddsServerUtil; +import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.server.events.Event; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.util.Time; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNull; + +/** + * Class to test the NodeStateManager, which is an internal class used by + * the SCMNodeManager. + */ + +public class TestNodeStateManager { + + private NodeStateManager nsm; + private Configuration conf; + private MockEventPublisher eventPublisher; + + @Before + public void setUp() { + conf = new Configuration(); + eventPublisher = new MockEventPublisher(); + nsm = new NodeStateManager(conf, eventPublisher); + } + + @After + public void tearDown() { + } + + @Test + public void testNodeCanBeAddedAndRetrieved() + throws NodeAlreadyExistsException, NodeNotFoundException { + // Create a datanode, then add and retrieve it + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + assertEquals(dn.getUuid(), nsm.getNode(dn).getUuid()); + // Now get the status of the newly added node and it should be + // IN_SERVICE and HEALTHY + NodeStatus expectedState = NodeStatus.inServiceHealthy(); + assertEquals(expectedState, nsm.getNodeStatus(dn)); + } + + @Test + public void testGetAllNodesReturnsCorrectly() + throws NodeAlreadyExistsException { + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + dn = generateDatanode(); + nsm.addNode(dn); + assertEquals(2, nsm.getAllNodes().size()); + assertEquals(2, nsm.getTotalNodeCount()); + } + + @Test + public void testGetNodeCountReturnsCorrectly() + throws NodeAlreadyExistsException { + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + assertEquals(1, nsm.getNodes(NodeStatus.inServiceHealthy()).size()); + assertEquals(0, nsm.getNodes(NodeStatus.inServiceStale()).size()); + } + + @Test + public void testGetNodeCount() throws NodeAlreadyExistsException { + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + assertEquals(1, nsm.getNodeCount(NodeStatus.inServiceHealthy())); + assertEquals(0, nsm.getNodeCount(NodeStatus.inServiceStale())); + } + + @Test + public void testNodesMarkedDeadAndStale() + throws NodeAlreadyExistsException, NodeNotFoundException { + long now = Time.monotonicNow(); + + // Set the dead and stale limits to be 1 second larger than configured + long staleLimit = HddsServerUtil.getStaleNodeInterval(conf) + 1000; + long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; + + DatanodeDetails staleDn = generateDatanode(); + nsm.addNode(staleDn); + nsm.getNode(staleDn).updateLastHeartbeatTime(now - staleLimit); + + DatanodeDetails deadDn = generateDatanode(); + nsm.addNode(deadDn); + nsm.getNode(deadDn).updateLastHeartbeatTime(now - deadLimit); + + DatanodeDetails healthyDn = generateDatanode(); + nsm.addNode(healthyDn); + nsm.getNode(healthyDn).updateLastHeartbeatTime(); + + nsm.checkNodesHealth(); + assertEquals(healthyDn, nsm.getHealthyNodes().get(0)); + // A node cannot go directly to dead. It must be marked stale first + // due to the allowed state transitions. Therefore we will initially have 2 + // stale nodesCheck it is in stale nodes + assertEquals(2, nsm.getStaleNodes().size()); + // Now check health again and it should be in deadNodes() + nsm.checkNodesHealth(); + assertEquals(staleDn, nsm.getStaleNodes().get(0)); + assertEquals(deadDn, nsm.getDeadNodes().get(0)); + } + + @Test + public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() + throws NodeAlreadyExistsException, NodeNotFoundException { + long now = Time.monotonicNow(); + + // Set the dead and stale limits to be 1 second larger than configured + long staleLimit = HddsServerUtil.getStaleNodeInterval(conf) + 1000; + long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; + + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + assertEquals("New_Node", eventPublisher.getLastEvent().getName()); + DatanodeInfo dni = nsm.getNode(dn); + dni.updateLastHeartbeatTime(); + + // Ensure node is initially healthy + eventPublisher.clearEvents(); + nsm.checkNodesHealth(); + assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); + assertNull(eventPublisher.getLastEvent()); + + // Set the heartbeat old enough to make it stale + dni.updateLastHeartbeatTime(now - staleLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); + assertEquals("Stale_Node", eventPublisher.getLastEvent().getName()); + + // Now make it dead + dni.updateLastHeartbeatTime(now - deadLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); + assertEquals("Dead_Node", eventPublisher.getLastEvent().getName()); + + // Transition back to healthy from dead + dni.updateLastHeartbeatTime(); + nsm.checkNodesHealth(); + assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); + assertEquals("NON_HEALTHY_TO_HEALTHY_NODE", + eventPublisher.getLastEvent().getName()); + + // Make the node stale again, and transition to healthy. + dni.updateLastHeartbeatTime(now - staleLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); + assertEquals("Stale_Node", eventPublisher.getLastEvent().getName()); + dni.updateLastHeartbeatTime(); + nsm.checkNodesHealth(); + assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); + assertEquals("NON_HEALTHY_TO_HEALTHY_NODE", + eventPublisher.getLastEvent().getName()); + } + + private DatanodeDetails generateDatanode() { + String uuid = UUID.randomUUID().toString(); + return DatanodeDetails.newBuilder().setUuid(uuid).build(); + } + + static class MockEventPublisher implements EventPublisher { + + private List events = new ArrayList<>(); + private List payloads = new ArrayList<>(); + + public void clearEvents() { + events.clear(); + payloads.clear(); + } + + public List getEvents() { + return events; + } + + public Event getLastEvent() { + if (events.size() == 0) { + return null; + } else { + return events.get(events.size()-1); + } + } + + @Override + public > void + fireEvent(EVENT_TYPE event, PAYLOAD payload) { + events.add(event); + payloads.add(payload); + } + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index d02885116857..a37142fee002 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -72,9 +72,6 @@ .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_STALENODE_INTERVAL; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState - .HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND; import static org.junit.Assert.assertEquals; @@ -238,7 +235,8 @@ public void testScmHealthyNodeCount() } //TODO: wait for heartbeat to be processed Thread.sleep(4 * 1000); - assertEquals(count, nodeManager.getNodeCount(HEALTHY)); + assertEquals(count, nodeManager.getNodeCount( + NodeStatus.inServiceHealthy())); } } @@ -312,9 +310,10 @@ public void testScmDetectStaleAndDeadNode() // Wait for 2 seconds, wait a total of 4 seconds to make sure that the // node moves into stale state. Thread.sleep(2 * 1000); - List staleNodeList = nodeManager.getNodes(STALE); + List staleNodeList = + nodeManager.getNodes(NodeStatus.inServiceStale()); assertEquals("Expected to find 1 stale node", - 1, nodeManager.getNodeCount(STALE)); + 1, nodeManager.getNodeCount(NodeStatus.inServiceStale())); assertEquals("Expected to find 1 stale node", 1, staleNodeList.size()); assertEquals("Stale node is not the expected ID", staleNode @@ -331,16 +330,17 @@ public void testScmDetectStaleAndDeadNode() Thread.sleep(2 * 1000); // the stale node has been removed - staleNodeList = nodeManager.getNodes(STALE); + staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale()); assertEquals("Expected to find 1 stale node", - 0, nodeManager.getNodeCount(STALE)); + 0, nodeManager.getNodeCount(NodeStatus.inServiceStale())); assertEquals("Expected to find 1 stale node", 0, staleNodeList.size()); // Check for the dead node now. - List deadNodeList = nodeManager.getNodes(DEAD); + List deadNodeList = + nodeManager.getNodes(NodeStatus.inServiceDead()); assertEquals("Expected to find 1 dead node", 1, - nodeManager.getNodeCount(DEAD)); + nodeManager.getNodeCount(NodeStatus.inServiceDead())); assertEquals("Expected to find 1 dead node", 1, deadNodeList.size()); assertEquals("Dead node is not the expected ID", staleNode @@ -388,8 +388,8 @@ public void testScmHandleJvmPause() //Assert all nodes are healthy. assertEquals(2, nodeManager.getAllNodes().size()); - assertEquals(2, nodeManager.getNodeCount(HEALTHY)); - + assertEquals(2, + nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); /** * Simulate a JVM Pause and subsequent handling in following steps: * Step 1 : stop heartbeat check process for stale node interval @@ -424,7 +424,7 @@ public void testScmHandleJvmPause() // Step 4 : all nodes should still be HEALTHY assertEquals(2, nodeManager.getAllNodes().size()); - assertEquals(2, nodeManager.getNodeCount(HEALTHY)); + assertEquals(2, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); // Step 5 : heartbeat for node1 nodeManager.processHeartbeat(node1); @@ -433,8 +433,8 @@ public void testScmHandleJvmPause() Thread.sleep(1000); // Step 7 : node2 should transition to STALE - assertEquals(1, nodeManager.getNodeCount(HEALTHY)); - assertEquals(1, nodeManager.getNodeCount(STALE)); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceStale())); } } @@ -533,7 +533,7 @@ public void testScmClusterIsInExpectedState1() //Assert all nodes are healthy. assertEquals(3, nodeManager.getAllNodes().size()); - assertEquals(3, nodeManager.getNodeCount(HEALTHY)); + assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); /** * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which @@ -541,7 +541,7 @@ public void testScmClusterIsInExpectedState1() */ Thread.sleep(3 * 1000); assertEquals(3, nodeManager.getAllNodes().size()); - assertEquals(3, nodeManager.getNodeCount(STALE)); + assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceStale())); /** @@ -559,18 +559,19 @@ public void testScmClusterIsInExpectedState1() Thread.sleep(1500); nodeManager.processHeartbeat(healthyNode); Thread.sleep(2 * 1000); - assertEquals(1, nodeManager.getNodeCount(HEALTHY)); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); // 3.5 seconds from last heartbeat for the stale and deadNode. So those // 2 nodes must move to Stale state and the healthy node must // remain in the healthy State. - List healthyList = nodeManager.getNodes(HEALTHY); + List healthyList = nodeManager.getNodes( + NodeStatus.inServiceHealthy()); assertEquals("Expected one healthy node", 1, healthyList.size()); assertEquals("Healthy node is not the expected ID", healthyNode .getUuid(), healthyList.get(0).getUuid()); - assertEquals(2, nodeManager.getNodeCount(STALE)); + assertEquals(2, nodeManager.getNodeCount(NodeStatus.inServiceStale())); /** * Cluster State: Allow healthyNode to remain in healthy state and @@ -586,14 +587,16 @@ public void testScmClusterIsInExpectedState1() // 3.5 seconds have elapsed for stale node, so it moves into Stale. // 7 seconds have elapsed for dead node, so it moves into dead. // 2 Seconds have elapsed for healthy node, so it stays in healthy state. - healthyList = nodeManager.getNodes(HEALTHY); - List staleList = nodeManager.getNodes(STALE); - List deadList = nodeManager.getNodes(DEAD); + healthyList = nodeManager.getNodes((NodeStatus.inServiceHealthy())); + List staleList = + nodeManager.getNodes(NodeStatus.inServiceStale()); + List deadList = + nodeManager.getNodes(NodeStatus.inServiceDead()); assertEquals(3, nodeManager.getAllNodes().size()); - assertEquals(1, nodeManager.getNodeCount(HEALTHY)); - assertEquals(1, nodeManager.getNodeCount(STALE)); - assertEquals(1, nodeManager.getNodeCount(DEAD)); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceStale())); + assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceDead())); assertEquals("Expected one healthy node", 1, healthyList.size()); @@ -619,7 +622,7 @@ public void testScmClusterIsInExpectedState1() Thread.sleep(500); //Assert all nodes are healthy. assertEquals(3, nodeManager.getAllNodes().size()); - assertEquals(3, nodeManager.getNodeCount(HEALTHY)); + assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); } } @@ -668,7 +671,7 @@ private List createNodeSet(SCMNodeManager nodeManager, int */ private boolean findNodes(NodeManager nodeManager, int count, HddsProtos.NodeState state) { - return count == nodeManager.getNodeCount(state); + return count == nodeManager.getNodeCount(NodeStatus.inServiceStale()); } /** @@ -741,11 +744,14 @@ public void testScmClusterIsInExpectedState2() // Assert all healthy nodes are healthy now, this has to be a greater // than check since Stale nodes can be healthy when we check the state. - assertTrue(nodeManager.getNodeCount(HEALTHY) >= healthyCount); + assertTrue(nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) + >= healthyCount); - assertEquals(deadCount, nodeManager.getNodeCount(DEAD)); + assertEquals(deadCount, + nodeManager.getNodeCount(NodeStatus.inServiceDead())); - List deadList = nodeManager.getNodes(DEAD); + List deadList = + nodeManager.getNodes(NodeStatus.inServiceDead()); for (DatanodeDetails node : deadList) { assertTrue(deadNodeList.contains(node)); @@ -861,7 +867,8 @@ public void testScmStatsFromNodeReport() } //TODO: wait for heartbeat to be processed Thread.sleep(4 * 1000); - assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); + assertEquals(nodeCount, + nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); assertEquals(capacity * nodeCount, (long) nodeManager.getStats() .getCapacity().get()); assertEquals(used * nodeCount, (long) nodeManager.getStats() @@ -953,7 +960,7 @@ public void testScmNodeReportUpdate() // Wait up to 4s so that the node becomes stale // Verify the usage info should be unchanged. GenericTestUtils.waitFor( - () -> nodeManager.getNodeCount(STALE) == 1, 100, + () -> nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1, 100, 4 * 1000); assertEquals(nodeCount, nodeManager.getNodeStats().size()); @@ -971,7 +978,7 @@ public void testScmNodeReportUpdate() // Wait up to 4 more seconds so the node becomes dead // Verify usage info should be updated. GenericTestUtils.waitFor( - () -> nodeManager.getNodeCount(DEAD) == 1, 100, + () -> nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1, 100, 4 * 1000); assertEquals(0, nodeManager.getNodeStats().size()); @@ -989,7 +996,7 @@ public void testScmNodeReportUpdate() // Wait up to 5 seconds so that the dead node becomes healthy // Verify usage info should be updated. GenericTestUtils.waitFor( - () -> nodeManager.getNodeCount(HEALTHY) == 1, + () -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5 * 1000); GenericTestUtils.waitFor( () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed, @@ -1100,7 +1107,8 @@ public void testScmRegisterNodeWith4LayerNetworkTopology() // verify network topology cluster has all the registered nodes Thread.sleep(4 * 1000); NetworkTopology clusterMap = scm.getClusterMap(); - assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); + assertEquals(nodeCount, + nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); assertEquals(nodeCount, clusterMap.getNumOfLeafNode("")); assertEquals(4, clusterMap.getMaxLevel()); List nodeList = nodeManager.getAllNodes(); @@ -1142,7 +1150,8 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) // verify network topology cluster has all the registered nodes Thread.sleep(4 * 1000); NetworkTopology clusterMap = scm.getClusterMap(); - assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY)); + assertEquals(nodeCount, + nodeManager.getNodeCount(NodeStatus.inServiceHealthy())); assertEquals(nodeCount, clusterMap.getNumOfLeafNode("")); assertEquals(3, clusterMap.getMaxLevel()); List nodeList = nodeManager.getAllNodes(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java new file mode 100644 index 000000000000..482f444e3c3c --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node.states; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.UUID; + +import static junit.framework.TestCase.assertEquals; + +/** + * Class to test the NodeStateMap class, which is an internal class used by + * NodeStateManager. + */ + +public class TestNodeStateMap { + + private NodeStateMap map; + + @Before + public void setUp() { + map = new NodeStateMap(); + } + + @After + public void tearDown() { + } + + @Test + public void testNodeCanBeAddedAndRetrieved() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + NodeStatus status = NodeStatus.inServiceHealthy(); + map.addNode(dn, status); + assertEquals(dn, map.getNodeInfo(dn.getUuid())); + assertEquals(status, map.getNodeStatus(dn.getUuid())); + } + + @Test + public void testNodeHealthStateCanBeUpdated() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + NodeStatus status = NodeStatus.inServiceHealthy(); + map.addNode(dn, status); + + NodeStatus expectedStatus = NodeStatus.inServiceStale(); + NodeStatus returnedStatus = + map.updateNodeHealthState(dn.getUuid(), expectedStatus.getHealth()); + assertEquals(expectedStatus, returnedStatus); + assertEquals(returnedStatus, map.getNodeStatus(dn.getUuid())); + } + + @Test + public void testNodeOperationalStateCanBeUpdated() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + NodeStatus status = NodeStatus.inServiceHealthy(); + map.addNode(dn, status); + + NodeStatus expectedStatus = new NodeStatus( + NodeOperationalState.DECOMMISSIONING, + NodeState.HEALTHY); + NodeStatus returnedStatus = map.updateNodeOperationalState( + dn.getUuid(), expectedStatus.getOperationalState()); + assertEquals(expectedStatus, returnedStatus); + assertEquals(returnedStatus, map.getNodeStatus(dn.getUuid())); + } + + @Test + public void testGetNodeMethodsReturnCorrectCountsAndStates() + throws NodeAlreadyExistsException { + // Add one node for all possible states + int nodeCount = 0; + for(NodeOperationalState op : NodeOperationalState.values()) { + for(NodeState health : NodeState.values()) { + addRandomNodeWithState(op, health); + nodeCount++; + } + } + NodeStatus requestedState = NodeStatus.inServiceStale(); + List nodes = map.getNodes(requestedState); + assertEquals(1, nodes.size()); + assertEquals(1, map.getNodeCount(requestedState)); + assertEquals(nodeCount, map.getTotalNodeCount()); + assertEquals(nodeCount, map.getAllNodes().size()); + assertEquals(nodeCount, map.getAllDatanodeInfos().size()); + + // Checks for the getNodeCount(opstate, health) method + assertEquals(nodeCount, map.getNodeCount(null, null)); + assertEquals(1, + map.getNodeCount(NodeOperationalState.DECOMMISSIONING, + NodeState.STALE)); + assertEquals(5, map.getNodeCount(null, NodeState.HEALTHY)); + assertEquals(3, + map.getNodeCount(NodeOperationalState.DECOMMISSIONING, null)); + } + + private void addNodeWithState(DatanodeDetails dn, + NodeOperationalState opState, NodeState health) + throws NodeAlreadyExistsException { + NodeStatus status = new NodeStatus(opState, health); + map.addNode(dn, status); + } + + private void addRandomNodeWithState( + NodeOperationalState opState, NodeState health) + throws NodeAlreadyExistsException { + DatanodeDetails dn = generateDatanode(); + addNodeWithState(dn, opState, health); + } + + private DatanodeDetails generateDatanode() { + String uuid = UUID.randomUUID().toString(); + return DatanodeDetails.newBuilder().setUuid(uuid).build(); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java index f0b1cbb146b7..2ac5f7041438 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/placement/TestContainerPlacement.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.ozone.OzoneConsts; import org.junit.Assert; import org.junit.Test; @@ -34,8 +35,6 @@ import java.util.List; import java.util.Random; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState - .HEALTHY; import static org.junit.Assert.assertEquals; /** @@ -45,7 +44,8 @@ public class TestContainerPlacement { private DescriptiveStatistics computeStatistics(NodeManager nodeManager) { DescriptiveStatistics descriptiveStatistics = new DescriptiveStatistics(); - for (DatanodeDetails dd : nodeManager.getNodes(HEALTHY)) { + for (DatanodeDetails dd : + nodeManager.getNodes(NodeStatus.inServiceHealthy())) { float weightedValue = nodeManager.getNodeStat(dd).get().getScmUsed().get() / (float) nodeManager.getNodeStat(dd).get().getCapacity().get(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index 30a75efb1942..b584f3fde3e0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -17,9 +17,11 @@ package org.apache.hadoop.ozone.container.testutils; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; @@ -81,22 +83,48 @@ public Map getNodeInfo() { /** * Gets all Live Datanodes that is currently communicating with SCM. * - * @param nodestate - State of the node + * @param nodestatus - State of the node * @return List of Datanodes that are Heartbeating SCM. */ @Override - public List getNodes(NodeState nodestate) { + public List getNodes(NodeStatus nodestatus) { return null; } + /** + * Gets all Live Datanodes that is currently communicating with SCM. + * + * @param opState - Operational state of the node + * @param health - Health of the node + * @return List of Datanodes that are Heartbeating SCM. + */ + @Override + public List getNodes( + HddsProtos.NodeOperationalState opState, NodeState health) { + return null; + } + + /** + * Returns the Number of Datanodes that are communicating with SCM. + * + * @param nodestatus - State of the node + * @return int -- count + */ + @Override + public int getNodeCount(NodeStatus nodestatus) { + return 0; + } + /** * Returns the Number of Datanodes that are communicating with SCM. * - * @param nodestate - State of the node + * @param opState - Operational state of the node + * @param health - Health of the node * @return int -- count */ @Override - public int getNodeCount(NodeState nodestate) { + public int getNodeCount( + HddsProtos.NodeOperationalState opState, NodeState health) { return 0; } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index 7de2e4be9c45..42773f82308f 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -25,8 +25,6 @@ import picocli.CommandLine; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONED; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DECOMMISSIONING; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; @@ -57,8 +55,6 @@ public class TopologySubcommand implements Callable { stateArray.add(HEALTHY); stateArray.add(STALE); stateArray.add(DEAD); - stateArray.add(DECOMMISSIONING); - stateArray.add(DECOMMISSIONED); } @CommandLine.Option(names = {"-o", "--order"}, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index ba072f81e9fd..225281a434b8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto; @@ -66,6 +65,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.DatanodeInfo; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -366,8 +366,7 @@ public void testBlockDeletingThrottling() throws Exception { NodeManager nodeManager = cluster.getStorageContainerManager() .getScmNodeManager(); List commands = nodeManager.processHeartbeat( - nodeManager.getNodes(NodeState.HEALTHY).get(0)); - + nodeManager.getNodes(NodeStatus.inServiceHealthy()).get(0)); if (commands != null) { for (SCMCommand cmd : commands) { if (cmd.getType() == SCMCommandProto.Type.deleteBlocksCommand) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index 65a6357de9d5..98a22a269951 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -149,10 +149,6 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissioningNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DiskCapacity", 100L, getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DiskUsed", 10L, From 1be2b033bd8cde8347531fc0c38a932c3e9302fd Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 23 Sep 2019 18:23:47 +0100 Subject: [PATCH 02/78] Create admin commands and protobuf messages to allow decommission / recommission and maintenance commands to be sent from the CLI and update the node status in a skeleton decommission manager --- .../scm/client/ContainerOperationClient.java | 16 + .../hadoop/hdds/scm/client/ScmClient.java | 29 ++ .../StorageContainerLocationProtocol.java | 7 + ...ocationProtocolClientSideTranslatorPB.java | 58 ++++ .../StorageContainerLocationProtocol.proto | 45 ++- .../scm/node/InvalidHostStringException.java | 34 +++ .../scm/node/NodeDecommissionManager.java | 286 ++++++++++++++++++ .../hadoop/hdds/scm/node/NodeManager.java | 14 +- .../hdds/scm/node/NodeStateManager.java | 15 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 19 +- ...ocationProtocolServerSideTranslatorPB.java | 54 ++++ .../scm/server/SCMClientProtocolServer.java | 37 +++ .../scm/server/StorageContainerManager.java | 15 + .../hdds/scm/container/MockNodeManager.java | 11 +- .../scm/node/TestNodeDecommissionManager.java | 253 ++++++++++++++++ .../hdds/scm/node/TestNodeStateManager.java | 17 ++ .../testutils/ReplicationNodeManagerMock.java | 34 ++- .../apache/hadoop/hdds/scm/cli/SCMCLI.java | 4 +- .../scm/cli/node/DatanodeAdminCommands.java | 55 ++++ .../DatanodeAdminDecommissionSubCommand.java | 58 ++++ .../DatanodeAdminMaintenanceSubCommand.java | 63 ++++ .../DatanodeAdminRecommissionSubCommand.java | 58 ++++ .../hdds/scm/cli/node/package-info.java | 23 ++ .../node/TestDecommissionAndMaintenance.java | 137 +++++++++ 24 files changed, 1322 insertions(+), 20 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidHostStringException.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminDecommissionSubCommand.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminMaintenanceSubCommand.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminRecommissionSubCommand.java create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/package-info.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java index 982fb8ea1eec..93b09d9ce591 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java @@ -212,6 +212,22 @@ public List queryNode(HddsProtos.NodeState poolName); } + @Override + public void decommissionNodes(List hosts) throws IOException { + storageContainerLocationClient.decommissionNodes(hosts); + } + + @Override + public void recommissionNodes(List hosts) throws IOException { + storageContainerLocationClient.recommissionNodes(hosts); + } + + @Override + public void startMaintenanceNodes(List hosts, int endHours) + throws IOException { + storageContainerLocationClient.startMaintenanceNodes(hosts, endHours); + } + /** * Creates a specified replication pipeline. */ diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 226ceda9255a..8238e8469a9f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -161,6 +161,35 @@ ContainerWithPipeline createContainer(HddsProtos.ReplicationType type, List queryNode(HddsProtos.NodeState nodeStatuses, HddsProtos.QueryScope queryScope, String poolName) throws IOException; + /** + * Allows a list of hosts to be decommissioned. The hosts are identified + * by their hostname and optionally port in the format foo.com:port. + * @param hosts A list of hostnames, optionally with port + * @throws IOException + */ + void decommissionNodes(List hosts) throws IOException; + + /** + * Allows a list of hosts in maintenance or decommission states to be placed + * back in service. The hosts are identified by their hostname and optionally + * port in the format foo.com:port. + * @param hosts A list of hostnames, optionally with port + * @throws IOException + */ + void recommissionNodes(List hosts) throws IOException; + + /** + * Place the list of datanodes into maintenance mode. If a non-null endDtm + * is passed, the hosts will automatically exit maintenance mode after the + * given time has passed. The hosts are identified by their hostname and + * optionally port in the format foo.com:port. + * @param hosts A list of hostnames, optionally with port + * @param endHours The number of hours from now which maintenance will end + * @throws IOException + */ + void startMaintenanceNodes(List hosts, int endHours) + throws IOException; + /** * Creates a specified replication pipeline. * @param type - Type diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 88db8205a408..91cd40dc3f5a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -112,6 +112,13 @@ List listContainer(long startContainerID, int count) List queryNode(HddsProtos.NodeState state, HddsProtos.QueryScope queryScope, String poolName) throws IOException; + void decommissionNodes(List nodes) throws IOException; + + void recommissionNodes(List nodes) throws IOException; + + void startMaintenanceNodes(List nodes, int endInHours) + throws IOException; + /** * Notify from client when begin or finish creating objects like pipeline * or containers on datanodes. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 01db597dfae1..99941d1c3346 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -53,6 +53,9 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ScmContainerLocationResponse; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartReplicationManagerRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartMaintenanceNodesRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -256,6 +259,61 @@ public List queryNode(HddsProtos.NodeState } + /** + * Attempts to decommission the list of nodes. + * @param nodes The list of hostnames or hostname:ports to decommission + * @throws IOException + */ + @Override + public void decommissionNodes(List nodes) throws IOException { + Preconditions.checkNotNull(nodes); + DecommissionNodesRequestProto request = + DecommissionNodesRequestProto.newBuilder() + .addAllHosts(nodes) + .build(); + submitRequest(Type.DecommissionNodes, + builder -> builder.setDecommissionNodesRequest(request)); + } + + /** + * Attempts to recommission the list of nodes. + * @param nodes The list of hostnames or hostname:ports to recommission + * @throws IOException + */ + @Override + public void recommissionNodes(List nodes) throws IOException { + Preconditions.checkNotNull(nodes); + RecommissionNodesRequestProto request = + RecommissionNodesRequestProto.newBuilder() + .addAllHosts(nodes) + .build(); + submitRequest(Type.RecommissionNodes, + builder -> builder.setRecommissionNodesRequest(request)); + } + + /** + * Attempts to put the list of nodes into maintenance mode. + * + * @param nodes The list of hostnames or hostname:ports to put into + * maintenance + * @param endInHours A number of hours from now where the nodes will be taken + * out of maintenance automatically. Passing zero will + * allow the nodes to stay in maintenance indefinitely + * @throws IOException + */ + @Override + public void startMaintenanceNodes(List nodes, int endInHours) + throws IOException { + Preconditions.checkNotNull(nodes); + StartMaintenanceNodesRequestProto request = + StartMaintenanceNodesRequestProto.newBuilder() + .addAllHosts(nodes) + .setEndInHours(endInHours) + .build(); + submitRequest(Type.StartMaintenanceNodes, + builder -> builder.setStartMaintenanceNodesRequest(request)); + } + /** * Notify from client that creates object on datanodes. * diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto index 8ea72b6cd178..00e58c0bc851 100644 --- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto @@ -59,7 +59,9 @@ message ScmContainerLocationRequest { optional StartReplicationManagerRequestProto startReplicationManagerRequest = 21; optional StopReplicationManagerRequestProto stopReplicationManagerRequest = 22; optional ReplicationManagerStatusRequestProto seplicationManagerStatusRequest = 23; - + optional DecommissionNodesRequestProto decommissionNodesRequest = 24; + optional RecommissionNodesRequestProto recommissionNodesRequest = 25; + optional StartMaintenanceNodesRequestProto startMaintenanceNodesRequest = 26; } message ScmContainerLocationResponse { @@ -91,6 +93,9 @@ message ScmContainerLocationResponse { optional StartReplicationManagerResponseProto startReplicationManagerResponse = 21; optional StopReplicationManagerResponseProto stopReplicationManagerResponse = 22; optional ReplicationManagerStatusResponseProto replicationManagerStatusResponse = 23; + optional DecommissionNodesResponseProto decommissionNodesResponse = 24; + optional RecommissionNodesResponseProto recommissionNodesResponse = 25; + optional StartMaintenanceNodesResponseProto startMaintenanceNodesResponse = 26; enum Status { OK = 1; CONTAINER_ALREADY_EXISTS = 2; @@ -118,6 +123,9 @@ enum Type { StartReplicationManager = 16; StopReplicationManager = 17; GetReplicationManagerStatus = 18; + DecommissionNodes = 19; + RecommissionNodes = 20; + StartMaintenanceNodes = 21; } /** @@ -225,6 +233,40 @@ message NodeQueryResponseProto { repeated Node datanodes = 1; } +/* + Decommission a list of hosts +*/ +message DecommissionNodesRequestProto { + repeated string hosts = 1; +} + +message DecommissionNodesResponseProto { + // empty response +} + +/* + Recommission a list of hosts in maintenance or decommission states +*/ +message RecommissionNodesRequestProto { + repeated string hosts = 1; +} + +message RecommissionNodesResponseProto { + // empty response +} + +/* + Place a list of hosts into maintenance mode +*/ +message StartMaintenanceNodesRequestProto { + repeated string hosts = 1; + optional int64 endInHours = 2; +} + +message StartMaintenanceNodesResponseProto { + // empty response +} + /** Request to create a replication pipeline. */ @@ -326,5 +368,4 @@ message ReplicationManagerStatusResponseProto { */ service StorageContainerLocationProtocolService { rpc submitRequest (ScmContainerLocationRequest) returns (ScmContainerLocationResponse); - } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidHostStringException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidHostStringException.java new file mode 100644 index 000000000000..c4046c1456e9 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidHostStringException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import java.io.IOException; + +/** + * Exception thrown by the NodeDecommissionManager when it encounters + * host strings it does not expect or understand. + */ + +public class InvalidHostStringException extends IOException { + public InvalidHostStringException(String msg) { + super(msg); + } + + public InvalidHostStringException(String msg, Exception e) { + super(msg, e); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java new file mode 100644 index 000000000000..60813ddd4b71 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -0,0 +1,286 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.LinkedList; +import java.util.List; + +/** + * Class used to manage datanodes scheduled for maintenance or decommission. + */ +public class NodeDecommissionManager { + + private NodeManager nodeManager; + private PipelineManager pipeLineManager; + private ContainerManager containerManager; + private OzoneConfiguration conf; + private boolean useHostnames; + + private List pendingNodes = new LinkedList<>(); + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminManager.class); + + + static class HostDefinition { + private String rawHostname; + private String hostname; + private int port; + + HostDefinition(String hostname) throws InvalidHostStringException { + this.rawHostname = hostname; + parseHostname(); + } + + public String getRawHostname() { + return rawHostname; + } + + public String getHostname() { + return hostname; + } + + public int getPort() { + return port; + } + + private void parseHostname() throws InvalidHostStringException{ + try { + // A URI *must* have a scheme, so just create a fake one + URI uri = new URI("my://"+rawHostname.trim()); + this.hostname = uri.getHost(); + this.port = uri.getPort(); + + if (this.hostname == null) { + throw new InvalidHostStringException("The string "+rawHostname+ + " does not contain a value hostname or hostname:port definition"); + } + } catch (URISyntaxException e) { + throw new InvalidHostStringException( + "Unable to parse the hoststring "+rawHostname, e); + } + } + } + + private List mapHostnamesToDatanodes(List hosts) + throws InvalidHostStringException { + List results = new LinkedList<>(); + for (String hostString : hosts) { + HostDefinition host = new HostDefinition(hostString); + InetAddress addr; + try { + addr = InetAddress.getByName(host.getHostname()); + } catch (UnknownHostException e) { + throw new InvalidHostStringException("Unable to resolve the host " + +host.getRawHostname(), e); + } + String dnsName; + if (useHostnames) { + dnsName = addr.getHostName(); + } else { + dnsName = addr.getHostAddress(); + } + List found = nodeManager.getNodesByAddress(dnsName); + if (found.size() == 0) { + throw new InvalidHostStringException("The string " + + host.getRawHostname()+" resolved to "+dnsName + + " is not found in SCM"); + } else if (found.size() == 1) { + if (host.getPort() != -1 && + !validateDNPortMatch(host.getPort(), found.get(0))) { + throw new InvalidHostStringException("The string "+ + host.getRawHostname()+" matched a single datanode, but the "+ + "given port is not used by that Datanode"); + } + results.add(found.get(0)); + } else if (found.size() > 1) { + DatanodeDetails match = null; + for(DatanodeDetails dn : found) { + if (validateDNPortMatch(host.getPort(), dn)) { + match = dn; + break; + } + } + if (match == null) { + throw new InvalidHostStringException("The string " + + host.getRawHostname()+ "matched multiple Datanodes, but no "+ + "datanode port matched the given port"); + } + results.add(match); + } + } + return results; + } + + /** + * Check if the passed port is used by the given DatanodeDetails object. If + * it is, return true, otherwise return false. + * @param port Port number to check if it is used by the datanode + * @param dn Datanode to check if it is using the given port + * @return True if port is used by the datanode. False otherwise. + */ + private boolean validateDNPortMatch(int port, DatanodeDetails dn) { + for (DatanodeDetails.Port p : dn.getPorts()) { + if (p.getValue() == port) { + return true; + } + } + return false; + } + + public NodeDecommissionManager(OzoneConfiguration conf, + NodeManager nodeManager, PipelineManager pipelineManager, + ContainerManager containerManager) { + this.conf = conf; + this.nodeManager = nodeManager; + this.pipeLineManager = pipelineManager; + this.containerManager = containerManager; + + useHostnames = conf.getBoolean( + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + } + + public synchronized void decommissionNodes(List nodes) + throws InvalidHostStringException { + List dns = mapHostnamesToDatanodes(nodes); + for (DatanodeDetails dn : dns) { + try { + startDecommission(dn); + } catch (NodeNotFoundException e) { + // We already validated the host strings and retrieved the DnDetails + // object from the node manager. Therefore we should never get a + // NodeNotFoundException here expect if the node is remove in the + // very short window between validation and starting decom. Therefore + // log a warning and ignore the exception + LOG.warn("The host {} was not found in SCM. Ignoring the request to "+ + "decommission it", dn.getHostName()); + } + } + } + + public synchronized void startDecommission(DatanodeDetails dn) + throws NodeNotFoundException { + NodeStatus nodeStatus = getNodeStatus(dn); + NodeOperationalState opState = nodeStatus.getOperationalState(); + LOG.info("In decommission the op state is {}", opState); + if (opState != NodeOperationalState.DECOMMISSIONING + && opState != NodeOperationalState.DECOMMISSIONED) { + LOG.info("Starting Decommission for node {}", dn); + nodeManager.setNodeOperationalState( + dn, NodeOperationalState.DECOMMISSIONING); + pendingNodes.add(dn); + } else { + LOG.info("Start Decommission called on node {} in state {}. Nothing to "+ + "do.", dn, opState); + } + } + + public synchronized void recommissionNodes(List nodes) + throws InvalidHostStringException { + List dns = mapHostnamesToDatanodes(nodes); + for (DatanodeDetails dn : dns) { + try { + recommission(dn); + } catch (NodeNotFoundException e) { + // We already validated the host strings and retrieved the DnDetails + // object from the node manager. Therefore we should never get a + // NodeNotFoundException here expect if the node is remove in the + // very short window between validation and starting decom. Therefore + // log a warning and ignore the exception + LOG.warn("The host {} was not found in SCM. Ignoring the request to "+ + "recommission it", dn.getHostName()); + } + } + } + + public synchronized void recommission(DatanodeDetails dn) + throws NodeNotFoundException{ + NodeStatus nodeStatus = getNodeStatus(dn); + NodeOperationalState opState = nodeStatus.getOperationalState(); + if (opState != NodeOperationalState.IN_SERVICE) { + nodeManager.setNodeOperationalState( + dn, NodeOperationalState.IN_SERVICE); + pendingNodes.remove(dn); + LOG.info("Recommissioned node {}", dn); + } else { + LOG.info("Recommission called on node {} with state {}. "+ + "Nothing to do.", dn, opState); + } + } + + public synchronized void startMaintenanceNodes(List nodes, int endInHours) + throws InvalidHostStringException { + List dns = mapHostnamesToDatanodes(nodes); + for (DatanodeDetails dn : dns) { + try { + startMaintenance(dn, endInHours); + } catch (NodeNotFoundException e) { + // We already validated the host strings and retrieved the DnDetails + // object from the node manager. Therefore we should never get a + // NodeNotFoundException here expect if the node is remove in the + // very short window between validation and starting decom. Therefore + // log a warning and ignore the exception + LOG.warn("The host {} was not found in SCM. Ignoring the request to "+ + "start maintenance on it", dn.getHostName()); + } + } + } + + // TODO - If startMaintenance is called on a host already in maintenance, + // then we should update the end time? + public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) + throws NodeNotFoundException { + NodeStatus nodeStatus = getNodeStatus(dn); + NodeOperationalState opState = nodeStatus.getOperationalState(); + if (opState != NodeOperationalState.ENTERING_MAINTENANCE && + opState != NodeOperationalState.IN_MAINTENANCE) { + nodeManager.setNodeOperationalState( + dn, NodeOperationalState.ENTERING_MAINTENANCE); + pendingNodes.add(dn); + LOG.info("Starting Maintenance for node {}", dn); + } else { + LOG.info("Starting Maintenance called on node {} with state {}. "+ + "Nothing to do.", dn, opState); + } + } + + private NodeStatus getNodeStatus(DatanodeDetails dn) + throws NodeNotFoundException { + NodeStatus nodeStatus = nodeManager.getNodeStatus(dn); + if (nodeStatus == null) { + throw new NodeNotFoundException(); + } + return nodeStatus; + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 205f2e1b22b2..252c38fe1c25 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -125,11 +125,19 @@ int getNodeCount( SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails); /** - * Returns the node state of a specific node. + * Returns the node status of a specific node. * @param datanodeDetails DatanodeDetails - * @return Healthy/Stale/Dead. + * @return NodeStatus for the node */ - NodeState getNodeState(DatanodeDetails datanodeDetails); + NodeStatus getNodeStatus(DatanodeDetails datanodeDetails); + + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + */ + void setNodeOperationalState(DatanodeDetails datanodeDetails, + NodeOperationalState newState) throws NodeNotFoundException; /** * Get set of pipelines a datanode is part of. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index 1e1a50cd3db0..4177b63dd396 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -394,6 +394,21 @@ public List getAllNodes() { return nodeStateMap.getAllDatanodeInfos(); } + /** + * Sets the operational state of the given node. Intended to be called when + * a node is being decommissioned etc. + * + * @param dn The datanode having its state set + * @param newState The new operational State of the node. + */ + public void setNodeOperationalState(DatanodeDetails dn, + NodeOperationalState newState) throws NodeNotFoundException { + DatanodeInfo dni = nodeStateMap.getNodeInfo(dn.getUuid()); + if (dni.getNodeStatus().getOperationalState() != newState) { + nodeStateMap.updateNodeOperationalState(dn.getUuid(), newState); + } + } + /** * Gets set of pipelineID a datanode belongs to. * @param dnId - Datanode ID diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index e5301468cdec..407f026ad9ea 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -213,21 +213,32 @@ public int getNodeCount(NodeOperationalState nodeOpState, NodeState health) { } /** - * Returns the node state of a specific node. + * Returns the node status of a specific node. * * @param datanodeDetails Datanode Details - * @return Healthy/Stale/Dead/Unknown. + * @return NodeStatus for the node */ @Override - public NodeState getNodeState(DatanodeDetails datanodeDetails) { + public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) { try { - return nodeStateManager.getNodeStatus(datanodeDetails).getHealth(); + return nodeStateManager.getNodeStatus(datanodeDetails); } catch (NodeNotFoundException e) { // TODO: should we throw NodeNotFoundException? return null; } } + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + */ + @Override + public void setNodeOperationalState(DatanodeDetails datanodeDetails, + NodeOperationalState newState) throws NodeNotFoundException{ + nodeStateManager.setNodeOperationalState(datanodeDetails, newState); + } + /** * Closes this stream and releases any system resources associated with it. If * the stream is already closed then invoking this method has no effect. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 0d2f47000038..53bd95d55a4d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -57,6 +57,18 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartReplicationManagerResponseProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerResponseProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.DecommissionNodesRequestProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.DecommissionNodesResponseProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.RecommissionNodesRequestProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.RecommissionNodesResponseProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.StartMaintenanceNodesRequestProto; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerLocationProtocolProtos.StartMaintenanceNodesResponseProto; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -214,6 +226,27 @@ public ScmContainerLocationResponse processRequest( .setReplicationManagerStatusResponse(getReplicationManagerStatus( request.getSeplicationManagerStatusRequest())) .build(); + case DecommissionNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setDecommissionNodesResponse(decommissionNodes( + request.getDecommissionNodesRequest())) + .build(); + case RecommissionNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setRecommissionNodesResponse(recommissionNodes( + request.getRecommissionNodesRequest())) + .build(); + case StartMaintenanceNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setStartMaintenanceNodesResponse(startMaintenanceNodes( + request.getStartMaintenanceNodesRequest())) + .build(); default: throw new IllegalArgumentException( "Unknown command type: " + request.getCmdType()); @@ -390,4 +423,25 @@ public ReplicationManagerStatusResponseProto getReplicationManagerStatus( .setIsRunning(impl.getReplicationManagerStatus()).build(); } + public DecommissionNodesResponseProto decommissionNodes( + DecommissionNodesRequestProto request) throws IOException { + impl.decommissionNodes(request.getHostsList()); + return DecommissionNodesResponseProto.newBuilder() + .build(); + } + + public RecommissionNodesResponseProto recommissionNodes( + RecommissionNodesRequestProto request) throws IOException { + impl.recommissionNodes(request.getHostsList()); + return RecommissionNodesResponseProto.newBuilder().build(); + } + + public StartMaintenanceNodesResponseProto startMaintenanceNodes( + StartMaintenanceNodesRequestProto request) throws IOException { + impl.startMaintenanceNodes(request.getHostsList(), + (int)request.getEndInHours()); + return StartMaintenanceNodesResponseProto.newBuilder() + .build(); + } + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index d9825074b218..38d846fb5fbd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -365,6 +365,43 @@ public List queryNode(HddsProtos.NodeState state, } + @Override + public void decommissionNodes(List nodes) throws IOException { + String remoteUser = getRpcRemoteUsername(); + try { + getScm().checkAdminAccess(remoteUser); + scm.getScmDecommissionManager().decommissionNodes(nodes); + } catch (Exception ex) { + LOG.error("Failed to decommission nodes", ex); + throw ex; + } + } + + @Override + public void recommissionNodes(List nodes) throws IOException { + String remoteUser = getRpcRemoteUsername(); + try { + getScm().checkAdminAccess(remoteUser); + scm.getScmDecommissionManager().recommissionNodes(nodes); + } catch (Exception ex) { + LOG.error("Failed to recommission nodes", ex); + throw ex; + } + } + + @Override + public void startMaintenanceNodes(List nodes, int endInHours) + throws IOException { + String remoteUser = getRpcRemoteUsername(); + try { + getScm().checkAdminAccess(remoteUser); + scm.getScmDecommissionManager().startMaintenanceNodes(nodes, endInHours); + } catch (Exception ex) { + LOG.error("Failed to place nodes into maintenance mode", ex); + throw ex; + } + } + @Override public void notifyObjectStageChange(StorageContainerLocationProtocolProtos .ObjectStageChangeRequestProto.Type type, long id, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 028f2eee1828..24572c7ab18c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdds.scm.node.NodeReportHandler; import org.apache.hadoop.hdds.scm.node.SCMNodeManager; import org.apache.hadoop.hdds.scm.node.StaleNodeHandler; +import org.apache.hadoop.hdds.scm.node.NodeDecommissionManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler; @@ -160,6 +161,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private ContainerManager containerManager; private BlockManager scmBlockManager; private final SCMStorageConfig scmStorageConfig; + private NodeDecommissionManager scmDecommissionManager; private SCMMetadataStore scmMetadataStore; @@ -335,6 +337,9 @@ public StorageContainerManager(OzoneConfiguration conf, clientProtocolServer, scmBlockManager, replicationManager, pipelineManager); + scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, + pipelineManager, containerManager); + eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler); @@ -929,6 +934,16 @@ public int getNodeCount(NodeState nodestate) { return scmNodeManager.getNodeCount(null, nodestate); } + /** + * Returns the node decommission manager. + * + * @return NodeDecommissionManager The decommission manger for the used by + * scm + */ + public NodeDecommissionManager getScmDecommissionManager() { + return scmDecommissionManager; + } + /** * Returns SCM container manager. */ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 20a8b747c950..25cf5049515c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -262,10 +262,19 @@ public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) { * @return Healthy/Stale/Dead. */ @Override - public HddsProtos.NodeState getNodeState(DatanodeDetails dd) { + public NodeStatus getNodeStatus(DatanodeDetails dd) { return null; } + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + */ + public void setNodeOperationalState(DatanodeDetails datanodeDetails, + HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { + } + /** * Get set of pipelines a datanode is part of. * @param dnId - datanodeID diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java new file mode 100644 index 000000000000..4492a6e3ceef --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.Arrays; +import java.util.ArrayList; +import static junit.framework.TestCase.assertEquals; +import static org.assertj.core.api.Fail.fail; + +/** + * Unit tests for the decommision manager. + */ + +public class TestNodeDecommissionManager { + + private NodeDecommissionManager decom; + private StorageContainerManager scm; + private NodeManager nodeManager; + private OzoneConfiguration conf; + private String storageDir; + + @Before + public void setup() throws Exception { + conf = new OzoneConfiguration(); + storageDir = GenericTestUtils.getTempPath( + TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + nodeManager = createNodeManager(conf); + decom = new NodeDecommissionManager(conf, nodeManager, null, null); + } + + @Test + public void testHostStringsParseCorrectly() + throws InvalidHostStringException { + NodeDecommissionManager.HostDefinition def = + new NodeDecommissionManager.HostDefinition("foobar"); + assertEquals("foobar", def.getHostname()); + assertEquals(-1, def.getPort()); + + def = new NodeDecommissionManager.HostDefinition(" foobar "); + assertEquals("foobar", def.getHostname()); + assertEquals(-1, def.getPort()); + + def = new NodeDecommissionManager.HostDefinition("foobar:1234"); + assertEquals("foobar", def.getHostname()); + assertEquals(1234, def.getPort()); + + def = new NodeDecommissionManager.HostDefinition( + "foobar.mycompany.com:1234"); + assertEquals("foobar.mycompany.com", def.getHostname()); + assertEquals(1234, def.getPort()); + + try { + def = new NodeDecommissionManager.HostDefinition("foobar:abcd"); + fail("InvalidHostStringException should have been thrown"); + } catch (InvalidHostStringException e) { + } + } + + @Test + public void testAnyInvalidHostThrowsException() + throws InvalidHostStringException{ + List dns = generateDatanodes(); + + // Try to decommission a host that does exist, but give incorrect port + try { + decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress()+":10")); + fail("InvalidHostStringException expected"); + } catch (InvalidHostStringException e) { + } + + // Try to decommission a host that does not exist + try { + decom.decommissionNodes(Arrays.asList("123.123.123.123")); + fail("InvalidHostStringException expected"); + } catch (InvalidHostStringException e) { + } + + // Try to decommission a host that does exist and a host that does not + try { + decom.decommissionNodes(Arrays.asList( + dns.get(1).getIpAddress(), "123,123,123,123")); + fail("InvalidHostStringException expected"); + } catch (InvalidHostStringException e) { + } + + // Try to decommission a host with many DNs on the address with no port + try { + decom.decommissionNodes(Arrays.asList( + dns.get(0).getIpAddress())); + fail("InvalidHostStringException expected"); + } catch (InvalidHostStringException e) { + } + + // Try to decommission a host with many DNs on the address with a port + // that does not exist + try { + decom.decommissionNodes(Arrays.asList( + dns.get(0).getIpAddress()+":10")); + fail("InvalidHostStringException expected"); + } catch (InvalidHostStringException e) { + } + } + + @Test + public void testNodesCanBeDecommissionedAndRecommissioned() + throws InvalidHostStringException { + List dns = generateDatanodes(); + + // Decommission 2 valid nodes + decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress(), + dns.get(2).getIpAddress())); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + + // Running the command again gives no error - nodes already decommissioning + // are silently ignored. + decom.decommissionNodes(Arrays.asList(dns.get(1).getIpAddress(), + dns.get(2).getIpAddress())); + + // Attempt to decommission dn(10) which has multiple hosts on the same IP + // and we hardcoded ports to 3456, 4567, 5678 + DatanodeDetails multiDn = dns.get(10); + String multiAddr = + multiDn.getIpAddress()+":"+multiDn.getPorts().get(0).getValue(); + decom.decommissionNodes(Arrays.asList(multiAddr)); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(multiDn).getOperationalState()); + + // Recommission all 3 hosts + decom.recommissionNodes(Arrays.asList( + multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress())); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(10)).getOperationalState()); + } + + @Test + public void testNodesCanBePutIntoMaintenanceAndRecommissioned() + throws InvalidHostStringException { + List dns = generateDatanodes(); + + // Put 2 valid nodes into maintenance + decom.startMaintenanceNodes(Arrays.asList(dns.get(1).getIpAddress(), + dns.get(2).getIpAddress()), 100); + assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + + // Running the command again gives no error - nodes already decommissioning + // are silently ignored. + decom.startMaintenanceNodes(Arrays.asList(dns.get(1).getIpAddress(), + dns.get(2).getIpAddress()), 100); + + // Attempt to decommission dn(10) which has multiple hosts on the same IP + // and we hardcoded ports to 3456, 4567, 5678 + DatanodeDetails multiDn = dns.get(10); + String multiAddr = + multiDn.getIpAddress()+":"+multiDn.getPorts().get(0).getValue(); + decom.startMaintenanceNodes(Arrays.asList(multiAddr), 100); + assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + nodeManager.getNodeStatus(multiDn).getOperationalState()); + + // Recommission all 3 hosts + decom.recommissionNodes(Arrays.asList( + multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress())); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dns.get(10)).getOperationalState()); + } + + private SCMNodeManager createNodeManager(OzoneConfiguration config) + throws IOException, AuthenticationException { + scm = HddsTestUtils.getScm(config); + return (SCMNodeManager) scm.getScmNodeManager(); + } + + /** + * Generate a list of random DNs and return the list. A total of 11 DNs will + * be generated and registered with the node manager. Index 0 and 10 will + * have the same IP and host and the rest will have unique IPs and Hosts. + * The DN at index 10, has 3 hard coded ports of 3456, 4567, 5678. All other + * DNs will have ports set to 0. + * @return The list of DatanodeDetails Generated + */ + private List generateDatanodes() { + List dns = new ArrayList<>(); + for (int i=0; i<10; i++) { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + dns.add(dn); + nodeManager.register(dn, null, null); + } + // We have 10 random DNs, we want to create another one that is on the same + // host as some of the others. + DatanodeDetails multiDn = dns.get(0); + + DatanodeDetails.Builder builder = DatanodeDetails.newBuilder(); + builder.setUuid(UUID.randomUUID().toString()) + .setHostName(multiDn.getHostName()) + .setIpAddress(multiDn.getIpAddress()) + .addPort(DatanodeDetails.newPort( + DatanodeDetails.Port.Name.STANDALONE, 3456)) + .addPort(DatanodeDetails.newPort( + DatanodeDetails.Port.Name.RATIS, 4567)) + .addPort(DatanodeDetails.newPort( + DatanodeDetails.Port.Name.REST, 5678)) + .setNetworkLocation(multiDn.getNetworkLocation()); + + DatanodeDetails dn = builder.build(); + nodeManager.register(dn, null, null); + dns.add(dn); + return dns; + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index bc28a438892a..9fbf2519f5e1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.HddsServerUtil; import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException; @@ -185,6 +186,22 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() eventPublisher.getLastEvent().getName()); } + @Test + public void testNodeOpStateCanBeSet() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + + nsm.setNodeOperationalState(dn, + HddsProtos.NodeOperationalState.DECOMMISSIONED); + + NodeStatus newStatus = nsm.getNodeStatus(dn); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + newStatus.getOperationalState()); + assertEquals(NodeState.HEALTHY, + newStatus.getHealth()); + } + private DatanodeDetails generateDatanode() { String uuid = UUID.randomUUID().toString(); return DatanodeDetails.newBuilder().setUuid(uuid).build(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index a48b2a05ebb2..5a2e3c2109ac 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -52,17 +52,17 @@ * A Node Manager to test replication. */ public class ReplicationNodeManagerMock implements NodeManager { - private final Map nodeStateMap; + private final Map nodeStateMap; private final CommandQueue commandQueue; /** * A list of Datanodes and current states. - * @param nodeState A node state map. + * @param nodeStatus A node state map. */ - public ReplicationNodeManagerMock(Map nodeState, + public ReplicationNodeManagerMock(Map nodeStatus, CommandQueue commandQueue) { - Preconditions.checkNotNull(nodeState); - this.nodeStateMap = nodeState; + Preconditions.checkNotNull(nodeStatus); + this.nodeStateMap = nodeStatus; this.commandQueue = commandQueue; } @@ -179,10 +179,26 @@ public SCMNodeMetric getNodeStat(DatanodeDetails dd) { * @return Healthy/Stale/Dead. */ @Override - public NodeState getNodeState(DatanodeDetails dd) { + public NodeStatus getNodeStatus(DatanodeDetails dd) { return nodeStateMap.get(dd); } + /** + * Set the operation state of a node. + * @param dd The datanode to set the new state for + * @param newState The new operational state for the node + */ + @Override + public void setNodeOperationalState(DatanodeDetails dd, + HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { + NodeStatus currentStatus = nodeStateMap.get(dd); + if (currentStatus != null) { + nodeStateMap.put(dd, new NodeStatus(newState, currentStatus.getHealth())); + } else { + throw new NodeNotFoundException(); + } + } + /** * Get set of pipelines a datanode is part of. * @param dnId - datanodeID @@ -313,10 +329,10 @@ public void clearMap() { * Adds a node to the existing Node manager. This is used only for test * purposes. * @param id DatanodeDetails - * @param state State you want to put that node to. + * @param status State you want to put that node to. */ - public void addNode(DatanodeDetails id, NodeState state) { - nodeStateMap.put(id, state); + public void addNode(DatanodeDetails id, NodeStatus status) { + nodeStateMap.put(id, status); } @Override diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java index 0b5c18e8205c..8649992971fb 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; +import org.apache.hadoop.hdds.scm.cli.node.DatanodeAdminCommands; import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.ContainerInfo; @@ -80,7 +81,8 @@ ContainerCommands.class, PipelineCommands.class, TopologySubcommand.class, - ReplicationManagerCommands.class + ReplicationManagerCommands.class, + DatanodeAdminCommands.class }, mixinStandardHelpOptions = true) public class SCMCLI extends GenericCli { diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java new file mode 100644 index 000000000000..dc7a6f4e4f1e --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.node; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.cli.MissingSubcommandException; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; +import org.apache.hadoop.hdds.scm.cli.SCMCLI; + +import java.util.concurrent.Callable; + +/** + * Subcommand to group datanode admin related operations. + */ +@Command( + name = "dnadmin", + description = "Datanode Administration specific operations", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class, + subcommands = { + DatanodeAdminDecommissionSubCommand.class, + DatanodeAdminMaintenanceSubCommand.class, + DatanodeAdminRecommissionSubCommand.class + }) +public class DatanodeAdminCommands implements Callable { + + @ParentCommand + private SCMCLI parent; + + public SCMCLI getParent() { + return parent; + } + + @Override + public Void call() throws Exception { + throw new MissingSubcommandException( + this.parent.getCmd().getSubcommands().get("nodeadmin")); + } +} \ No newline at end of file diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminDecommissionSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminDecommissionSubCommand.java new file mode 100644 index 000000000000..1406603eed34 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminDecommissionSubCommand.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.node; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Decommission one or more datanodes. + */ +@Command( + name = "decommission", + description = "Decommission a datanode", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class DatanodeAdminDecommissionSubCommand implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminDecommissionSubCommand.class); + + @CommandLine.Parameters(description = "List of fully qualified host names") + private List hosts = new ArrayList(); + + @ParentCommand + private DatanodeAdminCommands parent; + + @Override + public Void call() throws Exception { + try (ScmClient scmClient = parent.getParent().createScmClient()) { + scmClient.decommissionNodes(hosts); + return null; + } + } +} diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminMaintenanceSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminMaintenanceSubCommand.java new file mode 100644 index 000000000000..2e4b2b51fd4f --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminMaintenanceSubCommand.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.node; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Place one or more datanodes into Maintenance Mode. + */ +@Command( + name = "maintenance", + description = "Put a datanode into Maintenance Mode", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class DatanodeAdminMaintenanceSubCommand implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminMaintenanceSubCommand.class); + + @CommandLine.Parameters(description = "List of fully qualified host names") + private List hosts = new ArrayList(); + + @CommandLine.Option(names = {"--end"}, + description = "Automatically end maintenance after the given hours. "+ + "By default, maintenance must be ended manually.") + private int endInHours = 0; + + @ParentCommand + private DatanodeAdminCommands parent; + + @Override + public Void call() throws Exception { + try (ScmClient scmClient = parent.getParent().createScmClient()) { + scmClient.startMaintenanceNodes(hosts, endInHours); + return null; + } + } +} \ No newline at end of file diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminRecommissionSubCommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminRecommissionSubCommand.java new file mode 100644 index 000000000000..eaa1280ae49d --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminRecommissionSubCommand.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.node; + +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ParentCommand; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +/** + * Place decommissioned or maintenance nodes back into service. + */ +@Command( + name = "recommission", + description = "Return a datanode to service", + mixinStandardHelpOptions = true, + versionProvider = HddsVersionProvider.class) +public class DatanodeAdminRecommissionSubCommand implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminRecommissionSubCommand.class); + + @CommandLine.Parameters(description = "List of fully qualified host names") + private List hosts = new ArrayList(); + + @ParentCommand + private DatanodeAdminCommands parent; + + @Override + public Void call() throws Exception { + try (ScmClient scmClient = parent.getParent().createScmClient()) { + scmClient.recommissionNodes(hosts); + return null; + } + } +} \ No newline at end of file diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/package-info.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/package-info.java new file mode 100644 index 000000000000..dfb04b828765 --- /dev/null +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *

+ * SCM related cli tools. + */ +/** + * SCM related cli tools for Datanode Admin. + */ +package org.apache.hadoop.hdds.scm.cli.node; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java new file mode 100644 index 000000000000..bada595f9963 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.node; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static junit.framework.TestCase.assertEquals; +import static org.apache.hadoop.hdds.HddsConfigKeys.*; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*; + +/** + * Test from the scmclient for decommission and maintenance. + */ + +public class TestDecommissionAndMaintenance { + private static final Logger LOG = + LoggerFactory.getLogger(TestDecommissionAndMaintenance.class); + + private static int numOfDatanodes = 5; + private MiniOzoneCluster cluster; + + private ContainerOperationClient scmClient; + + @Before + public void setUp() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + final int interval = 100; + + conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, + interval, TimeUnit.MILLISECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); + conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(numOfDatanodes) + .build(); + cluster.waitForClusterToBeReady(); + scmClient = new ContainerOperationClient(cluster + .getStorageContainerLocationClient(), + new XceiverClientManager(conf)); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testNodeCanBeDecommMaintAndRecommissioned() + throws IOException { + NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager(); + + List dns = nm.getAllNodes(); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // Ensure one node is decommissioning + List decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(1, decomNodes.size()); + + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // Ensure zero nodes are now decommissioning + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + + scmClient.startMaintenanceNodes(Arrays.asList( + getDNHostAndPort(dns.get(0))), 10); + + // None are decommissioning + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + + // One is in Maintenance + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY); + assertEquals(1, decomNodes.size()); + + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // None are in maintenance + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + } + + private String getDNHostAndPort(DatanodeDetails dn) { + return dn.getHostName()+":"+dn.getPorts().get(0).getValue(); + } + +} From de928c99ad4d7f7d9a64bda58e7305cf9f3a8557 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 15 Oct 2019 16:36:47 +0100 Subject: [PATCH 03/78] Updates after review comments --- .../hadoop/hdds/scm/client/ScmClient.java | 11 ++++-- .../scm/node/InvalidNodeStateException.java | 34 +++++++++++++++++ .../scm/node/NodeDecommissionManager.java | 38 ++++++++++++++----- .../scm/node/TestNodeDecommissionManager.java | 35 +++++++++++++++++ .../scm/cli/node/DatanodeAdminCommands.java | 2 +- 5 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidNodeStateException.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index 8238e8469a9f..f96234cb14e0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -179,12 +179,15 @@ List queryNode(HddsProtos.NodeState nodeStatuses, void recommissionNodes(List hosts) throws IOException; /** - * Place the list of datanodes into maintenance mode. If a non-null endDtm + * Place the list of datanodes into maintenance mode. If a non-zero endDtm * is passed, the hosts will automatically exit maintenance mode after the - * given time has passed. The hosts are identified by their hostname and - * optionally port in the format foo.com:port. + * given time has passed. Passing an end time of zero means the hosts will + * remain in maintenance indefinitely. + * The hosts are identified by their hostname and optionally port in the + * format foo.com:port. * @param hosts A list of hostnames, optionally with port - * @param endHours The number of hours from now which maintenance will end + * @param endHours The number of hours from now which maintenance will end or + * zero if maintenance must be manually ended. * @throws IOException */ void startMaintenanceNodes(List hosts, int endHours) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidNodeStateException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidNodeStateException.java new file mode 100644 index 000000000000..9c82398d34d5 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/InvalidNodeStateException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import java.io.IOException; + +/** + * Exception thrown by the NodeDecommissionManager when it encounters + * host strings it does not expect or understand. + */ + +public class InvalidNodeStateException extends IOException { + public InvalidNodeStateException(String msg) { + super(msg); + } + + public InvalidNodeStateException(String msg, Exception e) { + super(msg, e); + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index 60813ddd4b71..a001683f5ea1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -18,6 +18,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; @@ -76,7 +77,7 @@ public int getPort() { private void parseHostname() throws InvalidHostStringException{ try { // A URI *must* have a scheme, so just create a fake one - URI uri = new URI("my://"+rawHostname.trim()); + URI uri = new URI("empty://"+rawHostname.trim()); this.hostname = uri.getHost(); this.port = uri.getPort(); @@ -184,24 +185,32 @@ public synchronized void decommissionNodes(List nodes) // log a warning and ignore the exception LOG.warn("The host {} was not found in SCM. Ignoring the request to "+ "decommission it", dn.getHostName()); + } catch (InvalidNodeStateException e) { + // TODO - decide how to handle this. We may not want to fail all nodes + // only one is in a bad state, as some nodes may have been OK + // and already processed. Perhaps we should return a list of + // error and feed that all the way back to the client? } } } public synchronized void startDecommission(DatanodeDetails dn) - throws NodeNotFoundException { + throws NodeNotFoundException, InvalidNodeStateException { NodeStatus nodeStatus = getNodeStatus(dn); NodeOperationalState opState = nodeStatus.getOperationalState(); - LOG.info("In decommission the op state is {}", opState); - if (opState != NodeOperationalState.DECOMMISSIONING - && opState != NodeOperationalState.DECOMMISSIONED) { + if (opState == NodeOperationalState.IN_SERVICE) { LOG.info("Starting Decommission for node {}", dn); nodeManager.setNodeOperationalState( dn, NodeOperationalState.DECOMMISSIONING); pendingNodes.add(dn); - } else { + } else if (opState == NodeOperationalState.DECOMMISSIONING + || opState == NodeOperationalState.DECOMMISSIONED) { LOG.info("Start Decommission called on node {} in state {}. Nothing to "+ "do.", dn, opState); + } else { + LOG.error("Cannot decommission node {} in state {}", dn, opState); + throw new InvalidNodeStateException("Cannot decommission node "+ + dn +" in state "+ opState); } } @@ -252,6 +261,11 @@ public synchronized void startMaintenanceNodes(List nodes, int endInHours) // log a warning and ignore the exception LOG.warn("The host {} was not found in SCM. Ignoring the request to "+ "start maintenance on it", dn.getHostName()); + } catch (InvalidNodeStateException e) { + // TODO - decide how to handle this. We may not want to fail all nodes + // only one is in a bad state, as some nodes may have been OK + // and already processed. Perhaps we should return a list of + // error and feed that all the way back to the client? } } } @@ -259,18 +273,22 @@ public synchronized void startMaintenanceNodes(List nodes, int endInHours) // TODO - If startMaintenance is called on a host already in maintenance, // then we should update the end time? public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) - throws NodeNotFoundException { + throws NodeNotFoundException, InvalidNodeStateException { NodeStatus nodeStatus = getNodeStatus(dn); NodeOperationalState opState = nodeStatus.getOperationalState(); - if (opState != NodeOperationalState.ENTERING_MAINTENANCE && - opState != NodeOperationalState.IN_MAINTENANCE) { + if (opState == NodeOperationalState.IN_SERVICE) { nodeManager.setNodeOperationalState( dn, NodeOperationalState.ENTERING_MAINTENANCE); pendingNodes.add(dn); LOG.info("Starting Maintenance for node {}", dn); - } else { + } else if (opState == NodeOperationalState.ENTERING_MAINTENANCE || + opState == NodeOperationalState.IN_MAINTENANCE) { LOG.info("Starting Maintenance called on node {} with state {}. "+ "Nothing to do.", dn, opState); + } else { + LOG.error("Cannot start maintenance on node {} in state {}", dn, opState); + throw new InvalidNodeStateException("Cannot start maintenance on node "+ + dn +" in state "+ opState); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index 4492a6e3ceef..82bd26b59854 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -207,6 +207,41 @@ public void testNodesCanBePutIntoMaintenanceAndRecommissioned() nodeManager.getNodeStatus(dns.get(10)).getOperationalState()); } + @Test + public void testNodesCannotTransitionFromDecomToMaint() throws Exception { + List dns = generateDatanodes(); + + // Put 1 node into maintenance and another into decom + decom.startMaintenance(dns.get(1), 100); + decom.startDecommission(dns.get(2)); + assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + + // Try to go from maint to decom: + try { + decom.startDecommission(dns.get(1)); + fail("Expected InvalidNodeStateException"); + } catch (InvalidNodeStateException e) { + } + + // Try to go from decom to maint: + try { + decom.startMaintenance(dns.get(2), 100); + fail("Expected InvalidNodeStateException"); + } catch (InvalidNodeStateException e) { + } + + // Ensure the states are still as before + assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + } + + + private SCMNodeManager createNodeManager(OzoneConfiguration config) throws IOException, AuthenticationException { scm = HddsTestUtils.getScm(config); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java index dc7a6f4e4f1e..2eba44e51db7 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java @@ -30,7 +30,7 @@ */ @Command( name = "dnadmin", - description = "Datanode Administration specific operations", + description = "Datanode Administration operations", mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class, subcommands = { From 4f58c81f678e97a06324803bd2096a9af9d9c72a Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 18 Oct 2019 14:09:50 +0200 Subject: [PATCH 04/78] HDDS-2197. Extend SCMCLI Topology command to print node Operational States Closes #37 --- .../scm/client/ContainerOperationClient.java | 15 +++-- .../hadoop/hdds/scm/client/ScmClient.java | 13 ++-- .../StorageContainerLocationProtocol.java | 11 ++-- ...ocationProtocolClientSideTranslatorPB.java | 29 ++++++--- .../StorageContainerLocationProtocol.proto | 3 +- hadoop-hdds/common/src/main/proto/hdds.proto | 1 + .../scm/node/NodeDecommissionManager.java | 13 ++-- ...ocationProtocolServerSideTranslatorPB.java | 4 +- .../scm/server/SCMClientProtocolServer.java | 22 ++++--- .../hdds/scm/cli/TopologySubcommand.java | 22 ++++--- .../src/main/smoketest/topology/scmcli.robot | 4 +- .../hadoop/ozone/scm/node/TestQueryNode.java | 61 +++++++++++++++++-- .../apache/hadoop/ozone/om/OzoneManager.java | 4 +- 13 files changed, 145 insertions(+), 57 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java index 93b09d9ce591..0839f3af9f93 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java @@ -198,18 +198,23 @@ public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type, /** * Returns a set of Nodes that meet a query criteria. * - * @param nodeStatuses - Criteria that we want the node to have. + * @param opState - The operational state we want the node to have + * eg IN_SERVICE, DECOMMISSIONED, etc + * @param nodeState - The health we want the node to have, eg HEALTHY, STALE, + * etc * @param queryScope - Query scope - Cluster or pool. * @param poolName - if it is pool, a pool name is required. * @return A set of nodes that meet the requested criteria. * @throws IOException */ @Override - public List queryNode(HddsProtos.NodeState - nodeStatuses, HddsProtos.QueryScope queryScope, String poolName) + public List queryNode( + HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState nodeState, + HddsProtos.QueryScope queryScope, String poolName) throws IOException { - return storageContainerLocationClient.queryNode(nodeStatuses, queryScope, - poolName); + return storageContainerLocationClient.queryNode(opState, nodeState, + queryScope, poolName); } @Override diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index f96234cb14e0..88ddbe6a1380 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@ -151,15 +151,20 @@ ContainerWithPipeline createContainer(HddsProtos.ReplicationType type, String owner) throws IOException; /** - * Returns a set of Nodes that meet a query criteria. - * @param nodeStatuses - Criteria that we want the node to have. + * Returns a set of Nodes that meet a query criteria. Passing null for opState + * or nodeState acts like a wild card, returning all nodes in that state. + * @param opState - Operational State of the node, eg IN_SERVICE, + * DECOMMISSIONED, etc + * @param nodeState - Health of the nodeCriteria that we want the node to + * have, eg HEALTHY, STALE etc * @param queryScope - Query scope - Cluster or pool. * @param poolName - if it is pool, a pool name is required. * @return A set of nodes that meet the requested criteria. * @throws IOException */ - List queryNode(HddsProtos.NodeState nodeStatuses, - HddsProtos.QueryScope queryScope, String poolName) throws IOException; + List queryNode(HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState nodeState, HddsProtos.QueryScope queryScope, + String poolName) throws IOException; /** * Allows a list of hosts to be decommissioned. The hosts are identified diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 91cd40dc3f5a..8b37c57ee1e0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -105,12 +105,15 @@ List listContainer(long startContainerID, int count) void deleteContainer(long containerID) throws IOException; /** - * Queries a list of Node Statuses. - * @param state + * Queries a list of Node Statuses. Passing a null for either opState or + * state acts like a wildcard returning all nodes in that state. + * @param opState The node operational state + * @param state The node health * @return List of Datanodes. */ - List queryNode(HddsProtos.NodeState state, - HddsProtos.QueryScope queryScope, String poolName) throws IOException; + List queryNode(HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState state, HddsProtos.QueryScope queryScope, + String poolName) throws IOException; void decommissionNodes(List nodes) throws IOException; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 99941d1c3346..0765a5608578 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -240,23 +240,34 @@ public void deleteContainer(long containerID) } /** - * Queries a list of Node Statuses. + * Queries a list of Nodes based on their operational state or health state. + * Passing a null for either value acts as a wildcard for that state. + * + * @param opState The operation state of the node + * @param nodeState The health of the node + * @return List of Datanodes. */ @Override - public List queryNode(HddsProtos.NodeState - nodeStatuses, HddsProtos.QueryScope queryScope, String poolName) + public List queryNode( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState + nodeState, HddsProtos.QueryScope queryScope, String poolName) throws IOException { // TODO : We support only cluster wide query right now. So ignoring checking // queryScope and poolName - Preconditions.checkNotNull(nodeStatuses); - NodeQueryRequestProto request = NodeQueryRequestProto.newBuilder() - .setState(nodeStatuses) + NodeQueryRequestProto.Builder builder = NodeQueryRequestProto.newBuilder() .setTraceID(TracingUtil.exportCurrentSpan()) - .setScope(queryScope).setPoolName(poolName).build(); + .setScope(queryScope).setPoolName(poolName); + if (opState != null) { + builder.setOpState(opState); + } + if (nodeState != null) { + builder.setState(nodeState); + } + NodeQueryRequestProto request = builder.build(); NodeQueryResponseProto response = submitRequest(Type.QueryNode, - builder -> builder.setNodeQueryRequest(request)).getNodeQueryResponse(); + builder1 -> builder1.setNodeQueryRequest(request)) + .getNodeQueryResponse(); return response.getDatanodesList(); - } /** diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto index 00e58c0bc851..01779652118c 100644 --- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto @@ -223,10 +223,11 @@ message ObjectStageChangeResponseProto { match the NodeState that we are requesting. */ message NodeQueryRequestProto { - required NodeState state = 1; + optional NodeState state = 1; required QueryScope scope = 2; optional string poolName = 3; // if scope is pool, then pool name is needed. optional string traceID = 4; + optional NodeOperationalState opState = 5; } message NodeQueryResponseProto { diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index 294f2b769512..f9ed49181491 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -118,6 +118,7 @@ enum QueryScope { message Node { required DatanodeDetailsProto nodeID = 1; repeated NodeState nodeStates = 2; + repeated NodeOperationalState nodeOperationalStates = 3; } message NodePool { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index a001683f5ea1..9806fbb430c8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -18,7 +18,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; @@ -41,9 +40,9 @@ public class NodeDecommissionManager { private NodeManager nodeManager; - private PipelineManager pipeLineManager; - private ContainerManager containerManager; - private OzoneConfiguration conf; + // private PipelineManager pipeLineManager; + // private ContainerManager containerManager; + // private OzoneConfiguration conf; private boolean useHostnames; private List pendingNodes = new LinkedList<>(); @@ -161,10 +160,10 @@ private boolean validateDNPortMatch(int port, DatanodeDetails dn) { public NodeDecommissionManager(OzoneConfiguration conf, NodeManager nodeManager, PipelineManager pipelineManager, ContainerManager containerManager) { - this.conf = conf; this.nodeManager = nodeManager; - this.pipeLineManager = pipelineManager; - this.containerManager = containerManager; + //this.conf = conf; + //this.pipeLineManager = pipelineManager; + //this.containerManager = containerManager; useHostnames = conf.getBoolean( DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 53bd95d55a4d..2c55f04ce5bc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -322,12 +322,12 @@ public NodeQueryResponseProto queryNode( throws IOException { HddsProtos.NodeState nodeState = request.getState(); - List datanodes = impl.queryNode(nodeState, + HddsProtos.NodeOperationalState opState = request.getOpState(); + List datanodes = impl.queryNode(opState, nodeState, request.getScope(), request.getPoolName()); return NodeQueryResponseProto.newBuilder() .addAllDatanodes(datanodes) .build(); - } public ObjectStageChangeResponseProto notifyObjectStageChange( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 38d846fb5fbd..69c1d2dd3d08 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -347,7 +347,8 @@ public void deleteContainer(long containerID) throws IOException { } @Override - public List queryNode(HddsProtos.NodeState state, + public List queryNode( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState state, HddsProtos.QueryScope queryScope, String poolName) throws IOException { @@ -356,9 +357,11 @@ public List queryNode(HddsProtos.NodeState state, } List result = new ArrayList<>(); - queryNode(state).forEach(node -> result.add(HddsProtos.Node.newBuilder() + queryNode(opState, state) + .forEach(node -> result.add(HddsProtos.Node.newBuilder() .setNodeID(node.getProtoBufMessage()) .addNodeStates(state) + .addNodeOperationalStates(opState) .build())); return result; @@ -565,12 +568,14 @@ public boolean getReplicationManagerStatus() { * operation between the * operators. * - * @param state - NodeStates. + * @param opState - NodeOperational State + * @param state - NodeState. * @return List of Datanodes. */ - public List queryNode(HddsProtos.NodeState state) { + public List queryNode( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState state) { Preconditions.checkNotNull(state, "Node Query set cannot be null"); - return new ArrayList<>(queryNodeState(state)); + return new ArrayList<>(queryNodeState(opState, state)); } @VisibleForTesting @@ -589,14 +594,15 @@ public boolean getSafeModeStatus() { /** * Query the System for Nodes. * + * @params opState - The node operational state * @param nodeState - NodeState that we are interested in matching. * @return Set of Datanodes that match the NodeState. */ - private Set queryNodeState(HddsProtos.NodeState nodeState) { + private Set queryNodeState( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState nodeState) { Set returnSet = new TreeSet<>(); - // TODO - decomm states needed List tmp = scm.getScmNodeManager() - .getNodes(null, nodeState); + .getNodes(opState, nodeState); if ((tmp != null) && (tmp.size() > 0)) { returnSet.addAll(tmp); } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index 42773f82308f..731a617b34c3 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -65,7 +65,7 @@ public class TopologySubcommand implements Callable { public Void call() throws Exception { try (ScmClient scmClient = parent.createScmClient()) { for (HddsProtos.NodeState state : stateArray) { - List nodes = scmClient.queryNode(state, + List nodes = scmClient.queryNode(null, state, HddsProtos.QueryScope.CLUSTER, ""); if (nodes != null && nodes.size() > 0) { // show node state @@ -83,36 +83,40 @@ public Void call() throws Exception { // Format // Location: rack1 - // ipAddress(hostName) + // ipAddress(hostName) OperationalState private void printOrderedByLocation(List nodes) { HashMap> tree = new HashMap<>(); + HashMap state = + new HashMap<>(); + for (HddsProtos.Node node : nodes) { String location = node.getNodeID().getNetworkLocation(); if (location != null && !tree.containsKey(location)) { tree.put(location, new TreeSet<>()); } - tree.get(location).add(DatanodeDetails.getFromProtoBuf(node.getNodeID())); + DatanodeDetails dn = DatanodeDetails.getFromProtoBuf(node.getNodeID()); + tree.get(location).add(dn); + state.put(dn, node.getNodeOperationalStates(0)); } ArrayList locations = new ArrayList<>(tree.keySet()); Collections.sort(locations); locations.forEach(location -> { System.out.println("Location: " + location); - tree.get(location).forEach(node -> { - System.out.println(" " + node.getIpAddress() + "(" + node.getHostName() - + ")"); + tree.get(location).forEach(n -> { + System.out.println(" " + n.getIpAddress() + "(" + n.getHostName() + + ") "+state.get(n)); }); }); } - - // Format "ipAddress(hostName) networkLocation" + // Format "ipAddress(hostName) OperationalState networkLocation" private void printNodesWithLocation(Collection nodes) { nodes.forEach(node -> { System.out.print(" " + node.getNodeID().getIpAddress() + "(" + node.getNodeID().getHostName() + ")"); - System.out.println(" " + + System.out.println(" " + node.getNodeOperationalStates(0) + " " + (node.getNodeID().getNetworkLocation() != null ? node.getNodeID().getNetworkLocation() : "NA")); }); diff --git a/hadoop-ozone/dist/src/main/smoketest/topology/scmcli.robot b/hadoop-ozone/dist/src/main/smoketest/topology/scmcli.robot index 823981d15f90..6267e09b3dbb 100644 --- a/hadoop-ozone/dist/src/main/smoketest/topology/scmcli.robot +++ b/hadoop-ozone/dist/src/main/smoketest/topology/scmcli.robot @@ -25,8 +25,8 @@ Resource ../commonlib.robot *** Test Cases *** Run printTopology ${output} = Execute ozone scmcli printTopology - Should contain ${output} 10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net) /rack2 + Should contain ${output} 10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net) IN_SERVICE /rack2 Run printTopology -o ${output} = Execute ozone scmcli printTopology -o Should contain ${output} Location: /rack2 - Should contain ${output} 10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net) + Should contain ${output} 10.5.0.7(ozone-topology_datanode_4_1.ozone-topology_net) IN_SERVICE diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java index c9b8c89e04da..eae76411e6fb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java @@ -16,6 +16,9 @@ */ package org.apache.hadoop.ozone.scm.node; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -45,6 +48,10 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos. + NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos. + NodeOperationalState.IN_MAINTENANCE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_DEADNODE_INTERVAL; @@ -96,7 +103,7 @@ public void tearDown() throws Exception { @Test public void testHealthyNodesCount() throws Exception { - List nodes = scmClient.queryNode(HEALTHY, + List nodes = scmClient.queryNode(null, HEALTHY, HddsProtos.QueryScope.CLUSTER, ""); assertEquals("Expected live nodes", numOfDatanodes, nodes.size()); @@ -111,7 +118,7 @@ public void testStaleNodesCount() throws Exception { cluster.getStorageContainerManager().getNodeCount(STALE) == 2, 100, 4 * 1000); - int nodeCount = scmClient.queryNode(STALE, + int nodeCount = scmClient.queryNode(null, STALE, HddsProtos.QueryScope.CLUSTER, "").size(); assertEquals("Mismatch of expected nodes count", 2, nodeCount); @@ -120,13 +127,59 @@ public void testStaleNodesCount() throws Exception { 100, 4 * 1000); // Assert that we don't find any stale nodes. - nodeCount = scmClient.queryNode(STALE, + nodeCount = scmClient.queryNode(null, STALE, HddsProtos.QueryScope.CLUSTER, "").size(); assertEquals("Mismatch of expected nodes count", 0, nodeCount); // Assert that we find the expected number of dead nodes. - nodeCount = scmClient.queryNode(DEAD, + nodeCount = scmClient.queryNode(null, DEAD, HddsProtos.QueryScope.CLUSTER, "").size(); assertEquals("Mismatch of expected nodes count", 2, nodeCount); } + + @Test + public void testNodeOperationalStates() throws Exception { + StorageContainerManager scm = cluster.getStorageContainerManager(); + NodeManager nm = scm.getScmNodeManager(); + + // All nodes should be returned as they are all in service + int nodeCount = scmClient.queryNode(IN_SERVICE, HEALTHY, + HddsProtos.QueryScope.CLUSTER, "").size(); + assertEquals(numOfDatanodes, nodeCount); + + // null acts as wildcard for opState + nodeCount = scmClient.queryNode(null, HEALTHY, + HddsProtos.QueryScope.CLUSTER, "").size(); + assertEquals(numOfDatanodes, nodeCount); + + // null acts as wildcard for nodeState + nodeCount = scmClient.queryNode(IN_SERVICE, null, + HddsProtos.QueryScope.CLUSTER, "").size(); + assertEquals(numOfDatanodes, nodeCount); + + // Both null - should return all nodes + nodeCount = scmClient.queryNode(null, null, + HddsProtos.QueryScope.CLUSTER, "").size(); + assertEquals(numOfDatanodes, nodeCount); + + // No node should be returned + nodeCount = scmClient.queryNode(IN_MAINTENANCE, HEALTHY, + HddsProtos.QueryScope.CLUSTER, "").size(); + assertEquals(0, nodeCount); + + // Test all operational states by looping over them all and setting the + // state manually. + DatanodeDetails node = nm.getAllNodes().get(0); + for (HddsProtos.NodeOperationalState s : + HddsProtos.NodeOperationalState.values()) { + nm.setNodeOperationalState(node, s); + nodeCount = scmClient.queryNode(s, HEALTHY, + HddsProtos.QueryScope.CLUSTER, "").size(); + if (s == IN_SERVICE) { + assertEquals(5, nodeCount); + } else { + assertEquals(1, nodeCount); + } + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 0cd087eee236..902fe6aed803 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2394,8 +2394,8 @@ public List getServiceList() throws IOException { .setType(ServicePort.Type.RPC) .setValue(scmAddr.getPort()).build()); services.add(scmServiceInfoBuilder.build()); - - List nodes = scmContainerClient.queryNode(HEALTHY, + List nodes = scmContainerClient.queryNode( + HddsProtos.NodeOperationalState.IN_SERVICE, HEALTHY, HddsProtos.QueryScope.CLUSTER, ""); for (HddsProtos.Node node : nodes) { From f4e3fd1742d84d09c46015d5bf9e6d9afd210d0a Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 23 Oct 2019 21:53:34 +0100 Subject: [PATCH 05/78] HDDS-2349. QueryNode does not respect null values for opState or state --- ...LocationProtocolServerSideTranslatorPB.java | 10 ++++++++-- .../scm/server/SCMClientProtocolServer.java | 18 +++++++++--------- .../hadoop/ozone/scm/node/TestQueryNode.java | 12 +++++++++--- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 2c55f04ce5bc..47e2c929d94f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -321,8 +321,14 @@ public NodeQueryResponseProto queryNode( StorageContainerLocationProtocolProtos.NodeQueryRequestProto request) throws IOException { - HddsProtos.NodeState nodeState = request.getState(); - HddsProtos.NodeOperationalState opState = request.getOpState(); + HddsProtos.NodeOperationalState opState = null; + HddsProtos.NodeState nodeState = null; + if (request.hasState()) { + nodeState = request.getState(); + } + if (request.hasOpState()) { + opState = request.getOpState(); + } List datanodes = impl.queryNode(opState, nodeState, request.getScope(), request.getPoolName()); return NodeQueryResponseProto.newBuilder() diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 69c1d2dd3d08..d55816216504 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -22,7 +22,6 @@ package org.apache.hadoop.hdds.scm.server; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.BlockingService; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -35,6 +34,7 @@ import org.apache.hadoop.hdds.scm.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.ScmUtils; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -358,14 +358,15 @@ public List queryNode( List result = new ArrayList<>(); queryNode(opState, state) - .forEach(node -> result.add(HddsProtos.Node.newBuilder() - .setNodeID(node.getProtoBufMessage()) - .addNodeStates(state) - .addNodeOperationalStates(opState) - .build())); - + .forEach(node -> { + NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node); + result.add(HddsProtos.Node.newBuilder() + .setNodeID(node.getProtoBufMessage()) + .addNodeStates(ns.getHealth()) + .addNodeOperationalStates(ns.getOperationalState()) + .build()); + }); return result; - } @Override @@ -574,7 +575,6 @@ public boolean getReplicationManagerStatus() { */ public List queryNode( HddsProtos.NodeOperationalState opState, HddsProtos.NodeState state) { - Preconditions.checkNotNull(state, "Node Query set cannot be null"); return new ArrayList<>(queryNodeState(opState, state)); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java index eae76411e6fb..ff8dce59a53e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestQueryNode.java @@ -50,6 +50,8 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos. NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos. + NodeOperationalState.DECOMMISSIONING; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos. NodeOperationalState.IN_MAINTENANCE; @@ -142,10 +144,14 @@ public void testNodeOperationalStates() throws Exception { StorageContainerManager scm = cluster.getStorageContainerManager(); NodeManager nm = scm.getScmNodeManager(); + // Set one node to be something other than IN_SERVICE + DatanodeDetails node = nm.getAllNodes().get(0); + nm.setNodeOperationalState(node, DECOMMISSIONING); + // All nodes should be returned as they are all in service int nodeCount = scmClient.queryNode(IN_SERVICE, HEALTHY, HddsProtos.QueryScope.CLUSTER, "").size(); - assertEquals(numOfDatanodes, nodeCount); + assertEquals(numOfDatanodes - 1, nodeCount); // null acts as wildcard for opState nodeCount = scmClient.queryNode(null, HEALTHY, @@ -155,7 +161,7 @@ public void testNodeOperationalStates() throws Exception { // null acts as wildcard for nodeState nodeCount = scmClient.queryNode(IN_SERVICE, null, HddsProtos.QueryScope.CLUSTER, "").size(); - assertEquals(numOfDatanodes, nodeCount); + assertEquals(numOfDatanodes - 1, nodeCount); // Both null - should return all nodes nodeCount = scmClient.queryNode(null, null, @@ -169,7 +175,7 @@ public void testNodeOperationalStates() throws Exception { // Test all operational states by looping over them all and setting the // state manually. - DatanodeDetails node = nm.getAllNodes().get(0); + node = nm.getAllNodes().get(0); for (HddsProtos.NodeOperationalState s : HddsProtos.NodeOperationalState.values()) { nm.setNodeOperationalState(node, s); From 278699e4d112c777d8a9e301b48f8acafabaffb5 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 18 Oct 2019 22:05:01 +0100 Subject: [PATCH 06/78] HDDS-2329. Destroy pipelines on any decommission or maintenance nodes Signed-off-by: Anu Engineer --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 + .../src/main/resources/ozone-default.xml | 11 + .../hadoop/hdds/scm/events/SCMEvents.java | 6 + .../hdds/scm/node/DatanodeAdminMonitor.java | 282 ++++++++++++++++++ .../node/DatanodeAdminMonitorInterface.java | 41 +++ .../scm/node/DatanodeAdminNodeDetails.java | 157 ++++++++++ .../scm/node/NodeDecommissionManager.java | 69 ++++- .../scm/node/StartDatanodeAdminHandler.java | 68 +++++ .../scm/server/StorageContainerManager.java | 7 +- .../scm/node/TestDatanodeAdminMonitor.java | 163 ++++++++++ .../node/TestDatanodeAdminNodeDetails.java | 168 +++++++++++ .../scm/node/TestNodeDecommissionManager.java | 2 +- 12 files changed, 961 insertions(+), 18 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 161780668ab0..a27d44e87009 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -366,6 +366,11 @@ public final class ScmConfigKeys { public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled"; public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true; + public static final String OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL = + "ozone.scm.datanode.admin.monitor.interval"; + public static final String OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT = + "30s"; + /** * Never constructed. */ diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index b0a59fa209cc..62878a4a0763 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2501,4 +2501,15 @@ The number of Recon Tasks that are waiting on updates from OM. + + ozone.scm.datanode.admin.monitor.interval + 30s + SCM + + This sets how frequently the datanode admin monitor runs to check for + nodes added to the admin workflow or removed from it. The progress + of decommissioning and entering maintenance nodes is also checked to see + if they have completed. + + diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java index 43d396e0cb12..97e998c90c46 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java @@ -169,6 +169,12 @@ public final class SCMEvents { public static final TypedEvent DEAD_NODE = new TypedEvent<>(DatanodeDetails.class, "Dead_Node"); + /** + * This event will be triggered whenever a datanode is moved into maintenance. + */ + public static final TypedEvent START_ADMIN_ON_NODE = + new TypedEvent<>(DatanodeDetails.class, "START_ADMIN_ON_NODE"); + /** * This event will be triggered whenever a datanode is moved from non-healthy * state to healthy state. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java new file mode 100644 index 000000000000..3f3521766056 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; +import org.apache.hadoop.ozone.common.statemachine.StateMachine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Queue; +import java.util.ArrayDeque; +import java.util.HashSet; +import java.util.Set; + +/** + * Monitor thread which watches for nodes to be decommissioned, recommissioned + * or placed into maintenance. Newly added nodes are queued in pendingNodes + * and recommissoned nodes are queued in cancelled nodes. On each monitor + * 'tick', the cancelled nodes are processed and removed from the monitor. + * Then any pending nodes are added to the trackedNodes set, where they stay + * until decommission or maintenance has ended. + * + * Once an node is placed into tracked nodes, it goes through a workflow where + * the following happens: + * + * 1. First an event is fired to close any pipelines on the node, which will + * also close any containers. + * 2. Next the containers on the node are obtained and checked to see if new + * replicas are needed. If so, the new replicas are scheduled. + * 3. After scheduling replication, the node remains pending until replication + * has completed. + * 4. At this stage the node will complete decommission or enter maintenance. + * 5. Maintenance nodes will remain tracked by this monitor until maintenance + * is manually ended, or the maintenance window expires. + */ +public class DatanodeAdminMonitor implements DatanodeAdminMonitorInterface { + + private OzoneConfiguration conf; + private EventPublisher eventQueue; + private NodeManager nodeManager; + private PipelineManager pipelineManager; + private Queue pendingNodes = new ArrayDeque(); + private Queue cancelledNodes = new ArrayDeque(); + private Set trackedNodes = new HashSet<>(); + private StateMachine workflowSM; + + /** + * States that a node must pass through when being decommissioned or placed + * into maintenance. + */ + public enum States { + CLOSE_PIPELINES(1), + GET_CONTAINERS(2), + REPLICATE_CONTAINERS(3), + AWAIT_MAINTENANCE_END(4), + COMPLETE(5); + + private int sequenceNumber; + + States(int sequenceNumber) { + this.sequenceNumber = sequenceNumber; + } + + public int getSequenceNumber() { + return sequenceNumber; + } + } + + /** + * Transition events that occur to move a node from one state to the next. + */ + public enum Transitions { + COMPLETE_DECOM_STAGE, COMPLETE_MAINT_STAGE, UNEXPECTED_NODE_STATE + } + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminMonitor.class); + + public DatanodeAdminMonitor(OzoneConfiguration config) { + conf = config; + initializeStateMachine(); + } + + @Override + public void setConf(OzoneConfiguration config) { + conf = config; + } + + @Override + public void setEventQueue(EventPublisher eventQueue) { + this.eventQueue = eventQueue; + } + + @Override + public void setNodeManager(NodeManager nm) { + nodeManager = nm; + } + + @Override + public void setPipelineManager(PipelineManager pm) { + pipelineManager = pm; + } + + /** + * Add a node to the decommission or maintenance workflow. The node will be + * queued and added to the workflow after a defined interval. + * + * @param dn The datanode to move into an admin state + * @param endInHours For nodes going into maintenance, the number of hours + * from now for maintenance to automatically end. Ignored + * for decommissioning nodes. + */ + @Override + public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { + DatanodeAdminNodeDetails nodeDetails = + new DatanodeAdminNodeDetails(dn, workflowSM.getInitialState(), + endInHours); + cancelledNodes.remove(nodeDetails); + pendingNodes.add(nodeDetails); + } + + /** + * Remove a node from the decommission or maintenance workflow, and return it + * to service. The node will be queued and removed from decommission or + * maintenance after a defined interval. + * @param dn The datanode for which to stop decommission or maintenance. + */ + @Override + public synchronized void stopMonitoring(DatanodeDetails dn) { + DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, + workflowSM.getInitialState(), 0); + pendingNodes.remove(nodeDetails); + cancelledNodes.add(nodeDetails); + } + + /** + * Run an iteration of the monitor. This is the main run loop, and performs + * the following checks: + * + * 1. Check for any cancelled nodes and process them + * 2. Check for any newly added nodes and add them to the workflow + * 3. Wait for any nodes which have completed closing pipelines + */ + @Override + public void run() { + try { + synchronized (this) { + processCancelledNodes(); + processPendingNodes(); + } + checkPipelinesClosed(); + if (trackedNodes.size() > 0 || pendingNodes.size() > 0) { + LOG.info("There are {} nodes tracked for decommission and "+ + "maintenance. {} pending nodes.", + trackedNodes.size(), pendingNodes.size()); + } + } catch (Exception e) { + LOG.error("Caught an error in the DatanodeAdminMonitor", e); + } + } + + @Override + public int getPendingCount() { + return pendingNodes.size(); + } + + @Override + public int getCancelledCount() { + return cancelledNodes.size(); + } + + @Override + public int getTrackedNodeCount() { + return trackedNodes.size(); + } + + @VisibleForTesting + public Set getTrackedNodes() { + return trackedNodes; + } + + /** + * Return the state machine used to transition a node through the admin + * workflow. + * @return The StateMachine used by the admin workflow + */ + @VisibleForTesting + public StateMachine getWorkflowStateMachine() { + return workflowSM; + } + + private void processCancelledNodes() { + while(!cancelledNodes.isEmpty()) { + DatanodeAdminNodeDetails dn = cancelledNodes.poll(); + trackedNodes.remove(dn); + // TODO - fire event to bring node back into service? + } + } + + private void processPendingNodes() { + while(!pendingNodes.isEmpty()) { + DatanodeAdminNodeDetails dn = pendingNodes.poll(); + // Trigger event to async close the node pipelines. + eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, + dn.getDatanodeDetails()); + trackedNodes.add(dn); + } + } + + private void checkPipelinesClosed() { + for (DatanodeAdminNodeDetails dn : trackedNodes) { + if (dn.getCurrentState() != States.CLOSE_PIPELINES) { + continue; + } + DatanodeDetails dnd = dn.getDatanodeDetails(); + Set pipelines = nodeManager.getPipelines(dnd); + if (pipelines == null || pipelines.size() == 0) { + NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); + try { + dn.transitionState(workflowSM, nodeStatus.getOperationalState()); + } catch (InvalidStateTransitionException e) { + LOG.warn("Unexpected state transition", e); + // TODO - how to handle this? This means the node is not in + // an expected state, eg it is IN_SERVICE when it should be + // decommissioning, so should we abort decom altogether for it? + // This could happen if a node is queued for cancel and not yet + // processed. + } + } else { + LOG.info("Waiting for pipelines to close for {}. There are {} "+ + "pipelines", dnd, pipelines.size()); + } + } + } + + /** + * Setup the state machine with the allowed transitions for a node to move + * through the maintenance workflow. + */ + private void initializeStateMachine() { + Set finalStates = new HashSet<>(); + workflowSM = new StateMachine<>(States.CLOSE_PIPELINES, finalStates); + workflowSM.addTransition(States.CLOSE_PIPELINES, + States.GET_CONTAINERS, Transitions.COMPLETE_DECOM_STAGE); + workflowSM.addTransition(States.GET_CONTAINERS, States.REPLICATE_CONTAINERS, + Transitions.COMPLETE_DECOM_STAGE); + workflowSM.addTransition(States.REPLICATE_CONTAINERS, States.COMPLETE, + Transitions.COMPLETE_DECOM_STAGE); + + workflowSM.addTransition(States.CLOSE_PIPELINES, + States.GET_CONTAINERS, Transitions.COMPLETE_MAINT_STAGE); + workflowSM.addTransition(States.GET_CONTAINERS, States.REPLICATE_CONTAINERS, + Transitions.COMPLETE_MAINT_STAGE); + workflowSM.addTransition(States.REPLICATE_CONTAINERS, + States.AWAIT_MAINTENANCE_END, Transitions.COMPLETE_MAINT_STAGE); + workflowSM.addTransition(States.AWAIT_MAINTENANCE_END, + States.COMPLETE, Transitions.COMPLETE_MAINT_STAGE); + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java new file mode 100644 index 000000000000..d15162b70357 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Interface used by the DatanodeAdminMonitor, which can be used to + * decommission or recommission nodes and take them in and out of maintenance. + */ +public interface DatanodeAdminMonitorInterface extends Runnable { + + void setConf(OzoneConfiguration conf); + void setEventQueue(EventPublisher scm); + void setNodeManager(NodeManager nm); + void setPipelineManager(PipelineManager pm); + void startMonitoring(DatanodeDetails dn, int endInHours); + void stopMonitoring(DatanodeDetails dn); + + int getPendingCount(); + int getCancelledCount(); + int getTrackedNodeCount(); +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java new file mode 100644 index 000000000000..a0607e97679a --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; +import org.apache.hadoop.ozone.common.statemachine.StateMachine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is used by the DatanodeAdminMonitor to track the state and + * details for Datanode decommission and maintenance. It provides a wrapper + * around a DatanodeDetails object adding some additional states and helper + * methods related to the admin workflow. + */ +public class DatanodeAdminNodeDetails { + private DatanodeDetails datanodeDetails; + private long maintenanceEndTime; + private DatanodeAdminMonitor.States currentState; + private long enteredStateAt = 0; + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminNodeDetails.class); + + + /** + * Create a new object given the DatanodeDetails and the maintenance endtime. + * @param dn The datanode going through the admin workflow + * @param maintenanceEnd The number of hours from 'now', when maintenance + * should end automatically. Passing zero indicates + * indicates maintenance will never end automatically. + */ + DatanodeAdminNodeDetails(DatanodeDetails dn, + DatanodeAdminMonitor.States initialState, long maintenanceEnd) { + datanodeDetails = dn; + setMaintenanceEnd(maintenanceEnd); + currentState = initialState; + enteredStateAt = System.currentTimeMillis(); + } + + public boolean shouldMaintenanceEnd() { + if (0 == maintenanceEndTime) { + return false; + } + return System.currentTimeMillis() >= maintenanceEndTime; + } + + public DatanodeDetails getDatanodeDetails() { + return datanodeDetails; + } + + /** + * Get the current admin workflow state for this node. + * @return The current Admin workflow state for this node + */ + public DatanodeAdminMonitor.States getCurrentState() { + return currentState; + } + + /** + * Set the number of hours after which maintenance should end. Passing zero + * indicates maintenance will never end automatically. It is possible to pass + * a negative number of hours can be passed for testing purposes. + * @param hoursFromNow The number of hours from now when maintenance should + * end, or zero for it to never end. + */ + @VisibleForTesting + public void setMaintenanceEnd(long hoursFromNow) { + if (0 == hoursFromNow) { + maintenanceEndTime = 0; + return; + } + // Convert hours to ms + long msFromNow = hoursFromNow * 60L * 60L * 1000L; + maintenanceEndTime = System.currentTimeMillis() + msFromNow; + } + + /** + * Given the workflow stateMachine and the current node status + * (DECOMMISSIONING or ENTERING_MAINTENANCE) move the node to the next + * admin workflow state. + * @param sm The stateMachine which controls the state flow + * @param nodeOperationalState The current operational state for the node, eg + * decommissioning or entering_maintenance + * @return + * @throws InvalidStateTransitionException + */ + public DatanodeAdminMonitor.States transitionState( + StateMachine sm, + NodeOperationalState nodeOperationalState) + throws InvalidStateTransitionException { + + DatanodeAdminMonitor.States newState = sm.getNextState(currentState, + getTransition(nodeOperationalState)); + long currentTime = System.currentTimeMillis(); + LOG.info("Datanode {} moved from admin workflow state {} to {} after {} "+ + "seconds", datanodeDetails, currentState, newState, + (currentTime - enteredStateAt)/1000L); + currentState = newState; + enteredStateAt = currentTime; + return currentState; + } + + private DatanodeAdminMonitor.Transitions getTransition( + NodeOperationalState nodeState) { + if (nodeState == NodeOperationalState.DECOMMISSIONED || + nodeState == NodeOperationalState.DECOMMISSIONING) { + return DatanodeAdminMonitor.Transitions.COMPLETE_DECOM_STAGE; + } else if (nodeState == + NodeOperationalState.ENTERING_MAINTENANCE || + nodeState == NodeOperationalState.IN_MAINTENANCE) { + return DatanodeAdminMonitor.Transitions.COMPLETE_MAINT_STAGE; + } else { + return DatanodeAdminMonitor.Transitions.UNEXPECTED_NODE_STATE; + } + } + + /** + * Matches only on the DatanodeDetails field, which compares only the UUID + * of the node to determine of they are the same object or not. + * + * @param o The object to compare this with + * @return True if the object match, otherwise false + * + */ + @Override + public boolean equals(Object o) { + return o instanceof DatanodeAdminNodeDetails && + datanodeDetails.equals( + ((DatanodeAdminNodeDetails) o).getDatanodeDetails()); + } + + @Override + public int hashCode() { + return datanodeDetails.hashCode(); + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index 9806fbb430c8..fbc5981eeea7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -16,14 +16,16 @@ */ package org.apache.hadoop.hdds.scm.node; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,23 +35,28 @@ import java.net.UnknownHostException; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; /** * Class used to manage datanodes scheduled for maintenance or decommission. */ public class NodeDecommissionManager { + private ScheduledExecutorService executor; + private DatanodeAdminMonitorInterface monitor; + private NodeManager nodeManager; - // private PipelineManager pipeLineManager; + private PipelineManager pipelineManager; // private ContainerManager containerManager; - // private OzoneConfiguration conf; + private EventPublisher eventQueue; + private OzoneConfiguration conf; private boolean useHostnames; - - private List pendingNodes = new LinkedList<>(); + private long monitorInterval; private static final Logger LOG = - LoggerFactory.getLogger(DatanodeAdminManager.class); - + LoggerFactory.getLogger(NodeDecommissionManager.class); static class HostDefinition { private String rawHostname; @@ -157,17 +164,47 @@ private boolean validateDNPortMatch(int port, DatanodeDetails dn) { return false; } - public NodeDecommissionManager(OzoneConfiguration conf, - NodeManager nodeManager, PipelineManager pipelineManager, - ContainerManager containerManager) { - this.nodeManager = nodeManager; - //this.conf = conf; - //this.pipeLineManager = pipelineManager; + public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, + PipelineManager pm, ContainerManager containerManager, + EventPublisher eventQueue) { + this.nodeManager = nm; + conf = config; + this.pipelineManager = pm; //this.containerManager = containerManager; + this.eventQueue = eventQueue; + + executor = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder().setNameFormat("DatanodeAdminManager-%d") + .setDaemon(true).build()); useHostnames = conf.getBoolean( DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + + monitorInterval = conf.getTimeDuration( + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT, + TimeUnit.SECONDS); + if (monitorInterval <= 0) { + LOG.warn("{} must be greater than zero, defaulting to {}", + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT); + monitorInterval = conf.getTimeDuration( + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, + ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL_DEFAULT, + TimeUnit.SECONDS); + } + + monitor = new DatanodeAdminMonitor(conf); + monitor.setConf(conf); + monitor.setEventQueue(this.eventQueue); + monitor.setNodeManager(nodeManager); + monitor.setPipelineManager(pipelineManager); + + executor.scheduleAtFixedRate(monitor, monitorInterval, monitorInterval, + TimeUnit.SECONDS); } public synchronized void decommissionNodes(List nodes) @@ -201,7 +238,7 @@ public synchronized void startDecommission(DatanodeDetails dn) LOG.info("Starting Decommission for node {}", dn); nodeManager.setNodeOperationalState( dn, NodeOperationalState.DECOMMISSIONING); - pendingNodes.add(dn); + monitor.startMonitoring(dn, 0); } else if (opState == NodeOperationalState.DECOMMISSIONING || opState == NodeOperationalState.DECOMMISSIONED) { LOG.info("Start Decommission called on node {} in state {}. Nothing to "+ @@ -238,7 +275,7 @@ public synchronized void recommission(DatanodeDetails dn) if (opState != NodeOperationalState.IN_SERVICE) { nodeManager.setNodeOperationalState( dn, NodeOperationalState.IN_SERVICE); - pendingNodes.remove(dn); + monitor.stopMonitoring(dn); LOG.info("Recommissioned node {}", dn); } else { LOG.info("Recommission called on node {} with state {}. "+ @@ -278,7 +315,7 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) if (opState == NodeOperationalState.IN_SERVICE) { nodeManager.setNodeOperationalState( dn, NodeOperationalState.ENTERING_MAINTENANCE); - pendingNodes.add(dn); + monitor.startMonitoring(dn, endInHours); LOG.info("Starting Maintenance for node {}", dn); } else if (opState == NodeOperationalState.ENTERING_MAINTENANCE || opState == NodeOperationalState.IN_MAINTENANCE) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java new file mode 100644 index 000000000000..9418a7a3e362 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StartDatanodeAdminHandler.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Set; + +/** + * Handler which is fired when a datanode starts admin (decommission or + * maintenance). + */ +public class StartDatanodeAdminHandler + implements EventHandler { + private static final Logger LOG = + LoggerFactory.getLogger(StartDatanodeAdminHandler.class); + + private final NodeManager nodeManager; + private final PipelineManager pipelineManager; + + public StartDatanodeAdminHandler(NodeManager nodeManager, + PipelineManager pipelineManager) { + this.nodeManager = nodeManager; + this.pipelineManager = pipelineManager; + } + + @Override + public void onMessage(DatanodeDetails datanodeDetails, + EventPublisher publisher) { + Set pipelineIds = + nodeManager.getPipelines(datanodeDetails); + LOG.info("Admin start on datanode {}. Finalizing its pipelines {}", + datanodeDetails, pipelineIds); + for (PipelineID pipelineID : pipelineIds) { + try { + Pipeline pipeline = pipelineManager.getPipeline(pipelineID); + pipelineManager.finalizeAndDestroyPipeline(pipeline, false); + } catch (IOException e) { + LOG.info("Could not finalize pipeline={} for dn={}", pipelineID, + datanodeDetails); + } + } + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 24572c7ab18c..db0f7cfe5f8c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; import org.apache.hadoop.hdds.scm.node.DeadNodeHandler; import org.apache.hadoop.hdds.scm.node.NewNodeHandler; +import org.apache.hadoop.hdds.scm.node.StartDatanodeAdminHandler; import org.apache.hadoop.hdds.scm.node.NonHealthyToHealthyNodeHandler; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeReportHandler; @@ -296,6 +297,8 @@ public StorageContainerManager(OzoneConfiguration conf, new StaleNodeHandler(scmNodeManager, pipelineManager, conf); DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager, pipelineManager, containerManager); + StartDatanodeAdminHandler datanodeStartAdminHandler = + new StartDatanodeAdminHandler(scmNodeManager, pipelineManager); NonHealthyToHealthyNodeHandler nonHealthyToHealthyNodeHandler = new NonHealthyToHealthyNodeHandler(pipelineManager, conf); ContainerActionsHandler actionsHandler = new ContainerActionsHandler(); @@ -338,7 +341,7 @@ public StorageContainerManager(OzoneConfiguration conf, pipelineManager); scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, - pipelineManager, containerManager); + pipelineManager, containerManager, eventQueue); eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); @@ -353,6 +356,8 @@ public StorageContainerManager(OzoneConfiguration conf, eventQueue.addHandler(SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE, nonHealthyToHealthyNodeHandler); eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler); + eventQueue.addHandler(SCMEvents.START_ADMIN_ON_NODE, + datanodeStartAdminHandler); eventQueue.addHandler(SCMEvents.CMD_STATUS_REPORT, cmdStatusReportHandler); eventQueue .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java new file mode 100644 index 000000000000..0aa02218dfca --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; +import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; +import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + +import static junit.framework.TestCase.assertEquals; + +/** + * Tests to ensure the DatanodeAdminMonitor is working correctly. + */ +public class TestDatanodeAdminMonitor { + + private StorageContainerManager scm; + private NodeManager nodeManager; + private ContainerManager containerManager; + private SCMPipelineManager pipelineManager; + private OzoneConfiguration conf; + private DatanodeAdminMonitor monitor; + private DatanodeDetails datanode1; + private DatanodeDetails datanode2; + private DatanodeDetails datanode3; + + @Before + public void setup() throws IOException, AuthenticationException { + // This creates a mocked cluster of 6 nodes, where there are mock pipelines + // etc. Borrows heavily from TestDeadNodeHandler. + conf = new OzoneConfiguration(); + String storageDir = GenericTestUtils.getTempPath( + TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + + scm = HddsTestUtils.getScm(conf); + nodeManager = scm.getScmNodeManager(); + pipelineManager = (SCMPipelineManager)scm.getPipelineManager(); + containerManager = scm.getContainerManager(); + + monitor = new DatanodeAdminMonitor(conf); + monitor.setEventQueue(scm.getEventQueue()); + monitor.setNodeManager(nodeManager); + monitor.setPipelineManager(pipelineManager); + + PipelineProvider mockRatisProvider = + new MockRatisPipelineProvider(nodeManager, + pipelineManager.getStateManager(), conf); + pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, + mockRatisProvider); + + datanode1 = TestUtils.randomDatanodeDetails(); + datanode2 = TestUtils.randomDatanodeDetails(); + datanode3 = TestUtils.randomDatanodeDetails(); + + String storagePath = GenericTestUtils.getRandomizedTempPath() + .concat("/" + datanode1.getUuidString()); + + StorageContainerDatanodeProtocolProtos.StorageReportProto + storageOne = TestUtils.createStorageReport( + datanode1.getUuid(), storagePath, 100, 10, 90, null); + + nodeManager.register(datanode1, + TestUtils.createNodeReport(storageOne), null); + nodeManager.register(datanode2, + TestUtils.createNodeReport(storageOne), null); + nodeManager.register(datanode3, + TestUtils.createNodeReport(storageOne), null); + nodeManager.register(TestUtils.randomDatanodeDetails(), + TestUtils.createNodeReport(storageOne), null); + nodeManager.register(TestUtils.randomDatanodeDetails(), + TestUtils.createNodeReport(storageOne), null); + nodeManager.register(TestUtils.randomDatanodeDetails(), + TestUtils.createNodeReport(storageOne), null); + } + + @After + public void teardown() { + } + + @Test + public void testNodeCanBeQueuedAndCancelled() { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + monitor.startMonitoring(dn, 0); + assertEquals(1, monitor.getPendingCount()); + + monitor.stopMonitoring(dn); + assertEquals(0, monitor.getPendingCount()); + assertEquals(1, monitor.getCancelledCount()); + + monitor.startMonitoring(dn, 0); + assertEquals(1, monitor.getPendingCount()); + assertEquals(0, monitor.getCancelledCount()); + + } + + @Test + public void testMonitoredNodeHasPipelinesClosed() + throws NodeNotFoundException, TimeoutException, InterruptedException { + + GenericTestUtils.waitFor(() -> nodeManager + .getPipelines(datanode1).size() == 2, 100, 20000); + + nodeManager.setNodeOperationalState(datanode1, + HddsProtos.NodeOperationalState.DECOMMISSIONING); + monitor.startMonitoring(datanode1, 0); + monitor.run(); + // Ensure the node moves from pending to tracked + assertEquals(0, monitor.getPendingCount()); + assertEquals(1, monitor.getTrackedNodeCount()); + + // Ensure the pipelines are closed, as this is the first step in the admin + // workflow + GenericTestUtils.waitFor(() -> nodeManager + .getPipelines(datanode1).size() == 0, 100, 20000); + + // Run the run loop again and ensure the tracked node is moved to the next + // state + monitor.run(); + for (DatanodeAdminNodeDetails node : monitor.getTrackedNodes()) { + assertEquals( + DatanodeAdminMonitor.States.GET_CONTAINERS, node.getCurrentState()); + } + // Finally, cancel decommission and see the node is removed from tracking + monitor.stopMonitoring(datanode1); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java new file mode 100644 index 000000000000..3b5177e47697 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import + org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static junit.framework.TestCase.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + +/** + * Tests to validate the DatanodeAdminNodeDetails class. + */ +public class TestDatanodeAdminNodeDetails { + + private OzoneConfiguration conf; + private DatanodeAdminMonitor monitor; + private final DatanodeAdminMonitor.States initialState = + DatanodeAdminMonitor.States.CLOSE_PIPELINES; + + @Before + public void setup() { + conf = new OzoneConfiguration(); + monitor = new DatanodeAdminMonitor(conf); + } + + @After + public void teardown() { + } + + @Test + public void testEqualityBasedOnDatanodeDetails() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + DatanodeDetails dn2 = TestUtils.randomDatanodeDetails(); + DatanodeAdminNodeDetails details1 = + new DatanodeAdminNodeDetails(dn1, initialState, 0); + DatanodeAdminNodeDetails details2 = + new DatanodeAdminNodeDetails(dn2, initialState, 0); + + assertNotEquals(details1, details2); + assertEquals(details1, + new DatanodeAdminNodeDetails(dn1, initialState, 0)); + assertNotEquals(details1, dn1); + } + + @Test + public void testUnexpectedNodeStateGivesBadTransition() { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + DatanodeAdminNodeDetails details = + new DatanodeAdminNodeDetails(dn, initialState, 0); + + try { + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.IN_SERVICE); + fail("InvalidStateTransitionException should be thrown"); + } catch (InvalidStateTransitionException e) { + + } + } + + @Test + public void testWorkflowStatesTransitionCorrectlyForDecom() + throws InvalidStateTransitionException { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, + initialState, 0); + + // Initial state should be CLOSE_PIPELINES + assertEquals(DatanodeAdminMonitor.States.CLOSE_PIPELINES, + details.getCurrentState()); + + // Next State is GET_CONTAINERS + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.DECOMMISSIONING); + assertEquals(DatanodeAdminMonitor.States.GET_CONTAINERS, + details.getCurrentState()); + + // Next State is REPLICATE_CONTAINERS + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.DECOMMISSIONING); + assertEquals(DatanodeAdminMonitor.States.REPLICATE_CONTAINERS, + details.getCurrentState()); + + // Next State is COMPLETE + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.DECOMMISSIONING); + assertEquals(DatanodeAdminMonitor.States.COMPLETE, + details.getCurrentState()); + } + + @Test + public void testWorkflowStatesTransitionCorrectlyForMaint() + throws InvalidStateTransitionException { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, + initialState, 0); + + // Initial state should be CLOSE_PIPELINES + assertEquals(DatanodeAdminMonitor.States.CLOSE_PIPELINES, + details.getCurrentState()); + + // Next State is GET_CONTAINERS + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.ENTERING_MAINTENANCE); + assertEquals(DatanodeAdminMonitor.States.GET_CONTAINERS, + details.getCurrentState()); + + // Next State is REPLICATE_CONTAINER + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.ENTERING_MAINTENANCE); + assertEquals(DatanodeAdminMonitor.States.REPLICATE_CONTAINERS, + details.getCurrentState()); + + // Next State is AWAIT_MAINTENANCE_END + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.ENTERING_MAINTENANCE); + assertEquals(DatanodeAdminMonitor.States.AWAIT_MAINTENANCE_END, + details.getCurrentState()); + + // Next State is COMPLETE + details.transitionState(monitor.getWorkflowStateMachine(), + NodeOperationalState.ENTERING_MAINTENANCE); + assertEquals(DatanodeAdminMonitor.States.COMPLETE, + details.getCurrentState()); + } + + @Test + public void testMaintenanceEnd() { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + // End in zero hours - should never end. + DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, + initialState, 0); + assertFalse(details.shouldMaintenanceEnd()); + + // End 1 hour - maintenance should not end yet. + details.setMaintenanceEnd(1); + assertFalse(details.shouldMaintenanceEnd()); + + // End 1 hour ago - maintenance should end. + details.setMaintenanceEnd(-1); + assertTrue(details.shouldMaintenanceEnd()); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index 82bd26b59854..e3b92084d82a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -55,7 +55,7 @@ public void setup() throws Exception { TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); nodeManager = createNodeManager(conf); - decom = new NodeDecommissionManager(conf, nodeManager, null, null); + decom = new NodeDecommissionManager(conf, nodeManager, null, null, null); } @Test From 920b8c5eddc6f25138692636a66270ee22c8dfa4 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 Nov 2019 09:32:13 +0100 Subject: [PATCH 07/78] HDDS-2459. Refactor ReplicationManager to consider maintenance states Closes #262 --- .../StorageContainerDatanodeProtocol.proto | 2 + .../scm/container/ContainerReplicaCount.java | 241 ++++++++++ .../scm/container/ReplicationManager.java | 224 ++++++--- .../hadoop/hdds/scm/node/NodeStatus.java | 69 +++ .../scm/server/StorageContainerManager.java | 3 +- .../scm/container/TestReplicationManager.java | 425 +++++++++++++++++- .../TestSCMContainerPlacementRackAware.java | 6 - .../states/TestContainerReplicaCount.java | 333 ++++++++++++++ .../scm/node/TestDatanodeAdminMonitor.java | 3 + .../scm/safemode/TestSafeModeHandler.java | 6 +- .../node/TestDecommissionAndMaintenance.java | 1 - 11 files changed, 1229 insertions(+), 84 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto index 45a1db681542..6c30ba17a529 100644 --- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -185,6 +185,8 @@ message ContainerReplicaProto { CLOSED = 4; UNHEALTHY = 5; INVALID = 6; + DECOMMISSIONED = 7; + MAINTENANCE = 8; } required int64 containerID = 1; required State state = 2; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java new file mode 100644 index 000000000000..a7ea56d8e018 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import java.util.Set; + +/** + * Immutable object that is created with a set of ContainerReplica objects and + * the number of in flight replica add and deletes, the container replication + * factor and the min count which must be available for maintenance. This + * information can be used to determine if the container is over or under + * replicated and also how many additional replicas need created or removed. + */ +public class ContainerReplicaCount { + + private int healthyCount = 0; + private int decommissionCount = 0; + private int maintenanceCount = 0; + private int inFlightAdd = 0; + private int inFlightDel = 0; + private int repFactor; + private int minHealthyForMaintenance; + private Set replica; + + public ContainerReplicaCount(Set replica, int inFlightAdd, + int inFlightDelete, int replicationFactor, + int minHealthyForMaintenance) { + this.healthyCount = 0; + this.decommissionCount = 0; + this.maintenanceCount = 0; + this.inFlightAdd = inFlightAdd; + this.inFlightDel = inFlightDelete; + this.repFactor = replicationFactor; + this.replica = replica; + this.minHealthyForMaintenance + = Math.min(this.repFactor, minHealthyForMaintenance); + + for (ContainerReplica cr : this.replica) { + ContainerReplicaProto.State state = cr.getState(); + if (state == ContainerReplicaProto.State.DECOMMISSIONED) { + decommissionCount++; + } else if (state == ContainerReplicaProto.State.MAINTENANCE) { + maintenanceCount++; + } else { + healthyCount++; + } + } + } + + public int getHealthyCount() { + return healthyCount; + } + + public int getDecommissionCount() { + return decommissionCount; + } + + public int getMaintenanceCount() { + return maintenanceCount; + } + + public int getReplicationFactor() { + return repFactor; + } + + public Set getReplica() { + return replica; + } + + @Override + public String toString() { + return "Replica Count: "+replica.size()+ + " Healthy Count: "+healthyCount+ + " Decommission Count: "+decommissionCount+ + " Maintenance Count: "+maintenanceCount+ + " inFlightAdd Count: "+inFlightAdd+ + " inFightDel Count: "+inFlightDel+ + " ReplicationFactor: "+repFactor+ + " minMaintenance Count: "+minHealthyForMaintenance; + } + + /** + * Calculates the the delta of replicas which need to be created or removed + * to ensure the container is correctly replicated when considered inflight + * adds and deletes. + * + * When considering inflight operations, it is assumed any operation will + * fail. However, to consider the worst case and avoid data loss, we always + * assume a delete will succeed and and add will fail. In this way, we will + * avoid scheduling too many deletes which could result in dataloss. + * + * Decisions around over-replication are made only on healthy replicas, + * ignoring any in maintenance and also any inflight adds. InFlight adds are + * ignored, as they may not complete, so if we have: + * + * H, H, H, IN_FLIGHT_ADD + * + * And then schedule a delete, we could end up under-replicated (add fails, + * delete completes). It is better to let the inflight operations complete + * and then deal with any further over or under replication. + * + * For maintenance replicas, assuming replication factor 3, and minHealthy + * 2, it is possible for all 3 hosts to be put into maintenance, leaving the + * following (H = healthy, M = maintenance): + * + * H, H, M, M, M + * + * Even though we are tracking 5 replicas, this is not over replicated as we + * ignore the maintenance copies. Later, the replicas could look like: + * + * H, H, H, H, M + * + * At this stage, the container is over replicated by 1, so one replica can be + * removed. + * + * For containers which have replication factor healthy replica, we ignore any + * inflight add or deletes, as they may fail. Instead, wait for them to + * complete and then deal with any excess or deficit. + * + * For under replicated containers we do consider inflight add and delete to + * avoid scheduling more adds than needed. There is additional logic around + * containers with maintenance replica to ensure minHealthyForMaintenance + * replia are maintained. + * + * @return Delta of replicas needed. Negative indicates over replication and + * containers should be removed. Positive indicates over replication + * and zero indicates the containers has replicationFactor healthy + * replica + */ + public int additionalReplicaNeeded() { + int delta = missingReplicas(); + + if (delta < 0) { + // Over replicated, so may need to remove a container. Do not consider + // inFlightAdds, as they may fail, but do consider inFlightDel which + // will reduce the over-replication if it completes. + // Note this could make the delta positive if there are too many in flight + // deletes, which will result in an additional being scheduled. + return delta + inFlightDel; + } else { + // May be under or perfectly replicated. + // We must consider in flight add and delete when calculating the new + // containers needed, but we bound the lower limit at zero to allow + // inflight operations to complete before handling any potential over + // replication + return Math.max(0, delta - inFlightAdd + inFlightDel); + } + } + + /** + * Returns the count of replicas which need to be created or removed to + * ensure the container is perfectly replicate. Inflight operations are not + * considered here, but the logic to determine the missing or excess counts + * for maintenance is present. + * + * Decisions around over-replication are made only on healthy replicas, + * ignoring any in maintenance. For example, if we have: + * + * H, H, H, M, M + * + * This will not be consider over replicated until one of the Maintenance + * replicas moves to Healthy. + * + * If the container is perfectly replicated, zero will be return. + * + * If it is under replicated a positive value will be returned, indicating + * how many replicas must be added. + * + * If it is over replicated a negative value will be returned, indicating now + * many replicas to remove. + * + * @return Zero if the container is perfectly replicated, a positive value + * for under replicated and a negative value for over replicated. + */ + private int missingReplicas() { + int delta = repFactor - healthyCount; + + if (delta < 0) { + // Over replicated, so may need to remove a container. + return delta; + } else if (delta > 0) { + // May be under-replicated, depending on maintenance. + delta = Math.max(0, delta - maintenanceCount); + int neededHealthy = + Math.max(0, minHealthyForMaintenance - healthyCount); + delta = Math.max(neededHealthy, delta); + return delta; + } else { // delta == 0 + // We have exactly the number of healthy replicas needed. + return delta; + } + } + + /** + * Return true if the container is sufficiently replicated. Decommissioning + * and Decommissioned containers are ignored in this check, assuming they will + * eventually be removed from the cluster. + * This check ignores inflight additions, as those replicas have not yet been + * created and the create could fail for some reason. + * The check does consider inflight deletes as there may be 3 healthy replicas + * now, but once the delete completes it will reduce to 2. + * We also assume a replica in Maintenance state cannot be removed, so the + * pending delete would affect only the healthy replica count. + * + * @return True if the container is sufficiently replicated and False + * otherwise. + */ + public boolean isSufficientlyReplicated() { + return missingReplicas() + inFlightDel <= 0; + } + + /** + * Return true is the container is over replicated. Decommission and + * maintenance containers are ignored for this check. + * The check ignores inflight additions, as they may fail, but it does + * consider inflight deletes, as they would reduce the over replication when + * they complete. + * + * @return True if the container is over replicated, false otherwise. + */ + public boolean isOverReplicated() { + return missingReplicas() + inFlightDel < 0; + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 37afd36da019..58e38a26708c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; @@ -97,6 +98,11 @@ public class ReplicationManager implements MetricsSource { */ private final LockManager lockManager; + /** + * Used to lookup the health of a nodes or the nodes operational state. + */ + private final NodeManager nodeManager; + /** * This is used for tracking container replication commands which are issued * by ReplicationManager and not yet complete. @@ -126,6 +132,11 @@ public class ReplicationManager implements MetricsSource { */ private volatile boolean running; + /** + * Minimum number of replica in a healthy state for maintenance. + */ + private int minHealthyForMaintenance; + /** * Constructs ReplicationManager instance with the given configuration. * @@ -138,15 +149,18 @@ public ReplicationManager(final ReplicationManagerConfiguration conf, final ContainerManager containerManager, final ContainerPlacementPolicy containerPlacement, final EventPublisher eventPublisher, - final LockManager lockManager) { + final LockManager lockManager, + final NodeManager nodeManager) { this.containerManager = containerManager; this.containerPlacement = containerPlacement; this.eventPublisher = eventPublisher; this.lockManager = lockManager; + this.nodeManager = nodeManager; this.conf = conf; this.running = false; this.inflightReplication = new ConcurrentHashMap<>(); this.inflightDeletion = new ConcurrentHashMap<>(); + this.minHealthyForMaintenance = conf.getMaintenanceReplicaMinimum(); } /** @@ -241,7 +255,7 @@ private synchronized void run() { * @param id ContainerID */ private void processContainer(ContainerID id) { - lockManager.lock(id); + lockManager.writeLock(id); try { final ContainerInfo container = containerManager.getContainer(id); final Set replicas = containerManager @@ -291,24 +305,15 @@ private void processContainer(ContainerID id) { action -> replicas.stream() .noneMatch(r -> r.getDatanodeDetails().equals(action.datanode))); - - /* - * We don't have to take any action if the container is healthy. - * - * According to ReplicationMonitor container is considered healthy if - * the container is either in QUASI_CLOSED or in CLOSED state and has - * exact number of replicas in the same state. - */ - if (isContainerHealthy(container, replicas)) { - return; - } + ContainerReplicaCount replicaSet = + getContainerReplicaCount(container, replicas); /* * Check if the container is under replicated and take appropriate * action. */ - if (isContainerUnderReplicated(container, replicas)) { - handleUnderReplicatedContainer(container, replicas); + if (!replicaSet.isSufficientlyReplicated()) { + handleUnderReplicatedContainer(container, replicaSet); return; } @@ -316,22 +321,24 @@ private void processContainer(ContainerID id) { * Check if the container is over replicated and take appropriate * action. */ - if (isContainerOverReplicated(container, replicas)) { - handleOverReplicatedContainer(container, replicas); + if (replicaSet.isOverReplicated()) { + handleOverReplicatedContainer(container, replicaSet); return; } /* - * The container is neither under nor over replicated and the container - * is not healthy. This means that the container has unhealthy/corrupted - * replica. + If we get here, the container is not over replicated or under replicated + but it may be "unhealthy", which means it has one or more replica which + are not in the same state as the container itself. */ - handleUnstableContainer(container, replicas); + if (!isContainerHealthy(container, replicas)) { + handleUnstableContainer(container, replicas); + } } catch (ContainerNotFoundException ex) { LOG.warn("Missing container {}.", id); } finally { - lockManager.unlock(id); + lockManager.writeUnlock(id); } } @@ -361,7 +368,8 @@ private void updateInflightAction(final ContainerInfo container, * Returns true if the container is healthy according to ReplicationMonitor. * * According to ReplicationMonitor container is considered healthy if - * it has exact number of replicas in the same state as the container. + * all replica which are not in a decommission or maintenance state are in + * the same state as the container and in QUASI_CLOSED or in CLOSED state. * * @param container Container to check * @param replicas Set of ContainerReplicas @@ -369,50 +377,76 @@ private void updateInflightAction(final ContainerInfo container, */ private boolean isContainerHealthy(final ContainerInfo container, final Set replicas) { - return container.getReplicationFactor().getNumber() == replicas.size() && - replicas.stream().allMatch( - r -> compareState(container.getState(), r.getState())); + return (container.getState() == LifeCycleState.CLOSED + || container.getState() == LifeCycleState.QUASI_CLOSED) + && replicas.stream() + .filter(r -> r.getState() != State.DECOMMISSIONED) + .filter(r -> r.getState() != State.MAINTENANCE) + .allMatch(r -> compareState(container.getState(), r.getState())); } /** - * Checks if the container is under replicated or not. - * - * @param container Container to check - * @param replicas Set of ContainerReplicas - * @return true if the container is under replicated, false otherwise + * Returns the number replica which are pending creation for the given + * container ID. + * @param id The ContainerID for which to check the pending replica + * @return The number of inflight additions or zero if none */ - private boolean isContainerUnderReplicated(final ContainerInfo container, - final Set replicas) { - return container.getReplicationFactor().getNumber() > - getReplicaCount(container.containerID(), replicas); + private int getInflightAdd(final ContainerID id) { + return inflightReplication.getOrDefault(id, Collections.emptyList()).size(); } /** - * Checks if the container is over replicated or not. - * - * @param container Container to check - * @param replicas Set of ContainerReplicas - * @return true if the container if over replicated, false otherwise + * Returns the number replica which are pending delete for the given + * container ID. + * @param id The ContainerID for which to check the pending replica + * @return The number of inflight deletes or zero if none */ - private boolean isContainerOverReplicated(final ContainerInfo container, - final Set replicas) { - return container.getReplicationFactor().getNumber() < - getReplicaCount(container.containerID(), replicas); + private int getInflightDel(final ContainerID id) { + return inflightDeletion.getOrDefault(id, Collections.emptyList()).size(); } /** - * Returns the replication count of the given container. This also - * considers inflight replication and deletion. + * Given a container, obtain the set of known replica for it, and return a + * ContainerReplicaCount object. This object will contain the set of replica + * as well as all information required to determine if the container is over + * or under replicated, including the delta of replica required to repair the + * over or under replication. * - * @param id ContainerID - * @param replicas Set of existing replicas - * @return number of estimated replicas for this container - */ - private int getReplicaCount(final ContainerID id, - final Set replicas) { - return replicas.size() - + inflightReplication.getOrDefault(id, Collections.emptyList()).size() - - inflightDeletion.getOrDefault(id, Collections.emptyList()).size(); + * @param container The container to create a ContainerReplicaCount for + * @return ContainerReplicaCount representing the replicated state of the + * container. + * @throws ContainerNotFoundException + */ + public ContainerReplicaCount getContainerReplicaCount(ContainerInfo container) + throws ContainerNotFoundException { + lockManager.readLock(container.containerID()); + try { + final Set replica = containerManager + .getContainerReplicas(container.containerID()); + return getContainerReplicaCount(container, replica); + } finally { + lockManager.readUnlock(container.containerID()); + } + } + + /** + * Given a container and its set of replicas, create and return a + * ContainerReplicaCount representing the container. + * + * @param container The container for which to construct a + * ContainerReplicaCount + * @param replica The set of existing replica for this container + * @return ContainerReplicaCount representing the current state of the + * container + */ + private ContainerReplicaCount getContainerReplicaCount( + ContainerInfo container, Set replica) { + return new ContainerReplicaCount( + replica, + getInflightAdd(container.containerID()), + getInflightDel(container.containerID()), + container.getReplicationFactor().getNumber(), + minHealthyForMaintenance); } /** @@ -478,13 +512,29 @@ private void forceCloseContainer(final ContainerInfo container, * and send replicate container command to the identified datanode(s). * * @param container ContainerInfo - * @param replicas Set of ContainerReplicas + * @param replicaSet An instance of ContainerReplicaCount, containing the + * current replica count and inflight adds and deletes */ private void handleUnderReplicatedContainer(final ContainerInfo container, - final Set replicas) { - LOG.debug("Handling underreplicated container: {}", + final ContainerReplicaCount replicaSet) { + LOG.debug("Handling under replicated container: {}", container.getContainerID()); + Set replicas = replicaSet.getReplica(); try { + + if (replicaSet.isSufficientlyReplicated()) { + LOG.info("The container {} with replicas {} is sufficiently "+ + "replicated", container.getContainerID(), replicaSet); + return; + } + int repDelta = replicaSet.additionalReplicaNeeded(); + if (repDelta <= 0) { + LOG.info("The container {} with {} is not sufficiently " + + "replicated but no further replicas will be scheduled until "+ + "in-flight operations complete", + container.getContainerID(), replicaSet); + return; + } final ContainerID id = container.containerID(); final List deletionInFlight = inflightDeletion .getOrDefault(id, Collections.emptyList()) @@ -494,15 +544,19 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, final List source = replicas.stream() .filter(r -> r.getState() == State.QUASI_CLOSED || - r.getState() == State.CLOSED) + r.getState() == State.CLOSED || + r.getState() == State.DECOMMISSIONED || + r.getState() == State.MAINTENANCE) + // Exclude stale and dead nodes. This is particularly important for + // maintenance nodes, as the replicas will remain present in the + // container manager, even when they go dead. + .filter(r -> + nodeManager.getNodeStatus(r.getDatanodeDetails()).isHealthy()) .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails())) .sorted((r1, r2) -> r2.getSequenceId().compareTo(r1.getSequenceId())) .map(ContainerReplica::getDatanodeDetails) .collect(Collectors.toList()); if (source.size() > 0) { - final int replicationFactor = container - .getReplicationFactor().getNumber(); - final int delta = replicationFactor - getReplicaCount(id, replicas); final List excludeList = replicas.stream() .map(ContainerReplica::getDatanodeDetails) .collect(Collectors.toList()); @@ -511,13 +565,14 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, actionList.stream().map(r -> r.datanode) .forEach(excludeList::add); } + // At this point we have all live source nodes and we have consider final List selectedDatanodes = containerPlacement - .chooseDatanodes(excludeList, null, delta, + .chooseDatanodes(excludeList, null, repDelta, container.getUsedBytes()); LOG.info("Container {} is under replicated. Expected replica count" + - " is {}, but found {}.", id, replicationFactor, - replicationFactor - delta); + " is {}, but found {}. An additional {} replica are needed", + id, replicaSet.getReplicationFactor(), replicaSet, repDelta); for (DatanodeDetails datanode : selectedDatanodes) { sendReplicateCommand(container, datanode, source); @@ -538,17 +593,16 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, * identified datanode(s). * * @param container ContainerInfo - * @param replicas Set of ContainerReplicas + * @param replicaSet An instance of ContainerReplicaCount, containing the + * current replica count and inflight adds and deletes */ private void handleOverReplicatedContainer(final ContainerInfo container, - final Set replicas) { + final ContainerReplicaCount replicaSet) { + final Set replicas = replicaSet.getReplica(); final ContainerID id = container.containerID(); final int replicationFactor = container.getReplicationFactor().getNumber(); - // Dont consider inflight replication while calculating excess here. - final int excess = replicas.size() - replicationFactor - - inflightDeletion.getOrDefault(id, Collections.emptyList()).size(); - + final int excess = replicaSet.additionalReplicaNeeded() * -1; if (excess > 0) { LOG.info("Container {} is over replicated. Expected replica count" + @@ -566,6 +620,11 @@ private void handleOverReplicatedContainer(final ContainerInfo container, // Retain one healthy replica per origin node Id. final List eligibleReplicas = new ArrayList<>(replicas); eligibleReplicas.removeAll(uniqueReplicas.values()); + // Replica which are maintenance or decommissioned are not eligible to + // be removed, as they do not count toward over-replication and they also + // many not be available + eligibleReplicas.removeIf(r -> (r.getState() == State.MAINTENANCE + || r.getState() == State.DECOMMISSIONED)); final List unhealthyReplicas = eligibleReplicas .stream() @@ -801,6 +860,12 @@ public static class ReplicationManagerConfiguration { */ private long eventTimeout = 10 * 60 * 1000; + /** + * The number of container replica which must be available for a node to + * enter maintenance. + */ + private int maintenanceReplicaMinimum = 2; + @Config(key = "thread.interval", type = ConfigType.TIME, defaultValue = "300s", @@ -825,6 +890,19 @@ public void setEventTimeout(long eventTimeout) { this.eventTimeout = eventTimeout; } + @Config(key = "maintenance.replica.minimum", + type = ConfigType.INT, + defaultValue = "2", + tags = {SCM, OZONE}, + description = "The minimum number of container replicas which must " + + " be available for a node to enter maintenance. If putting a " + + " node into maintenance reduces the available replicas for any " + + " container below this level, the node will remain in the " + + " entering maintenance state until a new replica is created.") + public void setMaintenanceReplicaMinimum(int replicaCount) { + this.maintenanceReplicaMinimum = replicaCount; + } + public long getInterval() { return interval; } @@ -832,6 +910,10 @@ public long getInterval() { public long getEventTimeout() { return eventTimeout; } + + public int getMaintenanceReplicaMinimum() { + return maintenanceReplicaMinimum; + } } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java index 0776c2894e15..5c1adf7ad526 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java @@ -61,6 +61,75 @@ public HddsProtos.NodeOperationalState getOperationalState() { return operationalState; } + /** + * Returns true if the nodeStatus indicates the node is in any decommission + * state. + * + * @return True if the node is in any decommission state, false otherwise + */ + public boolean isDecommission() { + return operationalState == HddsProtos.NodeOperationalState.DECOMMISSIONING + || operationalState == HddsProtos.NodeOperationalState.DECOMMISSIONED; + } + + /** + * Returns true if the node is currently decommissioning. + * + * @return True if the node is decommissioning, false otherwise + */ + public boolean isDecommissioning() { + return operationalState == HddsProtos.NodeOperationalState.DECOMMISSIONING; + } + + /** + * Returns true if the node is decommissioned. + * + * @return True if the node is decommissioned, false otherwise + */ + public boolean isDecommissioned() { + return operationalState == HddsProtos.NodeOperationalState.DECOMMISSIONED; + } + + /** + * Returns true if the node is in any maintenance state. + * + * @return True if the node is in any maintenance state, false otherwise + */ + public boolean isMaintenance() { + return operationalState + == HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE + || operationalState == HddsProtos.NodeOperationalState.IN_MAINTENANCE; + } + + /** + * Returns true if the node is currently entering maintenance. + * + * @return True if the node is entering maintenance, false otherwise + */ + public boolean isEnteringMaintenance() { + return operationalState + == HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE; + } + + /** + * Returns true if the node is currently in maintenance. + * + * @return True if the node is in maintenance, false otherwise. + */ + public boolean isInMaintenance() { + return operationalState == HddsProtos.NodeOperationalState.IN_MAINTENANCE; + } + + /** + * Returns true if the nodeStatus is healthy (ie not stale or dead) and false + * otherwise. + * + * @return True if the node is Healthy, false otherwise + */ + public boolean isHealthy() { + return health == HddsProtos.NodeState.HEALTHY; + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 4df38a3af323..d285e192bed1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -434,7 +434,8 @@ private void initializeSystemManagers(OzoneConfiguration conf, containerManager, containerPlacementPolicy, eventQueue, - new LockManager<>(conf)); + new LockManager<>(conf), + scmNodeManager); } if(configurator.getScmSafeModeManager() != null) { scmSafeModeManager = configurator.getScmSafeModeManager(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java index 1631447af1f5..158b1bd437de 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java @@ -21,21 +21,36 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMCommandProto; -import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; +import org.apache.hadoop.hdds.scm.container.ReplicationManager + .ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.container.placement.algorithms .ContainerPlacementPolicy; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.lock.LockManager; +import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; +import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; +import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -69,12 +84,16 @@ public class TestReplicationManager { private ContainerPlacementPolicy containerPlacementPolicy; private EventQueue eventQueue; private DatanodeCommandHandler datanodeCommandHandler; + private SimpleNodeManager nodeManager; + private ContainerManager containerManager; + private Configuration conf; @Before public void setup() throws IOException, InterruptedException { - final Configuration conf = new OzoneConfiguration(); - final ContainerManager containerManager = + conf = new OzoneConfiguration(); + containerManager = Mockito.mock(ContainerManager.class); + nodeManager = new SimpleNodeManager(); eventQueue = new EventQueue(); containerStateManager = new ContainerStateManager(conf); @@ -106,12 +125,27 @@ public void setup() throws IOException, InterruptedException { .collect(Collectors.toList()); }); + createReplicationManager(new ReplicationManagerConfiguration()); replicationManager = new ReplicationManager( new ReplicationManagerConfiguration(), containerManager, containerPlacementPolicy, eventQueue, - new LockManager<>(conf)); + new LockManager<>(conf), + nodeManager); + replicationManager.start(); + Thread.sleep(100L); + } + + private void createReplicationManager(ReplicationManagerConfiguration rmConf) + throws InterruptedException { + replicationManager = new ReplicationManager( + rmConf, + containerManager, + containerPlacementPolicy, + eventQueue, + new LockManager<>(conf), + nodeManager); replicationManager.start(); Thread.sleep(100L); } @@ -606,6 +640,213 @@ public void testGeneratedConfig() { } + /** + * ReplicationManager should replicate an additional replica if there are + * decommissioned replicas. + */ + @Test + public void testUnderReplicatedDueToDecommission() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.CLOSED, State.DECOMMISSIONED, State.DECOMMISSIONED); + assertReplicaScheduled(2); + } + + /** + * ReplicationManager should replicate an additional replica when all copies + * are decommissioning. + */ + @Test + public void testUnderReplicatedDueToAllDecommission() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.DECOMMISSIONED, State.DECOMMISSIONED, State.DECOMMISSIONED); + assertReplicaScheduled(3); + } + + /** + * ReplicationManager should not take any action when the container is + * correctly replicated with decommissioned replicas still present. + */ + @Test + public void testCorrectlyReplicatedWithDecommission() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.CLOSED, State.CLOSED, State.CLOSED, State.DECOMMISSIONED); + assertReplicaScheduled(0); + } + + /** + * ReplicationManager should replicate an additional replica when min rep + * is not met for maintenance. + */ + @Test + public void testUnderReplicatedDueToMaintenance() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + assertReplicaScheduled(1); + } + + /** + * ReplicationManager should not replicate an additional replica when if + * min replica for maintenance is 1 and another replica is available. + */ + @Test + public void testNotUnderReplicatedDueToMaintenanceMinRepOne() throws + SCMException, ContainerNotFoundException, InterruptedException { + replicationManager.stop(); + ReplicationManagerConfiguration newConf = + new ReplicationManagerConfiguration(); + newConf.setMaintenanceReplicaMinimum(1); + createReplicationManager(newConf); + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + assertReplicaScheduled(0); + } + + /** + * ReplicationManager should replicate an additional replica when all copies + * are going off line and min rep is 1. + */ + @Test + public void testUnderReplicatedDueToMaintenanceMinRepOne() throws + SCMException, ContainerNotFoundException, InterruptedException { + replicationManager.stop(); + ReplicationManagerConfiguration newConf = + new ReplicationManagerConfiguration(); + newConf.setMaintenanceReplicaMinimum(1); + createReplicationManager(newConf); + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.MAINTENANCE, State.MAINTENANCE, State.MAINTENANCE); + assertReplicaScheduled(1); + } + + /** + * ReplicationManager should replicate additional replica when all copies + * are going into maintenance. + */ + @Test + public void testUnderReplicatedDueToAllMaintenance() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.MAINTENANCE, State.MAINTENANCE, State.MAINTENANCE); + assertReplicaScheduled(2); + } + + /** + * ReplicationManager should not replicate additional replica sufficient + * replica are available. + */ + @Test + public void testCorrectlyReplicatedWithMaintenance() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.CLOSED, State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + assertReplicaScheduled(0); + } + + /** + * ReplicationManager should replicate additional replica when all copies + * are decommissioning or maintenance. + */ + @Test + public void testUnderReplicatedWithDecommissionAndMaintenance() throws + SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.DECOMMISSIONED, State.DECOMMISSIONED, State.MAINTENANCE, + State.MAINTENANCE); + assertReplicaScheduled(2); + } + + /** + * When a CLOSED container is over replicated, ReplicationManager + * deletes the excess replicas. While choosing the replica for deletion + * ReplicationManager should not attempt to remove a DECOMMISSION or + * MAINTENANCE replica. + */ + @Test + public void testOverReplicatedClosedContainerWithDecomAndMaint() + throws SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + State.DECOMMISSIONED, State.MAINTENANCE, + State.CLOSED, State.CLOSED, State.CLOSED, State.CLOSED); + + final int currentDeleteCommandCount = datanodeCommandHandler + .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand); + + replicationManager.processContainersNow(); + // Wait for EventQueue to call the event handler + Thread.sleep(100L); + Assert.assertEquals(currentDeleteCommandCount + 1, datanodeCommandHandler + .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand)); + // Get the DECOM and Maint replica and ensure none of them are scheduled + // for removal + Set decom = + containerStateManager.getContainerReplicas(container.containerID()) + .stream() + .filter(r -> r.getState() != State.CLOSED) + .collect(Collectors.toSet()); + for (ContainerReplica r : decom) { + Assert.assertFalse(datanodeCommandHandler.received( + SCMCommandProto.Type.deleteContainerCommand, + r.getDatanodeDetails())); + } + } + + /** + * Replication Manager should not attempt to replicate from an unhealthy + * (stale or dead) node. To test this, setup a scenario where a replia needs + * to be created, but mark all nodes stale. That way, no new replica will be + * scheduled. + */ + @Test + public void testUnderReplicatedNotHealthySource() + throws SCMException, ContainerNotFoundException, InterruptedException { + final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, + NodeStatus.inServiceStale(), + State.CLOSED, State.DECOMMISSIONED, State.DECOMMISSIONED); + // There should be replica scheduled, but as all nodes are stale, nothing + // gets scheduled. + assertReplicaScheduled(0); + } + + private ContainerInfo setupReplicas( + LifeCycleState containerState, State... states) + throws SCMException, ContainerNotFoundException { + return setupReplicas(containerState, NodeStatus.inServiceHealthy(), states); + } + + private ContainerInfo setupReplicas( + LifeCycleState containerState, NodeStatus allNodesStatus, State... states) + throws SCMException, ContainerNotFoundException { + final ContainerInfo container = getContainer(containerState); + final ContainerID id = container.containerID(); + containerStateManager.loadContainer(container); + final UUID originNodeId = UUID.randomUUID(); + + for (State s : states) { + DatanodeDetails dn = randomDatanodeDetails(); + nodeManager.register(dn, allNodesStatus); + final ContainerReplica replica = getReplicas( + id, s, 1000L, originNodeId, dn); + containerStateManager.updateContainerReplica(id, replica); + } + return container; + } + + private void assertReplicaScheduled(int delta) throws InterruptedException { + final int currentReplicateCommandCount = datanodeCommandHandler + .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand); + + replicationManager.processContainersNow(); + // Wait for EventQueue to call the event handler + Thread.sleep(100L); + Assert.assertEquals(currentReplicateCommandCount + delta, + datanodeCommandHandler.getInvocationCount( + SCMCommandProto.Type.replicateContainerCommand)); + } + @After public void teardown() throws IOException { containerStateManager.close(); @@ -659,4 +900,180 @@ private boolean received(final SCMCommandProto.Type type, } } + private class SimpleNodeManager implements NodeManager { + + private Map nodeMap = new HashMap(); + + public void register(DatanodeDetails dd, NodeStatus status) { + nodeMap.put(dd.getUuid(), new DatanodeInfo(dd, status)); + } + + /** + * If the given node was registed with the nodeManager, return the + * NodeStatus for the node. Otherwise return a NodeStatus of "In Service + * and Healthy". + * @param datanodeDetails DatanodeDetails + * @return The NodeStatus of the node if it is registered, otherwise an + * Inservice and Healthy NodeStatus. + */ + @Override + public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) { + DatanodeInfo dni = nodeMap.get(datanodeDetails.getUuid()); + if (dni != null) { + return dni.getNodeStatus(); + } else { + return NodeStatus.inServiceHealthy(); + } + } + + /** + * Below here, are all auto-generate placeholder methods to implement the + * interface. + */ + @Override + public List getNodes(NodeStatus nodeStatus) { + return null; + } + + @Override + public List getNodes( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState health) { + return null; + } + + @Override + public int getNodeCount(NodeStatus nodeStatus) { + return 0; + } + + @Override + public int getNodeCount(HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState health) { + return 0; + } + + @Override + public List getAllNodes() { + return null; + } + + @Override + public SCMNodeStat getStats() { + return null; + } + + @Override + public Map getNodeStats() { + return null; + } + + @Override + public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) { + return null; + } + + @Override + public void setNodeOperationalState(DatanodeDetails datanodeDetails, + HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { + } + + @Override + public Set getPipelines(DatanodeDetails datanodeDetails) { + return null; + } + + @Override + public void addPipeline(Pipeline pipeline) { + } + + @Override + public void removePipeline(Pipeline pipeline) { + } + + @Override + public void addContainer(DatanodeDetails datanodeDetails, + ContainerID containerId) throws NodeNotFoundException { + } + + @Override + public void setContainers(DatanodeDetails datanodeDetails, + Set containerIds) throws NodeNotFoundException { + } + + @Override + public Set getContainers(DatanodeDetails datanodeDetails) + throws NodeNotFoundException { + return null; + } + + @Override + public void addDatanodeCommand(UUID dnId, SCMCommand command) { + } + + @Override + public void processNodeReport(DatanodeDetails datanodeDetails, + StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport) { + } + + @Override + public List getCommandQueue(UUID dnID) { + return null; + } + + @Override + public DatanodeDetails getNodeByUuid(String uuid) { + return null; + } + + @Override + public List getNodesByAddress(String address) { + return null; + } + + @Override + public void close() throws IOException { + + } + + @Override + public Map getNodeCount() { + return null; + } + + @Override + public Map getNodeInfo() { + return null; + } + + @Override + public void onMessage(CommandForDatanode commandForDatanode, + EventPublisher publisher) { + } + + @Override + public VersionResponse getVersion( + StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto + versionRequest) { + return null; + } + + @Override + public RegisteredCommand register(DatanodeDetails datanodeDetails, + StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport, + StorageContainerDatanodeProtocolProtos.PipelineReportsProto + pipelineReport) { + return null; + } + + @Override + public List processHeartbeat(DatanodeDetails datanodeDetails) { + return null; + } + + @Override + public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) { + return null; + } + } + } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index 003035cf0694..abf7f9ff22cb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@ -47,17 +47,11 @@ import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA; import org.hamcrest.MatcherAssert; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import org.junit.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import static org.mockito.Matchers.anyObject; -import org.mockito.Mockito; import static org.mockito.Mockito.when; /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java new file mode 100644 index 000000000000..9a502327792e --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java @@ -0,0 +1,333 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.container.states; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; +import org.junit.Before; +import org.junit.Test; +import java.util.*; +import static junit.framework.TestCase.assertEquals; +import static org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; +import static org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State + .DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State + .MAINTENANCE; + +/** + * Class used to test the ContainerReplicaCount class. + */ +public class TestContainerReplicaCount { + + @Before + public void setup() { + } + + @Test + public void testThreeHealthyReplica() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, true, 0, false); + } + + @Test + public void testTwoHealthyReplica() { + Set replica = registerNodes(CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testOneHealthyReplica() { + Set replica = registerNodes(CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 2, false); + } + + @Test + public void testTwoHealthyAndInflightAdd() { + Set replica = registerNodes(CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + validate(rcnt, false, 0, false); + } + + @Test + /** + * This does not schedule a container to be removed, as the inFlight add may + * fail and then the delete would make things under-replicated. Once the add + * completes there will be 4 healthy and it will get taken care of then. + */ + public void testThreeHealthyAndInflightAdd() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + validate(rcnt, true, 0, false); + } + + @Test + /** + * As the inflight delete may fail, but as it will make the the container + * under replicated, we go ahead and schedule another replica to be added. + */ + public void testThreeHealthyAndInflightDelete() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + /** + * This is NOT sufficiently replicated as the inflight add may fail and the + * inflight del could succeed, leaving only 2 healthy replicas. + */ + public void testThreeHealthyAndInflightAddAndInFlightDelete() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 1, 3, 2); + validate(rcnt, false, 0, false); + } + + @Test + public void testFourHealthyReplicas() { + Set replica = + registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, true, -1, true); + } + + @Test + public void testFourHealthyReplicasAndInFlightDelete() { + Set replica = + registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 3, 2); + validate(rcnt, true, 0, false); + } + + @Test + public void testFourHealthyReplicasAndTwoInFlightDelete() { + Set replica = + registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 2, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testOneHealthyReplicaRepFactorOne() { + Set replica = registerNodes(CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + validate(rcnt, true, 0, false); + } + + @Test + public void testOneHealthyReplicaRepFactorOneInFlightDelete() { + Set replica = registerNodes(CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 1, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testTwoHealthyReplicaTwoInflightAdd() { + Set replica = registerNodes(CLOSED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 2, 0, 3, 2); + validate(rcnt, false, 0, false); + } + + /** + * From here consider decommission replicas. + */ + + @Test + public void testThreeHealthyAndTwoDecommission() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED, + DECOMMISSIONED, DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, true, 0, false); + } + + @Test + public void testOneDecommissionedReplica() { + Set replica = + registerNodes(CLOSED, CLOSED, DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testTwoHealthyOneDecommissionedneInFlightAdd() { + Set replica = + registerNodes(CLOSED, CLOSED, DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + validate(rcnt, false, 0, false); + } + + @Test + public void testAllDecommissioned() { + Set replica = + registerNodes(DECOMMISSIONED, DECOMMISSIONED, DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 3, false); + } + + @Test + public void testAllDecommissionedRepFactorOne() { + Set replica = registerNodes(DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testAllDecommissionedRepFactorOneInFlightAdd() { + Set replica = registerNodes(DECOMMISSIONED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 1, 2); + validate(rcnt, false, 0, false); + } + + @Test + public void testOneHealthyOneDecommissioningRepFactorOne() { + Set replica = registerNodes(DECOMMISSIONED, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + validate(rcnt, true, 0, false); + } + + /** + * Maintenance tests from here. + */ + + @Test + public void testOneHealthyTwoMaintenanceMinRepOfTwo() { + Set replica = + registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testOneHealthyThreeMaintenanceMinRepOfTwo() { + Set replica = registerNodes(CLOSED, + MAINTENANCE, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testOneHealthyTwoMaintenanceMinRepOfOne() { + Set replica = + registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 1); + validate(rcnt, true, 0, false); + } + + @Test + public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() { + Set replica = registerNodes(CLOSED, + MAINTENANCE, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + validate(rcnt, false, 0, false); + } + + @Test + public void testAllMaintenance() { + Set replica = + registerNodes(MAINTENANCE, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, false, 2, false); + } + + @Test + /** + * As we have exactly 3 healthy, but then an excess of maintenance copies + * we ignore the over-replication caused by the maintenance copies until they + * come back online, and then deal with them. + */ + public void testThreeHealthyTwoInMaintenance() { + Set replica = registerNodes(CLOSED, CLOSED, CLOSED, + MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, true, 0, false); + } + + @Test + /** + * This is somewhat similar to testThreeHealthyTwoInMaintenance() except now + * one of the maintenance copies has become healthy and we will need to remove + * the over-replicated healthy container. + */ + public void testFourHealthyOneInMaintenance() { + Set replica = + registerNodes(CLOSED, CLOSED, CLOSED, CLOSED, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + validate(rcnt, true, -1, true); + } + + @Test + public void testOneMaintenanceMinRepOfTwoRepFactorOne() { + Set replica = registerNodes(MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + validate(rcnt, false, 1, false); + } + + @Test + public void testOneMaintenanceMinRepOfTwoRepFactorOneInFlightAdd() { + Set replica = registerNodes(MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 1, 2); + validate(rcnt, false, 0, false); + } + + @Test + public void testOneHealthyOneMaintenanceRepFactorOne() { + Set replica = registerNodes(MAINTENANCE, CLOSED); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + validate(rcnt, true, 0, false); + } + + @Test + public void testTwoDecomTwoMaintenanceOneInflightAdd() { + Set replica = + registerNodes(DECOMMISSIONED, DECOMMISSIONED, MAINTENANCE, MAINTENANCE); + ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + validate(rcnt, false, 1, false); + } + + private void validate(ContainerReplicaCount rcnt, + boolean sufficientlyReplicated, int replicaDelta, boolean overRelicated) { + assertEquals(sufficientlyReplicated, rcnt.isSufficientlyReplicated()); + assertEquals(replicaDelta, rcnt.additionalReplicaNeeded()); + } + + private Set registerNodes( + ContainerReplicaProto.State... states) { + Set replica = new HashSet<>(); + for (ContainerReplicaProto.State s : states) { + DatanodeDetails dn = TestUtils.randomDatanodeDetails(); + replica.add(new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(new ContainerID(1)) + .setContainerState(s) + .setDatanodeDetails(dn) + .setOriginNodeId(dn.getUuid()) + .setSequenceId(1) + .build()); + } + return replica; + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 0aa02218dfca..acd89937b5f5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -34,6 +34,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -127,7 +128,9 @@ public void testNodeCanBeQueuedAndCancelled() { } + @Test + @Ignore // HDDS-2631 public void testMonitoredNodeHasPipelinesClosed() throws NodeNotFoundException, TimeoutException, InterruptedException { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java index 5572e9aa1ef4..a2587a733a6a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java @@ -23,11 +23,13 @@ import org.apache.hadoop.hdds.scm.block.BlockManager; import org.apache.hadoop.hdds.scm.block.BlockManagerImpl; import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.container.ReplicationManager; import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.container.placement.algorithms .ContainerPlacementPolicy; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer; @@ -54,6 +56,7 @@ public class TestSafeModeHandler { private EventQueue eventQueue; private SCMSafeModeManager.SafeModeStatus safeModeStatus; private PipelineManager scmPipelineManager; + private NodeManager nodeManager; public void setup(boolean enabled) { configuration = new OzoneConfiguration(); @@ -68,10 +71,11 @@ public void setup(boolean enabled) { Mockito.mock(ContainerManager.class); Mockito.when(containerManager.getContainerIDs()) .thenReturn(new HashSet<>()); + nodeManager = new MockNodeManager(false, 0); replicationManager = new ReplicationManager( new ReplicationManagerConfiguration(), containerManager, Mockito.mock(ContainerPlacementPolicy.class), - eventQueue, new LockManager(configuration)); + eventQueue, new LockManager(configuration), nodeManager); scmPipelineManager = Mockito.mock(SCMPipelineManager.class); blockManager = Mockito.mock(BlockManagerImpl.class); safeModeHandler = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index 5cf086409f1f..159683c1768e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.ozone.MiniOzoneCluster; From af2efa54c2fa3c65050320afd7f9108f08a53ffb Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 13 Dec 2019 12:40:12 +0100 Subject: [PATCH 08/78] HDDS-2593. DatanodeAdminMonitor should track under replicated containers and complete the admin workflow accordingly Closes #309 --- .../scm/container/ContainerReplicaCount.java | 32 +- .../scm/container/ReplicationManager.java | 39 +- .../hdds/scm/node/DatanodeAdminMonitor.java | 312 +++++++---- .../node/DatanodeAdminMonitorInterface.java | 2 + .../scm/node/DatanodeAdminNodeDetails.java | 80 +-- .../scm/node/NodeDecommissionManager.java | 26 +- .../hadoop/hdds/scm/node/NodeStatus.java | 20 + .../scm/server/StorageContainerManager.java | 2 +- .../hdds/scm/block/TestBlockManager.java | 3 +- .../scm/container/SimpleMockNodeManager.java | 284 ++++++++++ .../scm/container/TestReplicationManager.java | 193 +------ .../states/TestContainerReplicaCount.java | 173 ++++-- .../scm/node/TestDatanodeAdminMonitor.java | 530 +++++++++++++++--- .../node/TestDatanodeAdminNodeDetails.java | 93 +-- .../scm/node/TestNodeDecommissionManager.java | 5 +- 15 files changed, 1207 insertions(+), 587 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java index a7ea56d8e018..b888fdfe7dca 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdds.scm.container; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import java.util.Set; @@ -37,9 +38,11 @@ public class ContainerReplicaCount { private int inFlightDel = 0; private int repFactor; private int minHealthyForMaintenance; + private ContainerInfo container; private Set replica; - public ContainerReplicaCount(Set replica, int inFlightAdd, + public ContainerReplicaCount(ContainerInfo container, + Set replica, int inFlightAdd, int inFlightDelete, int replicationFactor, int minHealthyForMaintenance) { this.healthyCount = 0; @@ -51,6 +54,7 @@ public ContainerReplicaCount(Set replica, int inFlightAdd, this.replica = replica; this.minHealthyForMaintenance = Math.min(this.repFactor, minHealthyForMaintenance); + this.container = container; for (ContainerReplica cr : this.replica) { ContainerReplicaProto.State state = cr.getState(); @@ -80,13 +84,18 @@ public int getReplicationFactor() { return repFactor; } + public ContainerInfo getContainer() { + return container; + } + public Set getReplica() { return replica; } @Override public String toString() { - return "Replica Count: "+replica.size()+ + return "Container State: " +container.getState()+ + " Replica Count: "+replica.size()+ " Healthy Count: "+healthyCount+ " Decommission Count: "+decommissionCount+ " Maintenance Count: "+maintenanceCount+ @@ -238,4 +247,21 @@ public boolean isSufficientlyReplicated() { public boolean isOverReplicated() { return missingReplicas() + inFlightDel < 0; } -} \ No newline at end of file + + /** + * Returns true if the container is healthy, meaning all replica which are not + * in a decommission or maintenance state are in the same state as the + * container and in QUASI_CLOSED or in CLOSED state. + * + * @return true if the container is healthy, false otherwise + */ + public boolean isHealthy() { + return (container.getState() == HddsProtos.LifeCycleState.CLOSED + || container.getState() == HddsProtos.LifeCycleState.QUASI_CLOSED) + && replica.stream() + .filter(r -> r.getState() != ContainerReplicaProto.State.DECOMMISSIONED) + .filter(r -> r.getState() != ContainerReplicaProto.State.MAINTENANCE) + .allMatch(r -> ReplicationManager.compareState( + container.getState(), r.getState())); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index 58e38a26708c..b18024e459ac 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -331,7 +331,7 @@ private void processContainer(ContainerID id) { but it may be "unhealthy", which means it has one or more replica which are not in the same state as the container itself. */ - if (!isContainerHealthy(container, replicas)) { + if (!replicaSet.isHealthy()) { handleUnstableContainer(container, replicas); } @@ -364,27 +364,6 @@ private void updateInflightAction(final ContainerInfo container, } } - /** - * Returns true if the container is healthy according to ReplicationMonitor. - * - * According to ReplicationMonitor container is considered healthy if - * all replica which are not in a decommission or maintenance state are in - * the same state as the container and in QUASI_CLOSED or in CLOSED state. - * - * @param container Container to check - * @param replicas Set of ContainerReplicas - * @return true if the container is healthy, false otherwise - */ - private boolean isContainerHealthy(final ContainerInfo container, - final Set replicas) { - return (container.getState() == LifeCycleState.CLOSED - || container.getState() == LifeCycleState.QUASI_CLOSED) - && replicas.stream() - .filter(r -> r.getState() != State.DECOMMISSIONED) - .filter(r -> r.getState() != State.MAINTENANCE) - .allMatch(r -> compareState(container.getState(), r.getState())); - } - /** * Returns the number replica which are pending creation for the given * container ID. @@ -405,6 +384,19 @@ private int getInflightDel(final ContainerID id) { return inflightDeletion.getOrDefault(id, Collections.emptyList()).size(); } + /** + * Given a ContainerID, lookup the ContainerInfo and then return a + * ContainerReplicaCount object for the container. + * @param containerID The ID of the container + * @return ContainerReplicaCount for the given container + * @throws ContainerNotFoundException + */ + public ContainerReplicaCount getContainerReplicaCount(ContainerID containerID) + throws ContainerNotFoundException { + ContainerInfo container = containerManager.getContainer(containerID); + return getContainerReplicaCount(container); + } + /** * Given a container, obtain the set of known replica for it, and return a * ContainerReplicaCount object. This object will contain the set of replica @@ -442,6 +434,7 @@ public ContainerReplicaCount getContainerReplicaCount(ContainerInfo container) private ContainerReplicaCount getContainerReplicaCount( ContainerInfo container, Set replica) { return new ContainerReplicaCount( + container, replica, getInflightAdd(container.containerID()), getInflightDel(container.containerID()), @@ -799,7 +792,7 @@ private void sendAndTrackDatanodeCommand( * @param replicaState ReplicaState * @return true if the state matches, false otherwise */ - private static boolean compareState(final LifeCycleState containerState, + public static boolean compareState(final LifeCycleState containerState, final State replicaState) { switch (containerState) { case OPEN: diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java index 3f3521766056..6aabd621cb34 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -20,19 +20,21 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; +import org.apache.hadoop.hdds.scm.container.ReplicationManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; -import org.apache.hadoop.ozone.common.statemachine.StateMachine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Queue; -import java.util.ArrayDeque; -import java.util.HashSet; -import java.util.Set; +import java.util.*; /** * Monitor thread which watches for nodes to be decommissioned, recommissioned @@ -61,46 +63,16 @@ public class DatanodeAdminMonitor implements DatanodeAdminMonitorInterface { private EventPublisher eventQueue; private NodeManager nodeManager; private PipelineManager pipelineManager; + private ReplicationManager replicationManager; private Queue pendingNodes = new ArrayDeque(); private Queue cancelledNodes = new ArrayDeque(); private Set trackedNodes = new HashSet<>(); - private StateMachine workflowSM; - - /** - * States that a node must pass through when being decommissioned or placed - * into maintenance. - */ - public enum States { - CLOSE_PIPELINES(1), - GET_CONTAINERS(2), - REPLICATE_CONTAINERS(3), - AWAIT_MAINTENANCE_END(4), - COMPLETE(5); - - private int sequenceNumber; - - States(int sequenceNumber) { - this.sequenceNumber = sequenceNumber; - } - - public int getSequenceNumber() { - return sequenceNumber; - } - } - - /** - * Transition events that occur to move a node from one state to the next. - */ - public enum Transitions { - COMPLETE_DECOM_STAGE, COMPLETE_MAINT_STAGE, UNEXPECTED_NODE_STATE - } private static final Logger LOG = LoggerFactory.getLogger(DatanodeAdminMonitor.class); public DatanodeAdminMonitor(OzoneConfiguration config) { conf = config; - initializeStateMachine(); } @Override @@ -123,6 +95,11 @@ public void setPipelineManager(PipelineManager pm) { pipelineManager = pm; } + @Override + public void setReplicationManager(ReplicationManager rm) { + replicationManager = rm; + } + /** * Add a node to the decommission or maintenance workflow. The node will be * queued and added to the workflow after a defined interval. @@ -135,8 +112,7 @@ public void setPipelineManager(PipelineManager pm) { @Override public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { DatanodeAdminNodeDetails nodeDetails = - new DatanodeAdminNodeDetails(dn, workflowSM.getInitialState(), - endInHours); + new DatanodeAdminNodeDetails(dn, endInHours); cancelledNodes.remove(nodeDetails); pendingNodes.add(nodeDetails); } @@ -149,8 +125,7 @@ public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { */ @Override public synchronized void stopMonitoring(DatanodeDetails dn) { - DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, - workflowSM.getInitialState(), 0); + DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, 0); pendingNodes.remove(nodeDetails); cancelledNodes.add(nodeDetails); } @@ -161,7 +136,8 @@ public synchronized void stopMonitoring(DatanodeDetails dn) { * * 1. Check for any cancelled nodes and process them * 2. Check for any newly added nodes and add them to the workflow - * 3. Wait for any nodes which have completed closing pipelines + * 3. Perform checks on the transitioning nodes and move them through the + * workflow until they have completed decommission or maintenance */ @Override public void run() { @@ -170,7 +146,7 @@ public void run() { processCancelledNodes(); processPendingNodes(); } - checkPipelinesClosed(); + processTransitioningNodes(); if (trackedNodes.size() > 0 || pendingNodes.size() > 0) { LOG.info("There are {} nodes tracked for decommission and "+ "maintenance. {} pending nodes.", @@ -178,6 +154,8 @@ public void run() { } } catch (Exception e) { LOG.error("Caught an error in the DatanodeAdminMonitor", e); + // Intentionally do not re-throw, as if we do the monitor thread + // will not get rescheduled. } } @@ -201,82 +179,218 @@ public Set getTrackedNodes() { return trackedNodes; } - /** - * Return the state machine used to transition a node through the admin - * workflow. - * @return The StateMachine used by the admin workflow - */ - @VisibleForTesting - public StateMachine getWorkflowStateMachine() { - return workflowSM; - } - private void processCancelledNodes() { while(!cancelledNodes.isEmpty()) { DatanodeAdminNodeDetails dn = cancelledNodes.poll(); - trackedNodes.remove(dn); + try { + stopTrackingNode(dn); + putNodeBackInService(dn); + LOG.info("Recommissioned node {}", dn.getDatanodeDetails()); + } catch (NodeNotFoundException e) { + LOG.warn("Failed processing the cancel admin request for {}", + dn.getDatanodeDetails(), e); + } // TODO - fire event to bring node back into service? } } private void processPendingNodes() { while(!pendingNodes.isEmpty()) { - DatanodeAdminNodeDetails dn = pendingNodes.poll(); - // Trigger event to async close the node pipelines. - eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, - dn.getDatanodeDetails()); - trackedNodes.add(dn); + startTrackingNode(pendingNodes.poll()); } } - private void checkPipelinesClosed() { - for (DatanodeAdminNodeDetails dn : trackedNodes) { - if (dn.getCurrentState() != States.CLOSE_PIPELINES) { - continue; + private void processTransitioningNodes() { + Iterator iterator = trackedNodes.iterator(); + while (iterator.hasNext()) { + DatanodeAdminNodeDetails dn = iterator.next(); + try { + NodeStatus status = getNodeStatus(dn.getDatanodeDetails()); + + if (!shouldContinueWorkflow(dn, status)) { + abortWorkflow(dn); + iterator.remove(); + continue; + } + + if (status.isMaintenance()) { + if (dn.shouldMaintenanceEnd()) { + completeMaintenance(dn); + iterator.remove(); + continue; + } + } + + if (status.isDecommissioning() || status.isEnteringMaintenance()) { + if (checkPipelinesClosedOnNode(dn) + && checkContainersReplicatedOnNode(dn)) { + // CheckContainersReplicatedOnNode may take a short time to run + // so after it completes, re-get the nodestatus to check the health + // and ensure the state is still good to continue + status = getNodeStatus(dn.getDatanodeDetails()); + if (status.isDead()) { + LOG.warn("Datanode {} is dead and the admin workflow cannot "+ + "continue. The node will be put back to IN_SERVICE and "+ + "handled as a dead node", dn); + putNodeBackInService(dn); + iterator.remove(); + } else if (status.isDecommissioning()) { + completeDecommission(dn); + iterator.remove(); + } else if (status.isEnteringMaintenance()) { + putIntoMaintenance(dn); + } + } + } + + } catch (NodeNotFoundException e) { + LOG.error("An unexpected error occurred processing datanode {}. " + + "Aborting the admin workflow", dn.getDatanodeDetails(), e); + abortWorkflow(dn); + iterator.remove(); } - DatanodeDetails dnd = dn.getDatanodeDetails(); - Set pipelines = nodeManager.getPipelines(dnd); - if (pipelines == null || pipelines.size() == 0) { - NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); - try { - dn.transitionState(workflowSM, nodeStatus.getOperationalState()); - } catch (InvalidStateTransitionException e) { - LOG.warn("Unexpected state transition", e); - // TODO - how to handle this? This means the node is not in - // an expected state, eg it is IN_SERVICE when it should be - // decommissioning, so should we abort decom altogether for it? - // This could happen if a node is queued for cancel and not yet - // processed. + } + } + + /** + * Checks if a node is in an unexpected state or has gone dead while + * decommissioning or entering maintenance. If the node is not in a valid + * state to continue the admin workflow, return false, otherwise return true. + * @param dn The Datanode for which to check the current state + * @param nodeStatus The current NodeStatus for the datanode + * @return True if admin can continue, false otherwise + */ + private boolean shouldContinueWorkflow(DatanodeAdminNodeDetails dn, + NodeStatus nodeStatus) { + if (!nodeStatus.isDecommission() && !nodeStatus.isMaintenance()) { + LOG.warn("Datanode {} has an operational state of {} when it should "+ + "be undergoing decommission or maintenance. Aborting admin for "+ + "this node.", + dn.getDatanodeDetails(), nodeStatus.getOperationalState()); + return false; + } + if (nodeStatus.isDead() && !nodeStatus.isInMaintenance()) { + LOG.error("Datanode {} is dead but is not IN_MAINTENANCE. Aborting the "+ + "admin workflow for this node", dn.getDatanodeDetails()); + return false; + } + return true; + } + + private boolean checkPipelinesClosedOnNode(DatanodeAdminNodeDetails dn) { + DatanodeDetails dnd = dn.getDatanodeDetails(); + Set pipelines = nodeManager.getPipelines(dnd); + if (pipelines == null || pipelines.size() == 0 + || dn.shouldMaintenanceEnd()) { + return true; + } else { + LOG.info("Waiting for pipelines to close for {}. There are {} "+ + "pipelines", dnd, pipelines.size()); + return false; + } + } + + private boolean checkContainersReplicatedOnNode(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + int sufficientlyReplicated = 0; + int underReplicated = 0; + int unhealthy = 0; + Set containers = + nodeManager.getContainers(dn.getDatanodeDetails()); + for(ContainerID cid : containers) { + try { + ContainerReplicaCount replicaSet = + replicationManager.getContainerReplicaCount(cid); + if (replicaSet.isSufficientlyReplicated()) { + sufficientlyReplicated++; + } else { + underReplicated++; } - } else { - LOG.info("Waiting for pipelines to close for {}. There are {} "+ - "pipelines", dnd, pipelines.size()); + if (!replicaSet.isHealthy()) { + unhealthy++; + } + } catch (ContainerNotFoundException e) { + LOG.warn("ContainerID {} present in node list for {} but not found "+ + "in containerManager", cid, dn.getDatanodeDetails()); } } + dn.setSufficientlyReplicatedContainers(sufficientlyReplicated); + dn.setUnderReplicatedContainers(underReplicated); + dn.setUnHealthyContainers(unhealthy); + + return underReplicated == 0 && unhealthy == 0; + } + + private void completeDecommission(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException{ + setNodeOpState(dn, NodeOperationalState.DECOMMISSIONED); + LOG.info("Datanode {} has completed the admin workflow. The operational "+ + "state has been set to {}", dn.getDatanodeDetails(), + NodeOperationalState.DECOMMISSIONED); + } + + private void putIntoMaintenance(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + LOG.info("Datanode {} has entered maintenance", dn.getDatanodeDetails()); + setNodeOpState(dn, NodeOperationalState.IN_MAINTENANCE); + } + + private void completeMaintenance(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + // The end state of Maintenance is to put the node back IN_SERVICE, whether + // it is dead or not. + // TODO - if the node is dead do we trigger a dead node event here or leave + // it to the heartbeat manager? + LOG.info("Datanode {} has ended maintenance automatically", + dn.getDatanodeDetails()); + putNodeBackInService(dn); + } + + private void startTrackingNode(DatanodeAdminNodeDetails dn) { + eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, + dn.getDatanodeDetails()); + trackedNodes.add(dn); + } + + private void stopTrackingNode(DatanodeAdminNodeDetails dn) { + trackedNodes.remove(dn); } /** - * Setup the state machine with the allowed transitions for a node to move - * through the maintenance workflow. + * If we encounter an unexpected condition in maintenance, we must abort the + * workflow by setting the node operationalState back to IN_SERVICE and then + * remove the node from tracking. + * @param dn The datanode for which to abort tracking */ - private void initializeStateMachine() { - Set finalStates = new HashSet<>(); - workflowSM = new StateMachine<>(States.CLOSE_PIPELINES, finalStates); - workflowSM.addTransition(States.CLOSE_PIPELINES, - States.GET_CONTAINERS, Transitions.COMPLETE_DECOM_STAGE); - workflowSM.addTransition(States.GET_CONTAINERS, States.REPLICATE_CONTAINERS, - Transitions.COMPLETE_DECOM_STAGE); - workflowSM.addTransition(States.REPLICATE_CONTAINERS, States.COMPLETE, - Transitions.COMPLETE_DECOM_STAGE); - - workflowSM.addTransition(States.CLOSE_PIPELINES, - States.GET_CONTAINERS, Transitions.COMPLETE_MAINT_STAGE); - workflowSM.addTransition(States.GET_CONTAINERS, States.REPLICATE_CONTAINERS, - Transitions.COMPLETE_MAINT_STAGE); - workflowSM.addTransition(States.REPLICATE_CONTAINERS, - States.AWAIT_MAINTENANCE_END, Transitions.COMPLETE_MAINT_STAGE); - workflowSM.addTransition(States.AWAIT_MAINTENANCE_END, - States.COMPLETE, Transitions.COMPLETE_MAINT_STAGE); + private void abortWorkflow(DatanodeAdminNodeDetails dn) { + try { + putNodeBackInService(dn); + } catch (NodeNotFoundException e) { + LOG.error("Unable to set the node OperationalState for {} while "+ + "aborting the datanode admin workflow", dn.getDatanodeDetails()); + } + } + + private void putNodeBackInService(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + setNodeOpState(dn, NodeOperationalState.IN_SERVICE); + } + + private void setNodeOpState(DatanodeAdminNodeDetails dn, + HddsProtos.NodeOperationalState state) throws NodeNotFoundException { + nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state); + } + + // TODO - The nodeManager.getNodeStatus call should really throw + // NodeNotFoundException rather than having to handle it here as all + // registered nodes must have a status. + private NodeStatus getNodeStatus(DatanodeDetails dnd) + throws NodeNotFoundException { + NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); + if (nodeStatus == null) { + throw new NodeNotFoundException("Unable to retrieve the nodeStatus"); + } + return nodeStatus; } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java index d15162b70357..cfbbffb8d977 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.container.ReplicationManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventPublisher; @@ -32,6 +33,7 @@ public interface DatanodeAdminMonitorInterface extends Runnable { void setEventQueue(EventPublisher scm); void setNodeManager(NodeManager nm); void setPipelineManager(PipelineManager pm); + void setReplicationManager(ReplicationManager rm); void startMonitoring(DatanodeDetails dn, int endInHours); void stopMonitoring(DatanodeDetails dn); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java index a0607e97679a..c23fcd28c69a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java @@ -19,9 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; -import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; -import org.apache.hadoop.ozone.common.statemachine.StateMachine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +31,10 @@ public class DatanodeAdminNodeDetails { private DatanodeDetails datanodeDetails; private long maintenanceEndTime; - private DatanodeAdminMonitor.States currentState; private long enteredStateAt = 0; + private int unHealthyContainers = 0; + private int underReplicatedContainers = 0; + private int sufficientlyReplicatedContainers = 0; private static final Logger LOG = LoggerFactory.getLogger(DatanodeAdminNodeDetails.class); @@ -48,11 +47,9 @@ public class DatanodeAdminNodeDetails { * should end automatically. Passing zero indicates * indicates maintenance will never end automatically. */ - DatanodeAdminNodeDetails(DatanodeDetails dn, - DatanodeAdminMonitor.States initialState, long maintenanceEnd) { + DatanodeAdminNodeDetails(DatanodeDetails dn, long maintenanceEnd) { datanodeDetails = dn; setMaintenanceEnd(maintenanceEnd); - currentState = initialState; enteredStateAt = System.currentTimeMillis(); } @@ -67,12 +64,28 @@ public DatanodeDetails getDatanodeDetails() { return datanodeDetails; } - /** - * Get the current admin workflow state for this node. - * @return The current Admin workflow state for this node - */ - public DatanodeAdminMonitor.States getCurrentState() { - return currentState; + public void setUnHealthyContainers(int val) { + this.unHealthyContainers = val; + } + + public void setUnderReplicatedContainers(int val) { + this.underReplicatedContainers = val; + } + + public void setSufficientlyReplicatedContainers(int val) { + this.sufficientlyReplicatedContainers = val; + } + + public int getUnHealthyContainers() { + return unHealthyContainers; + } + + public int getUnderReplicatedContainers() { + return underReplicatedContainers; + } + + public int getSufficientlyReplicatedContainers() { + return sufficientlyReplicatedContainers; } /** @@ -93,47 +106,6 @@ public void setMaintenanceEnd(long hoursFromNow) { maintenanceEndTime = System.currentTimeMillis() + msFromNow; } - /** - * Given the workflow stateMachine and the current node status - * (DECOMMISSIONING or ENTERING_MAINTENANCE) move the node to the next - * admin workflow state. - * @param sm The stateMachine which controls the state flow - * @param nodeOperationalState The current operational state for the node, eg - * decommissioning or entering_maintenance - * @return - * @throws InvalidStateTransitionException - */ - public DatanodeAdminMonitor.States transitionState( - StateMachine sm, - NodeOperationalState nodeOperationalState) - throws InvalidStateTransitionException { - - DatanodeAdminMonitor.States newState = sm.getNextState(currentState, - getTransition(nodeOperationalState)); - long currentTime = System.currentTimeMillis(); - LOG.info("Datanode {} moved from admin workflow state {} to {} after {} "+ - "seconds", datanodeDetails, currentState, newState, - (currentTime - enteredStateAt)/1000L); - currentState = newState; - enteredStateAt = currentTime; - return currentState; - } - - private DatanodeAdminMonitor.Transitions getTransition( - NodeOperationalState nodeState) { - if (nodeState == NodeOperationalState.DECOMMISSIONED || - nodeState == NodeOperationalState.DECOMMISSIONING) { - return DatanodeAdminMonitor.Transitions.COMPLETE_DECOM_STAGE; - } else if (nodeState == - NodeOperationalState.ENTERING_MAINTENANCE || - nodeState == NodeOperationalState.IN_MAINTENANCE) { - return DatanodeAdminMonitor.Transitions.COMPLETE_MAINT_STAGE; - } else { - return DatanodeAdminMonitor.Transitions.UNEXPECTED_NODE_STATE; - } - } - /** * Matches only on the DatanodeDetails field, which compares only the UUID * of the node to determine of they are the same object or not. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index fbc5981eeea7..f76929ef8514 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -16,12 +16,14 @@ */ package org.apache.hadoop.hdds.scm.node; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ReplicationManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventPublisher; @@ -49,8 +51,9 @@ public class NodeDecommissionManager { private NodeManager nodeManager; private PipelineManager pipelineManager; - // private ContainerManager containerManager; + //private ContainerManager containerManager; private EventPublisher eventQueue; + private ReplicationManager replicationManager; private OzoneConfiguration conf; private boolean useHostnames; private long monitorInterval; @@ -166,12 +169,13 @@ private boolean validateDNPortMatch(int port, DatanodeDetails dn) { public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, PipelineManager pm, ContainerManager containerManager, - EventPublisher eventQueue) { + EventPublisher eventQueue, ReplicationManager rm) { this.nodeManager = nm; conf = config; this.pipelineManager = pm; //this.containerManager = containerManager; this.eventQueue = eventQueue; + this.replicationManager = rm; executor = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("DatanodeAdminManager-%d") @@ -202,11 +206,17 @@ public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, monitor.setEventQueue(this.eventQueue); monitor.setNodeManager(nodeManager); monitor.setPipelineManager(pipelineManager); + monitor.setReplicationManager(replicationManager); executor.scheduleAtFixedRate(monitor, monitorInterval, monitorInterval, TimeUnit.SECONDS); } + @VisibleForTesting + public DatanodeAdminMonitorInterface getMonitor() { + return monitor; + } + public synchronized void decommissionNodes(List nodes) throws InvalidHostStringException { List dns = mapHostnamesToDatanodes(nodes); @@ -239,8 +249,7 @@ public synchronized void startDecommission(DatanodeDetails dn) nodeManager.setNodeOperationalState( dn, NodeOperationalState.DECOMMISSIONING); monitor.startMonitoring(dn, 0); - } else if (opState == NodeOperationalState.DECOMMISSIONING - || opState == NodeOperationalState.DECOMMISSIONED) { + } else if (nodeStatus.isDecommission()) { LOG.info("Start Decommission called on node {} in state {}. Nothing to "+ "do.", dn, opState); } else { @@ -273,10 +282,10 @@ public synchronized void recommission(DatanodeDetails dn) NodeStatus nodeStatus = getNodeStatus(dn); NodeOperationalState opState = nodeStatus.getOperationalState(); if (opState != NodeOperationalState.IN_SERVICE) { - nodeManager.setNodeOperationalState( - dn, NodeOperationalState.IN_SERVICE); + // The node will be set back to IN_SERVICE when it is processed by the + // monitor monitor.stopMonitoring(dn); - LOG.info("Recommissioned node {}", dn); + LOG.info("Queued node {} for recommission", dn); } else { LOG.info("Recommission called on node {} with state {}. "+ "Nothing to do.", dn, opState); @@ -317,8 +326,7 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) dn, NodeOperationalState.ENTERING_MAINTENANCE); monitor.startMonitoring(dn, endInHours); LOG.info("Starting Maintenance for node {}", dn); - } else if (opState == NodeOperationalState.ENTERING_MAINTENANCE || - opState == NodeOperationalState.IN_MAINTENANCE) { + } else if (nodeStatus.isMaintenance()) { LOG.info("Starting Maintenance called on node {} with state {}. "+ "Nothing to do.", dn, opState); } else { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java index 5c1adf7ad526..b8a92e5a6796 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java @@ -130,6 +130,26 @@ public boolean isHealthy() { return health == HddsProtos.NodeState.HEALTHY; } + /** + * Returns true if the nodeStatus is either healthy or stale and false + * otherwise. + * + * @return True is the node is Healthy or Stale, false otherwise. + */ + public boolean isAlive() { + return health == HddsProtos.NodeState.HEALTHY + || health == HddsProtos.NodeState.STALE; + } + + /** + * Returns true if the nodeStatus is dead and false otherwise. + * + * @return True is the node is Dead, false otherwise. + */ + public boolean isDead() { + return health == HddsProtos.NodeState.DEAD; + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index a4aa19d18d0d..651f259c3fe7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -340,7 +340,7 @@ public StorageContainerManager(OzoneConfiguration conf, pipelineManager); scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, - pipelineManager, containerManager, eventQueue); + pipelineManager, containerManager, eventQueue, replicationManager); eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index af66e00f687d..5e8294467d54 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -315,7 +315,8 @@ public void testMultipleBlockAllocationWithClosedContainer() // create pipelines for (int i = 0; - i < nodeManager.getNodes(NodeStatus.inServiceHealthy()).size(); i++) { + i < nodeManager.getNodes(NodeStatus.inServiceHealthy()).size() + / factor.getNumber(); i++) { pipelineManager.createPipeline(type, factor); } TestUtils.openAllRatisPipelines(pipelineManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java new file mode 100644 index 000000000000..78d576c55016 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -0,0 +1,284 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; +import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; +import org.apache.hadoop.hdds.scm.node.DatanodeInfo; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.protocol.VersionResponse; +import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; +import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; +import org.apache.hadoop.ozone.protocol.commands.SCMCommand; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Basic implementation of the NodeManager interface which can be used in tests. + * + * TODO - Merge the functionality with MockNodeManager, as it needs refactored + * after the introduction of decommission and maintenance states. + */ +public class SimpleMockNodeManager implements NodeManager { + + private Map nodeMap = new ConcurrentHashMap<>(); + private Map> pipelineMap = new ConcurrentHashMap<>(); + private Map> containerMap = new ConcurrentHashMap<>(); + + public void register(DatanodeDetails dd, NodeStatus status) { + nodeMap.put(dd.getUuid(), new DatanodeInfo(dd, status)); + } + + public void setNodeStatus(DatanodeDetails dd, NodeStatus status) { + DatanodeInfo dni = nodeMap.get(dd.getUuid()); + dni.setNodeStatus(status); + } + + /** + * Set the number of pipelines for the given node. This simply generates + * new PipelineID objects and places them in a set. No actual pipelines are + * created. + * + * Setting the count to zero effectively deletes the pipelines for the node + * + * @param dd The DatanodeDetails for which to create the pipelines + * @param count The number of pipelines to create or zero to delete all + * pipelines + */ + public void setPipelines(DatanodeDetails dd, int count) { + Set pipelines = new HashSet<>(); + for (int i=0; i getPipelines(DatanodeDetails datanodeDetails) { + Set p = pipelineMap.get(datanodeDetails.getUuid()); + if (p == null || p.size() == 0) { + return null; + } else { + return p; + } + } + + @Override + public void setContainers(DatanodeDetails dn, + Set containerIds) throws NodeNotFoundException { + containerMap.put(dn.getUuid(), containerIds); + } + + /** + * Return the set of ContainerID associated with the datanode. If there are no + * container present, an empty set is return to mirror the behaviour of + * SCMNodeManaer + * + * @param dn The datanodeDetails for which to return the containers + * @return A Set of ContainerID or an empty Set if none are present + * @throws NodeNotFoundException + */ + @Override + public Set getContainers(DatanodeDetails dn) + throws NodeNotFoundException { + // The concrete implementation of this method in SCMNodeManager will return + // an empty set if there are no containers, and will never return null. + return containerMap + .computeIfAbsent(dn.getUuid(), key -> new HashSet<>()); + } + + /** + * Below here, are all auto-generate placeholder methods to implement the + * interface. + */ + + @Override + public List getNodes(NodeStatus nodeStatus) { + return null; + } + + @Override + public List getNodes( + HddsProtos.NodeOperationalState opState, HddsProtos.NodeState health) { + return null; + } + + @Override + public int getNodeCount(NodeStatus nodeStatus) { + return 0; + } + + @Override + public int getNodeCount(HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState health) { + return 0; + } + + @Override + public List getAllNodes() { + return null; + } + + @Override + public SCMNodeStat getStats() { + return null; + } + + @Override + public Map getNodeStats() { + return null; + } + + @Override + public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) { + return null; + } + + @Override + public void addPipeline(Pipeline pipeline) { + } + + @Override + public void removePipeline(Pipeline pipeline) { + } + + @Override + public void addContainer(DatanodeDetails datanodeDetails, + ContainerID containerId) throws NodeNotFoundException { + } + + + + @Override + public void addDatanodeCommand(UUID dnId, SCMCommand command) { + } + + @Override + public void processNodeReport(DatanodeDetails datanodeDetails, + StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport) { + } + + @Override + public List getCommandQueue(UUID dnID) { + return null; + } + + @Override + public DatanodeDetails getNodeByUuid(String uuid) { + return null; + } + + @Override + public List getNodesByAddress(String address) { + return null; + } + + @Override + public void close() throws IOException { + + } + + @Override + public Map getNodeCount() { + return null; + } + + @Override + public Map getNodeInfo() { + return null; + } + + @Override + public void onMessage(CommandForDatanode commandForDatanode, + EventPublisher publisher) { + } + + @Override + public VersionResponse getVersion( + StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto + versionRequest) { + return null; + } + + @Override + public RegisteredCommand register(DatanodeDetails datanodeDetails, + StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport, + StorageContainerDatanodeProtocolProtos.PipelineReportsProto + pipelineReport) { + return null; + } + + @Override + public List processHeartbeat(DatanodeDetails datanodeDetails) { + return null; + } + + @Override + public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) { + return null; + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java index 158b1bd437de..8fe1d9a7bdc7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java @@ -21,10 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.protocol.proto @@ -33,24 +30,14 @@ .ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.container.placement.algorithms .ContainerPlacementPolicy; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; -import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.node.DatanodeInfo; -import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; -import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.lock.LockManager; -import org.apache.hadoop.ozone.protocol.VersionResponse; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; -import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; -import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -84,7 +71,7 @@ public class TestReplicationManager { private ContainerPlacementPolicy containerPlacementPolicy; private EventQueue eventQueue; private DatanodeCommandHandler datanodeCommandHandler; - private SimpleNodeManager nodeManager; + private SimpleMockNodeManager nodeManager; private ContainerManager containerManager; private Configuration conf; @@ -93,7 +80,7 @@ public void setup() throws IOException, InterruptedException { conf = new OzoneConfiguration(); containerManager = Mockito.mock(ContainerManager.class); - nodeManager = new SimpleNodeManager(); + nodeManager = new SimpleMockNodeManager(); eventQueue = new EventQueue(); containerStateManager = new ContainerStateManager(conf); @@ -900,180 +887,4 @@ private boolean received(final SCMCommandProto.Type type, } } - private class SimpleNodeManager implements NodeManager { - - private Map nodeMap = new HashMap(); - - public void register(DatanodeDetails dd, NodeStatus status) { - nodeMap.put(dd.getUuid(), new DatanodeInfo(dd, status)); - } - - /** - * If the given node was registed with the nodeManager, return the - * NodeStatus for the node. Otherwise return a NodeStatus of "In Service - * and Healthy". - * @param datanodeDetails DatanodeDetails - * @return The NodeStatus of the node if it is registered, otherwise an - * Inservice and Healthy NodeStatus. - */ - @Override - public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) { - DatanodeInfo dni = nodeMap.get(datanodeDetails.getUuid()); - if (dni != null) { - return dni.getNodeStatus(); - } else { - return NodeStatus.inServiceHealthy(); - } - } - - /** - * Below here, are all auto-generate placeholder methods to implement the - * interface. - */ - @Override - public List getNodes(NodeStatus nodeStatus) { - return null; - } - - @Override - public List getNodes( - HddsProtos.NodeOperationalState opState, HddsProtos.NodeState health) { - return null; - } - - @Override - public int getNodeCount(NodeStatus nodeStatus) { - return 0; - } - - @Override - public int getNodeCount(HddsProtos.NodeOperationalState opState, - HddsProtos.NodeState health) { - return 0; - } - - @Override - public List getAllNodes() { - return null; - } - - @Override - public SCMNodeStat getStats() { - return null; - } - - @Override - public Map getNodeStats() { - return null; - } - - @Override - public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) { - return null; - } - - @Override - public void setNodeOperationalState(DatanodeDetails datanodeDetails, - HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { - } - - @Override - public Set getPipelines(DatanodeDetails datanodeDetails) { - return null; - } - - @Override - public void addPipeline(Pipeline pipeline) { - } - - @Override - public void removePipeline(Pipeline pipeline) { - } - - @Override - public void addContainer(DatanodeDetails datanodeDetails, - ContainerID containerId) throws NodeNotFoundException { - } - - @Override - public void setContainers(DatanodeDetails datanodeDetails, - Set containerIds) throws NodeNotFoundException { - } - - @Override - public Set getContainers(DatanodeDetails datanodeDetails) - throws NodeNotFoundException { - return null; - } - - @Override - public void addDatanodeCommand(UUID dnId, SCMCommand command) { - } - - @Override - public void processNodeReport(DatanodeDetails datanodeDetails, - StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport) { - } - - @Override - public List getCommandQueue(UUID dnID) { - return null; - } - - @Override - public DatanodeDetails getNodeByUuid(String uuid) { - return null; - } - - @Override - public List getNodesByAddress(String address) { - return null; - } - - @Override - public void close() throws IOException { - - } - - @Override - public Map getNodeCount() { - return null; - } - - @Override - public Map getNodeInfo() { - return null; - } - - @Override - public void onMessage(CommandForDatanode commandForDatanode, - EventPublisher publisher) { - } - - @Override - public VersionResponse getVersion( - StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto - versionRequest) { - return null; - } - - @Override - public RegisteredCommand register(DatanodeDetails datanodeDetails, - StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport, - StorageContainerDatanodeProtocolProtos.PipelineReportsProto - pipelineReport) { - return null; - } - - @Override - public List processHeartbeat(DatanodeDetails datanodeDetails) { - return null; - } - - @Override - public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) { - return null; - } - } - } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java index 9a502327792e..6c239c355667 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java @@ -18,24 +18,30 @@ package org.apache.hadoop.hdds.scm.container.states; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; import org.junit.Before; import org.junit.Test; import java.util.*; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; import static org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; +import static org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN; import static org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State .DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State .MAINTENANCE; +import static org.junit.Assert.assertFalse; /** * Class used to test the ContainerReplicaCount class. @@ -49,28 +55,37 @@ public void setup() { @Test public void testThreeHealthyReplica() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, true, 0, false); } @Test public void testTwoHealthyReplica() { Set replica = registerNodes(CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 1, false); } @Test public void testOneHealthyReplica() { Set replica = registerNodes(CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 2, false); } @Test public void testTwoHealthyAndInflightAdd() { + Set replica = registerNodes(CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 3, 2); validate(rcnt, false, 0, false); } @@ -82,7 +97,9 @@ public void testTwoHealthyAndInflightAdd() { */ public void testThreeHealthyAndInflightAdd() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 3, 2); validate(rcnt, true, 0, false); } @@ -93,7 +110,9 @@ public void testThreeHealthyAndInflightAdd() { */ public void testThreeHealthyAndInflightDelete() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 1, 3, 2); validate(rcnt, false, 1, false); } @@ -104,7 +123,9 @@ public void testThreeHealthyAndInflightDelete() { */ public void testThreeHealthyAndInflightAddAndInFlightDelete() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 1, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 1, 3, 2); validate(rcnt, false, 0, false); } @@ -112,7 +133,9 @@ public void testThreeHealthyAndInflightAddAndInFlightDelete() { public void testFourHealthyReplicas() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, true, -1, true); } @@ -120,7 +143,9 @@ public void testFourHealthyReplicas() { public void testFourHealthyReplicasAndInFlightDelete() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 1, 3, 2); validate(rcnt, true, 0, false); } @@ -128,28 +153,36 @@ public void testFourHealthyReplicasAndInFlightDelete() { public void testFourHealthyReplicasAndTwoInFlightDelete() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 2, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 2, 3, 2); validate(rcnt, false, 1, false); } @Test public void testOneHealthyReplicaRepFactorOne() { Set replica = registerNodes(CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 1, 2); validate(rcnt, true, 0, false); } @Test public void testOneHealthyReplicaRepFactorOneInFlightDelete() { Set replica = registerNodes(CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 1, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 1, 1, 2); validate(rcnt, false, 1, false); } @Test public void testTwoHealthyReplicaTwoInflightAdd() { Set replica = registerNodes(CLOSED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 2, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 2, 0, 3, 2); validate(rcnt, false, 0, false); } @@ -161,7 +194,9 @@ public void testTwoHealthyReplicaTwoInflightAdd() { public void testThreeHealthyAndTwoDecommission() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, DECOMMISSIONED, DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, true, 0, false); } @@ -169,7 +204,9 @@ public void testThreeHealthyAndTwoDecommission() { public void testOneDecommissionedReplica() { Set replica = registerNodes(CLOSED, CLOSED, DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 1, false); } @@ -177,7 +214,9 @@ public void testOneDecommissionedReplica() { public void testTwoHealthyOneDecommissionedneInFlightAdd() { Set replica = registerNodes(CLOSED, CLOSED, DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 3, 2); validate(rcnt, false, 0, false); } @@ -185,28 +224,37 @@ public void testTwoHealthyOneDecommissionedneInFlightAdd() { public void testAllDecommissioned() { Set replica = registerNodes(DECOMMISSIONED, DECOMMISSIONED, DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 3, false); } @Test public void testAllDecommissionedRepFactorOne() { Set replica = registerNodes(DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 1, 2); validate(rcnt, false, 1, false); + } @Test public void testAllDecommissionedRepFactorOneInFlightAdd() { Set replica = registerNodes(DECOMMISSIONED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 1, 2); validate(rcnt, false, 0, false); } @Test public void testOneHealthyOneDecommissioningRepFactorOne() { Set replica = registerNodes(DECOMMISSIONED, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 1, 2); validate(rcnt, true, 0, false); } @@ -218,7 +266,9 @@ public void testOneHealthyOneDecommissioningRepFactorOne() { public void testOneHealthyTwoMaintenanceMinRepOfTwo() { Set replica = registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 1, false); } @@ -226,7 +276,9 @@ public void testOneHealthyTwoMaintenanceMinRepOfTwo() { public void testOneHealthyThreeMaintenanceMinRepOfTwo() { Set replica = registerNodes(CLOSED, MAINTENANCE, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 1, false); } @@ -234,7 +286,9 @@ public void testOneHealthyThreeMaintenanceMinRepOfTwo() { public void testOneHealthyTwoMaintenanceMinRepOfOne() { Set replica = registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 1); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 1); validate(rcnt, true, 0, false); } @@ -242,7 +296,9 @@ public void testOneHealthyTwoMaintenanceMinRepOfOne() { public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() { Set replica = registerNodes(CLOSED, MAINTENANCE, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 3, 2); validate(rcnt, false, 0, false); } @@ -250,7 +306,9 @@ public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() { public void testAllMaintenance() { Set replica = registerNodes(MAINTENANCE, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, false, 2, false); } @@ -263,7 +321,9 @@ public void testAllMaintenance() { public void testThreeHealthyTwoInMaintenance() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, true, 0, false); } @@ -276,28 +336,36 @@ public void testThreeHealthyTwoInMaintenance() { public void testFourHealthyOneInMaintenance() { Set replica = registerNodes(CLOSED, CLOSED, CLOSED, CLOSED, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); validate(rcnt, true, -1, true); } @Test public void testOneMaintenanceMinRepOfTwoRepFactorOne() { Set replica = registerNodes(MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 1, 2); validate(rcnt, false, 1, false); } @Test public void testOneMaintenanceMinRepOfTwoRepFactorOneInFlightAdd() { Set replica = registerNodes(MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 1, 2); validate(rcnt, false, 0, false); } @Test public void testOneHealthyOneMaintenanceRepFactorOne() { Set replica = registerNodes(MAINTENANCE, CLOSED); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 0, 0, 1, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 1, 2); validate(rcnt, true, 0, false); } @@ -305,13 +373,47 @@ public void testOneHealthyOneMaintenanceRepFactorOne() { public void testTwoDecomTwoMaintenanceOneInflightAdd() { Set replica = registerNodes(DECOMMISSIONED, DECOMMISSIONED, MAINTENANCE, MAINTENANCE); - ContainerReplicaCount rcnt = new ContainerReplicaCount(replica, 1, 0, 3, 2); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 1, 0, 3, 2); validate(rcnt, false, 1, false); } + @Test + public void testHealthyContainerIsHealthy() { + Set replica = + registerNodes(CLOSED, CLOSED, CLOSED); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); + assertTrue(rcnt.isHealthy()); + } + + @Test + public void testIsHealthyWithDifferentReplicaStateNotHealthy() { + Set replica = + registerNodes(CLOSED, CLOSED, OPEN); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); + assertFalse(rcnt.isHealthy()); + } + + @Test + public void testIsHealthyWithMaintReplicaIsHealthy() { + Set replica = + registerNodes(CLOSED, CLOSED, MAINTENANCE, MAINTENANCE); + ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); + ContainerReplicaCount rcnt = + new ContainerReplicaCount(container, replica, 0, 0, 3, 2); + assertTrue(rcnt.isHealthy()); + } + private void validate(ContainerReplicaCount rcnt, - boolean sufficientlyReplicated, int replicaDelta, boolean overRelicated) { + boolean sufficientlyReplicated, int replicaDelta, + boolean overReplicated) { assertEquals(sufficientlyReplicated, rcnt.isSufficientlyReplicated()); + assertEquals(overReplicated, rcnt.isOverReplicated()); assertEquals(replicaDelta, rcnt.additionalReplicaNeeded()); } @@ -330,4 +432,11 @@ private Set registerNodes( } return replica; } + + private ContainerInfo createContainer(HddsProtos.LifeCycleState state) { + return new ContainerInfo.Builder() + .setContainerID(new ContainerID(1).getId()) + .setState(state) + .build(); + } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index acd89937b5f5..98ad037f092a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -17,95 +17,62 @@ */ package org.apache.hadoop.hdds.scm.node; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.*; +import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; - +import org.mockito.Mockito; import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.TimeoutException; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; +import static org.mockito.Mockito.reset; /** - * Tests to ensure the DatanodeAdminMonitor is working correctly. + * Tests to ensure the DatanodeAdminMonitor is working correctly. This class + * uses mocks or basic implementations of the key classes outside of the + * datanodeAdminMonitor to allow it to be tested in isolation. */ public class TestDatanodeAdminMonitor { - private StorageContainerManager scm; - private NodeManager nodeManager; - private ContainerManager containerManager; - private SCMPipelineManager pipelineManager; + private SimpleMockNodeManager nodeManager; private OzoneConfiguration conf; private DatanodeAdminMonitor monitor; - private DatanodeDetails datanode1; - private DatanodeDetails datanode2; - private DatanodeDetails datanode3; + private DatanodeAdminHandler startAdminHandler; + private ReplicationManager repManager; + private EventQueue eventQueue; @Before public void setup() throws IOException, AuthenticationException { - // This creates a mocked cluster of 6 nodes, where there are mock pipelines - // etc. Borrows heavily from TestDeadNodeHandler. conf = new OzoneConfiguration(); - String storageDir = GenericTestUtils.getTempPath( - TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); - conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - scm = HddsTestUtils.getScm(conf); - nodeManager = scm.getScmNodeManager(); - pipelineManager = (SCMPipelineManager)scm.getPipelineManager(); - containerManager = scm.getContainerManager(); + eventQueue = new EventQueue(); + startAdminHandler = new DatanodeAdminHandler(); + eventQueue.addHandler(SCMEvents.START_ADMIN_ON_NODE, startAdminHandler); + + nodeManager = new SimpleMockNodeManager(); + + repManager = Mockito.mock(ReplicationManager.class); monitor = new DatanodeAdminMonitor(conf); - monitor.setEventQueue(scm.getEventQueue()); + monitor.setEventQueue(eventQueue); monitor.setNodeManager(nodeManager); - monitor.setPipelineManager(pipelineManager); - - PipelineProvider mockRatisProvider = - new MockRatisPipelineProvider(nodeManager, - pipelineManager.getStateManager(), conf); - pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, - mockRatisProvider); - - datanode1 = TestUtils.randomDatanodeDetails(); - datanode2 = TestUtils.randomDatanodeDetails(); - datanode3 = TestUtils.randomDatanodeDetails(); - - String storagePath = GenericTestUtils.getRandomizedTempPath() - .concat("/" + datanode1.getUuidString()); - - StorageContainerDatanodeProtocolProtos.StorageReportProto - storageOne = TestUtils.createStorageReport( - datanode1.getUuid(), storagePath, 100, 10, 90, null); - - nodeManager.register(datanode1, - TestUtils.createNodeReport(storageOne), null); - nodeManager.register(datanode2, - TestUtils.createNodeReport(storageOne), null); - nodeManager.register(datanode3, - TestUtils.createNodeReport(storageOne), null); - nodeManager.register(TestUtils.randomDatanodeDetails(), - TestUtils.createNodeReport(storageOne), null); - nodeManager.register(TestUtils.randomDatanodeDetails(), - TestUtils.createNodeReport(storageOne), null); - nodeManager.register(TestUtils.randomDatanodeDetails(), - TestUtils.createNodeReport(storageOne), null); + monitor.setReplicationManager(repManager); } @After @@ -125,42 +92,437 @@ public void testNodeCanBeQueuedAndCancelled() { monitor.startMonitoring(dn, 0); assertEquals(1, monitor.getPendingCount()); assertEquals(0, monitor.getCancelledCount()); + } + + /** + * In this test we ensure there are some pipelines for the node being + * decommissioned, but there are no containers. Therefore the workflow + * must wait until the pipelines have closed before completing the flow. + */ + @Test + public void testClosePipelinesEventFiredWhenAdminStarted() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); + // Ensure the node has some pipelines + nodeManager.setPipelines(dn1, 2); + // Add the node to the monitor + monitor.startMonitoring(dn1, 0); + monitor.run(); + // Ensure a StartAdmin event was fired + eventQueue.processAll(20000); + assertEquals(1, startAdminHandler.getInvocation()); + // Ensure a node is now tracked for decommission + assertEquals(1, monitor.getTrackedNodeCount()); + // Ensure the node remains decommissioning + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); + // Run the monitor again, and it should remain decommissioning + monitor.run(); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); + // Clear the pipelines and the node should transition to DECOMMISSIONED + nodeManager.setPipelines(dn1, 0); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + nodeManager.getNodeStatus(dn1).getOperationalState()); } + /** + * In this test, there are no open pipelines and no containers on the node. + * Therefore, we expect the decommission flow to finish on the first run + * on the monitor, leaving zero nodes tracked and the node in DECOMMISSIONED + * state. + */ + @Test + public void testDecommissionNodeTransitionsToCompleteWhenNoContainers() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); + + // Add the node to the monitor. By default we have zero pipelines and + // zero containers in the test setup, so the node should immediately + // transition to COMPLETED state + monitor.startMonitoring(dn1, 0); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + NodeStatus newStatus = nodeManager.getNodeStatus(dn1); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + newStatus.getOperationalState()); + } @Test - @Ignore // HDDS-2631 - public void testMonitoredNodeHasPipelinesClosed() - throws NodeNotFoundException, TimeoutException, InterruptedException { + public void testDecommissionNodeWaitsForContainersToReplicate() + throws NodeNotFoundException, ContainerNotFoundException { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); - GenericTestUtils.waitFor(() -> nodeManager - .getPipelines(datanode1).size() == 2, 100, 20000); + nodeManager.setContainers(dn1, generateContainers(3)); + // Mock Replication Manager to return ContainerReplicaCount's which + // always have a DECOMMISSIONED replica. + mockGetContainerReplicaCount( + HddsProtos.LifeCycleState.CLOSED, + ContainerReplicaProto.State.DECOMMISSIONED, + ContainerReplicaProto.State.CLOSED, + ContainerReplicaProto.State.CLOSED); - nodeManager.setNodeOperationalState(datanode1, - HddsProtos.NodeOperationalState.DECOMMISSIONING); - monitor.startMonitoring(datanode1, 0); + // Run the monitor for the first time and the node will transition to + // REPLICATE_CONTAINERS as there are no pipelines to close. + monitor.startMonitoring(dn1, 0); monitor.run(); - // Ensure the node moves from pending to tracked - assertEquals(0, monitor.getPendingCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); assertEquals(1, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); - // Ensure the pipelines are closed, as this is the first step in the admin - // workflow - GenericTestUtils.waitFor(() -> nodeManager - .getPipelines(datanode1).size() == 0, 100, 20000); + // Running the monitor again causes it to remain DECOMMISSIONING + // as nothing has changed. + monitor.run(); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); + assertEquals(0, node.getSufficientlyReplicatedContainers()); + assertEquals(0, node.getUnHealthyContainers()); + assertEquals(3, node.getUnderReplicatedContainers()); + + // Now change the replicationManager mock to return 3 CLOSED replicas + // and the node should complete the REPLICATE_CONTAINERS step, moving to + // complete which will end the decommission workflow + mockGetContainerReplicaCount( + HddsProtos.LifeCycleState.CLOSED, + ContainerReplicaProto.State.CLOSED, + ContainerReplicaProto.State.CLOSED, + ContainerReplicaProto.State.CLOSED); - // Run the run loop again and ensure the tracked node is moved to the next - // state monitor.run(); - for (DatanodeAdminNodeDetails node : monitor.getTrackedNodes()) { - assertEquals( - DatanodeAdminMonitor.States.GET_CONTAINERS, node.getCurrentState()); - } - // Finally, cancel decommission and see the node is removed from tracking - monitor.stopMonitoring(datanode1); + + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(3, node.getSufficientlyReplicatedContainers()); + assertEquals(0, node.getUnHealthyContainers()); + assertEquals(0, node.getUnderReplicatedContainers()); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testDecommissionAbortedWhenNodeInUnexpectedState() + throws NodeNotFoundException, ContainerNotFoundException { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); + + nodeManager.setContainers(dn1, generateContainers(3)); + mockGetContainerReplicaCount( + HddsProtos.LifeCycleState.CLOSED, + ContainerReplicaProto.State.DECOMMISSIONED, + ContainerReplicaProto.State.CLOSED, + ContainerReplicaProto.State.CLOSED); + + // Add the node to the monitor, it should have 3 under-replicated containers + // after the first run + monitor.startMonitoring(dn1, 0); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); + assertEquals(3, node.getUnderReplicatedContainers()); + + // Set the node to dead, and then the workflow should get aborted, setting + // the node state back to IN_SERVICE on the next run. + nodeManager.setNodeStatus(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE, + HddsProtos.NodeState.HEALTHY)); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testDecommissionAbortedWhenNodeGoesDead() + throws NodeNotFoundException, ContainerNotFoundException { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY)); + + nodeManager.setContainers(dn1, generateContainers(3)); + mockGetContainerReplicaCount( + HddsProtos.LifeCycleState.CLOSED, + ContainerReplicaProto.State.DECOMMISSIONED, + ContainerReplicaProto.State.CLOSED, + ContainerReplicaProto.State.CLOSED); + + // Add the node to the monitor, it should have 3 under-replicated containers + // after the first run + monitor.startMonitoring(dn1, 0); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, + nodeManager.getNodeStatus(dn1).getOperationalState()); + assertEquals(3, node.getUnderReplicatedContainers()); + + // Set the node to dead, and then the workflow should get aborted, setting + // the node state back to IN_SERVICE. + nodeManager.setNodeStatus(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.DEAD)); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testMaintenanceWaitsForMaintenanceToComplete() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY)); + + // Add the node to the monitor, it should transiting to + // IN_MAINTENANCE as there are no containers to replicate. + monitor.startMonitoring(dn1, 1); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertEquals(0, node.getUnderReplicatedContainers()); + assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); + + // Running the monitor again causes the node to remain in maintenance + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); + + // Set the maintenance end time to a time in the past and then the node + // should complete the workflow and transition to IN_SERVICE + node.setMaintenanceEnd(-1); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testMaintenanceEndsClosingPipelines() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY)); + // Ensure the node has some pipelines + nodeManager.setPipelines(dn1, 2); + // Add the node to the monitor + monitor.startMonitoring(dn1, 1); + monitor.run(); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertEquals(1, monitor.getTrackedNodeCount()); + assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance()); + + // Set the maintenance end time to the past and the node should complete + // the workflow and return to IN_SERVICE + node.setMaintenanceEnd(-1); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testMaintenanceEndsWhileReplicatingContainers() + throws ContainerNotFoundException, NodeNotFoundException { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY)); + + nodeManager.setContainers(dn1, generateContainers(3)); + mockGetContainerReplicaCount( + HddsProtos.LifeCycleState.CLOSED, + ContainerReplicaProto.State.MAINTENANCE, + ContainerReplicaProto.State.MAINTENANCE, + ContainerReplicaProto.State.MAINTENANCE); + + // Add the node to the monitor, it should transiting to + // REPLICATE_CONTAINERS as the containers are under-replicated for + // maintenance. + monitor.startMonitoring(dn1, 1); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance()); + assertEquals(3, node.getUnderReplicatedContainers()); + + node.setMaintenanceEnd(-1); + monitor.run(); + assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + @Test + public void testDeadMaintenanceNodeDoesNotAbortWorkflow() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY)); + + // Add the node to the monitor, it should transiting to + // AWAIT_MAINTENANCE_END as there are no under-replicated containers. + monitor.startMonitoring(dn1, 1); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); + assertEquals(0, node.getUnderReplicatedContainers()); + + // Set the node dead and ensure the workflow does not end + NodeStatus status = nodeManager.getNodeStatus(dn1); + nodeManager.setNodeStatus(dn1, new NodeStatus( + status.getOperationalState(), HddsProtos.NodeState.DEAD)); + + // Running the monitor again causes the node to remain in maintenance + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); + } + + @Test + public void testCancelledNodesMovedToInService() { + DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); + nodeManager.register(dn1, + new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY)); + + // Add the node to the monitor, it should transiting to + // AWAIT_MAINTENANCE_END as there are no under-replicated containers. + monitor.startMonitoring(dn1, 1); + monitor.run(); + assertEquals(1, monitor.getTrackedNodeCount()); + DatanodeAdminNodeDetails node = getFirstTrackedNode(); + assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); + assertEquals(0, node.getUnderReplicatedContainers()); + + // Now cancel the node and run the monitor, the node should be IN_SERVICE + monitor.stopMonitoring(dn1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); + assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + nodeManager.getNodeStatus(dn1).getOperationalState()); + } + + /** + * Generate a set of ContainerID, starting from an ID of zero up to the given + * count minus 1. + * @param count The number of ContainerID objects to generate. + * @return A Set of ContainerID objects. + */ + private Set generateContainers(int count) { + Set containers = new HashSet<>(); + for (int i=0; i replicas = new HashSet<>(); + for (ContainerReplicaProto.State s : states) { + replicas.add(generateReplica(containerID, s)); + } + ContainerInfo container = new ContainerInfo.Builder() + .setContainerID(containerID.getId()) + .setState(containerState) + .build(); + + return new ContainerReplicaCount(container, replicas, 0, 0, 3, 2); + } + + /** + * Generate a new ContainerReplica with the given containerID and State. + * @param containerID The ID the replica is associated with + * @param state The state of the generated replica. + * @return A containerReplica with the given ID and state + */ + private ContainerReplica generateReplica(ContainerID containerID, + ContainerReplicaProto.State state) { + return ContainerReplica.newBuilder() + .setContainerState(state) + .setContainerID(containerID) + .setSequenceId(1) + .setDatanodeDetails(TestUtils.randomDatanodeDetails()) + .build(); + } + + /** + * Helper method to get the first node from the set of trackedNodes within + * the monitor. + * @return DatanodeAdminNodeDetails for the first tracked node found. + */ + private DatanodeAdminNodeDetails getFirstTrackedNode() { + return + monitor.getTrackedNodes().toArray(new DatanodeAdminNodeDetails[0])[0]; + } + + /** + * The only interaction the DatanodeAdminMonitor has with the + * ReplicationManager, is to request a ContainerReplicaCount object for each + * container on nodes being deocmmissioned or moved to maintenance. This + * method mocks that interface to return a ContainerReplicaCount with a + * container in the given containerState and a set of replias in the given + * replicaStates. + * @param containerState + * @param replicaStates + * @throws ContainerNotFoundException + */ + private void mockGetContainerReplicaCount( + HddsProtos.LifeCycleState containerState, + ContainerReplicaProto.State... replicaStates) + throws ContainerNotFoundException { + reset(repManager); + Mockito.when(repManager.getContainerReplicaCount( + Mockito.any(ContainerID.class))) + .thenAnswer(invocation -> + generateReplicaCount((ContainerID)invocation.getArguments()[0], + containerState, replicaStates)); + } + + /** + * This simple internal class is used to track and handle any DatanodeAdmin + * events fired by the DatanodeAdminMonitor during tests. + */ + private class DatanodeAdminHandler implements + EventHandler { + + private AtomicInteger invocation = new AtomicInteger(0); + + @Override + public void onMessage(final DatanodeDetails dn, + final EventPublisher publisher) { + invocation.incrementAndGet(); + } + + public int getInvocation() { + return invocation.get(); + } + } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java index 3b5177e47697..916e383c0034 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java @@ -20,9 +20,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; -import - org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.junit.After; import org.junit.Before; @@ -39,8 +36,6 @@ public class TestDatanodeAdminNodeDetails { private OzoneConfiguration conf; private DatanodeAdminMonitor monitor; - private final DatanodeAdminMonitor.States initialState = - DatanodeAdminMonitor.States.CLOSE_PIPELINES; @Before public void setup() { @@ -57,103 +52,23 @@ public void testEqualityBasedOnDatanodeDetails() { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); DatanodeDetails dn2 = TestUtils.randomDatanodeDetails(); DatanodeAdminNodeDetails details1 = - new DatanodeAdminNodeDetails(dn1, initialState, 0); + new DatanodeAdminNodeDetails(dn1, 0); DatanodeAdminNodeDetails details2 = - new DatanodeAdminNodeDetails(dn2, initialState, 0); + new DatanodeAdminNodeDetails(dn2, 0); assertNotEquals(details1, details2); assertEquals(details1, - new DatanodeAdminNodeDetails(dn1, initialState, 0)); + new DatanodeAdminNodeDetails(dn1, 0)); assertNotEquals(details1, dn1); } - @Test - public void testUnexpectedNodeStateGivesBadTransition() { - DatanodeDetails dn = TestUtils.randomDatanodeDetails(); - DatanodeAdminNodeDetails details = - new DatanodeAdminNodeDetails(dn, initialState, 0); - - try { - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.IN_SERVICE); - fail("InvalidStateTransitionException should be thrown"); - } catch (InvalidStateTransitionException e) { - - } - } - - @Test - public void testWorkflowStatesTransitionCorrectlyForDecom() - throws InvalidStateTransitionException { - DatanodeDetails dn = TestUtils.randomDatanodeDetails(); - DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, - initialState, 0); - - // Initial state should be CLOSE_PIPELINES - assertEquals(DatanodeAdminMonitor.States.CLOSE_PIPELINES, - details.getCurrentState()); - - // Next State is GET_CONTAINERS - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.DECOMMISSIONING); - assertEquals(DatanodeAdminMonitor.States.GET_CONTAINERS, - details.getCurrentState()); - - // Next State is REPLICATE_CONTAINERS - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.DECOMMISSIONING); - assertEquals(DatanodeAdminMonitor.States.REPLICATE_CONTAINERS, - details.getCurrentState()); - // Next State is COMPLETE - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.DECOMMISSIONING); - assertEquals(DatanodeAdminMonitor.States.COMPLETE, - details.getCurrentState()); - } - - @Test - public void testWorkflowStatesTransitionCorrectlyForMaint() - throws InvalidStateTransitionException { - DatanodeDetails dn = TestUtils.randomDatanodeDetails(); - DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, - initialState, 0); - - // Initial state should be CLOSE_PIPELINES - assertEquals(DatanodeAdminMonitor.States.CLOSE_PIPELINES, - details.getCurrentState()); - - // Next State is GET_CONTAINERS - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.ENTERING_MAINTENANCE); - assertEquals(DatanodeAdminMonitor.States.GET_CONTAINERS, - details.getCurrentState()); - - // Next State is REPLICATE_CONTAINER - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.ENTERING_MAINTENANCE); - assertEquals(DatanodeAdminMonitor.States.REPLICATE_CONTAINERS, - details.getCurrentState()); - - // Next State is AWAIT_MAINTENANCE_END - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.ENTERING_MAINTENANCE); - assertEquals(DatanodeAdminMonitor.States.AWAIT_MAINTENANCE_END, - details.getCurrentState()); - - // Next State is COMPLETE - details.transitionState(monitor.getWorkflowStateMachine(), - NodeOperationalState.ENTERING_MAINTENANCE); - assertEquals(DatanodeAdminMonitor.States.COMPLETE, - details.getCurrentState()); - } @Test public void testMaintenanceEnd() { DatanodeDetails dn = TestUtils.randomDatanodeDetails(); // End in zero hours - should never end. - DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, - initialState, 0); + DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, 0); assertFalse(details.shouldMaintenanceEnd()); // End 1 hour - maintenance should not end yet. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index e3b92084d82a..2da7a611c37e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -55,7 +55,8 @@ public void setup() throws Exception { TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); nodeManager = createNodeManager(conf); - decom = new NodeDecommissionManager(conf, nodeManager, null, null, null); + decom = new NodeDecommissionManager( + conf, nodeManager, null, null, null, null); } @Test @@ -161,6 +162,7 @@ public void testNodesCanBeDecommissionedAndRecommissioned() // Recommission all 3 hosts decom.recommissionNodes(Arrays.asList( multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress())); + decom.getMonitor().run(); assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, @@ -199,6 +201,7 @@ public void testNodesCanBePutIntoMaintenanceAndRecommissioned() // Recommission all 3 hosts decom.recommissionNodes(Arrays.asList( multiAddr, dns.get(1).getIpAddress(), dns.get(2).getIpAddress())); + decom.getMonitor().run(); assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, From f66c40cce184dfaf099c6f1ffd3f3ec4d306d923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= Date: Fri, 13 Dec 2019 21:17:34 +0100 Subject: [PATCH 09/78] HDDS-2728. Remove methods of internal representation from DatanodeAdminMontor interface (#355) --- .../hdds/scm/node/DatanodeAdminMonitor.java | 382 +----------------- .../scm/node/DatanodeAdminMonitorImpl.java | 378 +++++++++++++++++ .../node/DatanodeAdminMonitorInterface.java | 43 -- .../scm/node/NodeDecommissionManager.java | 17 +- .../scm/server/StorageContainerManager.java | 2 +- .../scm/node/TestDatanodeAdminMonitor.java | 8 +- .../node/TestDatanodeAdminNodeDetails.java | 2 - .../scm/node/TestNodeDecommissionManager.java | 2 +- 8 files changed, 397 insertions(+), 437 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java index 6aabd621cb34..9ead66f4b4fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,380 +17,16 @@ */ package org.apache.hadoop.hdds.scm.node; -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; -import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; -import org.apache.hadoop.hdds.scm.container.ReplicationManager; -import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.pipeline.PipelineID; -import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; /** - * Monitor thread which watches for nodes to be decommissioned, recommissioned - * or placed into maintenance. Newly added nodes are queued in pendingNodes - * and recommissoned nodes are queued in cancelled nodes. On each monitor - * 'tick', the cancelled nodes are processed and removed from the monitor. - * Then any pending nodes are added to the trackedNodes set, where they stay - * until decommission or maintenance has ended. - * - * Once an node is placed into tracked nodes, it goes through a workflow where - * the following happens: - * - * 1. First an event is fired to close any pipelines on the node, which will - * also close any containers. - * 2. Next the containers on the node are obtained and checked to see if new - * replicas are needed. If so, the new replicas are scheduled. - * 3. After scheduling replication, the node remains pending until replication - * has completed. - * 4. At this stage the node will complete decommission or enter maintenance. - * 5. Maintenance nodes will remain tracked by this monitor until maintenance - * is manually ended, or the maintenance window expires. + * Interface used by the DatanodeAdminMonitor, which can be used to + * decommission or recommission nodes and take them in and out of maintenance. */ -public class DatanodeAdminMonitor implements DatanodeAdminMonitorInterface { - - private OzoneConfiguration conf; - private EventPublisher eventQueue; - private NodeManager nodeManager; - private PipelineManager pipelineManager; - private ReplicationManager replicationManager; - private Queue pendingNodes = new ArrayDeque(); - private Queue cancelledNodes = new ArrayDeque(); - private Set trackedNodes = new HashSet<>(); - - private static final Logger LOG = - LoggerFactory.getLogger(DatanodeAdminMonitor.class); - - public DatanodeAdminMonitor(OzoneConfiguration config) { - conf = config; - } - - @Override - public void setConf(OzoneConfiguration config) { - conf = config; - } - - @Override - public void setEventQueue(EventPublisher eventQueue) { - this.eventQueue = eventQueue; - } - - @Override - public void setNodeManager(NodeManager nm) { - nodeManager = nm; - } - - @Override - public void setPipelineManager(PipelineManager pm) { - pipelineManager = pm; - } - - @Override - public void setReplicationManager(ReplicationManager rm) { - replicationManager = rm; - } - - /** - * Add a node to the decommission or maintenance workflow. The node will be - * queued and added to the workflow after a defined interval. - * - * @param dn The datanode to move into an admin state - * @param endInHours For nodes going into maintenance, the number of hours - * from now for maintenance to automatically end. Ignored - * for decommissioning nodes. - */ - @Override - public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { - DatanodeAdminNodeDetails nodeDetails = - new DatanodeAdminNodeDetails(dn, endInHours); - cancelledNodes.remove(nodeDetails); - pendingNodes.add(nodeDetails); - } - - /** - * Remove a node from the decommission or maintenance workflow, and return it - * to service. The node will be queued and removed from decommission or - * maintenance after a defined interval. - * @param dn The datanode for which to stop decommission or maintenance. - */ - @Override - public synchronized void stopMonitoring(DatanodeDetails dn) { - DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, 0); - pendingNodes.remove(nodeDetails); - cancelledNodes.add(nodeDetails); - } - - /** - * Run an iteration of the monitor. This is the main run loop, and performs - * the following checks: - * - * 1. Check for any cancelled nodes and process them - * 2. Check for any newly added nodes and add them to the workflow - * 3. Perform checks on the transitioning nodes and move them through the - * workflow until they have completed decommission or maintenance - */ - @Override - public void run() { - try { - synchronized (this) { - processCancelledNodes(); - processPendingNodes(); - } - processTransitioningNodes(); - if (trackedNodes.size() > 0 || pendingNodes.size() > 0) { - LOG.info("There are {} nodes tracked for decommission and "+ - "maintenance. {} pending nodes.", - trackedNodes.size(), pendingNodes.size()); - } - } catch (Exception e) { - LOG.error("Caught an error in the DatanodeAdminMonitor", e); - // Intentionally do not re-throw, as if we do the monitor thread - // will not get rescheduled. - } - } - - @Override - public int getPendingCount() { - return pendingNodes.size(); - } - - @Override - public int getCancelledCount() { - return cancelledNodes.size(); - } - - @Override - public int getTrackedNodeCount() { - return trackedNodes.size(); - } - - @VisibleForTesting - public Set getTrackedNodes() { - return trackedNodes; - } - - private void processCancelledNodes() { - while(!cancelledNodes.isEmpty()) { - DatanodeAdminNodeDetails dn = cancelledNodes.poll(); - try { - stopTrackingNode(dn); - putNodeBackInService(dn); - LOG.info("Recommissioned node {}", dn.getDatanodeDetails()); - } catch (NodeNotFoundException e) { - LOG.warn("Failed processing the cancel admin request for {}", - dn.getDatanodeDetails(), e); - } - // TODO - fire event to bring node back into service? - } - } - - private void processPendingNodes() { - while(!pendingNodes.isEmpty()) { - startTrackingNode(pendingNodes.poll()); - } - } - - private void processTransitioningNodes() { - Iterator iterator = trackedNodes.iterator(); - while (iterator.hasNext()) { - DatanodeAdminNodeDetails dn = iterator.next(); - try { - NodeStatus status = getNodeStatus(dn.getDatanodeDetails()); - - if (!shouldContinueWorkflow(dn, status)) { - abortWorkflow(dn); - iterator.remove(); - continue; - } - - if (status.isMaintenance()) { - if (dn.shouldMaintenanceEnd()) { - completeMaintenance(dn); - iterator.remove(); - continue; - } - } - - if (status.isDecommissioning() || status.isEnteringMaintenance()) { - if (checkPipelinesClosedOnNode(dn) - && checkContainersReplicatedOnNode(dn)) { - // CheckContainersReplicatedOnNode may take a short time to run - // so after it completes, re-get the nodestatus to check the health - // and ensure the state is still good to continue - status = getNodeStatus(dn.getDatanodeDetails()); - if (status.isDead()) { - LOG.warn("Datanode {} is dead and the admin workflow cannot "+ - "continue. The node will be put back to IN_SERVICE and "+ - "handled as a dead node", dn); - putNodeBackInService(dn); - iterator.remove(); - } else if (status.isDecommissioning()) { - completeDecommission(dn); - iterator.remove(); - } else if (status.isEnteringMaintenance()) { - putIntoMaintenance(dn); - } - } - } - - } catch (NodeNotFoundException e) { - LOG.error("An unexpected error occurred processing datanode {}. " + - "Aborting the admin workflow", dn.getDatanodeDetails(), e); - abortWorkflow(dn); - iterator.remove(); - } - } - } - - /** - * Checks if a node is in an unexpected state or has gone dead while - * decommissioning or entering maintenance. If the node is not in a valid - * state to continue the admin workflow, return false, otherwise return true. - * @param dn The Datanode for which to check the current state - * @param nodeStatus The current NodeStatus for the datanode - * @return True if admin can continue, false otherwise - */ - private boolean shouldContinueWorkflow(DatanodeAdminNodeDetails dn, - NodeStatus nodeStatus) { - if (!nodeStatus.isDecommission() && !nodeStatus.isMaintenance()) { - LOG.warn("Datanode {} has an operational state of {} when it should "+ - "be undergoing decommission or maintenance. Aborting admin for "+ - "this node.", - dn.getDatanodeDetails(), nodeStatus.getOperationalState()); - return false; - } - if (nodeStatus.isDead() && !nodeStatus.isInMaintenance()) { - LOG.error("Datanode {} is dead but is not IN_MAINTENANCE. Aborting the "+ - "admin workflow for this node", dn.getDatanodeDetails()); - return false; - } - return true; - } - - private boolean checkPipelinesClosedOnNode(DatanodeAdminNodeDetails dn) { - DatanodeDetails dnd = dn.getDatanodeDetails(); - Set pipelines = nodeManager.getPipelines(dnd); - if (pipelines == null || pipelines.size() == 0 - || dn.shouldMaintenanceEnd()) { - return true; - } else { - LOG.info("Waiting for pipelines to close for {}. There are {} "+ - "pipelines", dnd, pipelines.size()); - return false; - } - } - - private boolean checkContainersReplicatedOnNode(DatanodeAdminNodeDetails dn) - throws NodeNotFoundException { - int sufficientlyReplicated = 0; - int underReplicated = 0; - int unhealthy = 0; - Set containers = - nodeManager.getContainers(dn.getDatanodeDetails()); - for(ContainerID cid : containers) { - try { - ContainerReplicaCount replicaSet = - replicationManager.getContainerReplicaCount(cid); - if (replicaSet.isSufficientlyReplicated()) { - sufficientlyReplicated++; - } else { - underReplicated++; - } - if (!replicaSet.isHealthy()) { - unhealthy++; - } - } catch (ContainerNotFoundException e) { - LOG.warn("ContainerID {} present in node list for {} but not found "+ - "in containerManager", cid, dn.getDatanodeDetails()); - } - } - dn.setSufficientlyReplicatedContainers(sufficientlyReplicated); - dn.setUnderReplicatedContainers(underReplicated); - dn.setUnHealthyContainers(unhealthy); - - return underReplicated == 0 && unhealthy == 0; - } - - private void completeDecommission(DatanodeAdminNodeDetails dn) - throws NodeNotFoundException{ - setNodeOpState(dn, NodeOperationalState.DECOMMISSIONED); - LOG.info("Datanode {} has completed the admin workflow. The operational "+ - "state has been set to {}", dn.getDatanodeDetails(), - NodeOperationalState.DECOMMISSIONED); - } - - private void putIntoMaintenance(DatanodeAdminNodeDetails dn) - throws NodeNotFoundException { - LOG.info("Datanode {} has entered maintenance", dn.getDatanodeDetails()); - setNodeOpState(dn, NodeOperationalState.IN_MAINTENANCE); - } - - private void completeMaintenance(DatanodeAdminNodeDetails dn) - throws NodeNotFoundException { - // The end state of Maintenance is to put the node back IN_SERVICE, whether - // it is dead or not. - // TODO - if the node is dead do we trigger a dead node event here or leave - // it to the heartbeat manager? - LOG.info("Datanode {} has ended maintenance automatically", - dn.getDatanodeDetails()); - putNodeBackInService(dn); - } - - private void startTrackingNode(DatanodeAdminNodeDetails dn) { - eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, - dn.getDatanodeDetails()); - trackedNodes.add(dn); - } - - private void stopTrackingNode(DatanodeAdminNodeDetails dn) { - trackedNodes.remove(dn); - } - - /** - * If we encounter an unexpected condition in maintenance, we must abort the - * workflow by setting the node operationalState back to IN_SERVICE and then - * remove the node from tracking. - * @param dn The datanode for which to abort tracking - */ - private void abortWorkflow(DatanodeAdminNodeDetails dn) { - try { - putNodeBackInService(dn); - } catch (NodeNotFoundException e) { - LOG.error("Unable to set the node OperationalState for {} while "+ - "aborting the datanode admin workflow", dn.getDatanodeDetails()); - } - } - - private void putNodeBackInService(DatanodeAdminNodeDetails dn) - throws NodeNotFoundException { - setNodeOpState(dn, NodeOperationalState.IN_SERVICE); - } +public interface DatanodeAdminMonitor extends Runnable { - private void setNodeOpState(DatanodeAdminNodeDetails dn, - HddsProtos.NodeOperationalState state) throws NodeNotFoundException { - nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state); - } + void startMonitoring(DatanodeDetails dn, int endInHours); - // TODO - The nodeManager.getNodeStatus call should really throw - // NodeNotFoundException rather than having to handle it here as all - // registered nodes must have a status. - private NodeStatus getNodeStatus(DatanodeDetails dnd) - throws NodeNotFoundException { - NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); - if (nodeStatus == null) { - throw new NodeNotFoundException("Unable to retrieve the nodeStatus"); - } - return nodeStatus; - } + void stopMonitoring(DatanodeDetails dn); -} \ No newline at end of file +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java new file mode 100644 index 000000000000..931a45e3bda4 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -0,0 +1,378 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.node; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; +import org.apache.hadoop.hdds.scm.container.ReplicationManager; +import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * Monitor thread which watches for nodes to be decommissioned, recommissioned + * or placed into maintenance. Newly added nodes are queued in pendingNodes + * and recommissoned nodes are queued in cancelled nodes. On each monitor + * 'tick', the cancelled nodes are processed and removed from the monitor. + * Then any pending nodes are added to the trackedNodes set, where they stay + * until decommission or maintenance has ended. + *

+ * Once an node is placed into tracked nodes, it goes through a workflow where + * the following happens: + *

+ * 1. First an event is fired to close any pipelines on the node, which will + * also close any containers. + * 2. Next the containers on the node are obtained and checked to see if new + * replicas are needed. If so, the new replicas are scheduled. + * 3. After scheduling replication, the node remains pending until replication + * has completed. + * 4. At this stage the node will complete decommission or enter maintenance. + * 5. Maintenance nodes will remain tracked by this monitor until maintenance + * is manually ended, or the maintenance window expires. + */ +public class DatanodeAdminMonitorImpl implements DatanodeAdminMonitor { + + private OzoneConfiguration conf; + private EventPublisher eventQueue; + private NodeManager nodeManager; + private ReplicationManager replicationManager; + private Queue pendingNodes = new ArrayDeque(); + private Queue cancelledNodes = new ArrayDeque(); + private Set trackedNodes = new HashSet<>(); + + private static final Logger LOG = + LoggerFactory.getLogger(DatanodeAdminMonitorImpl.class); + + public DatanodeAdminMonitorImpl( + OzoneConfiguration conf, + EventPublisher eventQueue, + NodeManager nodeManager, + ReplicationManager replicationManager) { + this.conf = conf; + this.eventQueue = eventQueue; + this.nodeManager = nodeManager; + this.replicationManager = replicationManager; + } + + /** + * Add a node to the decommission or maintenance workflow. The node will be + * queued and added to the workflow after a defined interval. + * + * @param dn The datanode to move into an admin state + * @param endInHours For nodes going into maintenance, the number of hours + * from now for maintenance to automatically end. Ignored + * for decommissioning nodes. + */ + @Override + public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { + DatanodeAdminNodeDetails nodeDetails = + new DatanodeAdminNodeDetails(dn, endInHours); + cancelledNodes.remove(nodeDetails); + pendingNodes.add(nodeDetails); + } + + /** + * Remove a node from the decommission or maintenance workflow, and return it + * to service. The node will be queued and removed from decommission or + * maintenance after a defined interval. + * + * @param dn The datanode for which to stop decommission or maintenance. + */ + @Override + public synchronized void stopMonitoring(DatanodeDetails dn) { + DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, 0); + pendingNodes.remove(nodeDetails); + cancelledNodes.add(nodeDetails); + } + + /** + * Run an iteration of the monitor. This is the main run loop, and performs + * the following checks: + *

+ * 1. Check for any cancelled nodes and process them + * 2. Check for any newly added nodes and add them to the workflow + * 3. Perform checks on the transitioning nodes and move them through the + * workflow until they have completed decommission or maintenance + */ + @Override + public void run() { + try { + synchronized (this) { + processCancelledNodes(); + processPendingNodes(); + } + processTransitioningNodes(); + if (trackedNodes.size() > 0 || pendingNodes.size() > 0) { + LOG.info("There are {} nodes tracked for decommission and " + + "maintenance. {} pending nodes.", + trackedNodes.size(), pendingNodes.size()); + } + } catch (Exception e) { + LOG.error("Caught an error in the DatanodeAdminMonitor", e); + // Intentionally do not re-throw, as if we do the monitor thread + // will not get rescheduled. + } + } + + public int getPendingCount() { + return pendingNodes.size(); + } + + public int getCancelledCount() { + return cancelledNodes.size(); + } + + public int getTrackedNodeCount() { + return trackedNodes.size(); + } + + @VisibleForTesting + public Set getTrackedNodes() { + return trackedNodes; + } + + private void processCancelledNodes() { + while (!cancelledNodes.isEmpty()) { + DatanodeAdminNodeDetails dn = cancelledNodes.poll(); + try { + stopTrackingNode(dn); + putNodeBackInService(dn); + LOG.info("Recommissioned node {}", dn.getDatanodeDetails()); + } catch (NodeNotFoundException e) { + LOG.warn("Failed processing the cancel admin request for {}", + dn.getDatanodeDetails(), e); + } + // TODO - fire event to bring node back into service? + } + } + + private void processPendingNodes() { + while (!pendingNodes.isEmpty()) { + startTrackingNode(pendingNodes.poll()); + } + } + + private void processTransitioningNodes() { + Iterator iterator = trackedNodes.iterator(); + while (iterator.hasNext()) { + DatanodeAdminNodeDetails dn = iterator.next(); + try { + NodeStatus status = getNodeStatus(dn.getDatanodeDetails()); + + if (!shouldContinueWorkflow(dn, status)) { + abortWorkflow(dn); + iterator.remove(); + continue; + } + + if (status.isMaintenance()) { + if (dn.shouldMaintenanceEnd()) { + completeMaintenance(dn); + iterator.remove(); + continue; + } + } + + if (status.isDecommissioning() || status.isEnteringMaintenance()) { + if (checkPipelinesClosedOnNode(dn) + && checkContainersReplicatedOnNode(dn)) { + // CheckContainersReplicatedOnNode may take a short time to run + // so after it completes, re-get the nodestatus to check the health + // and ensure the state is still good to continue + status = getNodeStatus(dn.getDatanodeDetails()); + if (status.isDead()) { + LOG.warn("Datanode {} is dead and the admin workflow cannot " + + "continue. The node will be put back to IN_SERVICE and " + + "handled as a dead node", dn); + putNodeBackInService(dn); + iterator.remove(); + } else if (status.isDecommissioning()) { + completeDecommission(dn); + iterator.remove(); + } else if (status.isEnteringMaintenance()) { + putIntoMaintenance(dn); + } + } + } + + } catch (NodeNotFoundException e) { + LOG.error("An unexpected error occurred processing datanode {}. " + + "Aborting the admin workflow", dn.getDatanodeDetails(), e); + abortWorkflow(dn); + iterator.remove(); + } + } + } + + /** + * Checks if a node is in an unexpected state or has gone dead while + * decommissioning or entering maintenance. If the node is not in a valid + * state to continue the admin workflow, return false, otherwise return true. + * + * @param dn The Datanode for which to check the current state + * @param nodeStatus The current NodeStatus for the datanode + * @return True if admin can continue, false otherwise + */ + private boolean shouldContinueWorkflow(DatanodeAdminNodeDetails dn, + NodeStatus nodeStatus) { + if (!nodeStatus.isDecommission() && !nodeStatus.isMaintenance()) { + LOG.warn("Datanode {} has an operational state of {} when it should " + + "be undergoing decommission or maintenance. Aborting admin for " + + "this node.", + dn.getDatanodeDetails(), nodeStatus.getOperationalState()); + return false; + } + if (nodeStatus.isDead() && !nodeStatus.isInMaintenance()) { + LOG.error("Datanode {} is dead but is not IN_MAINTENANCE. Aborting the " + + "admin workflow for this node", dn.getDatanodeDetails()); + return false; + } + return true; + } + + private boolean checkPipelinesClosedOnNode(DatanodeAdminNodeDetails dn) { + DatanodeDetails dnd = dn.getDatanodeDetails(); + Set pipelines = nodeManager.getPipelines(dnd); + if (pipelines == null || pipelines.size() == 0 + || dn.shouldMaintenanceEnd()) { + return true; + } else { + LOG.info("Waiting for pipelines to close for {}. There are {} " + + "pipelines", dnd, pipelines.size()); + return false; + } + } + + private boolean checkContainersReplicatedOnNode(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + int sufficientlyReplicated = 0; + int underReplicated = 0; + int unhealthy = 0; + Set containers = + nodeManager.getContainers(dn.getDatanodeDetails()); + for (ContainerID cid : containers) { + try { + ContainerReplicaCount replicaSet = + replicationManager.getContainerReplicaCount(cid); + if (replicaSet.isSufficientlyReplicated()) { + sufficientlyReplicated++; + } else { + underReplicated++; + } + if (!replicaSet.isHealthy()) { + unhealthy++; + } + } catch (ContainerNotFoundException e) { + LOG.warn("ContainerID {} present in node list for {} but not found " + + "in containerManager", cid, dn.getDatanodeDetails()); + } + } + dn.setSufficientlyReplicatedContainers(sufficientlyReplicated); + dn.setUnderReplicatedContainers(underReplicated); + dn.setUnHealthyContainers(unhealthy); + + return underReplicated == 0 && unhealthy == 0; + } + + private void completeDecommission(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + setNodeOpState(dn, NodeOperationalState.DECOMMISSIONED); + LOG.info("Datanode {} has completed the admin workflow. The operational " + + "state has been set to {}", dn.getDatanodeDetails(), + NodeOperationalState.DECOMMISSIONED); + } + + private void putIntoMaintenance(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + LOG.info("Datanode {} has entered maintenance", dn.getDatanodeDetails()); + setNodeOpState(dn, NodeOperationalState.IN_MAINTENANCE); + } + + private void completeMaintenance(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + // The end state of Maintenance is to put the node back IN_SERVICE, whether + // it is dead or not. + // TODO - if the node is dead do we trigger a dead node event here or leave + // it to the heartbeat manager? + LOG.info("Datanode {} has ended maintenance automatically", + dn.getDatanodeDetails()); + putNodeBackInService(dn); + } + + private void startTrackingNode(DatanodeAdminNodeDetails dn) { + eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, + dn.getDatanodeDetails()); + trackedNodes.add(dn); + } + + private void stopTrackingNode(DatanodeAdminNodeDetails dn) { + trackedNodes.remove(dn); + } + + /** + * If we encounter an unexpected condition in maintenance, we must abort the + * workflow by setting the node operationalState back to IN_SERVICE and then + * remove the node from tracking. + * + * @param dn The datanode for which to abort tracking + */ + private void abortWorkflow(DatanodeAdminNodeDetails dn) { + try { + putNodeBackInService(dn); + } catch (NodeNotFoundException e) { + LOG.error("Unable to set the node OperationalState for {} while " + + "aborting the datanode admin workflow", dn.getDatanodeDetails()); + } + } + + private void putNodeBackInService(DatanodeAdminNodeDetails dn) + throws NodeNotFoundException { + setNodeOpState(dn, NodeOperationalState.IN_SERVICE); + } + + private void setNodeOpState(DatanodeAdminNodeDetails dn, + HddsProtos.NodeOperationalState state) throws NodeNotFoundException { + nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state); + } + + // TODO - The nodeManager.getNodeStatus call should really throw + // NodeNotFoundException rather than having to handle it here as all + // registered nodes must have a status. + private NodeStatus getNodeStatus(DatanodeDetails dnd) + throws NodeNotFoundException { + NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); + if (nodeStatus == null) { + throw new NodeNotFoundException("Unable to retrieve the nodeStatus"); + } + return nodeStatus; + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java deleted file mode 100644 index cfbbffb8d977..000000000000 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorInterface.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdds.scm.node; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.container.ReplicationManager; -import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.server.events.EventPublisher; - -/** - * Interface used by the DatanodeAdminMonitor, which can be used to - * decommission or recommission nodes and take them in and out of maintenance. - */ -public interface DatanodeAdminMonitorInterface extends Runnable { - - void setConf(OzoneConfiguration conf); - void setEventQueue(EventPublisher scm); - void setNodeManager(NodeManager nm); - void setPipelineManager(PipelineManager pm); - void setReplicationManager(ReplicationManager rm); - void startMonitoring(DatanodeDetails dn, int endInHours); - void stopMonitoring(DatanodeDetails dn); - - int getPendingCount(); - int getCancelledCount(); - int getTrackedNodeCount(); -} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index f76929ef8514..4c9bf9c45169 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ReplicationManager; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.slf4j.Logger; @@ -47,10 +46,9 @@ public class NodeDecommissionManager { private ScheduledExecutorService executor; - private DatanodeAdminMonitorInterface monitor; + private DatanodeAdminMonitor monitor; private NodeManager nodeManager; - private PipelineManager pipelineManager; //private ContainerManager containerManager; private EventPublisher eventQueue; private ReplicationManager replicationManager; @@ -168,11 +166,10 @@ private boolean validateDNPortMatch(int port, DatanodeDetails dn) { } public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, - PipelineManager pm, ContainerManager containerManager, + ContainerManager containerManager, EventPublisher eventQueue, ReplicationManager rm) { this.nodeManager = nm; conf = config; - this.pipelineManager = pm; //this.containerManager = containerManager; this.eventQueue = eventQueue; this.replicationManager = rm; @@ -201,19 +198,15 @@ public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, TimeUnit.SECONDS); } - monitor = new DatanodeAdminMonitor(conf); - monitor.setConf(conf); - monitor.setEventQueue(this.eventQueue); - monitor.setNodeManager(nodeManager); - monitor.setPipelineManager(pipelineManager); - monitor.setReplicationManager(replicationManager); + monitor = new DatanodeAdminMonitorImpl(conf, eventQueue, nodeManager, + replicationManager); executor.scheduleAtFixedRate(monitor, monitorInterval, monitorInterval, TimeUnit.SECONDS); } @VisibleForTesting - public DatanodeAdminMonitorInterface getMonitor() { + public DatanodeAdminMonitor getMonitor() { return monitor; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 651f259c3fe7..a7a75ebceece 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -340,7 +340,7 @@ public StorageContainerManager(OzoneConfiguration conf, pipelineManager); scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, - pipelineManager, containerManager, eventQueue, replicationManager); + containerManager, eventQueue, replicationManager); eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 98ad037f092a..5b7475000ceb 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -52,7 +52,7 @@ public class TestDatanodeAdminMonitor { private SimpleMockNodeManager nodeManager; private OzoneConfiguration conf; - private DatanodeAdminMonitor monitor; + private DatanodeAdminMonitorImpl monitor; private DatanodeAdminHandler startAdminHandler; private ReplicationManager repManager; private EventQueue eventQueue; @@ -69,10 +69,8 @@ public void setup() throws IOException, AuthenticationException { repManager = Mockito.mock(ReplicationManager.class); - monitor = new DatanodeAdminMonitor(conf); - monitor.setEventQueue(eventQueue); - monitor.setNodeManager(nodeManager); - monitor.setReplicationManager(repManager); + monitor = + new DatanodeAdminMonitorImpl(conf, eventQueue, nodeManager, repManager); } @After diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java index 916e383c0034..c5310b9d4c1d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java @@ -35,12 +35,10 @@ public class TestDatanodeAdminNodeDetails { private OzoneConfiguration conf; - private DatanodeAdminMonitor monitor; @Before public void setup() { conf = new OzoneConfiguration(); - monitor = new DatanodeAdminMonitor(conf); } @After diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index 2da7a611c37e..47055f58eaf8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -56,7 +56,7 @@ public void setup() throws Exception { conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); nodeManager = createNodeManager(conf); decom = new NodeDecommissionManager( - conf, nodeManager, null, null, null, null); + conf, nodeManager, null, null, null); } @Test From ec1df698226adb574d44aeac565899b2364e9900 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Tue, 17 Dec 2019 17:40:18 +0000 Subject: [PATCH 10/78] HDDS-2607. DeadNodeHandler should not remove replica for a dead maintenance node (#343) --- .../scm/node/DatanodeAdminMonitorImpl.java | 3 - .../hadoop/hdds/scm/node/DeadNodeHandler.java | 8 +- .../hdds/scm/node/NodeStateManager.java | 79 ++++++------------- .../hdds/scm/node/TestDeadNodeHandler.java | 32 ++++++-- .../hdds/scm/node/TestNodeStateManager.java | 67 ++++++++++++++-- 5 files changed, 116 insertions(+), 73 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index 931a45e3bda4..8eb985ba3378 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -170,7 +170,6 @@ private void processCancelledNodes() { LOG.warn("Failed processing the cancel admin request for {}", dn.getDatanodeDetails(), e); } - // TODO - fire event to bring node back into service? } } @@ -320,8 +319,6 @@ private void completeMaintenance(DatanodeAdminNodeDetails dn) throws NodeNotFoundException { // The end state of Maintenance is to put the node back IN_SERVICE, whether // it is dead or not. - // TODO - if the node is dead do we trigger a dead node event here or leave - // it to the heartbeat manager? LOG.info("Datanode {} has ended maintenance automatically", dn.getDatanodeDetails()); putNodeBackInService(dn); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java index 17e1fedd9525..216f82cf87d2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java @@ -74,12 +74,14 @@ public void onMessage(final DatanodeDetails datanodeDetails, * To be on a safer side, we double check here and take appropriate * action. */ - destroyPipelines(datanodeDetails); closeContainers(datanodeDetails, publisher); - // Remove the container replicas associated with the dead node. - removeContainerReplicas(datanodeDetails); + // Remove the container replicas associated with the dead node unless it + // is IN_MAINTENANCE + if (!nodeManager.getNodeStatus(datanodeDetails).isInMaintenance()) { + removeContainerReplicas(datanodeDetails); + } } catch (NodeNotFoundException ex) { // This should not happen, we cannot get a dead node event for an diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index 4177b63dd396..b16e79c6ce59 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -80,11 +80,6 @@ private enum NodeLifeCycleEvent { TIMEOUT, RESTORE, RESURRECT } - private enum NodeOperationStateEvent { - START_DECOMMISSION, COMPLETE_DECOMMISSION, START_MAINTENANCE, - ENTER_MAINTENANCE, RETURN_TO_SERVICE - } - private static final Logger LOG = LoggerFactory .getLogger(NodeStateManager.class); @@ -92,11 +87,6 @@ private enum NodeOperationStateEvent { * StateMachine for node lifecycle. */ private final StateMachine nodeHealthSM; - /** - * StateMachine for node operational state. - */ - private final StateMachine nodeOpStateSM; /** * This is the map which maintains the current state of all datanodes. */ @@ -112,7 +102,7 @@ private enum NodeOperationStateEvent { /** * Maps the event to be triggered when a node state us updated. */ - private final Map> state2EventMap; + private final Map> state2EventMap; /** * ExecutorService used for scheduling heartbeat processing thread. */ @@ -162,10 +152,7 @@ public NodeStateManager(Configuration conf, EventPublisher eventPublisher) { this.state2EventMap = new HashMap<>(); initialiseState2EventMap(); Set finalStates = new HashSet<>(); - Set opStateFinalStates = new HashSet<>(); this.nodeHealthSM = new StateMachine<>(NodeState.HEALTHY, finalStates); - this.nodeOpStateSM = new StateMachine<>( - NodeOperationalState.IN_SERVICE, opStateFinalStates); initializeStateMachines(); heartbeatCheckerIntervalMs = HddsServerUtil .getScmheartbeatCheckerInterval(conf); @@ -190,12 +177,10 @@ public NodeStateManager(Configuration conf, EventPublisher eventPublisher) { * Populates state2event map. */ private void initialiseState2EventMap() { - state2EventMap.put(NodeStatus.inServiceStale(), SCMEvents.STALE_NODE); - state2EventMap.put(NodeStatus.inServiceDead(), SCMEvents.DEAD_NODE); - state2EventMap.put(NodeStatus.inServiceHealthy(), + state2EventMap.put(NodeState.STALE, SCMEvents.STALE_NODE); + state2EventMap.put(NodeState.DEAD, SCMEvents.DEAD_NODE); + state2EventMap.put(NodeState.HEALTHY, SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE); - // TODO - add whatever events are needed for decomm / maint to stale, dead, - // healthy } /* @@ -238,36 +223,6 @@ private void initializeStateMachines() { NodeState.STALE, NodeState.HEALTHY, NodeLifeCycleEvent.RESTORE); nodeHealthSM.addTransition( NodeState.DEAD, NodeState.HEALTHY, NodeLifeCycleEvent.RESURRECT); - - nodeOpStateSM.addTransition( - NodeOperationalState.IN_SERVICE, NodeOperationalState.DECOMMISSIONING, - NodeOperationStateEvent.START_DECOMMISSION); - nodeOpStateSM.addTransition( - NodeOperationalState.DECOMMISSIONING, NodeOperationalState.IN_SERVICE, - NodeOperationStateEvent.RETURN_TO_SERVICE); - nodeOpStateSM.addTransition( - NodeOperationalState.DECOMMISSIONING, - NodeOperationalState.DECOMMISSIONED, - NodeOperationStateEvent.COMPLETE_DECOMMISSION); - nodeOpStateSM.addTransition( - NodeOperationalState.DECOMMISSIONED, NodeOperationalState.IN_SERVICE, - NodeOperationStateEvent.RETURN_TO_SERVICE); - - nodeOpStateSM.addTransition( - NodeOperationalState.IN_SERVICE, - NodeOperationalState.ENTERING_MAINTENANCE, - NodeOperationStateEvent.START_MAINTENANCE); - nodeOpStateSM.addTransition( - NodeOperationalState.ENTERING_MAINTENANCE, - NodeOperationalState.IN_SERVICE, - NodeOperationStateEvent.RETURN_TO_SERVICE); - nodeOpStateSM.addTransition( - NodeOperationalState.ENTERING_MAINTENANCE, - NodeOperationalState.IN_MAINTENANCE, - NodeOperationStateEvent.ENTER_MAINTENANCE); - nodeOpStateSM.addTransition( - NodeOperationalState.IN_MAINTENANCE, NodeOperationalState.IN_SERVICE, - NodeOperationStateEvent.RETURN_TO_SERVICE); } /** @@ -280,7 +235,7 @@ private void initializeStateMachines() { public void addNode(DatanodeDetails datanodeDetails) throws NodeAlreadyExistsException { nodeStateMap.addNode(datanodeDetails, new NodeStatus( - nodeOpStateSM.getInitialState(), nodeHealthSM.getInitialState())); + NodeOperationalState.IN_SERVICE, nodeHealthSM.getInitialState())); eventPublisher.fireEvent(SCMEvents.NEW_NODE, datanodeDetails); } @@ -404,8 +359,18 @@ public List getAllNodes() { public void setNodeOperationalState(DatanodeDetails dn, NodeOperationalState newState) throws NodeNotFoundException { DatanodeInfo dni = nodeStateMap.getNodeInfo(dn.getUuid()); - if (dni.getNodeStatus().getOperationalState() != newState) { + NodeStatus oldStatus = dni.getNodeStatus(); + if (oldStatus.getOperationalState() != newState) { nodeStateMap.updateNodeOperationalState(dn.getUuid(), newState); + // This will trigger an event based on the nodes health when the + // operational state changes. Eg a node that was IN_MAINTENANCE goes + // to IN_SERVICE + HEALTHY. This will trigger the HEALTHY node event to + // create new pipelines. OTH, if the nodes goes IN_MAINTENANCE to + // IN_SERVICE + DEAD, it will trigger the dead node handler to remove its + // container replicas. Sometimes the event will do nothing, but it will + // not do any harm either. Eg DECOMMISSIONING -> DECOMMISSIONED + HEALTHY + // but the pipeline creation logic will ignore decommissioning nodes. + fireHealthStateEvent(oldStatus.getHealth(), dn); } } @@ -706,9 +671,7 @@ private void updateNodeState(DatanodeInfo node, Predicate condition, getNextState(status.getHealth(), lifeCycleEvent); NodeStatus newStatus = nodeStateMap.updateNodeHealthState(node.getUuid(), newHealthState); - if (state2EventMap.containsKey(newStatus)) { - eventPublisher.fireEvent(state2EventMap.get(newStatus), node); - } + fireHealthStateEvent(newStatus.getHealth(), node); } } catch (InvalidStateTransitionException e) { LOG.warn("Invalid state transition of node {}." + @@ -717,6 +680,14 @@ private void updateNodeState(DatanodeInfo node, Predicate condition, } } + private void fireHealthStateEvent(HddsProtos.NodeState health, + DatanodeDetails node) { + Event event = state2EventMap.get(health); + if (event != null) { + eventPublisher.fireEvent(event, node); + } + } + @Override public void close() { executorService.shutdown(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java index 1676af1c3b22..85ea68b8870a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@ -58,11 +58,11 @@ import org.apache.hadoop.security.authentication.client .AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; -import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.junit.Assert; +import org.junit.After; import org.mockito.Mockito; /** @@ -174,22 +174,44 @@ public void testOnMessage() throws IOException, NodeNotFoundException { TestUtils.closeContainer(containerManager, container2.containerID()); TestUtils.quasiCloseContainer(containerManager, container3.containerID()); + // First set the node to IN_MAINTENANCE and ensure the container replicas + // are not removed on the dead event + nodeManager.setNodeOperationalState(datanode1, + HddsProtos.NodeOperationalState.IN_MAINTENANCE); deadNodeHandler.onMessage(datanode1, publisher); Set container1Replicas = containerManager .getContainerReplicas(new ContainerID(container1.getContainerID())); + Assert.assertEquals(2, container1Replicas.size()); + + Set container2Replicas = containerManager + .getContainerReplicas(new ContainerID(container2.getContainerID())); + Assert.assertEquals(2, container2Replicas.size()); + + Set container3Replicas = containerManager + .getContainerReplicas(new ContainerID(container3.getContainerID())); + Assert.assertEquals(1, container3Replicas.size()); + + // Now set the node to anything other than IN_MAINTENANCE and the relevant + // replicas should be removed + nodeManager.setNodeOperationalState(datanode1, + HddsProtos.NodeOperationalState.IN_SERVICE); + deadNodeHandler.onMessage(datanode1, publisher); + + container1Replicas = containerManager + .getContainerReplicas(new ContainerID(container1.getContainerID())); Assert.assertEquals(1, container1Replicas.size()); Assert.assertEquals(datanode2, container1Replicas.iterator().next().getDatanodeDetails()); - Set container2Replicas = containerManager + container2Replicas = containerManager .getContainerReplicas(new ContainerID(container2.getContainerID())); Assert.assertEquals(1, container2Replicas.size()); Assert.assertEquals(datanode2, container2Replicas.iterator().next().getDatanodeDetails()); - Set container3Replicas = containerManager - .getContainerReplicas(new ContainerID(container3.getContainerID())); + container3Replicas = containerManager + .getContainerReplicas(new ContainerID(container3.getContainerID())); Assert.assertEquals(1, container3Replicas.size()); Assert.assertEquals(datanode3, container3Replicas.iterator().next().getDatanodeDetails()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index 9fbf2519f5e1..5d3ee5e83d31 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.HddsServerUtil; +import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.server.events.Event; @@ -145,7 +146,7 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() DatanodeDetails dn = generateDatanode(); nsm.addNode(dn); - assertEquals("New_Node", eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.NEW_NODE, eventPublisher.getLastEvent()); DatanodeInfo dni = nsm.getNode(dn); dni.updateLastHeartbeatTime(); @@ -159,31 +160,31 @@ public void testNodeCanTransitionThroughHealthStatesAndFiresEvents() dni.updateLastHeartbeatTime(now - staleLimit); nsm.checkNodesHealth(); assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); - assertEquals("Stale_Node", eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.STALE_NODE, eventPublisher.getLastEvent()); // Now make it dead dni.updateLastHeartbeatTime(now - deadLimit); nsm.checkNodesHealth(); assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); - assertEquals("Dead_Node", eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.DEAD_NODE, eventPublisher.getLastEvent()); // Transition back to healthy from dead dni.updateLastHeartbeatTime(); nsm.checkNodesHealth(); assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); - assertEquals("NON_HEALTHY_TO_HEALTHY_NODE", - eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE, + eventPublisher.getLastEvent()); // Make the node stale again, and transition to healthy. dni.updateLastHeartbeatTime(now - staleLimit); nsm.checkNodesHealth(); assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); - assertEquals("Stale_Node", eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.STALE_NODE, eventPublisher.getLastEvent()); dni.updateLastHeartbeatTime(); nsm.checkNodesHealth(); assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); - assertEquals("NON_HEALTHY_TO_HEALTHY_NODE", - eventPublisher.getLastEvent().getName()); + assertEquals(SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE, + eventPublisher.getLastEvent()); } @Test @@ -202,6 +203,56 @@ public void testNodeOpStateCanBeSet() newStatus.getHealth()); } + @Test + public void testHealthEventsFiredWhenOpStateChanged() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + nsm.addNode(dn); + + // First set the node to decommissioned, then run through all op states in + // order and ensure the non_healthy_to_healthy event gets fired + nsm.setNodeOperationalState(dn, + HddsProtos.NodeOperationalState.DECOMMISSIONED); + for (HddsProtos.NodeOperationalState s : + HddsProtos.NodeOperationalState.values()) { + eventPublisher.clearEvents(); + nsm.setNodeOperationalState(dn, s); + assertEquals(SCMEvents.NON_HEALTHY_TO_HEALTHY_NODE, + eventPublisher.getLastEvent()); + } + + // Now make the node stale and run through all states again ensuring the + // stale event gets fired + long now = Time.monotonicNow(); + long staleLimit = HddsServerUtil.getStaleNodeInterval(conf) + 1000; + long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; + DatanodeInfo dni = nsm.getNode(dn); + dni.updateLastHeartbeatTime(now - staleLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); + nsm.setNodeOperationalState(dn, + HddsProtos.NodeOperationalState.DECOMMISSIONED); + for (HddsProtos.NodeOperationalState s : + HddsProtos.NodeOperationalState.values()) { + eventPublisher.clearEvents(); + nsm.setNodeOperationalState(dn, s); + assertEquals(SCMEvents.STALE_NODE, eventPublisher.getLastEvent()); + } + + // Finally make the node dead and run through all the op states again + dni.updateLastHeartbeatTime(now - deadLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); + nsm.setNodeOperationalState(dn, + HddsProtos.NodeOperationalState.DECOMMISSIONED); + for (HddsProtos.NodeOperationalState s : + HddsProtos.NodeOperationalState.values()) { + eventPublisher.clearEvents(); + nsm.setNodeOperationalState(dn, s); + assertEquals(SCMEvents.DEAD_NODE, eventPublisher.getLastEvent()); + } + } + private DatanodeDetails generateDatanode() { String uuid = UUID.randomUUID().toString(); return DatanodeDetails.newBuilder().setUuid(uuid).build(); From 692420faeea70fe02ab805a448d8a08c33956c11 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Tue, 17 Dec 2019 18:22:00 +0000 Subject: [PATCH 11/78] HDDS-2671. Have NodeManager.getNodeStatus throw NodeNotFoundException (#328) --- .../scm/container/ReplicationManager.java | 20 ++++++++++++++-- .../hdds/scm/node/DatanodeAdminMonitor.java | 1 - .../scm/node/DatanodeAdminMonitorImpl.java | 9 +------- .../scm/node/NodeDecommissionManager.java | 6 +---- .../hadoop/hdds/scm/node/NodeManager.java | 4 +++- .../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +++----- .../scm/server/SCMClientProtocolServer.java | 23 +++++++++++-------- .../hdds/scm/container/MockNodeManager.java | 3 ++- .../scm/container/SimpleMockNodeManager.java | 3 ++- .../scm/node/TestDatanodeAdminMonitor.java | 18 ++++++++++----- .../scm/node/TestNodeDecommissionManager.java | 5 ++-- 11 files changed, 59 insertions(+), 43 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index b18024e459ac..dfb5961a4db4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -42,6 +42,8 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; @@ -544,7 +546,7 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, // maintenance nodes, as the replicas will remain present in the // container manager, even when they go dead. .filter(r -> - nodeManager.getNodeStatus(r.getDatanodeDetails()).isHealthy()) + getNodeStatus(r.getDatanodeDetails()).isHealthy()) .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails())) .sorted((r1, r2) -> r2.getSequenceId().compareTo(r1.getSequenceId())) .map(ContainerReplica::getDatanodeDetails) @@ -574,7 +576,7 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, LOG.warn("Cannot replicate container {}, no healthy replica found.", container.containerID()); } - } catch (IOException ex) { + } catch (IOException | IllegalStateException ex) { LOG.warn("Exception while replicating container {}.", container.getContainerID(), ex); } @@ -785,6 +787,20 @@ private void sendAndTrackDatanodeCommand( tracker.accept(new InflightAction(datanode, Time.monotonicNow())); } + /** + * Wrap the call to nodeManager.getNodeStatus, catching any + * NodeNotFoundException and instead throwing an IllegalStateException. + * @param dn The datanodeDetails to obtain the NodeStatus for + * @return NodeStatus corresponding to the given Datanode. + */ + private NodeStatus getNodeStatus(DatanodeDetails dn) { + try { + return nodeManager.getNodeStatus(dn); + } catch (NodeNotFoundException e) { + throw new IllegalStateException("Unable to find NodeStatus for "+dn, e); + } + } + /** * Compares the container state with the replica state. * diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java index 9ead66f4b4fd..e78eeffd84ad 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -26,7 +26,6 @@ public interface DatanodeAdminMonitor extends Runnable { void startMonitoring(DatanodeDetails dn, int endInHours); - void stopMonitoring(DatanodeDetails dn); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index 8eb985ba3378..4d2d895ec266 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -360,16 +360,9 @@ private void setNodeOpState(DatanodeAdminNodeDetails dn, nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state); } - // TODO - The nodeManager.getNodeStatus call should really throw - // NodeNotFoundException rather than having to handle it here as all - // registered nodes must have a status. private NodeStatus getNodeStatus(DatanodeDetails dnd) throws NodeNotFoundException { - NodeStatus nodeStatus = nodeManager.getNodeStatus(dnd); - if (nodeStatus == null) { - throw new NodeNotFoundException("Unable to retrieve the nodeStatus"); - } - return nodeStatus; + return nodeManager.getNodeStatus(dnd); } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index 4c9bf9c45169..06fe270e2164 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -331,11 +331,7 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) private NodeStatus getNodeStatus(DatanodeDetails dn) throws NodeNotFoundException { - NodeStatus nodeStatus = nodeManager.getNodeStatus(dn); - if (nodeStatus == null) { - throw new NodeNotFoundException(); - } - return nodeStatus; + return nodeManager.getNodeStatus(dn); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index 252c38fe1c25..b595d0023971 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -128,8 +128,10 @@ int getNodeCount( * Returns the node status of a specific node. * @param datanodeDetails DatanodeDetails * @return NodeStatus for the node + * @throws NodeNotFoundException if the node does not exist */ - NodeStatus getNodeStatus(DatanodeDetails datanodeDetails); + NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) + throws NodeNotFoundException; /** * Set the operation state of a node. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 131a0e44ed45..3c5071f1cde5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -218,13 +218,9 @@ public int getNodeCount(NodeOperationalState nodeOpState, NodeState health) { * @return NodeStatus for the node */ @Override - public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) { - try { - return nodeStateManager.getNodeStatus(datanodeDetails); - } catch (NodeNotFoundException e) { - // TODO: should we throw NodeNotFoundException? - return null; - } + public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) + throws NodeNotFoundException { + return nodeStateManager.getNodeStatus(datanodeDetails); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 76efb3ab204e..8f6b356c5617 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.ScmUtils; import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; import org.apache.hadoop.hdds.scm.safemode.SafeModePrecheck; import org.apache.hadoop.hdds.scm.container.ContainerID; @@ -357,15 +358,19 @@ public List queryNode( } List result = new ArrayList<>(); - queryNode(opState, state) - .forEach(node -> { - NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node); - result.add(HddsProtos.Node.newBuilder() - .setNodeID(node.getProtoBufMessage()) - .addNodeStates(ns.getHealth()) - .addNodeOperationalStates(ns.getOperationalState()) - .build()); - }); + for (DatanodeDetails node : queryNode(opState, state)) { + try { + NodeStatus ns = scm.getScmNodeManager().getNodeStatus(node); + result.add(HddsProtos.Node.newBuilder() + .setNodeID(node.getProtoBufMessage()) + .addNodeStates(ns.getHealth()) + .addNodeOperationalStates(ns.getOperationalState()) + .build()); + } catch (NodeNotFoundException e) { + throw new IOException( + "An unexpected error occurred querying the NodeStatus", e); + } + } return result; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 1a5db7f6d129..45be60bf97a7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -263,7 +263,8 @@ public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) { * @return Healthy/Stale/Dead. */ @Override - public NodeStatus getNodeStatus(DatanodeDetails dd) { + public NodeStatus getNodeStatus(DatanodeDetails dd) + throws NodeNotFoundException { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index 78d576c55016..883e910a4b29 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -87,7 +87,8 @@ public void setPipelines(DatanodeDetails dd, int count) { * Inservice and Healthy NodeStatus. */ @Override - public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) { + public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) + throws NodeNotFoundException { DatanodeInfo dni = nodeMap.get(datanodeDetails.getUuid()); if (dni != null) { return dni.getNodeStatus(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 5b7475000ceb..f3b7d8f9093a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -98,7 +98,8 @@ public void testNodeCanBeQueuedAndCancelled() { * must wait until the pipelines have closed before completing the flow. */ @Test - public void testClosePipelinesEventFiredWhenAdminStarted() { + public void testClosePipelinesEventFiredWhenAdminStarted() + throws NodeNotFoundException{ DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, @@ -136,7 +137,8 @@ public void testClosePipelinesEventFiredWhenAdminStarted() { * state. */ @Test - public void testDecommissionNodeTransitionsToCompleteWhenNoContainers() { + public void testDecommissionNodeTransitionsToCompleteWhenNoContainers() + throws NodeNotFoundException { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.DECOMMISSIONING, @@ -280,7 +282,8 @@ public void testDecommissionAbortedWhenNodeGoesDead() } @Test - public void testMaintenanceWaitsForMaintenanceToComplete() { + public void testMaintenanceWaitsForMaintenanceToComplete() + throws NodeNotFoundException { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, @@ -310,7 +313,8 @@ public void testMaintenanceWaitsForMaintenanceToComplete() { } @Test - public void testMaintenanceEndsClosingPipelines() { + public void testMaintenanceEndsClosingPipelines() + throws NodeNotFoundException { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, @@ -366,7 +370,8 @@ public void testMaintenanceEndsWhileReplicatingContainers() } @Test - public void testDeadMaintenanceNodeDoesNotAbortWorkflow() { + public void testDeadMaintenanceNodeDoesNotAbortWorkflow() + throws NodeNotFoundException { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, @@ -393,7 +398,8 @@ public void testDeadMaintenanceNodeDoesNotAbortWorkflow() { } @Test - public void testCancelledNodesMovedToInService() { + public void testCancelledNodesMovedToInService() + throws NodeNotFoundException { DatanodeDetails dn1 = TestUtils.randomDatanodeDetails(); nodeManager.register(dn1, new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index 47055f58eaf8..daf67312d728 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; @@ -134,7 +135,7 @@ public void testAnyInvalidHostThrowsException() @Test public void testNodesCanBeDecommissionedAndRecommissioned() - throws InvalidHostStringException { + throws InvalidHostStringException, NodeNotFoundException { List dns = generateDatanodes(); // Decommission 2 valid nodes @@ -173,7 +174,7 @@ public void testNodesCanBeDecommissionedAndRecommissioned() @Test public void testNodesCanBePutIntoMaintenanceAndRecommissioned() - throws InvalidHostStringException { + throws InvalidHostStringException, NodeNotFoundException { List dns = generateDatanodes(); // Put 2 valid nodes into maintenance From 5b228c039bd9715058e62bb450cb188566de1759 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 10 Feb 2020 11:34:09 +0100 Subject: [PATCH 12/78] HDDS-2860. Cluster disk space metrics should reflect decommission and maintenance states --- .../hdds/scm/node/NodeManagerMXBean.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 113 ++++++++++++------ .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 91 ++++++++------ .../hdds/scm/container/MockNodeManager.java | 20 +++- .../scm/container/SimpleMockNodeManager.java | 2 +- .../hdds/scm/node/TestSCMNodeManager.java | 72 +++++++++++ .../testutils/ReplicationNodeManagerMock.java | 2 +- .../ozone/scm/TestSCMNodeManagerMXBean.java | 30 ++++- .../ozone/scm/node/TestSCMNodeMetrics.java | 54 ++++++++- 9 files changed, 297 insertions(+), 89 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java index e1b51efc34cd..791e754e2c4a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java @@ -34,7 +34,7 @@ public interface NodeManagerMXBean { * * @return A state to number of nodes that in this state mapping */ - Map getNodeCount(); + Map> getNodeCount(); /** * Get the disk metrics like capacity, usage and remaining based on the diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 3c5071f1cde5..d89dac1cae35 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -482,62 +482,97 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) { } } - @Override - public Map getNodeCount() { - // TODO - This does not consider decom, maint etc. - Map nodeCountMap = new HashMap(); - for(NodeState state : NodeState.values()) { - // TODO - this iterate the node list once per state and needs - // fixed to only perform one pass. - nodeCountMap.put(state.toString(), getNodeCount(null, state)); + @Override // NodeManagerMXBean + public Map> getNodeCount() { + Map> nodes = new HashMap<>(); + for (NodeOperationalState opState : NodeOperationalState.values()) { + Map states = new HashMap<>(); + for (NodeState health : NodeState.values()) { + states.put(health.name(), 0); + } + nodes.put(opState.name(), states); + } + for (DatanodeInfo dni : nodeStateManager.getAllNodes()) { + NodeStatus status = dni.getNodeStatus(); + nodes.get(status.getOperationalState().name()) + .compute(status.getHealth().name(), (k, v) -> v+1); } - return nodeCountMap; + return nodes; } // We should introduce DISK, SSD, etc., notion in // SCMNodeStat and try to use it. - @Override + @Override // NodeManagerMXBean public Map getNodeInfo() { - long diskCapacity = 0L; - long diskUsed = 0L; - long diskRemaning = 0L; - - long ssdCapacity = 0L; - long ssdUsed = 0L; - long ssdRemaining = 0L; - - List healthyNodes = nodeStateManager.getHealthyNodes(); - List staleNodes = nodeStateManager.getStaleNodes(); - - List datanodes = new ArrayList<>(healthyNodes); - datanodes.addAll(staleNodes); + Map nodeInfo = new HashMap<>(); + // Compute all the possible stats from the enums, and default to zero: + for (UsageStates s : UsageStates.values()) { + for (UsageMetrics stat : UsageMetrics.values()) { + nodeInfo.put(s.label + stat.name(), 0L); + } + } - for (DatanodeInfo dnInfo : datanodes) { - List storageReportProtos = dnInfo.getStorageReports(); + for (DatanodeInfo node : nodeStateManager.getAllNodes()) { + String keyPrefix = ""; + NodeStatus status = node.getNodeStatus(); + if (status.isMaintenance()) { + keyPrefix = UsageStates.MAINT.getLabel(); + } else if (status.isDecommission()) { + keyPrefix = UsageStates.DECOM.getLabel(); + } else if (status.isAlive()) { + // Inservice but not dead + keyPrefix = UsageStates.ONLINE.getLabel(); + } else { + // dead inservice node, skip it + continue; + } + List storageReportProtos = node.getStorageReports(); for (StorageReportProto reportProto : storageReportProtos) { if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.DISK) { - diskCapacity += reportProto.getCapacity(); - diskRemaning += reportProto.getRemaining(); - diskUsed += reportProto.getScmUsed(); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskCapacity.name(), + (k, v) -> v + reportProto.getCapacity()); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskRemaining.name(), + (k, v) -> v + reportProto.getRemaining()); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskUsed.name(), + (k, v) -> v + reportProto.getScmUsed()); } else if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.SSD) { - ssdCapacity += reportProto.getCapacity(); - ssdRemaining += reportProto.getRemaining(); - ssdUsed += reportProto.getScmUsed(); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDCapacity.name(), + (k, v) -> v + reportProto.getCapacity()); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDRemaining.name(), + (k, v) -> v + reportProto.getRemaining()); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDUsed.name(), + (k, v) -> v + reportProto.getScmUsed()); } } } + return nodeInfo; + } - Map nodeInfo = new HashMap<>(); - nodeInfo.put("DISKCapacity", diskCapacity); - nodeInfo.put("DISKUsed", diskUsed); - nodeInfo.put("DISKRemaining", diskRemaning); + private enum UsageMetrics { + DiskCapacity, + DiskUsed, + DiskRemaining, + SSDCapacity, + SSDUsed, + SSDRemaining + } - nodeInfo.put("SSDCapacity", ssdCapacity); - nodeInfo.put("SSDUsed", ssdUsed); - nodeInfo.put("SSDRemaining", ssdRemaining); - return nodeInfo; + private enum UsageStates { + ONLINE(""), + MAINT("Maintenance"), + DECOM("Decommissioned"); + + private final String label; + + public String getLabel() { + return label; + } + + UsageStates(String label) { + this.label = label; + } } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index 5b872c10155b..ccb5d144ae4f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -18,15 +18,12 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; - import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; @@ -36,6 +33,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.util.StringUtils; /** * This class maintains Node related metrics. @@ -51,6 +49,7 @@ public final class SCMNodeMetrics implements MetricsSource { private @Metric MutableCounterLong numHBProcessingFailed; private @Metric MutableCounterLong numNodeReportProcessed; private @Metric MutableCounterLong numNodeReportProcessingFailed; + private @Metric String textMetric; private final MetricsRegistry registry; private final NodeManagerMXBean managerMXBean; @@ -61,6 +60,7 @@ public final class SCMNodeMetrics implements MetricsSource { private SCMNodeMetrics(NodeManagerMXBean managerMXBean) { this.managerMXBean = managerMXBean; this.registry = new MetricsRegistry(recordInfo); + this.textMetric = "my_test_metric"; } /** @@ -116,39 +116,58 @@ void incNumNodeReportProcessingFailed() { @Override @SuppressWarnings("SuspiciousMethodCalls") public void getMetrics(MetricsCollector collector, boolean all) { - Map nodeCount = managerMXBean.getNodeCount(); + Map> nodeCount = managerMXBean.getNodeCount(); Map nodeInfo = managerMXBean.getNodeInfo(); - registry.snapshot( - collector.addRecord(registry.info()) // Add annotated ones first - .addGauge(Interns.info( - "HealthyNodes", - "Number of healthy datanodes"), - nodeCount.get(HEALTHY.toString())) - .addGauge(Interns.info("StaleNodes", - "Number of stale datanodes"), - nodeCount.get(STALE.toString())) - .addGauge(Interns.info("DeadNodes", - "Number of dead datanodes"), - nodeCount.get(DEAD.toString())) - .addGauge(Interns.info("DiskCapacity", - "Total disk capacity"), - nodeInfo.get("DISKCapacity")) - .addGauge(Interns.info("DiskUsed", - "Total disk capacity used"), - nodeInfo.get("DISKUsed")) - .addGauge(Interns.info("DiskRemaining", - "Total disk capacity remaining"), - nodeInfo.get("DISKRemaining")) - .addGauge(Interns.info("SSDCapacity", - "Total ssd capacity"), - nodeInfo.get("SSDCapacity")) - .addGauge(Interns.info("SSDUsed", - "Total ssd capacity used"), - nodeInfo.get("SSDUsed")) - .addGauge(Interns.info("SSDRemaining", - "Total disk capacity remaining"), - nodeInfo.get("SSDRemaining")), - all); + /** + * Loop over the Node map and create a metric for the cross product of all + * Operational and health states, ie: + * InServiceHealthy + * InServiceStale + * ... + * EnteringMaintenanceHealthy + * ... + */ + MetricsRecordBuilder metrics = collector.addRecord(registry.info()); + for(Map.Entry> e : nodeCount.entrySet()) { + for(Map.Entry h : e.getValue().entrySet()) { + metrics.addGauge( + Interns.info( + StringUtils.camelize(e.getKey()+"_"+h.getKey()+"_nodes"), + "Number of "+e.getKey()+" "+h.getKey()+" datanodes"), + h.getValue()); + } + } + + for (Map.Entry e : nodeInfo.entrySet()) { + metrics.addGauge( + Interns.info(e.getKey(), diskMetricDescription(e.getKey())), + e.getValue()); + } + registry.snapshot(metrics, all); + } + + private String diskMetricDescription(String metric) { + StringBuilder sb = new StringBuilder(); + sb.append("Total"); + if (metric.indexOf("Maintenance") >= 0) { + sb.append(" maintenance"); + } else if (metric.indexOf("Decommissioned") >= 0) { + sb.append(" decommissioned"); + } + if (metric.indexOf("DiskCapacity") >= 0) { + sb.append(" disk capacity"); + } else if (metric.indexOf("DiskUsed") >= 0) { + sb.append(" disk capacity used"); + } else if (metric.indexOf("DiskRemaining") >= 0) { + sb.append(" disk capacity remaining"); + } else if (metric.indexOf("SSDCapacity") >= 0) { + sb.append(" SSD capacity"); + } else if (metric.indexOf("SSDUsed") >= 0) { + sb.append(" SSD capacity used"); + } else if (metric.indexOf("SSDRemaining") >= 0) { + sb.append(" SSD capacity remaining"); + } + return sb.toString(); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 8a48a68fcd08..d24cbfd5114d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto @@ -471,12 +472,23 @@ public Boolean isNodeRegistered( } @Override - public Map getNodeCount() { - Map nodeCountMap = new HashMap(); + public Map> getNodeCount() { + Map> nodes = new HashMap<>(); + for (NodeOperationalState opState : NodeOperationalState.values()) { + Map states = new HashMap<>(); + for (HddsProtos.NodeState health : HddsProtos.NodeState.values()) { + states.put(health.name(), 0); + } + nodes.put(opState.name(), states); + } + // At the moment MockNodeManager is not aware of decommission and + // maintenance states, therefore loop over all nodes and assume all nodes + // are IN_SERVICE. This will be fixed as part of HDDS-2673 for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) { - nodeCountMap.put(state.toString(), getNodeCount(null, state)); + nodes.get(NodeOperationalState.IN_SERVICE.name()) + .compute(state.name(), (k, v) -> v + 1); } - return nodeCountMap; + return nodes; } @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index 883e910a4b29..dad3448bdf38 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -243,7 +243,7 @@ public void close() throws IOException { } @Override - public Map getNodeCount() { + public Map> getNodeCount() { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 1af996cb3265..0917fa4019b6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -19,10 +19,12 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.net.NetworkTopology; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -53,6 +55,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; @@ -240,6 +243,11 @@ public void testScmHealthyNodeCount() Thread.sleep(4 * 1000); assertEquals(count, nodeManager.getNodeCount( NodeStatus.inServiceHealthy())); + + Map> nodeCounts = nodeManager.getNodeCount(); + assertEquals(count, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.HEALTHY.name()).intValue()); } } @@ -323,6 +331,11 @@ public void testScmDetectStaleAndDeadNode() .getUuid(), staleNodeList.get(0).getUuid()); Thread.sleep(1000); + Map> nodeCounts = nodeManager.getNodeCount(); + assertEquals(1, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.STALE.name()).intValue()); + // heartbeat good nodes again. for (DatanodeDetails dn : nodeList) { nodeManager.processHeartbeat(dn); @@ -334,10 +347,14 @@ public void testScmDetectStaleAndDeadNode() // the stale node has been removed staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale()); + nodeCounts = nodeManager.getNodeCount(); assertEquals("Expected to find 1 stale node", 0, nodeManager.getNodeCount(NodeStatus.inServiceStale())); assertEquals("Expected to find 1 stale node", 0, staleNodeList.size()); + assertEquals(0, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.STALE.name()).intValue()); // Check for the dead node now. List deadNodeList = @@ -346,6 +363,9 @@ public void testScmDetectStaleAndDeadNode() nodeManager.getNodeCount(NodeStatus.inServiceDead())); assertEquals("Expected to find 1 dead node", 1, deadNodeList.size()); + assertEquals(1, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.DEAD.name()).intValue()); assertEquals("Dead node is not the expected ID", staleNode .getUuid(), deadNodeList.get(0).getUuid()); } @@ -1195,6 +1215,58 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) } } + @Test + public void testGetNodeInfo() + throws IOException, InterruptedException, NodeNotFoundException, + AuthenticationException { + OzoneConfiguration conf = getConf(); + final int nodeCount = 6; + SCMNodeManager nodeManager = createNodeManager(conf); + + for (int i=0; i stats = nodeManager.getNodeInfo(); + // 3 IN_SERVICE nodes: + assertEquals(6000, stats.get("DiskCapacity").longValue()); + assertEquals(300, stats.get("DiskUsed").longValue()); + assertEquals(5700, stats.get("DiskRemaining").longValue()); + + // 2 Decommissioned nodes + assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue()); + assertEquals(200, stats.get("DecommissionedDiskUsed").longValue()); + assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue()); + + // 1 Maintenance node + assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue()); + assertEquals(100, stats.get("MaintenanceDiskUsed").longValue()); + assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue()); + } + /** * Test add node into a 4-layer network topology during node register. */ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index 5a2e3c2109ac..ee96565a4097 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -72,7 +72,7 @@ public ReplicationNodeManagerMock(Map nodeStatus, * @return A state to number of nodes that in this state mapping */ @Override - public Map getNodeCount() { + public Map> getNodeCount() { return null; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java index 43b9bf03a6ef..0f961095b9ad 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java @@ -92,10 +92,32 @@ public void testNodeCount() throws Exception { + "name=SCMNodeManagerInfo"); TabularData data = (TabularData) mbs.getAttribute(bean, "NodeCount"); - Map nodeCount = scm.getScmNodeManager().getNodeCount(); - Map nodeCountLong = new HashMap<>(); - nodeCount.forEach((k, v) -> nodeCountLong.put(k, new Long(v))); - verifyEquals(data, nodeCountLong); + Map> mbeanMap = convertNodeCountToMap(data); + Map> nodeMap = + scm.getScmNodeManager().getNodeCount(); + assertTrue(nodeMap.equals(mbeanMap)); + } + + private Map> convertNodeCountToMap( + TabularData data) { + Map> map = new HashMap<>(); + for (Object o : data.values()) { + CompositeData cds = (CompositeData) o; + Iterator it = cds.values().iterator(); + String opState = it.next().toString(); + TabularData states = (TabularData) it.next(); + + Map healthStates = new HashMap<>(); + for (Object obj : states.values()) { + CompositeData stateData = (CompositeData) obj; + Iterator stateIt = stateData.values().iterator(); + String health = stateIt.next().toString(); + Integer value = Integer.parseInt(stateIt.next().toString()); + healthStates.put(health, value); + } + map.put(opState, healthStates); + } + return map; } private void verifyEquals(TabularData actualData, Map diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index d528f437cee1..6192ff5bf315 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -146,11 +146,35 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { cluster.getStorageContainerManager().getScmNodeManager() .processNodeReport(datanode.getDatanodeDetails(), nodeReport); - assertGauge("HealthyNodes", 1, + assertGauge("InServiceHealthyNodes", 1, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("StaleNodes", 0, + assertGauge("InServiceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DeadNodes", 0, + assertGauge("InServiceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DiskCapacity", 100L, getMetrics(SCMNodeMetrics.class.getSimpleName())); @@ -164,6 +188,30 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("SSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); } @After From a251d45530656d1882406e109453315a3832c101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Mon, 10 Feb 2020 11:39:37 +0100 Subject: [PATCH 13/78] Revert "HDDS-2860. Cluster disk space metrics should reflect decommission and maintenance states" This reverts commit 5b228c039bd9715058e62bb450cb188566de1759. --- .../hdds/scm/node/NodeManagerMXBean.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 113 ++++++------------ .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 91 ++++++-------- .../hdds/scm/container/MockNodeManager.java | 20 +--- .../scm/container/SimpleMockNodeManager.java | 2 +- .../hdds/scm/node/TestSCMNodeManager.java | 72 ----------- .../testutils/ReplicationNodeManagerMock.java | 2 +- .../ozone/scm/TestSCMNodeManagerMXBean.java | 30 +---- .../ozone/scm/node/TestSCMNodeMetrics.java | 54 +-------- 9 files changed, 89 insertions(+), 297 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java index 791e754e2c4a..e1b51efc34cd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java @@ -34,7 +34,7 @@ public interface NodeManagerMXBean { * * @return A state to number of nodes that in this state mapping */ - Map> getNodeCount(); + Map getNodeCount(); /** * Get the disk metrics like capacity, usage and remaining based on the diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index d89dac1cae35..3c5071f1cde5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -482,97 +482,62 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) { } } - @Override // NodeManagerMXBean - public Map> getNodeCount() { - Map> nodes = new HashMap<>(); - for (NodeOperationalState opState : NodeOperationalState.values()) { - Map states = new HashMap<>(); - for (NodeState health : NodeState.values()) { - states.put(health.name(), 0); - } - nodes.put(opState.name(), states); - } - for (DatanodeInfo dni : nodeStateManager.getAllNodes()) { - NodeStatus status = dni.getNodeStatus(); - nodes.get(status.getOperationalState().name()) - .compute(status.getHealth().name(), (k, v) -> v+1); + @Override + public Map getNodeCount() { + // TODO - This does not consider decom, maint etc. + Map nodeCountMap = new HashMap(); + for(NodeState state : NodeState.values()) { + // TODO - this iterate the node list once per state and needs + // fixed to only perform one pass. + nodeCountMap.put(state.toString(), getNodeCount(null, state)); } - return nodes; + return nodeCountMap; } // We should introduce DISK, SSD, etc., notion in // SCMNodeStat and try to use it. - @Override // NodeManagerMXBean + @Override public Map getNodeInfo() { - Map nodeInfo = new HashMap<>(); - // Compute all the possible stats from the enums, and default to zero: - for (UsageStates s : UsageStates.values()) { - for (UsageMetrics stat : UsageMetrics.values()) { - nodeInfo.put(s.label + stat.name(), 0L); - } - } + long diskCapacity = 0L; + long diskUsed = 0L; + long diskRemaning = 0L; - for (DatanodeInfo node : nodeStateManager.getAllNodes()) { - String keyPrefix = ""; - NodeStatus status = node.getNodeStatus(); - if (status.isMaintenance()) { - keyPrefix = UsageStates.MAINT.getLabel(); - } else if (status.isDecommission()) { - keyPrefix = UsageStates.DECOM.getLabel(); - } else if (status.isAlive()) { - // Inservice but not dead - keyPrefix = UsageStates.ONLINE.getLabel(); - } else { - // dead inservice node, skip it - continue; - } - List storageReportProtos = node.getStorageReports(); + long ssdCapacity = 0L; + long ssdUsed = 0L; + long ssdRemaining = 0L; + + List healthyNodes = nodeStateManager.getHealthyNodes(); + List staleNodes = nodeStateManager.getStaleNodes(); + + List datanodes = new ArrayList<>(healthyNodes); + datanodes.addAll(staleNodes); + + for (DatanodeInfo dnInfo : datanodes) { + List storageReportProtos = dnInfo.getStorageReports(); for (StorageReportProto reportProto : storageReportProtos) { if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.DISK) { - nodeInfo.compute(keyPrefix + UsageMetrics.DiskCapacity.name(), - (k, v) -> v + reportProto.getCapacity()); - nodeInfo.compute(keyPrefix + UsageMetrics.DiskRemaining.name(), - (k, v) -> v + reportProto.getRemaining()); - nodeInfo.compute(keyPrefix + UsageMetrics.DiskUsed.name(), - (k, v) -> v + reportProto.getScmUsed()); + diskCapacity += reportProto.getCapacity(); + diskRemaning += reportProto.getRemaining(); + diskUsed += reportProto.getScmUsed(); } else if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.SSD) { - nodeInfo.compute(keyPrefix + UsageMetrics.SSDCapacity.name(), - (k, v) -> v + reportProto.getCapacity()); - nodeInfo.compute(keyPrefix + UsageMetrics.SSDRemaining.name(), - (k, v) -> v + reportProto.getRemaining()); - nodeInfo.compute(keyPrefix + UsageMetrics.SSDUsed.name(), - (k, v) -> v + reportProto.getScmUsed()); + ssdCapacity += reportProto.getCapacity(); + ssdRemaining += reportProto.getRemaining(); + ssdUsed += reportProto.getScmUsed(); } } } - return nodeInfo; - } - - private enum UsageMetrics { - DiskCapacity, - DiskUsed, - DiskRemaining, - SSDCapacity, - SSDUsed, - SSDRemaining - } - private enum UsageStates { - ONLINE(""), - MAINT("Maintenance"), - DECOM("Decommissioned"); - - private final String label; - - public String getLabel() { - return label; - } + Map nodeInfo = new HashMap<>(); + nodeInfo.put("DISKCapacity", diskCapacity); + nodeInfo.put("DISKUsed", diskUsed); + nodeInfo.put("DISKRemaining", diskRemaning); - UsageStates(String label) { - this.label = label; - } + nodeInfo.put("SSDCapacity", ssdCapacity); + nodeInfo.put("SSDUsed", ssdUsed); + nodeInfo.put("SSDRemaining", ssdRemaining); + return nodeInfo; } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index ccb5d144ae4f..5b872c10155b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -18,12 +18,15 @@ package org.apache.hadoop.hdds.scm.node; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; + import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; @@ -33,7 +36,6 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.util.StringUtils; /** * This class maintains Node related metrics. @@ -49,7 +51,6 @@ public final class SCMNodeMetrics implements MetricsSource { private @Metric MutableCounterLong numHBProcessingFailed; private @Metric MutableCounterLong numNodeReportProcessed; private @Metric MutableCounterLong numNodeReportProcessingFailed; - private @Metric String textMetric; private final MetricsRegistry registry; private final NodeManagerMXBean managerMXBean; @@ -60,7 +61,6 @@ public final class SCMNodeMetrics implements MetricsSource { private SCMNodeMetrics(NodeManagerMXBean managerMXBean) { this.managerMXBean = managerMXBean; this.registry = new MetricsRegistry(recordInfo); - this.textMetric = "my_test_metric"; } /** @@ -116,58 +116,39 @@ void incNumNodeReportProcessingFailed() { @Override @SuppressWarnings("SuspiciousMethodCalls") public void getMetrics(MetricsCollector collector, boolean all) { - Map> nodeCount = managerMXBean.getNodeCount(); + Map nodeCount = managerMXBean.getNodeCount(); Map nodeInfo = managerMXBean.getNodeInfo(); - /** - * Loop over the Node map and create a metric for the cross product of all - * Operational and health states, ie: - * InServiceHealthy - * InServiceStale - * ... - * EnteringMaintenanceHealthy - * ... - */ - MetricsRecordBuilder metrics = collector.addRecord(registry.info()); - for(Map.Entry> e : nodeCount.entrySet()) { - for(Map.Entry h : e.getValue().entrySet()) { - metrics.addGauge( - Interns.info( - StringUtils.camelize(e.getKey()+"_"+h.getKey()+"_nodes"), - "Number of "+e.getKey()+" "+h.getKey()+" datanodes"), - h.getValue()); - } - } - - for (Map.Entry e : nodeInfo.entrySet()) { - metrics.addGauge( - Interns.info(e.getKey(), diskMetricDescription(e.getKey())), - e.getValue()); - } - registry.snapshot(metrics, all); - } - - private String diskMetricDescription(String metric) { - StringBuilder sb = new StringBuilder(); - sb.append("Total"); - if (metric.indexOf("Maintenance") >= 0) { - sb.append(" maintenance"); - } else if (metric.indexOf("Decommissioned") >= 0) { - sb.append(" decommissioned"); - } - if (metric.indexOf("DiskCapacity") >= 0) { - sb.append(" disk capacity"); - } else if (metric.indexOf("DiskUsed") >= 0) { - sb.append(" disk capacity used"); - } else if (metric.indexOf("DiskRemaining") >= 0) { - sb.append(" disk capacity remaining"); - } else if (metric.indexOf("SSDCapacity") >= 0) { - sb.append(" SSD capacity"); - } else if (metric.indexOf("SSDUsed") >= 0) { - sb.append(" SSD capacity used"); - } else if (metric.indexOf("SSDRemaining") >= 0) { - sb.append(" SSD capacity remaining"); - } - return sb.toString(); + registry.snapshot( + collector.addRecord(registry.info()) // Add annotated ones first + .addGauge(Interns.info( + "HealthyNodes", + "Number of healthy datanodes"), + nodeCount.get(HEALTHY.toString())) + .addGauge(Interns.info("StaleNodes", + "Number of stale datanodes"), + nodeCount.get(STALE.toString())) + .addGauge(Interns.info("DeadNodes", + "Number of dead datanodes"), + nodeCount.get(DEAD.toString())) + .addGauge(Interns.info("DiskCapacity", + "Total disk capacity"), + nodeInfo.get("DISKCapacity")) + .addGauge(Interns.info("DiskUsed", + "Total disk capacity used"), + nodeInfo.get("DISKUsed")) + .addGauge(Interns.info("DiskRemaining", + "Total disk capacity remaining"), + nodeInfo.get("DISKRemaining")) + .addGauge(Interns.info("SSDCapacity", + "Total ssd capacity"), + nodeInfo.get("SSDCapacity")) + .addGauge(Interns.info("SSDUsed", + "Total ssd capacity used"), + nodeInfo.get("SSDUsed")) + .addGauge(Interns.info("SSDRemaining", + "Total disk capacity remaining"), + nodeInfo.get("SSDRemaining")), + all); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index d24cbfd5114d..8a48a68fcd08 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto @@ -472,23 +471,12 @@ public Boolean isNodeRegistered( } @Override - public Map> getNodeCount() { - Map> nodes = new HashMap<>(); - for (NodeOperationalState opState : NodeOperationalState.values()) { - Map states = new HashMap<>(); - for (HddsProtos.NodeState health : HddsProtos.NodeState.values()) { - states.put(health.name(), 0); - } - nodes.put(opState.name(), states); - } - // At the moment MockNodeManager is not aware of decommission and - // maintenance states, therefore loop over all nodes and assume all nodes - // are IN_SERVICE. This will be fixed as part of HDDS-2673 + public Map getNodeCount() { + Map nodeCountMap = new HashMap(); for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) { - nodes.get(NodeOperationalState.IN_SERVICE.name()) - .compute(state.name(), (k, v) -> v + 1); + nodeCountMap.put(state.toString(), getNodeCount(null, state)); } - return nodes; + return nodeCountMap; } @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index dad3448bdf38..883e910a4b29 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -243,7 +243,7 @@ public void close() throws IOException { } @Override - public Map> getNodeCount() { + public Map getNodeCount() { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 0917fa4019b6..1af996cb3265 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -19,12 +19,10 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.net.NetworkTopology; -import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -55,7 +53,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; @@ -243,11 +240,6 @@ public void testScmHealthyNodeCount() Thread.sleep(4 * 1000); assertEquals(count, nodeManager.getNodeCount( NodeStatus.inServiceHealthy())); - - Map> nodeCounts = nodeManager.getNodeCount(); - assertEquals(count, - nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) - .get(HddsProtos.NodeState.HEALTHY.name()).intValue()); } } @@ -331,11 +323,6 @@ public void testScmDetectStaleAndDeadNode() .getUuid(), staleNodeList.get(0).getUuid()); Thread.sleep(1000); - Map> nodeCounts = nodeManager.getNodeCount(); - assertEquals(1, - nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) - .get(HddsProtos.NodeState.STALE.name()).intValue()); - // heartbeat good nodes again. for (DatanodeDetails dn : nodeList) { nodeManager.processHeartbeat(dn); @@ -347,14 +334,10 @@ public void testScmDetectStaleAndDeadNode() // the stale node has been removed staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale()); - nodeCounts = nodeManager.getNodeCount(); assertEquals("Expected to find 1 stale node", 0, nodeManager.getNodeCount(NodeStatus.inServiceStale())); assertEquals("Expected to find 1 stale node", 0, staleNodeList.size()); - assertEquals(0, - nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) - .get(HddsProtos.NodeState.STALE.name()).intValue()); // Check for the dead node now. List deadNodeList = @@ -363,9 +346,6 @@ public void testScmDetectStaleAndDeadNode() nodeManager.getNodeCount(NodeStatus.inServiceDead())); assertEquals("Expected to find 1 dead node", 1, deadNodeList.size()); - assertEquals(1, - nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) - .get(HddsProtos.NodeState.DEAD.name()).intValue()); assertEquals("Dead node is not the expected ID", staleNode .getUuid(), deadNodeList.get(0).getUuid()); } @@ -1215,58 +1195,6 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) } } - @Test - public void testGetNodeInfo() - throws IOException, InterruptedException, NodeNotFoundException, - AuthenticationException { - OzoneConfiguration conf = getConf(); - final int nodeCount = 6; - SCMNodeManager nodeManager = createNodeManager(conf); - - for (int i=0; i stats = nodeManager.getNodeInfo(); - // 3 IN_SERVICE nodes: - assertEquals(6000, stats.get("DiskCapacity").longValue()); - assertEquals(300, stats.get("DiskUsed").longValue()); - assertEquals(5700, stats.get("DiskRemaining").longValue()); - - // 2 Decommissioned nodes - assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue()); - assertEquals(200, stats.get("DecommissionedDiskUsed").longValue()); - assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue()); - - // 1 Maintenance node - assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue()); - assertEquals(100, stats.get("MaintenanceDiskUsed").longValue()); - assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue()); - } - /** * Test add node into a 4-layer network topology during node register. */ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index ee96565a4097..5a2e3c2109ac 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -72,7 +72,7 @@ public ReplicationNodeManagerMock(Map nodeStatus, * @return A state to number of nodes that in this state mapping */ @Override - public Map> getNodeCount() { + public Map getNodeCount() { return null; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java index 0f961095b9ad..43b9bf03a6ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java @@ -92,32 +92,10 @@ public void testNodeCount() throws Exception { + "name=SCMNodeManagerInfo"); TabularData data = (TabularData) mbs.getAttribute(bean, "NodeCount"); - Map> mbeanMap = convertNodeCountToMap(data); - Map> nodeMap = - scm.getScmNodeManager().getNodeCount(); - assertTrue(nodeMap.equals(mbeanMap)); - } - - private Map> convertNodeCountToMap( - TabularData data) { - Map> map = new HashMap<>(); - for (Object o : data.values()) { - CompositeData cds = (CompositeData) o; - Iterator it = cds.values().iterator(); - String opState = it.next().toString(); - TabularData states = (TabularData) it.next(); - - Map healthStates = new HashMap<>(); - for (Object obj : states.values()) { - CompositeData stateData = (CompositeData) obj; - Iterator stateIt = stateData.values().iterator(); - String health = stateIt.next().toString(); - Integer value = Integer.parseInt(stateIt.next().toString()); - healthStates.put(health, value); - } - map.put(opState, healthStates); - } - return map; + Map nodeCount = scm.getScmNodeManager().getNodeCount(); + Map nodeCountLong = new HashMap<>(); + nodeCount.forEach((k, v) -> nodeCountLong.put(k, new Long(v))); + verifyEquals(data, nodeCountLong); } private void verifyEquals(TabularData actualData, Map diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index 6192ff5bf315..d528f437cee1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -146,35 +146,11 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { cluster.getStorageContainerManager().getScmNodeManager() .processNodeReport(datanode.getDatanodeDetails(), nodeReport); - assertGauge("InServiceHealthyNodes", 1, + assertGauge("HealthyNodes", 1, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("InServiceStaleNodes", 0, + assertGauge("StaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("InServiceDeadNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissioningHealthyNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissioningStaleNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissioningDeadNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedHealthyNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedStaleNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedDeadNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("EnteringMaintenanceHealthyNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("EnteringMaintenanceStaleNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("EnteringMaintenanceDeadNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("InMaintenanceHealthyNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("InMaintenanceStaleNodes", 0, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("InMaintenanceDeadNodes", 0, + assertGauge("DeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DiskCapacity", 100L, getMetrics(SCMNodeMetrics.class.getSimpleName())); @@ -188,30 +164,6 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("SSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceDiskCapacity", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceDiskUsed", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceDiskRemaining", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceSSDCapacity", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceSSDUsed", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("MaintenanceSSDRemaining", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedDiskCapacity", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedDiskUsed", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedDiskRemaining", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedSSDCapacity", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedSSDUsed", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DecommissionedSSDRemaining", 0L, - getMetrics(SCMNodeMetrics.class.getSimpleName())); } @After From 265fba41eec4aade2201f0a8561870fee5b01ed5 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 10 Feb 2020 11:40:14 +0100 Subject: [PATCH 14/78] HDDS-2113. Update JMX metrics for node count in SCMNodeMetrics for Decommission and Maintenance --- .../hdds/scm/node/NodeManagerMXBean.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 24 +++--- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 79 +++++++++++-------- .../hdds/scm/container/MockNodeManager.java | 20 ++++- .../scm/container/SimpleMockNodeManager.java | 2 +- .../hdds/scm/node/TestSCMNodeManager.java | 18 +++++ .../testutils/ReplicationNodeManagerMock.java | 2 +- .../ozone/scm/TestSCMNodeManagerMXBean.java | 30 ++++++- .../ozone/scm/node/TestSCMNodeMetrics.java | 30 ++++++- 9 files changed, 149 insertions(+), 58 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java index e1b51efc34cd..791e754e2c4a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java @@ -34,7 +34,7 @@ public interface NodeManagerMXBean { * * @return A state to number of nodes that in this state mapping */ - Map getNodeCount(); + Map> getNodeCount(); /** * Get the disk metrics like capacity, usage and remaining based on the diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 3c5071f1cde5..5f4e44594fdc 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -482,16 +482,22 @@ private SCMNodeStat getNodeStatInternal(DatanodeDetails datanodeDetails) { } } - @Override - public Map getNodeCount() { - // TODO - This does not consider decom, maint etc. - Map nodeCountMap = new HashMap(); - for(NodeState state : NodeState.values()) { - // TODO - this iterate the node list once per state and needs - // fixed to only perform one pass. - nodeCountMap.put(state.toString(), getNodeCount(null, state)); + @Override // NodeManagerMXBean + public Map> getNodeCount() { + Map> nodes = new HashMap<>(); + for (NodeOperationalState opState : NodeOperationalState.values()) { + Map states = new HashMap<>(); + for (NodeState health : NodeState.values()) { + states.put(health.name(), 0); + } + nodes.put(opState.name(), states); + } + for (DatanodeInfo dni : nodeStateManager.getAllNodes()) { + NodeStatus status = dni.getNodeStatus(); + nodes.get(status.getOperationalState().name()) + .compute(status.getHealth().name(), (k, v) -> v+1); } - return nodeCountMap; + return nodes; } // We should introduce DISK, SSD, etc., notion in diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index 5b872c10155b..517669e4199e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -18,15 +18,12 @@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; - import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; @@ -36,6 +33,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.util.StringUtils; /** * This class maintains Node related metrics. @@ -51,6 +49,7 @@ public final class SCMNodeMetrics implements MetricsSource { private @Metric MutableCounterLong numHBProcessingFailed; private @Metric MutableCounterLong numNodeReportProcessed; private @Metric MutableCounterLong numNodeReportProcessingFailed; + private @Metric String textMetric; private final MetricsRegistry registry; private final NodeManagerMXBean managerMXBean; @@ -61,6 +60,7 @@ public final class SCMNodeMetrics implements MetricsSource { private SCMNodeMetrics(NodeManagerMXBean managerMXBean) { this.managerMXBean = managerMXBean; this.registry = new MetricsRegistry(recordInfo); + this.textMetric = "my_test_metric"; } /** @@ -116,39 +116,48 @@ void incNumNodeReportProcessingFailed() { @Override @SuppressWarnings("SuspiciousMethodCalls") public void getMetrics(MetricsCollector collector, boolean all) { - Map nodeCount = managerMXBean.getNodeCount(); + Map> nodeCount = managerMXBean.getNodeCount(); Map nodeInfo = managerMXBean.getNodeInfo(); - registry.snapshot( - collector.addRecord(registry.info()) // Add annotated ones first - .addGauge(Interns.info( - "HealthyNodes", - "Number of healthy datanodes"), - nodeCount.get(HEALTHY.toString())) - .addGauge(Interns.info("StaleNodes", - "Number of stale datanodes"), - nodeCount.get(STALE.toString())) - .addGauge(Interns.info("DeadNodes", - "Number of dead datanodes"), - nodeCount.get(DEAD.toString())) - .addGauge(Interns.info("DiskCapacity", - "Total disk capacity"), - nodeInfo.get("DISKCapacity")) - .addGauge(Interns.info("DiskUsed", - "Total disk capacity used"), - nodeInfo.get("DISKUsed")) - .addGauge(Interns.info("DiskRemaining", - "Total disk capacity remaining"), - nodeInfo.get("DISKRemaining")) - .addGauge(Interns.info("SSDCapacity", - "Total ssd capacity"), - nodeInfo.get("SSDCapacity")) - .addGauge(Interns.info("SSDUsed", - "Total ssd capacity used"), - nodeInfo.get("SSDUsed")) - .addGauge(Interns.info("SSDRemaining", - "Total disk capacity remaining"), - nodeInfo.get("SSDRemaining")), + /** + * Loop over the Node map and create a metric for the cross product of all + * Operational and health states, ie: + * InServiceHealthy + * InServiceStale + * ... + * EnteringMaintenanceHealthy + * ... + */ + MetricsRecordBuilder metrics = collector.addRecord(registry.info()); + for(Map.Entry> e : nodeCount.entrySet()) { + for(Map.Entry h : e.getValue().entrySet()) { + metrics.addGauge( + Interns.info( + StringUtils.camelize(e.getKey()+"_"+h.getKey()+"_nodes"), + "Number of "+e.getKey()+" "+h.getKey()+" datanodes"), + h.getValue()); + } + } + + registry.snapshot(metrics + .addGauge(Interns.info("DiskCapacity", + "Total disk capacity"), + nodeInfo.get("DISKCapacity")) + .addGauge(Interns.info("DiskUsed", + "Total disk capacity used"), + nodeInfo.get("DISKUsed")) + .addGauge(Interns.info("DiskRemaining", + "Total disk capacity remaining"), + nodeInfo.get("DISKRemaining")) + .addGauge(Interns.info("SSDCapacity", + "Total ssd capacity"), + nodeInfo.get("SSDCapacity")) + .addGauge(Interns.info("SSDUsed", + "Total ssd capacity used"), + nodeInfo.get("SSDUsed")) + .addGauge(Interns.info("SSDRemaining", + "Total disk capacity remaining"), + nodeInfo.get("SSDRemaining")), all); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index 8a48a68fcd08..d24cbfd5114d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto @@ -471,12 +472,23 @@ public Boolean isNodeRegistered( } @Override - public Map getNodeCount() { - Map nodeCountMap = new HashMap(); + public Map> getNodeCount() { + Map> nodes = new HashMap<>(); + for (NodeOperationalState opState : NodeOperationalState.values()) { + Map states = new HashMap<>(); + for (HddsProtos.NodeState health : HddsProtos.NodeState.values()) { + states.put(health.name(), 0); + } + nodes.put(opState.name(), states); + } + // At the moment MockNodeManager is not aware of decommission and + // maintenance states, therefore loop over all nodes and assume all nodes + // are IN_SERVICE. This will be fixed as part of HDDS-2673 for (HddsProtos.NodeState state : HddsProtos.NodeState.values()) { - nodeCountMap.put(state.toString(), getNodeCount(null, state)); + nodes.get(NodeOperationalState.IN_SERVICE.name()) + .compute(state.name(), (k, v) -> v + 1); } - return nodeCountMap; + return nodes; } @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index 883e910a4b29..dad3448bdf38 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -243,7 +243,7 @@ public void close() throws IOException { } @Override - public Map getNodeCount() { + public Map> getNodeCount() { return null; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 1af996cb3265..11f346ffc916 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; @@ -240,6 +241,11 @@ public void testScmHealthyNodeCount() Thread.sleep(4 * 1000); assertEquals(count, nodeManager.getNodeCount( NodeStatus.inServiceHealthy())); + + Map> nodeCounts = nodeManager.getNodeCount(); + assertEquals(count, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.HEALTHY.name()).intValue()); } } @@ -323,6 +329,11 @@ public void testScmDetectStaleAndDeadNode() .getUuid(), staleNodeList.get(0).getUuid()); Thread.sleep(1000); + Map> nodeCounts = nodeManager.getNodeCount(); + assertEquals(1, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.STALE.name()).intValue()); + // heartbeat good nodes again. for (DatanodeDetails dn : nodeList) { nodeManager.processHeartbeat(dn); @@ -334,10 +345,14 @@ public void testScmDetectStaleAndDeadNode() // the stale node has been removed staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale()); + nodeCounts = nodeManager.getNodeCount(); assertEquals("Expected to find 1 stale node", 0, nodeManager.getNodeCount(NodeStatus.inServiceStale())); assertEquals("Expected to find 1 stale node", 0, staleNodeList.size()); + assertEquals(0, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.STALE.name()).intValue()); // Check for the dead node now. List deadNodeList = @@ -346,6 +361,9 @@ public void testScmDetectStaleAndDeadNode() nodeManager.getNodeCount(NodeStatus.inServiceDead())); assertEquals("Expected to find 1 dead node", 1, deadNodeList.size()); + assertEquals(1, + nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name()) + .get(HddsProtos.NodeState.DEAD.name()).intValue()); assertEquals("Dead node is not the expected ID", staleNode .getUuid(), deadNodeList.get(0).getUuid()); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index 5a2e3c2109ac..ee96565a4097 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -72,7 +72,7 @@ public ReplicationNodeManagerMock(Map nodeStatus, * @return A state to number of nodes that in this state mapping */ @Override - public Map getNodeCount() { + public Map> getNodeCount() { return null; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java index 43b9bf03a6ef..0f961095b9ad 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMNodeManagerMXBean.java @@ -92,10 +92,32 @@ public void testNodeCount() throws Exception { + "name=SCMNodeManagerInfo"); TabularData data = (TabularData) mbs.getAttribute(bean, "NodeCount"); - Map nodeCount = scm.getScmNodeManager().getNodeCount(); - Map nodeCountLong = new HashMap<>(); - nodeCount.forEach((k, v) -> nodeCountLong.put(k, new Long(v))); - verifyEquals(data, nodeCountLong); + Map> mbeanMap = convertNodeCountToMap(data); + Map> nodeMap = + scm.getScmNodeManager().getNodeCount(); + assertTrue(nodeMap.equals(mbeanMap)); + } + + private Map> convertNodeCountToMap( + TabularData data) { + Map> map = new HashMap<>(); + for (Object o : data.values()) { + CompositeData cds = (CompositeData) o; + Iterator it = cds.values().iterator(); + String opState = it.next().toString(); + TabularData states = (TabularData) it.next(); + + Map healthStates = new HashMap<>(); + for (Object obj : states.values()) { + CompositeData stateData = (CompositeData) obj; + Iterator stateIt = stateData.values().iterator(); + String health = stateIt.next().toString(); + Integer value = Integer.parseInt(stateIt.next().toString()); + healthStates.put(health, value); + } + map.put(opState, healthStates); + } + return map; } private void verifyEquals(TabularData actualData, Map diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index d528f437cee1..4d117c9a58ed 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -146,11 +146,35 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { cluster.getStorageContainerManager().getScmNodeManager() .processNodeReport(datanode.getDatanodeDetails(), nodeReport); - assertGauge("HealthyNodes", 1, + assertGauge("InServiceHealthyNodes", 1, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("StaleNodes", 0, + assertGauge("InServiceStaleNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); - assertGauge("DeadNodes", 0, + assertGauge("InServiceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceDeadNodes", 0, getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("DiskCapacity", 100L, getMetrics(SCMNodeMetrics.class.getSimpleName())); From 9506aa9554c4993870d20d10708d17336809e702 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 9 Jan 2020 17:21:45 +0000 Subject: [PATCH 15/78] HDDS-2860. Cluster disk space metrics should reflect decommission and maintenance states #433 --- .../hadoop/hdds/scm/node/SCMNodeManager.java | 89 ++++++++++++------- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 50 ++++++----- .../hdds/scm/node/TestSCMNodeManager.java | 54 +++++++++++ .../ozone/scm/node/TestSCMNodeMetrics.java | 24 +++++ 4 files changed, 167 insertions(+), 50 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 5f4e44594fdc..d89dac1cae35 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -502,48 +502,77 @@ public Map> getNodeCount() { // We should introduce DISK, SSD, etc., notion in // SCMNodeStat and try to use it. - @Override + @Override // NodeManagerMXBean public Map getNodeInfo() { - long diskCapacity = 0L; - long diskUsed = 0L; - long diskRemaning = 0L; - - long ssdCapacity = 0L; - long ssdUsed = 0L; - long ssdRemaining = 0L; - - List healthyNodes = nodeStateManager.getHealthyNodes(); - List staleNodes = nodeStateManager.getStaleNodes(); - - List datanodes = new ArrayList<>(healthyNodes); - datanodes.addAll(staleNodes); + Map nodeInfo = new HashMap<>(); + // Compute all the possible stats from the enums, and default to zero: + for (UsageStates s : UsageStates.values()) { + for (UsageMetrics stat : UsageMetrics.values()) { + nodeInfo.put(s.label + stat.name(), 0L); + } + } - for (DatanodeInfo dnInfo : datanodes) { - List storageReportProtos = dnInfo.getStorageReports(); + for (DatanodeInfo node : nodeStateManager.getAllNodes()) { + String keyPrefix = ""; + NodeStatus status = node.getNodeStatus(); + if (status.isMaintenance()) { + keyPrefix = UsageStates.MAINT.getLabel(); + } else if (status.isDecommission()) { + keyPrefix = UsageStates.DECOM.getLabel(); + } else if (status.isAlive()) { + // Inservice but not dead + keyPrefix = UsageStates.ONLINE.getLabel(); + } else { + // dead inservice node, skip it + continue; + } + List storageReportProtos = node.getStorageReports(); for (StorageReportProto reportProto : storageReportProtos) { if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.DISK) { - diskCapacity += reportProto.getCapacity(); - diskRemaning += reportProto.getRemaining(); - diskUsed += reportProto.getScmUsed(); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskCapacity.name(), + (k, v) -> v + reportProto.getCapacity()); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskRemaining.name(), + (k, v) -> v + reportProto.getRemaining()); + nodeInfo.compute(keyPrefix + UsageMetrics.DiskUsed.name(), + (k, v) -> v + reportProto.getScmUsed()); } else if (reportProto.getStorageType() == StorageContainerDatanodeProtocolProtos.StorageTypeProto.SSD) { - ssdCapacity += reportProto.getCapacity(); - ssdRemaining += reportProto.getRemaining(); - ssdUsed += reportProto.getScmUsed(); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDCapacity.name(), + (k, v) -> v + reportProto.getCapacity()); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDRemaining.name(), + (k, v) -> v + reportProto.getRemaining()); + nodeInfo.compute(keyPrefix + UsageMetrics.SSDUsed.name(), + (k, v) -> v + reportProto.getScmUsed()); } } } + return nodeInfo; + } - Map nodeInfo = new HashMap<>(); - nodeInfo.put("DISKCapacity", diskCapacity); - nodeInfo.put("DISKUsed", diskUsed); - nodeInfo.put("DISKRemaining", diskRemaning); + private enum UsageMetrics { + DiskCapacity, + DiskUsed, + DiskRemaining, + SSDCapacity, + SSDUsed, + SSDRemaining + } - nodeInfo.put("SSDCapacity", ssdCapacity); - nodeInfo.put("SSDUsed", ssdUsed); - nodeInfo.put("SSDRemaining", ssdRemaining); - return nodeInfo; + private enum UsageStates { + ONLINE(""), + MAINT("Maintenance"), + DECOM("Decommissioned"); + + private final String label; + + public String getLabel() { + return label; + } + + UsageStates(String label) { + this.label = label; + } } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index 517669e4199e..ccb5d144ae4f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -139,25 +139,35 @@ public void getMetrics(MetricsCollector collector, boolean all) { } } - registry.snapshot(metrics - .addGauge(Interns.info("DiskCapacity", - "Total disk capacity"), - nodeInfo.get("DISKCapacity")) - .addGauge(Interns.info("DiskUsed", - "Total disk capacity used"), - nodeInfo.get("DISKUsed")) - .addGauge(Interns.info("DiskRemaining", - "Total disk capacity remaining"), - nodeInfo.get("DISKRemaining")) - .addGauge(Interns.info("SSDCapacity", - "Total ssd capacity"), - nodeInfo.get("SSDCapacity")) - .addGauge(Interns.info("SSDUsed", - "Total ssd capacity used"), - nodeInfo.get("SSDUsed")) - .addGauge(Interns.info("SSDRemaining", - "Total disk capacity remaining"), - nodeInfo.get("SSDRemaining")), - all); + for (Map.Entry e : nodeInfo.entrySet()) { + metrics.addGauge( + Interns.info(e.getKey(), diskMetricDescription(e.getKey())), + e.getValue()); + } + registry.snapshot(metrics, all); + } + + private String diskMetricDescription(String metric) { + StringBuilder sb = new StringBuilder(); + sb.append("Total"); + if (metric.indexOf("Maintenance") >= 0) { + sb.append(" maintenance"); + } else if (metric.indexOf("Decommissioned") >= 0) { + sb.append(" decommissioned"); + } + if (metric.indexOf("DiskCapacity") >= 0) { + sb.append(" disk capacity"); + } else if (metric.indexOf("DiskUsed") >= 0) { + sb.append(" disk capacity used"); + } else if (metric.indexOf("DiskRemaining") >= 0) { + sb.append(" disk capacity remaining"); + } else if (metric.indexOf("SSDCapacity") >= 0) { + sb.append(" SSD capacity"); + } else if (metric.indexOf("SSDUsed") >= 0) { + sb.append(" SSD capacity used"); + } else if (metric.indexOf("SSDRemaining") >= 0) { + sb.append(" SSD capacity remaining"); + } + return sb.toString(); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 11f346ffc916..0917fa4019b6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -19,10 +19,12 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.net.NetworkTopology; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -1213,6 +1215,58 @@ private void testScmRegisterNodeWithNetworkTopology(boolean useHostname) } } + @Test + public void testGetNodeInfo() + throws IOException, InterruptedException, NodeNotFoundException, + AuthenticationException { + OzoneConfiguration conf = getConf(); + final int nodeCount = 6; + SCMNodeManager nodeManager = createNodeManager(conf); + + for (int i=0; i stats = nodeManager.getNodeInfo(); + // 3 IN_SERVICE nodes: + assertEquals(6000, stats.get("DiskCapacity").longValue()); + assertEquals(300, stats.get("DiskUsed").longValue()); + assertEquals(5700, stats.get("DiskRemaining").longValue()); + + // 2 Decommissioned nodes + assertEquals(4000, stats.get("DecommissionedDiskCapacity").longValue()); + assertEquals(200, stats.get("DecommissionedDiskUsed").longValue()); + assertEquals(3800, stats.get("DecommissionedDiskRemaining").longValue()); + + // 1 Maintenance node + assertEquals(2000, stats.get("MaintenanceDiskCapacity").longValue()); + assertEquals(100, stats.get("MaintenanceDiskUsed").longValue()); + assertEquals(1900, stats.get("MaintenanceDiskRemaining").longValue()); + } + /** * Test add node into a 4-layer network topology during node register. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index 4d117c9a58ed..6192ff5bf315 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -188,6 +188,30 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { getMetrics(SCMNodeMetrics.class.getSimpleName())); assertGauge("SSDRemaining", 0L, getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); } @After From d902586b7ca39083ba66ea19778b537212bc42d3 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Tue, 31 Mar 2020 22:58:40 +0100 Subject: [PATCH 16/78] HDDS-2592. Add Datanode command to allow the datanode to persist its admin state (#521) --- .../hadoop/hdds/protocol/DatanodeDetails.java | 98 ++++++++++- hadoop-hdds/common/src/main/proto/hdds.proto | 2 + .../common/helpers/DatanodeIdYaml.java | 35 ++++ .../statemachine/DatanodeStateMachine.java | 2 + ...SetNodeOperationalStateCommandHandler.java | 157 ++++++++++++++++++ .../endpoint/HeartbeatEndpointTask.java | 12 ++ .../SetNodeOperationalStateCommand.java | 89 ++++++++++ .../StorageContainerDatanodeProtocol.proto | 8 + .../scm/node/DatanodeAdminMonitorImpl.java | 3 +- .../scm/node/DatanodeAdminNodeDetails.java | 8 + .../scm/node/NodeDecommissionManager.java | 8 +- .../hadoop/hdds/scm/node/NodeManager.java | 12 ++ .../hdds/scm/node/NodeStateManager.java | 26 ++- .../hadoop/hdds/scm/node/NodeStatus.java | 32 +++- .../hadoop/hdds/scm/node/SCMNodeManager.java | 81 ++++++++- .../hdds/scm/node/states/NodeStateMap.java | 5 +- .../scm/server/SCMDatanodeProtocolServer.java | 8 + .../hdds/scm/container/MockNodeManager.java | 10 ++ .../scm/container/SimpleMockNodeManager.java | 10 +- .../scm/node/TestNodeDecommissionManager.java | 5 + .../hdds/scm/node/TestSCMNodeManager.java | 30 ++++ .../scm/node/states/TestNodeStateMap.java | 5 +- .../testutils/ReplicationNodeManagerMock.java | 15 +- 23 files changed, 643 insertions(+), 18 deletions(-) create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index 698a443fc6b4..a2584d8044e0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -49,6 +49,8 @@ public class DatanodeDetails extends NodeImpl implements private String hostName; private List ports; private String certSerialId; + private HddsProtos.NodeOperationalState persistedOpState; + private long persistedOpStateExpiryEpochSec = 0; /** * Constructs DatanodeDetails instance. DatanodeDetails.Builder is used @@ -59,15 +61,23 @@ public class DatanodeDetails extends NodeImpl implements * @param networkLocation DataNode's network location path * @param ports Ports used by the DataNode * @param certSerialId serial id from SCM issued certificate. + * @param persistedOpState Operational State stored on DN. + * @param persistedOpStateExpiryEpochSec Seconds after the epoch the stored + * state should expire. */ + @SuppressWarnings("checkstyle:ParameterNumber") private DatanodeDetails(String uuid, String ipAddress, String hostName, - String networkLocation, List ports, String certSerialId) { + String networkLocation, List ports, String certSerialId, + HddsProtos.NodeOperationalState persistedOpState, + long persistedOpStateExpiryEpochSec) { super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT); this.uuid = UUID.fromString(uuid); this.ipAddress = ipAddress; this.hostName = hostName; this.ports = ports; this.certSerialId = certSerialId; + this.persistedOpState = persistedOpState; + this.persistedOpStateExpiryEpochSec = persistedOpStateExpiryEpochSec; } protected DatanodeDetails(DatanodeDetails datanodeDetails) { @@ -78,6 +88,9 @@ protected DatanodeDetails(DatanodeDetails datanodeDetails) { this.hostName = datanodeDetails.hostName; this.ports = datanodeDetails.ports; this.setNetworkName(datanodeDetails.getNetworkName()); + this.persistedOpState = datanodeDetails.getPersistedOpState(); + this.persistedOpStateExpiryEpochSec = + datanodeDetails.getPersistedOpStateExpiryEpochSec(); } /** @@ -155,6 +168,46 @@ public List getPorts() { return ports; } + /** + * Return the persistedOpState. If the stored value is null, return the + * default value of IN_SERVICE. + * + * @return The OperationalState persisted on the datanode. + */ + public HddsProtos.NodeOperationalState getPersistedOpState() { + if (persistedOpState == null) { + return HddsProtos.NodeOperationalState.IN_SERVICE; + } else { + return persistedOpState; + } + } + + /** + * Set the persistedOpState for this instance. + * + * @param state The new operational state. + */ + public void setPersistedOpState(HddsProtos.NodeOperationalState state) { + this.persistedOpState = state; + } + + /** + * Get the persistedOpStateExpiryEpochSec for the instance. + * @return Seconds from the epoch when the operational state should expire. + */ + public long getPersistedOpStateExpiryEpochSec() { + return persistedOpStateExpiryEpochSec; + } + + /** + * Set persistedOpStateExpiryEpochSec. + * @param expiry The number of second after the epoch the operational state + * should expire. + */ + public void setPersistedOpStateExpiryEpochSec(long expiry) { + this.persistedOpStateExpiryEpochSec = expiry; + } + /** * Given the name returns port number, null if the asked port is not found. * @@ -200,6 +253,13 @@ public static DatanodeDetails getFromProtoBuf( if (datanodeDetailsProto.hasNetworkLocation()) { builder.setNetworkLocation(datanodeDetailsProto.getNetworkLocation()); } + if (datanodeDetailsProto.hasPersistedOpState()) { + builder.setPersistedOpState(datanodeDetailsProto.getPersistedOpState()); + } + if (datanodeDetailsProto.hasPersistedOpStateExpiry()) { + builder.setPersistedOpStateExpiry( + datanodeDetailsProto.getPersistedOpStateExpiry()); + } return builder.build(); } @@ -226,6 +286,10 @@ public HddsProtos.DatanodeDetailsProto getProtoBufMessage() { if (!Strings.isNullOrEmpty(getNetworkLocation())) { builder.setNetworkLocation(getNetworkLocation()); } + if (persistedOpState != null) { + builder.setPersistedOpState(persistedOpState); + } + builder.setPersistedOpStateExpiry(persistedOpStateExpiryEpochSec); for (Port port : ports) { builder.addPorts(HddsProtos.Port.newBuilder() @@ -246,6 +310,8 @@ public String toString() { ", networkLocation: " + getNetworkLocation() + ", certSerialId: " + certSerialId + + ", persistedOpState: " + persistedOpState + + ", persistedOpStateExpiryEpochSec: " + persistedOpStateExpiryEpochSec + "}"; } @@ -285,6 +351,8 @@ public static final class Builder { private String networkLocation; private List ports; private String certSerialId; + private HddsProtos.NodeOperationalState persistedOpState; + private long persistedOpStateExpiryEpochSec = 0; /** * Default private constructor. To create Builder instance use @@ -373,6 +441,31 @@ public Builder setCertSerialId(String certId) { return this; } + /** + * Adds persistedOpState. + * + * @param state The operational state persisted on the datanode + * + * @return DatanodeDetails.Builder + */ + public Builder setPersistedOpState(HddsProtos.NodeOperationalState state) { + this.persistedOpState = state; + return this; + } + + /** + * Adds persistedOpStateExpiryEpochSec. + * + * @param expiry The seconds after the epoch the operational state should + * expire. + * + * @return DatanodeDetails.Builder + */ + public Builder setPersistedOpStateExpiry(long expiry) { + this.persistedOpStateExpiryEpochSec = expiry; + return this; + } + /** * Builds and returns DatanodeDetails instance. * @@ -384,7 +477,8 @@ public DatanodeDetails build() { networkLocation = NetConstants.DEFAULT_RACK; } DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName, - networkLocation, ports, certSerialId); + networkLocation, ports, certSerialId, persistedOpState, + persistedOpStateExpiryEpochSec); if (networkName != null) { dn.setNetworkName(networkName); } diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index ad45d30713cd..f629560ff362 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -37,6 +37,8 @@ message DatanodeDetailsProto { // network name, can be Ip address or host name, depends optional string networkName = 6; optional string networkLocation = 7; // Network topology location + optional NodeOperationalState persistedOpState = 8; // The Operational state persisted in the datanode.id file + optional int64 persistedOpStateExpiry = 9; // The seconds after the epoch when the OpState should expire } /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java index d3efa98795a9..d016569a7946 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java @@ -29,6 +29,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; @@ -82,6 +83,12 @@ public static DatanodeDetails readDatanodeIdFile(File path) .setIpAddress(datanodeDetailsYaml.getIpAddress()) .setHostName(datanodeDetailsYaml.getHostName()) .setCertSerialId(datanodeDetailsYaml.getCertSerialId()); + if (datanodeDetailsYaml.getPersistedOpState() != null) { + builder.setPersistedOpState(HddsProtos.NodeOperationalState.valueOf( + datanodeDetailsYaml.getPersistedOpState())); + } + builder.setPersistedOpStateExpiry( + datanodeDetailsYaml.getPersistedOpStateExpiryEpochSec()); if (!MapUtils.isEmpty(datanodeDetailsYaml.getPortDetails())) { for (Map.Entry portEntry : @@ -105,6 +112,8 @@ public static class DatanodeDetailsYaml { private String ipAddress; private String hostName; private String certSerialId; + private String persistedOpState; + private long persistedOpStateExpiryEpochSec = 0; private Map portDetails; public DatanodeDetailsYaml() { @@ -113,11 +122,15 @@ public DatanodeDetailsYaml() { private DatanodeDetailsYaml(String uuid, String ipAddress, String hostName, String certSerialId, + String persistedOpState, + long persistedOpStateExpiryEpochSec, Map portDetails) { this.uuid = uuid; this.ipAddress = ipAddress; this.hostName = hostName; this.certSerialId = certSerialId; + this.persistedOpState = persistedOpState; + this.persistedOpStateExpiryEpochSec = persistedOpStateExpiryEpochSec; this.portDetails = portDetails; } @@ -137,6 +150,14 @@ public String getCertSerialId() { return certSerialId; } + public String getPersistedOpState() { + return persistedOpState; + } + + public long getPersistedOpStateExpiryEpochSec() { + return persistedOpStateExpiryEpochSec; + } + public Map getPortDetails() { return portDetails; } @@ -157,6 +178,14 @@ public void setCertSerialId(String certSerialId) { this.certSerialId = certSerialId; } + public void setPersistedOpState(String persistedOpState) { + this.persistedOpState = persistedOpState; + } + + public void setPersistedOpStateExpiryEpochSec(long opStateExpiryEpochSec) { + this.persistedOpStateExpiryEpochSec = opStateExpiryEpochSec; + } + public void setPortDetails(Map portDetails) { this.portDetails = portDetails; } @@ -172,11 +201,17 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml( } } + String persistedOpString = null; + if (datanodeDetails.getPersistedOpState() != null) { + persistedOpString = datanodeDetails.getPersistedOpState().name(); + } return new DatanodeDetailsYaml( datanodeDetails.getUuid().toString(), datanodeDetails.getIpAddress(), datanodeDetails.getHostName(), datanodeDetails.getCertSerialId(), + persistedOpString, + datanodeDetails.getPersistedOpStateExpiryEpochSec(), portDetails); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 5229ae84144e..6835ef67eb86 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -50,6 +50,7 @@ .DeleteContainerCommandHandler; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler .ReplicateContainerCommandHandler; +import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.SetNodeOperationalStateCommandHandler; import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.container.replication.ContainerReplicator; @@ -138,6 +139,7 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails, dnConf.getContainerDeleteThreads())) .addHandler(new ClosePipelineCommandHandler()) .addHandler(new CreatePipelineCommandHandler(conf)) + .addHandler(new SetNodeOperationalStateCommandHandler(conf)) .setConnectionManager(connectionManager) .setContainer(container) .setContext(context) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java new file mode 100644 index 000000000000..1fcf5a21d3d7 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.container.common.statemachine.commandhandler; + + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.scm.HddsServerUtil; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; +import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; +import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; +import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Handle the SetNodeOperationalStateCommand sent from SCM to the datanode + * to persist the current operational state. + */ +public class SetNodeOperationalStateCommandHandler implements CommandHandler { + + private static final Logger LOG = + LoggerFactory.getLogger(SetNodeOperationalStateCommandHandler.class); + private final Configuration conf; + private final AtomicInteger invocationCount = new AtomicInteger(0); + private final AtomicLong totalTime = new AtomicLong(0); + + /** + * Set Node State command handler. + * + * @param conf - Configuration for the datanode. + */ + public SetNodeOperationalStateCommandHandler(Configuration conf) { + this.conf = conf; + } + + /** + * Handles a given SCM command. + * + * @param command - SCM Command + * @param container - Ozone Container. + * @param context - Current Context. + * @param connectionManager - The SCMs that we are talking to. + */ + @Override + public void handle(SCMCommand command, OzoneContainer container, + StateContext context, SCMConnectionManager connectionManager) { + long startTime = Time.monotonicNow(); + invocationCount.incrementAndGet(); + StorageContainerDatanodeProtocolProtos.SetNodeOperationalStateCommandProto + setNodeCmdProto = null; + + if (command.getType() != Type.setNodeOperationalStateCommand) { + LOG.warn("Skipping handling command, expected command " + + "type {} but found {}", + Type.setNodeOperationalStateCommand, command.getType()); + return; + } + SetNodeOperationalStateCommand setNodeCmd = + (SetNodeOperationalStateCommand) command; + setNodeCmdProto = setNodeCmd.getProto(); + DatanodeDetails dni = context.getParent().getDatanodeDetails(); + dni.setPersistedOpState(setNodeCmdProto.getNodeOperationalState()); + dni.setPersistedOpStateExpiryEpochSec( + setNodeCmd.getStateExpiryEpochSeconds()); + try { + persistDatanodeDetails(dni); + } catch (IOException ioe) { + LOG.error("Failed to persist the datanode state", ioe); + // TODO - this should probably be raised, but it will break the command + // handler interface. + } + totalTime.addAndGet(Time.monotonicNow() - startTime); + } + + // TODO - this duplicates code in HddsDatanodeService and InitDatanodeState + // Need to refactor. + private void persistDatanodeDetails(DatanodeDetails dnDetails) + throws IOException { + String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf); + if (idFilePath == null || idFilePath.isEmpty()) { + LOG.error("A valid path is needed for config setting {}", + ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR); + throw new IllegalArgumentException( + ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR + + " must be defined. See" + + " https://wiki.apache.org/hadoop/Ozone#Configuration" + + " for details on configuring Ozone."); + } + + Preconditions.checkNotNull(idFilePath); + File idFile = new File(idFilePath); + ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile); + } + + /** + * Returns the command type that this command handler handles. + * + * @return Type + */ + @Override + public StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type + getCommandType() { + return Type.setNodeOperationalStateCommand; + } + + /** + * Returns number of times this handler has been invoked. + * + * @return int + */ + @Override + public int getInvocationCount() { + return invocationCount.intValue(); + } + + /** + * Returns the average time this function takes to run. + * + * @return long + */ + @Override + public long getAverageRunTime() { + final int invocations = invocationCount.get(); + return invocations == 0 ? + 0 : totalTime.get() / invocations; + } +} \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index a55d0d647d54..fa66a57b86de 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -52,6 +52,7 @@ import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand; import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand; +import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -331,6 +332,17 @@ private void processResponse(SCMHeartbeatResponseProto response, } this.context.addCommand(closePipelineCommand); break; + case setNodeOperationalStateCommand: + SetNodeOperationalStateCommand setNodeOperationalStateCommand = + SetNodeOperationalStateCommand.getFromProtobuf( + commandResponseProto.getSetNodeOperationalStateCommandProto()); + if (LOG.isDebugEnabled()) { + LOG.debug("Received SCM set operational state command. State: {} " + + "Expiry: {}", setNodeOperationalStateCommand.getOpState(), + setNodeOperationalStateCommand.getStateExpiryEpochSeconds()); + } + this.context.addCommand(setNodeOperationalStateCommand); + break; default: throw new IllegalArgumentException("Unknown response : " + commandResponseProto.getCommandType().name()); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java new file mode 100644 index 000000000000..3ff79498b182 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.protocol.commands; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SCMCommandProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SetNodeOperationalStateCommandProto; + +/** + * A command used to persist the current node operational state on the datanode. + */ +public class SetNodeOperationalStateCommand + extends SCMCommand { + + private final HddsProtos.NodeOperationalState opState; + private long stateExpiryEpochSeconds; + + /** + * Ctor that creates a SetNodeOperationalStateCommand. + * + * @param id - Command ID. Something a time stamp would suffice. + * @param state - OperationalState that want the node to be set into. + * @param stateExpiryEpochSeconds The epoch time when the state should + * expire, or zero for the state to remain + * indefinitely. + */ + public SetNodeOperationalStateCommand(long id, + HddsProtos.NodeOperationalState state, long stateExpiryEpochSeconds) { + super(id); + this.opState = state; + this.stateExpiryEpochSeconds = stateExpiryEpochSeconds; + } + + /** + * Returns the type of this command. + * + * @return Type - This is setNodeOperationalStateCommand. + */ + @Override + public SCMCommandProto.Type getType() { + return SCMCommandProto.Type.setNodeOperationalStateCommand; + } + + /** + * Gets the protobuf message of this object. + * + * @return A protobuf message. + */ + @Override + public SetNodeOperationalStateCommandProto getProto() { + return SetNodeOperationalStateCommandProto.newBuilder() + .setCmdId(getId()) + .setNodeOperationalState(opState) + .setStateExpiryEpochSeconds(stateExpiryEpochSeconds).build(); + } + + public HddsProtos.NodeOperationalState getOpState() { + return opState; + } + + public long getStateExpiryEpochSeconds() { + return stateExpiryEpochSeconds; + } + + public static SetNodeOperationalStateCommand getFromProtobuf( + SetNodeOperationalStateCommandProto cmdProto) { + Preconditions.checkNotNull(cmdProto); + return new SetNodeOperationalStateCommand(cmdProto.getCmdId(), + cmdProto.getNodeOperationalState(), + cmdProto.getStateExpiryEpochSeconds()); + } +} \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto index 171ea8655259..4a977e1cb221 100644 --- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -287,6 +287,7 @@ message SCMCommandProto { replicateContainerCommand = 5; createPipelineCommand = 6; closePipelineCommand = 7; + setNodeOperationalStateCommand = 8; } // TODO: once we start using protoc 3.x, refactor this message using "oneof" required Type commandType = 1; @@ -297,6 +298,7 @@ message SCMCommandProto { optional ReplicateContainerCommandProto replicateContainerCommandProto = 6; optional CreatePipelineCommandProto createPipelineCommandProto = 7; optional ClosePipelineCommandProto closePipelineCommandProto = 8; + optional SetNodeOperationalStateCommandProto setNodeOperationalStateCommandProto = 9; } /** @@ -384,6 +386,12 @@ message ClosePipelineCommandProto { required int64 cmdId = 2; } +message SetNodeOperationalStateCommandProto { + required int64 cmdId = 1; + required NodeOperationalState nodeOperationalState = 2; + required int64 stateExpiryEpochSeconds = 3; +} + /** * Protocol used from a datanode to StorageContainerManager. * diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index 4d2d895ec266..a1d47fdc2194 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -357,7 +357,8 @@ private void putNodeBackInService(DatanodeAdminNodeDetails dn) private void setNodeOpState(DatanodeAdminNodeDetails dn, HddsProtos.NodeOperationalState state) throws NodeNotFoundException { - nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state); + nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state, + dn.getMaintenanceEnd() / 1000); } private NodeStatus getNodeStatus(DatanodeDetails dnd) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java index c23fcd28c69a..9c8a905bb996 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java @@ -106,6 +106,14 @@ public void setMaintenanceEnd(long hoursFromNow) { maintenanceEndTime = System.currentTimeMillis() + msFromNow; } + /** + * Returns the maintenance end time as milli seconds from the epoch. + * @return The maintenance end time, or zero if no end time is set. + */ + public long getMaintenanceEnd() { + return maintenanceEndTime; + } + /** * Matches only on the DatanodeDetails field, which compares only the UUID * of the node to determine of they are the same object or not. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index 06fe270e2164..3258fefa8fd7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -314,9 +314,15 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) throws NodeNotFoundException, InvalidNodeStateException { NodeStatus nodeStatus = getNodeStatus(dn); NodeOperationalState opState = nodeStatus.getOperationalState(); + + long maintenanceEnd = 0; + if (endInHours != 0) { + maintenanceEnd = + (System.currentTimeMillis() / 1000L) + (endInHours * 60L * 60L); + } if (opState == NodeOperationalState.IN_SERVICE) { nodeManager.setNodeOperationalState( - dn, NodeOperationalState.ENTERING_MAINTENANCE); + dn, NodeOperationalState.ENTERING_MAINTENANCE, maintenanceEnd); monitor.startMonitoring(dn, endInHours); LOG.info("Starting Maintenance for node {}", dn); } else if (nodeStatus.isMaintenance()) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java index b595d0023971..44ab5815f494 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java @@ -141,6 +141,18 @@ NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) void setNodeOperationalState(DatanodeDetails datanodeDetails, NodeOperationalState newState) throws NodeNotFoundException; + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + * @param opStateExpiryEpocSec Seconds from the epoch when the operational + * state should end. Zero indicates the state + * never end. + */ + void setNodeOperationalState(DatanodeDetails datanodeDetails, + NodeOperationalState newState, + long opStateExpiryEpocSec) throws NodeNotFoundException; + /** * Get set of pipelines a datanode is part of. * @param datanodeDetails DatanodeDetails diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index b16e79c6ce59..f18ac602ea53 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -358,10 +358,28 @@ public List getAllNodes() { */ public void setNodeOperationalState(DatanodeDetails dn, NodeOperationalState newState) throws NodeNotFoundException { + setNodeOperationalState(dn, newState, 0); + } + + /** + * Sets the operational state of the given node. Intended to be called when + * a node is being decommissioned etc. + * + * @param dn The datanode having its state set + * @param newState The new operational State of the node. + * @param stateExpiryEpochSec The number of seconds from the epoch when the + * operational state should expire. Passing zero + * indicates the state will never expire + */ + public void setNodeOperationalState(DatanodeDetails dn, + NodeOperationalState newState, + long stateExpiryEpochSec) throws NodeNotFoundException { DatanodeInfo dni = nodeStateMap.getNodeInfo(dn.getUuid()); NodeStatus oldStatus = dni.getNodeStatus(); - if (oldStatus.getOperationalState() != newState) { - nodeStateMap.updateNodeOperationalState(dn.getUuid(), newState); + if (oldStatus.getOperationalState() != newState || + oldStatus.getOpStateExpiryEpochSeconds() != stateExpiryEpochSec) { + nodeStateMap.updateNodeOperationalState( + dn.getUuid(), newState, stateExpiryEpochSec); // This will trigger an event based on the nodes health when the // operational state changes. Eg a node that was IN_MAINTENANCE goes // to IN_SERVICE + HEALTHY. This will trigger the HEALTHY node event to @@ -370,7 +388,9 @@ public void setNodeOperationalState(DatanodeDetails dn, // container replicas. Sometimes the event will do nothing, but it will // not do any harm either. Eg DECOMMISSIONING -> DECOMMISSIONED + HEALTHY // but the pipeline creation logic will ignore decommissioning nodes. - fireHealthStateEvent(oldStatus.getHealth(), dn); + if (oldStatus.getOperationalState() != newState) { + fireHealthStateEvent(oldStatus.getHealth(), dn); + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java index b8a92e5a6796..72ca0153e2ef 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStatus.java @@ -25,17 +25,28 @@ /** * This class is used to capture the current status of a datanode. This * includes its health (healthy, stale or dead) and its operation status ( - * in_service, decommissioned and maintenance mode. + * in_service, decommissioned and maintenance mode) along with the expiry time + * for the operational state (used with maintenance mode). */ public class NodeStatus { private HddsProtos.NodeOperationalState operationalState; private HddsProtos.NodeState health; + private long opStateExpiryEpochSeconds; public NodeStatus(HddsProtos.NodeOperationalState operationalState, HddsProtos.NodeState health) { this.operationalState = operationalState; this.health = health; + this.opStateExpiryEpochSeconds = 0; + } + + public NodeStatus(HddsProtos.NodeOperationalState operationalState, + HddsProtos.NodeState health, + long opStateExpireEpocSeconds) { + this.operationalState = operationalState; + this.health = health; + this.opStateExpiryEpochSeconds = opStateExpireEpocSeconds; } public static NodeStatus inServiceHealthy() { @@ -61,6 +72,17 @@ public HddsProtos.NodeOperationalState getOperationalState() { return operationalState; } + public long getOpStateExpiryEpochSeconds() { + return opStateExpiryEpochSeconds; + } + + public boolean operationalStateExpired() { + if (0 == opStateExpiryEpochSeconds) { + return false; + } + return System.currentTimeMillis() / 1000 >= opStateExpiryEpochSeconds; + } + /** * Returns true if the nodeStatus indicates the node is in any decommission * state. @@ -163,7 +185,8 @@ public boolean equals(Object obj) { } NodeStatus other = (NodeStatus) obj; if (this.operationalState == other.operationalState && - this.health == other.health) { + this.health == other.health + && this.opStateExpiryEpochSeconds == other.opStateExpiryEpochSeconds) { return true; } return false; @@ -171,12 +194,13 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(health, operationalState); + return Objects.hash(health, operationalState, opStateExpiryEpochSeconds); } @Override public String toString() { - return "OperationalState: "+operationalState+" Health: "+health; + return "OperationalState: "+operationalState+" Health: "+health+ + " OperastionStateExpiry: "+opStateExpiryEpochSeconds; } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index d89dac1cae35..3f17e32bb071 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -34,6 +34,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; @@ -64,11 +65,13 @@ import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; import org.apache.hadoop.util.ReflectionUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -231,7 +234,23 @@ public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) @Override public void setNodeOperationalState(DatanodeDetails datanodeDetails, NodeOperationalState newState) throws NodeNotFoundException{ - nodeStateManager.setNodeOperationalState(datanodeDetails, newState); + setNodeOperationalState(datanodeDetails, newState, 0); + } + + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + * @param opStateExpiryEpocSec Seconds from the epoch when the operational + * state should end. Zero indicates the state + * never end. + */ + @Override + public void setNodeOperationalState(DatanodeDetails datanodeDetails, + NodeOperationalState newState, long opStateExpiryEpocSec) + throws NodeNotFoundException{ + nodeStateManager.setNodeOperationalState( + datanodeDetails, newState, opStateExpiryEpocSec); } /** @@ -312,6 +331,7 @@ public RegisteredCommand register( datanodeDetails.toString()); } } + registerInitialDatanodeOpState(datanodeDetails); return RegisteredCommand.newBuilder().setErrorCode(ErrorCode.success) .setDatanode(datanodeDetails) @@ -319,6 +339,29 @@ public RegisteredCommand register( .build(); } + /** + * When a node registers with SCM, the operational state stored on the + * datanode is the source of truth. Therefore, if the datanode reports + * anything other than IN_SERVICE on registration, the state in SCM should be + * updated to reflect the datanode state. + * @param dn + */ + private void registerInitialDatanodeOpState(DatanodeDetails dn) { + try { + HddsProtos.NodeOperationalState dnOpState = dn.getPersistedOpState(); + if (dnOpState != NodeOperationalState.IN_SERVICE) { + LOG.info("Updating nodeOperationalState on registration as the " + + "datanode has a persisted state of {} and expiry of {}", + dnOpState, dn.getPersistedOpStateExpiryEpochSec()); + setNodeOperationalState(dn, dnOpState, + dn.getPersistedOpStateExpiryEpochSec()); + } + } catch (NodeNotFoundException e) { + LOG.error("Unable to find the node when setting the operational state", + e); + } + } + /** * Add an entry to the dnsToUuidMap, which maps hostname / IP to the DNs * running on that host. As each address can have many DNs running on it, @@ -352,6 +395,7 @@ public List processHeartbeat(DatanodeDetails datanodeDetails) { try { nodeStateManager.updateLastHeartbeatTime(datanodeDetails); metrics.incNumHBProcessed(); + updateDatanodeOpState(datanodeDetails); } catch (NodeNotFoundException e) { metrics.incNumHBProcessingFailed(); LOG.error("SCM trying to process heartbeat from an " + @@ -360,6 +404,41 @@ public List processHeartbeat(DatanodeDetails datanodeDetails) { return commandQueue.getCommand(datanodeDetails.getUuid()); } + /** + * If the operational state or expiry reported in the datanode heartbeat do + * not match those store in SCM, queue a command to update the state persisted + * on the datanode. Additionally, ensure the datanodeDetails stored in SCM + * match those reported in the heartbeat. + * This method should only be called when processing the + * heartbeat, and for a registered node, the information stored in SCM is the + * source of truth. + * @param reportedDn The DatanodeDetails taken from the node heartbeat. + * @throws NodeNotFoundException + */ + private void updateDatanodeOpState(DatanodeDetails reportedDn) + throws NodeNotFoundException { + NodeStatus scmStatus = getNodeStatus(reportedDn); + if (scmStatus.getOperationalState() != reportedDn.getPersistedOpState() + || scmStatus.getOpStateExpiryEpochSeconds() + != reportedDn.getPersistedOpStateExpiryEpochSec()) { + LOG.info("Scheduling a command to update the operationalState " + + "persisted on the datanode as the reported value ({}, {}) does not " + + "match the value stored in SCM ({}, {})", + reportedDn.getPersistedOpState(), + reportedDn.getPersistedOpStateExpiryEpochSec(), + scmStatus.getOperationalState(), + scmStatus.getOpStateExpiryEpochSeconds()); + commandQueue.addCommand(reportedDn.getUuid(), + new SetNodeOperationalStateCommand( + Time.monotonicNow(), scmStatus.getOperationalState(), + scmStatus.getOpStateExpiryEpochSeconds())); + } + DatanodeDetails scmDnd = nodeStateManager.getNode(reportedDn); + scmDnd.setPersistedOpStateExpiryEpochSec( + reportedDn.getPersistedOpStateExpiryEpochSec()); + scmDnd.setPersistedOpState(reportedDn.getPersistedOpState()); + } + @Override public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) { try { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java index 6565e8117c73..3cf232af392b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java @@ -113,13 +113,14 @@ public NodeStatus updateNodeHealthState(UUID nodeId, NodeState newHealth) * @throws NodeNotFoundException if the node is not present */ public NodeStatus updateNodeOperationalState(UUID nodeId, - NodeOperationalState newOpState) throws NodeNotFoundException { + NodeOperationalState newOpState, long opStateExpiryEpochSeconds) + throws NodeNotFoundException { try { lock.writeLock().lock(); DatanodeInfo dn = getNodeInfo(nodeId); NodeStatus oldStatus = dn.getNodeStatus(); NodeStatus newStatus = new NodeStatus( - newOpState, oldStatus.getHealth()); + newOpState, oldStatus.getHealth(), opStateExpiryEpochSeconds); dn.setNodeStatus(newStatus); return newStatus; } finally { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index d5690448829c..98a25d25f32e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -68,6 +68,7 @@ import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; import org.apache.hadoop.ozone.protocolPB.ProtocolMessageMetrics; import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB; import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolServerSideTranslatorPB; @@ -87,6 +88,7 @@ import static org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type .closePipelineCommand; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.setNodeOperationalStateCommand; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY; @@ -350,6 +352,12 @@ public SCMCommandProto getCommandResponse(SCMCommand cmd) .setClosePipelineCommandProto( ((ClosePipelineCommand)cmd).getProto()) .build(); + case setNodeOperationalStateCommand: + return builder + .setCommandType(setNodeOperationalStateCommand) + .setSetNodeOperationalStateCommandProto( + ((SetNodeOperationalStateCommand)cmd).getProto()) + .build(); default: throw new IllegalArgumentException("Scm command " + cmd.getType().toString() + " is not implemented"); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index d24cbfd5114d..da04b717447c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -278,6 +278,16 @@ public void setNodeOperationalState(DatanodeDetails datanodeDetails, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { } + /** + * Set the operation state of a node. + * @param datanodeDetails The datanode to set the new state for + * @param newState The new operational state for the node + */ + public void setNodeOperationalState(DatanodeDetails datanodeDetails, + HddsProtos.NodeOperationalState newState, long opStateExpiryEpocSec) + throws NodeNotFoundException { + } + /** * Get set of pipelines a datanode is part of. * @param dnId - datanodeID diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index dad3448bdf38..6982d03910a7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -100,12 +100,20 @@ public NodeStatus getNodeStatus(DatanodeDetails datanodeDetails) @Override public void setNodeOperationalState(DatanodeDetails dn, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { + setNodeOperationalState(dn, newState, 0); + } + + @Override + public void setNodeOperationalState(DatanodeDetails dn, + HddsProtos.NodeOperationalState newState, long opStateExpiryEpocSec) + throws NodeNotFoundException { DatanodeInfo dni = nodeMap.get(dn.getUuid()); if (dni == null) { throw new NodeNotFoundException(); } dni.setNodeStatus( - new NodeStatus(newState, dni.getNodeStatus().getHealth())); + new NodeStatus( + newState, dni.getNodeStatus().getHealth(), opStateExpiryEpocSec)); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java index df624386471f..a8bcf5402dda 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionManager.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import static junit.framework.TestCase.assertEquals; import static org.assertj.core.api.Fail.fail; +import static org.junit.Assert.assertNotEquals; /** * Unit tests for the decommision manager. @@ -182,8 +183,12 @@ public void testNodesCanBePutIntoMaintenanceAndRecommissioned() dns.get(2).getIpAddress()), 100); assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, nodeManager.getNodeStatus(dns.get(1)).getOperationalState()); + assertNotEquals(0, nodeManager.getNodeStatus( + dns.get(1)).getOpStateExpiryEpochSeconds()); assertEquals(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, nodeManager.getNodeStatus(dns.get(2)).getOperationalState()); + assertNotEquals(0, nodeManager.getNodeStatus( + dns.get(2)).getOpStateExpiryEpochSeconds()); // Running the command again gives no error - nodes already decommissioning // are silently ignored. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 0917fa4019b6..b78a76fd471e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.apache.hadoop.ozone.protocol.commands.SetNodeOperationalStateCommand; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; @@ -274,6 +275,35 @@ public void testScmSanityOfUserConfig2() createNodeManager(conf).close(); } + /** + * Ensure that a change to the operationalState of a node fires a datanode + * event of type SetNodeOperationalStateCommand. + */ + @Test + @Ignore // TODO - this test is no longer valid as the heartbeat processing + // now generates the command message. + public void testSetNodeOpStateAndCommandFired() + throws IOException, NodeNotFoundException, AuthenticationException { + final int interval = 100; + + OzoneConfiguration conf = getConf(); + conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval, + MILLISECONDS); + + try (SCMNodeManager nodeManager = createNodeManager(conf)) { + DatanodeDetails dn = TestUtils.createRandomDatanodeAndRegister( + nodeManager); + long expiry = System.currentTimeMillis() / 1000 + 1000; + nodeManager.setNodeOperationalState(dn, + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, expiry); + List commands = nodeManager.getCommandQueue(dn.getUuid()); + + Assert.assertTrue(commands.get(0).getClass().equals( + SetNodeOperationalStateCommand.class)); + assertEquals(1, commands.size()); + } + } + /** * Asserts that a single node moves from Healthy to stale node, then from * stale node to dead node if it misses enough heartbeats. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java index 482f444e3c3c..ad7139f4bf42 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNodeStateMap.java @@ -82,11 +82,12 @@ public void testNodeOperationalStateCanBeUpdated() NodeStatus expectedStatus = new NodeStatus( NodeOperationalState.DECOMMISSIONING, - NodeState.HEALTHY); + NodeState.HEALTHY, 999); NodeStatus returnedStatus = map.updateNodeOperationalState( - dn.getUuid(), expectedStatus.getOperationalState()); + dn.getUuid(), expectedStatus.getOperationalState(), 999); assertEquals(expectedStatus, returnedStatus); assertEquals(returnedStatus, map.getNodeStatus(dn.getUuid())); + assertEquals(999, returnedStatus.getOpStateExpiryEpochSeconds()); } @Test diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java index ee96565a4097..9654715b88ac 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java @@ -191,9 +191,22 @@ public NodeStatus getNodeStatus(DatanodeDetails dd) { @Override public void setNodeOperationalState(DatanodeDetails dd, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException { + setNodeOperationalState(dd, newState, 0); + } + + /** + * Set the operation state of a node. + * @param dd The datanode to set the new state for + * @param newState The new operational state for the node + */ + @Override + public void setNodeOperationalState(DatanodeDetails dd, + HddsProtos.NodeOperationalState newState, long opStateExpiryEpocSec) + throws NodeNotFoundException { NodeStatus currentStatus = nodeStateMap.get(dd); if (currentStatus != null) { - nodeStateMap.put(dd, new NodeStatus(newState, currentStatus.getHealth())); + nodeStateMap.put(dd, new NodeStatus(newState, currentStatus.getHealth(), + opStateExpiryEpocSec)); } else { throw new NodeNotFoundException(); } From 637853d170c3bf7ebdef287cb5529b8011b5b164 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 3 Apr 2020 22:32:05 +0100 Subject: [PATCH 17/78] Address issues cause by merge. --- .../hdds/scm/cli/node/DatanodeAdminCommands.java | 16 ++++++++++------ .../scm/node/TestDecommissionAndMaintenance.java | 2 ++ .../ozone/recon/api/ClusterStateEndpoint.java | 5 +++-- .../hadoop/ozone/recon/api/NodeEndpoint.java | 7 ++++++- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java index 2eba44e51db7..b7752a37f352 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/node/DatanodeAdminCommands.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.hdds.scm.cli.node; +import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.cli.MissingSubcommandException; +import org.apache.hadoop.hdds.scm.cli.container.WithScmClient; +import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.ParentCommand; -import org.apache.hadoop.hdds.scm.cli.SCMCLI; import java.util.concurrent.Callable; @@ -40,16 +41,19 @@ }) public class DatanodeAdminCommands implements Callable { + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + @ParentCommand - private SCMCLI parent; + private WithScmClient parent; - public SCMCLI getParent() { + public WithScmClient getParent() { return parent; } @Override public Void call() throws Exception { - throw new MissingSubcommandException( - this.parent.getCmd().getSubcommands().get("nodeadmin")); + GenericCli.missingSubcommand(spec); + return null; } } \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index fb03edd21222..f39af441e3c2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,6 +83,7 @@ public void tearDown() throws Exception { } @Test + @Ignore // HDDS-3345 public void testNodeCanBeDecommMaintAndRecommissioned() throws IOException { NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java index e43dbf8238d4..263c301cd022 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java @@ -19,8 +19,8 @@ package org.apache.hadoop.ozone.recon.api; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport; @@ -74,7 +74,8 @@ public Response getClusterState() { List datanodeDetails = nodeManager.getAllNodes(); int containers = this.containerManager.getContainerIDs().size(); int pipelines = this.pipelineManager.getPipelines().size(); - int healthyDatanodes = nodeManager.getNodeCount(NodeState.HEALTHY); + int healthyDatanodes = + nodeManager.getNodeCount(NodeStatus.inServiceHealthy()); SCMNodeStat stats = nodeManager.getStats(); DatanodeStorageReport storageReport = new DatanodeStorageReport(stats.getCapacity().get(), diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java index d59bee62da99..0558eeaf63ab 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java @@ -77,7 +77,12 @@ public Response getDatanodes() { datanodeDetails.forEach(datanode -> { DatanodeStorageReport storageReport = getStorageReport(datanode); - NodeState nodeState = nodeManager.getNodeState(datanode); + NodeState nodeState = null; + try { + nodeState = nodeManager.getNodeStatus(datanode).getHealth(); + } catch (NodeNotFoundException e) { + LOG.warn("Cannot get nodeState for datanode {}", datanode, e); + } String hostname = datanode.getHostName(); Set pipelineIDs = nodeManager.getPipelines(datanode); List pipelines = new ArrayList<>(); From 6fc79045346150825906b30b3cb683fc6febb86c Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Thu, 9 Apr 2020 12:05:35 +0100 Subject: [PATCH 18/78] HDDS-3361. Remove ContainerReplica states representing decomission and maintenance (#789) --- .../hdds/protocol/MockDatanodeDetails.java | 4 + .../StorageContainerDatanodeProtocol.proto | 2 - .../scm/container/ContainerReplicaCount.java | 18 ++- .../scm/container/ReplicationManager.java | 11 +- .../scm/node/DatanodeAdminMonitorImpl.java | 4 + .../scm/container/SimpleMockNodeManager.java | 4 + .../scm/container/TestReplicationManager.java | 130 +++++++++++------- .../states/TestContainerReplicaCount.java | 119 +++++++++------- .../scm/node/TestDatanodeAdminMonitor.java | 83 ++++++----- 9 files changed, 228 insertions(+), 147 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java index 0cf3b2520dc6..f66f1ae77cff 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/protocol/MockDatanodeDetails.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdds.protocol; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import java.io.IOException; import java.net.ServerSocket; import java.util.Random; @@ -101,6 +103,8 @@ public static DatanodeDetails createDatanodeDetails(String uuid, .addPort(ratisPort) .addPort(restPort) .setNetworkLocation(networkLocation) + .setPersistedOpState(HddsProtos.NodeOperationalState.IN_SERVICE) + .setPersistedOpStateExpiry(0) .build(); } diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto index fc6249108bd5..17da79b8ffc9 100644 --- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -185,8 +185,6 @@ message ContainerReplicaProto { CLOSED = 4; UNHEALTHY = 5; INVALID = 6; - DECOMMISSIONED = 7; - MAINTENANCE = 8; } required int64 containerID = 1; required State state = 2; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java index b888fdfe7dca..bf8c3b933b22 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaCount.java @@ -18,10 +18,14 @@ package org.apache.hadoop.hdds.scm.container; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import java.util.Set; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; + /** * Immutable object that is created with a set of ContainerReplica objects and * the number of in flight replica add and deletes, the container replication @@ -57,10 +61,11 @@ public ContainerReplicaCount(ContainerInfo container, this.container = container; for (ContainerReplica cr : this.replica) { - ContainerReplicaProto.State state = cr.getState(); - if (state == ContainerReplicaProto.State.DECOMMISSIONED) { + HddsProtos.NodeOperationalState state = + cr.getDatanodeDetails().getPersistedOpState(); + if (state == DECOMMISSIONED || state == DECOMMISSIONING) { decommissionCount++; - } else if (state == ContainerReplicaProto.State.MAINTENANCE) { + } else if (state == IN_MAINTENANCE || state == ENTERING_MAINTENANCE) { maintenanceCount++; } else { healthyCount++; @@ -259,8 +264,7 @@ public boolean isHealthy() { return (container.getState() == HddsProtos.LifeCycleState.CLOSED || container.getState() == HddsProtos.LifeCycleState.QUASI_CLOSED) && replica.stream() - .filter(r -> r.getState() != ContainerReplicaProto.State.DECOMMISSIONED) - .filter(r -> r.getState() != ContainerReplicaProto.State.MAINTENANCE) + .filter(r -> r.getDatanodeDetails().getPersistedOpState() == IN_SERVICE) .allMatch(r -> ReplicationManager.compareState( container.getState(), r.getState())); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index ae7242a0176c..eb10ae68decf 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.ConfigType; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; @@ -544,9 +545,7 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, final List source = replicas.stream() .filter(r -> r.getState() == State.QUASI_CLOSED || - r.getState() == State.CLOSED || - r.getState() == State.DECOMMISSIONED || - r.getState() == State.MAINTENANCE) + r.getState() == State.CLOSED) // Exclude stale and dead nodes. This is particularly important for // maintenance nodes, as the replicas will remain present in the // container manager, even when they go dead. @@ -623,9 +622,9 @@ private void handleOverReplicatedContainer(final ContainerInfo container, // Replica which are maintenance or decommissioned are not eligible to // be removed, as they do not count toward over-replication and they also // many not be available - eligibleReplicas.removeIf(r -> (r.getState() == State.MAINTENANCE - || r.getState() == State.DECOMMISSIONED)); - + eligibleReplicas.removeIf(r -> + r.getDatanodeDetails().getPersistedOpState() != + HddsProtos.NodeOperationalState.IN_SERVICE); final List unhealthyReplicas = eligibleReplicas .stream() .filter(r -> !compareState(container.getState(), r.getState())) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index a1d47fdc2194..f9d1a3216de5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -202,6 +202,10 @@ private void processTransitioningNodes() { if (status.isDecommissioning() || status.isEnteringMaintenance()) { if (checkPipelinesClosedOnNode(dn) + // Ensure the DN has received and persisted the current maint + // state. + && status.getOperationalState() + == dn.getDatanodeDetails().getPersistedOpState() && checkContainersReplicatedOnNode(dn)) { // CheckContainersReplicatedOnNode may take a short time to run // so after it completes, re-get the nodestatus to check the health diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java index 7e6bf0c048a0..19ca69e84d88 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/SimpleMockNodeManager.java @@ -52,10 +52,14 @@ public class SimpleMockNodeManager implements NodeManager { private Map> containerMap = new ConcurrentHashMap<>(); public void register(DatanodeDetails dd, NodeStatus status) { + dd.setPersistedOpState(status.getOperationalState()); + dd.setPersistedOpStateExpiryEpochSec(status.getOpStateExpiryEpochSeconds()); nodeMap.put(dd.getUuid(), new DatanodeInfo(dd, status)); } public void setNodeStatus(DatanodeDetails dd, NodeStatus status) { + dd.setPersistedOpState(status.getOperationalState()); + dd.setPersistedOpStateExpiryEpochSec(status.getOpStateExpiryEpochSeconds()); DatanodeInfo dni = nodeMap.get(dd.getUuid()); dni.setNodeStatus(status); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java index 1c83231ac5e8..3ed9f38c1466 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.container; +import com.google.common.primitives.Longs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -56,6 +57,13 @@ import java.util.stream.IntStream; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.createDatanodeDetails; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; import static org.apache.hadoop.hdds.scm.TestUtils.getContainer; import static org.apache.hadoop.hdds.scm.TestUtils.getReplicas; import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; @@ -596,7 +604,7 @@ public void testHealthyClosedContainer() throws SCMException, ContainerNotFoundException, InterruptedException { final ContainerInfo container = getContainer(LifeCycleState.CLOSED); final ContainerID id = container.containerID(); - final Set replicas = getReplicas(id, State.CLOSED, + final Set replicas = getReplicas(id, CLOSED, randomDatanodeDetails(), randomDatanodeDetails(), randomDatanodeDetails()); @@ -633,8 +641,10 @@ public void testGeneratedConfig() { @Test public void testUnderReplicatedDueToDecommission() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.CLOSED, State.DECOMMISSIONED, State.DECOMMISSIONED); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); assertReplicaScheduled(2); } @@ -645,8 +655,10 @@ public void testUnderReplicatedDueToDecommission() throws @Test public void testUnderReplicatedDueToAllDecommission() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.DECOMMISSIONED, State.DECOMMISSIONED, State.DECOMMISSIONED); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); assertReplicaScheduled(3); } @@ -657,8 +669,11 @@ public void testUnderReplicatedDueToAllDecommission() throws @Test public void testCorrectlyReplicatedWithDecommission() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.CLOSED, State.CLOSED, State.CLOSED, State.DECOMMISSIONED); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONING, HEALTHY), CLOSED); assertReplicaScheduled(0); } @@ -669,8 +684,10 @@ public void testCorrectlyReplicatedWithDecommission() throws @Test public void testUnderReplicatedDueToMaintenance() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(1); } @@ -686,8 +703,10 @@ public void testNotUnderReplicatedDueToMaintenanceMinRepOne() throws new ReplicationManagerConfiguration(); newConf.setMaintenanceReplicaMinimum(1); createReplicationManager(newConf); - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(0); } @@ -703,8 +722,10 @@ public void testUnderReplicatedDueToMaintenanceMinRepOne() throws new ReplicationManagerConfiguration(); newConf.setMaintenanceReplicaMinimum(1); createReplicationManager(newConf); - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.MAINTENANCE, State.MAINTENANCE, State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(1); } @@ -715,8 +736,10 @@ public void testUnderReplicatedDueToMaintenanceMinRepOne() throws @Test public void testUnderReplicatedDueToAllMaintenance() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.MAINTENANCE, State.MAINTENANCE, State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(2); } @@ -727,8 +750,11 @@ public void testUnderReplicatedDueToAllMaintenance() throws @Test public void testCorrectlyReplicatedWithMaintenance() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.CLOSED, State.CLOSED, State.MAINTENANCE, State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_SERVICE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(0); } @@ -739,9 +765,11 @@ public void testCorrectlyReplicatedWithMaintenance() throws @Test public void testUnderReplicatedWithDecommissionAndMaintenance() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.DECOMMISSIONED, State.DECOMMISSIONED, State.MAINTENANCE, - State.MAINTENANCE); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); assertReplicaScheduled(2); } @@ -754,9 +782,14 @@ public void testUnderReplicatedWithDecommissionAndMaintenance() throws @Test public void testOverReplicatedClosedContainerWithDecomAndMaint() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - State.DECOMMISSIONED, State.MAINTENANCE, - State.CLOSED, State.CLOSED, State.CLOSED, State.CLOSED); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONED, HEALTHY), CLOSED); + addReplica(container, new NodeStatus(IN_MAINTENANCE, HEALTHY), CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), CLOSED); + addReplica(container, NodeStatus.inServiceHealthy(), CLOSED); final int currentDeleteCommandCount = datanodeCommandHandler .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand); @@ -764,14 +797,14 @@ public void testOverReplicatedClosedContainerWithDecomAndMaint() replicationManager.processContainersNow(); // Wait for EventQueue to call the event handler Thread.sleep(100L); - Assert.assertEquals(currentDeleteCommandCount + 1, datanodeCommandHandler + Assert.assertEquals(currentDeleteCommandCount + 2, datanodeCommandHandler .getInvocationCount(SCMCommandProto.Type.deleteContainerCommand)); // Get the DECOM and Maint replica and ensure none of them are scheduled // for removal Set decom = containerStateManager.getContainerReplicas(container.containerID()) .stream() - .filter(r -> r.getState() != State.CLOSED) + .filter(r -> r.getDatanodeDetails().getPersistedOpState() != IN_SERVICE) .collect(Collectors.toSet()); for (ContainerReplica r : decom) { Assert.assertFalse(datanodeCommandHandler.received( @@ -789,38 +822,43 @@ public void testOverReplicatedClosedContainerWithDecomAndMaint() @Test public void testUnderReplicatedNotHealthySource() throws SCMException, ContainerNotFoundException, InterruptedException { - final ContainerInfo container = setupReplicas(LifeCycleState.CLOSED, - NodeStatus.inServiceStale(), - State.CLOSED, State.DECOMMISSIONED, State.DECOMMISSIONED); + final ContainerInfo container = createContainer(LifeCycleState.CLOSED); + addReplica(container, NodeStatus.inServiceStale(), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED); + addReplica(container, new NodeStatus(DECOMMISSIONED, STALE), CLOSED); // There should be replica scheduled, but as all nodes are stale, nothing // gets scheduled. assertReplicaScheduled(0); } - private ContainerInfo setupReplicas( - LifeCycleState containerState, State... states) - throws SCMException, ContainerNotFoundException { - return setupReplicas(containerState, NodeStatus.inServiceHealthy(), states); - } - - private ContainerInfo setupReplicas( - LifeCycleState containerState, NodeStatus allNodesStatus, State... states) - throws SCMException, ContainerNotFoundException { + private ContainerInfo createContainer(LifeCycleState containerState) + throws SCMException { final ContainerInfo container = getContainer(containerState); final ContainerID id = container.containerID(); containerStateManager.loadContainer(container); - final UUID originNodeId = UUID.randomUUID(); - - for (State s : states) { - DatanodeDetails dn = randomDatanodeDetails(); - nodeManager.register(dn, allNodesStatus); - final ContainerReplica replica = getReplicas( - id, s, 1000L, originNodeId, dn); - containerStateManager.updateContainerReplica(id, replica); - } return container; } + private ContainerReplica addReplica(ContainerInfo container, + NodeStatus nodeStatus, State replicaState) + throws ContainerNotFoundException { + DatanodeDetails dn = randomDatanodeDetails(); + dn.setPersistedOpState(nodeStatus.getOperationalState()); + dn.setPersistedOpStateExpiryEpochSec( + nodeStatus.getOpStateExpiryEpochSeconds()); + nodeManager.register(dn, nodeStatus); + // Using the same originID for all replica in the container set. If each + // replica has a unique originID, it causes problems in ReplicationManager + // when processing over-replicated containers. + final UUID originNodeId = + UUID.nameUUIDFromBytes(Longs.toByteArray(container.getContainerID())); + final ContainerReplica replica = getReplicas( + container.containerID(), CLOSED, 1000L, originNodeId, dn); + containerStateManager + .updateContainerReplica(container.containerID(), replica); + return replica; + } + private void assertReplicaScheduled(int delta) throws InterruptedException { final int currentReplicateCommandCount = datanodeCommandHandler .getInvocationCount(SCMCommandProto.Type.replicateContainerCommand); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java index f6e2187a8b48..3c7c952c0180 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestContainerReplicaCount.java @@ -20,8 +20,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -31,16 +29,19 @@ import java.util.*; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .NodeOperationalState.ENTERING_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .NodeOperationalState.IN_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos + .NodeOperationalState.IN_SERVICE; import static org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; -import static org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN; -import static org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State - .DECOMMISSIONED; -import static org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State - .MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN; import static org.junit.Assert.assertFalse; /** @@ -54,7 +55,8 @@ public void setup() { @Test public void testThreeHealthyReplica() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + Set replica = + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -63,7 +65,7 @@ public void testThreeHealthyReplica() { @Test public void testTwoHealthyReplica() { - Set replica = registerNodes(CLOSED, CLOSED); + Set replica = registerNodes(IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -72,7 +74,7 @@ public void testTwoHealthyReplica() { @Test public void testOneHealthyReplica() { - Set replica = registerNodes(CLOSED); + Set replica = registerNodes(IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -82,7 +84,7 @@ public void testOneHealthyReplica() { @Test public void testTwoHealthyAndInflightAdd() { - Set replica = registerNodes(CLOSED, CLOSED); + Set replica = registerNodes(IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 3, 2); @@ -96,7 +98,8 @@ public void testTwoHealthyAndInflightAdd() { * completes there will be 4 healthy and it will get taken care of then. */ public void testThreeHealthyAndInflightAdd() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + Set replica = + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 3, 2); @@ -109,7 +112,8 @@ public void testThreeHealthyAndInflightAdd() { * under replicated, we go ahead and schedule another replica to be added. */ public void testThreeHealthyAndInflightDelete() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + Set replica = + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 1, 3, 2); @@ -122,7 +126,8 @@ public void testThreeHealthyAndInflightDelete() { * inflight del could succeed, leaving only 2 healthy replicas. */ public void testThreeHealthyAndInflightAddAndInFlightDelete() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED); + Set replica = + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 1, 3, 2); @@ -132,7 +137,7 @@ public void testThreeHealthyAndInflightAddAndInFlightDelete() { @Test public void testFourHealthyReplicas() { Set replica = - registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -142,7 +147,7 @@ public void testFourHealthyReplicas() { @Test public void testFourHealthyReplicasAndInFlightDelete() { Set replica = - registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 1, 3, 2); @@ -152,7 +157,7 @@ public void testFourHealthyReplicasAndInFlightDelete() { @Test public void testFourHealthyReplicasAndTwoInFlightDelete() { Set replica = - registerNodes(CLOSED, CLOSED, CLOSED, CLOSED); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 2, 3, 2); @@ -161,7 +166,7 @@ public void testFourHealthyReplicasAndTwoInFlightDelete() { @Test public void testOneHealthyReplicaRepFactorOne() { - Set replica = registerNodes(CLOSED); + Set replica = registerNodes(IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 1, 2); @@ -170,7 +175,7 @@ public void testOneHealthyReplicaRepFactorOne() { @Test public void testOneHealthyReplicaRepFactorOneInFlightDelete() { - Set replica = registerNodes(CLOSED); + Set replica = registerNodes(IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 1, 1, 2); @@ -179,7 +184,7 @@ public void testOneHealthyReplicaRepFactorOneInFlightDelete() { @Test public void testTwoHealthyReplicaTwoInflightAdd() { - Set replica = registerNodes(CLOSED, CLOSED); + Set replica = registerNodes(IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 2, 0, 3, 2); @@ -192,8 +197,8 @@ public void testTwoHealthyReplicaTwoInflightAdd() { @Test public void testThreeHealthyAndTwoDecommission() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED, - DECOMMISSIONED, DECOMMISSIONED); + Set replica = registerNodes(IN_SERVICE, IN_SERVICE, + IN_SERVICE, DECOMMISSIONING, DECOMMISSIONING); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -203,7 +208,7 @@ public void testThreeHealthyAndTwoDecommission() { @Test public void testOneDecommissionedReplica() { Set replica = - registerNodes(CLOSED, CLOSED, DECOMMISSIONED); + registerNodes(IN_SERVICE, IN_SERVICE, DECOMMISSIONING); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -213,7 +218,7 @@ public void testOneDecommissionedReplica() { @Test public void testTwoHealthyOneDecommissionedneInFlightAdd() { Set replica = - registerNodes(CLOSED, CLOSED, DECOMMISSIONED); + registerNodes(IN_SERVICE, IN_SERVICE, DECOMMISSIONED); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 3, 2); @@ -251,7 +256,7 @@ public void testAllDecommissionedRepFactorOneInFlightAdd() { @Test public void testOneHealthyOneDecommissioningRepFactorOne() { - Set replica = registerNodes(DECOMMISSIONED, CLOSED); + Set replica = registerNodes(DECOMMISSIONED, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 1, 2); @@ -265,7 +270,7 @@ public void testOneHealthyOneDecommissioningRepFactorOne() { @Test public void testOneHealthyTwoMaintenanceMinRepOfTwo() { Set replica = - registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); + registerNodes(IN_SERVICE, IN_MAINTENANCE, IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -274,8 +279,8 @@ public void testOneHealthyTwoMaintenanceMinRepOfTwo() { @Test public void testOneHealthyThreeMaintenanceMinRepOfTwo() { - Set replica = registerNodes(CLOSED, - MAINTENANCE, MAINTENANCE, MAINTENANCE); + Set replica = registerNodes(IN_SERVICE, + IN_MAINTENANCE, IN_MAINTENANCE, ENTERING_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -285,7 +290,7 @@ public void testOneHealthyThreeMaintenanceMinRepOfTwo() { @Test public void testOneHealthyTwoMaintenanceMinRepOfOne() { Set replica = - registerNodes(CLOSED, MAINTENANCE, MAINTENANCE); + registerNodes(IN_SERVICE, IN_MAINTENANCE, ENTERING_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 1); @@ -294,8 +299,8 @@ public void testOneHealthyTwoMaintenanceMinRepOfOne() { @Test public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() { - Set replica = registerNodes(CLOSED, - MAINTENANCE, MAINTENANCE, MAINTENANCE); + Set replica = registerNodes(IN_SERVICE, + IN_MAINTENANCE, ENTERING_MAINTENANCE, IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 3, 2); @@ -305,7 +310,7 @@ public void testOneHealthyThreeMaintenanceMinRepOfTwoInFlightAdd() { @Test public void testAllMaintenance() { Set replica = - registerNodes(MAINTENANCE, MAINTENANCE, MAINTENANCE); + registerNodes(IN_MAINTENANCE, ENTERING_MAINTENANCE, IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -319,8 +324,8 @@ public void testAllMaintenance() { * come back online, and then deal with them. */ public void testThreeHealthyTwoInMaintenance() { - Set replica = registerNodes(CLOSED, CLOSED, CLOSED, - MAINTENANCE, MAINTENANCE); + Set replica = registerNodes(IN_SERVICE, IN_SERVICE, + IN_SERVICE, IN_MAINTENANCE, ENTERING_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -335,7 +340,8 @@ public void testThreeHealthyTwoInMaintenance() { */ public void testFourHealthyOneInMaintenance() { Set replica = - registerNodes(CLOSED, CLOSED, CLOSED, CLOSED, MAINTENANCE); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE, IN_SERVICE, + IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -344,7 +350,7 @@ public void testFourHealthyOneInMaintenance() { @Test public void testOneMaintenanceMinRepOfTwoRepFactorOne() { - Set replica = registerNodes(MAINTENANCE); + Set replica = registerNodes(IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 1, 2); @@ -353,7 +359,7 @@ public void testOneMaintenanceMinRepOfTwoRepFactorOne() { @Test public void testOneMaintenanceMinRepOfTwoRepFactorOneInFlightAdd() { - Set replica = registerNodes(MAINTENANCE); + Set replica = registerNodes(IN_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 1, 2); @@ -362,7 +368,7 @@ public void testOneMaintenanceMinRepOfTwoRepFactorOneInFlightAdd() { @Test public void testOneHealthyOneMaintenanceRepFactorOne() { - Set replica = registerNodes(MAINTENANCE, CLOSED); + Set replica = registerNodes(IN_MAINTENANCE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 1, 2); @@ -372,7 +378,8 @@ public void testOneHealthyOneMaintenanceRepFactorOne() { @Test public void testTwoDecomTwoMaintenanceOneInflightAdd() { Set replica = - registerNodes(DECOMMISSIONED, DECOMMISSIONED, MAINTENANCE, MAINTENANCE); + registerNodes(DECOMMISSIONED, DECOMMISSIONING, + IN_MAINTENANCE, ENTERING_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 1, 0, 3, 2); @@ -382,7 +389,7 @@ public void testTwoDecomTwoMaintenanceOneInflightAdd() { @Test public void testHealthyContainerIsHealthy() { Set replica = - registerNodes(CLOSED, CLOSED, CLOSED); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -392,7 +399,21 @@ public void testHealthyContainerIsHealthy() { @Test public void testIsHealthyWithDifferentReplicaStateNotHealthy() { Set replica = - registerNodes(CLOSED, CLOSED, OPEN); + registerNodes(IN_SERVICE, IN_SERVICE, IN_SERVICE); + for (ContainerReplica r : replica) { + DatanodeDetails dn = r.getDatanodeDetails(); + + ContainerReplica replace = new ContainerReplica.ContainerReplicaBuilder() + .setContainerID(new ContainerID(1)) + .setContainerState(OPEN) + .setDatanodeDetails(dn) + .setOriginNodeId(dn.getUuid()) + .setSequenceId(1) + .build(); + replica.remove(r); + replica.add(replace); + break; + } ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -402,7 +423,8 @@ public void testIsHealthyWithDifferentReplicaStateNotHealthy() { @Test public void testIsHealthyWithMaintReplicaIsHealthy() { Set replica = - registerNodes(CLOSED, CLOSED, MAINTENANCE, MAINTENANCE); + registerNodes(IN_SERVICE, IN_SERVICE, IN_MAINTENANCE, + ENTERING_MAINTENANCE); ContainerInfo container = createContainer(HddsProtos.LifeCycleState.CLOSED); ContainerReplicaCount rcnt = new ContainerReplicaCount(container, replica, 0, 0, 3, 2); @@ -418,13 +440,14 @@ private void validate(ContainerReplicaCount rcnt, } private Set registerNodes( - ContainerReplicaProto.State... states) { + HddsProtos.NodeOperationalState... states) { Set replica = new HashSet<>(); - for (ContainerReplicaProto.State s : states) { + for (HddsProtos.NodeOperationalState s : states) { DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + dn.setPersistedOpState(s); replica.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(new ContainerID(1)) - .setContainerState(s) + .setContainerState(CLOSED) .setDatanodeDetails(dn) .setOriginNodeId(dn.getUuid()) .setSequenceId(1) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index cb787032c061..0592f94a6105 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -41,6 +41,11 @@ import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED; import static org.mockito.Mockito.reset; /** @@ -126,7 +131,7 @@ public void testClosePipelinesEventFiredWhenAdminStarted() nodeManager.setPipelines(dn1, 0); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + assertEquals(DECOMMISSIONED, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -151,7 +156,7 @@ public void testDecommissionNodeTransitionsToCompleteWhenNoContainers() monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); NodeStatus newStatus = nodeManager.getNodeStatus(dn1); - assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + assertEquals(DECOMMISSIONED, newStatus.getOperationalState()); } @@ -168,9 +173,9 @@ public void testDecommissionNodeWaitsForContainersToReplicate() // always have a DECOMMISSIONED replica. mockGetContainerReplicaCount( HddsProtos.LifeCycleState.CLOSED, - ContainerReplicaProto.State.DECOMMISSIONED, - ContainerReplicaProto.State.CLOSED, - ContainerReplicaProto.State.CLOSED); + DECOMMISSIONED, + IN_SERVICE, + IN_SERVICE); // Run the monitor for the first time and the node will transition to // REPLICATE_CONTAINERS as there are no pipelines to close. @@ -195,9 +200,9 @@ public void testDecommissionNodeWaitsForContainersToReplicate() // complete which will end the decommission workflow mockGetContainerReplicaCount( HddsProtos.LifeCycleState.CLOSED, - ContainerReplicaProto.State.CLOSED, - ContainerReplicaProto.State.CLOSED, - ContainerReplicaProto.State.CLOSED); + IN_SERVICE, + IN_SERVICE, + IN_SERVICE); monitor.run(); @@ -205,7 +210,7 @@ public void testDecommissionNodeWaitsForContainersToReplicate() assertEquals(3, node.getSufficientlyReplicatedContainers()); assertEquals(0, node.getUnHealthyContainers()); assertEquals(0, node.getUnderReplicatedContainers()); - assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, + assertEquals(DECOMMISSIONED, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -220,9 +225,9 @@ public void testDecommissionAbortedWhenNodeInUnexpectedState() nodeManager.setContainers(dn1, generateContainers(3)); mockGetContainerReplicaCount( HddsProtos.LifeCycleState.CLOSED, - ContainerReplicaProto.State.DECOMMISSIONED, - ContainerReplicaProto.State.CLOSED, - ContainerReplicaProto.State.CLOSED); + DECOMMISSIONED, + IN_SERVICE, + IN_SERVICE); // Add the node to the monitor, it should have 3 under-replicated containers // after the first run @@ -237,11 +242,11 @@ public void testDecommissionAbortedWhenNodeInUnexpectedState() // Set the node to dead, and then the workflow should get aborted, setting // the node state back to IN_SERVICE on the next run. nodeManager.setNodeStatus(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.IN_SERVICE, + new NodeStatus(IN_SERVICE, HddsProtos.NodeState.HEALTHY)); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -256,9 +261,7 @@ public void testDecommissionAbortedWhenNodeGoesDead() nodeManager.setContainers(dn1, generateContainers(3)); mockGetContainerReplicaCount( HddsProtos.LifeCycleState.CLOSED, - ContainerReplicaProto.State.DECOMMISSIONED, - ContainerReplicaProto.State.CLOSED, - ContainerReplicaProto.State.CLOSED); + DECOMMISSIONED, IN_SERVICE, IN_SERVICE); // Add the node to the monitor, it should have 3 under-replicated containers // after the first run @@ -277,7 +280,7 @@ public void testDecommissionAbortedWhenNodeGoesDead() HddsProtos.NodeState.DEAD)); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -286,7 +289,7 @@ public void testMaintenanceWaitsForMaintenanceToComplete() throws NodeNotFoundException { DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + new NodeStatus(ENTERING_MAINTENANCE, HddsProtos.NodeState.HEALTHY)); // Add the node to the monitor, it should transiting to @@ -308,7 +311,7 @@ public void testMaintenanceWaitsForMaintenanceToComplete() node.setMaintenanceEnd(-1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -317,7 +320,7 @@ public void testMaintenanceEndsClosingPipelines() throws NodeNotFoundException { DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + new NodeStatus(ENTERING_MAINTENANCE, HddsProtos.NodeState.HEALTHY)); // Ensure the node has some pipelines nodeManager.setPipelines(dn1, 2); @@ -333,7 +336,7 @@ public void testMaintenanceEndsClosingPipelines() node.setMaintenanceEnd(-1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -342,15 +345,15 @@ public void testMaintenanceEndsWhileReplicatingContainers() throws ContainerNotFoundException, NodeNotFoundException { DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + new NodeStatus(ENTERING_MAINTENANCE, HddsProtos.NodeState.HEALTHY)); nodeManager.setContainers(dn1, generateContainers(3)); mockGetContainerReplicaCount( HddsProtos.LifeCycleState.CLOSED, - ContainerReplicaProto.State.MAINTENANCE, - ContainerReplicaProto.State.MAINTENANCE, - ContainerReplicaProto.State.MAINTENANCE); + IN_MAINTENANCE, + ENTERING_MAINTENANCE, + IN_MAINTENANCE); // Add the node to the monitor, it should transiting to // REPLICATE_CONTAINERS as the containers are under-replicated for @@ -365,7 +368,7 @@ public void testMaintenanceEndsWhileReplicatingContainers() node.setMaintenanceEnd(-1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -374,7 +377,7 @@ public void testDeadMaintenanceNodeDoesNotAbortWorkflow() throws NodeNotFoundException { DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + new NodeStatus(ENTERING_MAINTENANCE, HddsProtos.NodeState.HEALTHY)); // Add the node to the monitor, it should transiting to @@ -402,7 +405,7 @@ public void testCancelledNodesMovedToInService() throws NodeNotFoundException { DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); nodeManager.register(dn1, - new NodeStatus(HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + new NodeStatus(ENTERING_MAINTENANCE, HddsProtos.NodeState.HEALTHY)); // Add the node to the monitor, it should transiting to @@ -418,7 +421,7 @@ public void testCancelledNodesMovedToInService() monitor.stopMonitoring(dn1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(HddsProtos.NodeOperationalState.IN_SERVICE, + assertEquals(IN_SERVICE, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -448,10 +451,10 @@ private Set generateContainers(int count) { */ private ContainerReplicaCount generateReplicaCount(ContainerID containerID, HddsProtos.LifeCycleState containerState, - ContainerReplicaProto.State... states) { + HddsProtos.NodeOperationalState...states) { Set replicas = new HashSet<>(); - for (ContainerReplicaProto.State s : states) { - replicas.add(generateReplica(containerID, s)); + for (HddsProtos.NodeOperationalState s : states) { + replicas.add(generateReplica(containerID, s, CLOSED)); } ContainerInfo container = new ContainerInfo.Builder() .setContainerID(containerID.getId()) @@ -464,16 +467,20 @@ private ContainerReplicaCount generateReplicaCount(ContainerID containerID, /** * Generate a new ContainerReplica with the given containerID and State. * @param containerID The ID the replica is associated with - * @param state The state of the generated replica. + * @param nodeState The persistedOpState stored in datanodeDetails. + * @param replicaState The state of the generated replica. * @return A containerReplica with the given ID and state */ private ContainerReplica generateReplica(ContainerID containerID, - ContainerReplicaProto.State state) { + HddsProtos.NodeOperationalState nodeState, + ContainerReplicaProto.State replicaState) { + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + dn.setPersistedOpState(nodeState); return ContainerReplica.newBuilder() - .setContainerState(state) + .setContainerState(replicaState) .setContainerID(containerID) .setSequenceId(1) - .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) + .setDatanodeDetails(dn) .build(); } @@ -500,7 +507,7 @@ private DatanodeAdminNodeDetails getFirstTrackedNode() { */ private void mockGetContainerReplicaCount( HddsProtos.LifeCycleState containerState, - ContainerReplicaProto.State... replicaStates) + HddsProtos.NodeOperationalState...replicaStates) throws ContainerNotFoundException { reset(repManager); Mockito.when(repManager.getContainerReplicaCount( From 2300b9f33aad21a422f435f3e5abfe72a51a6170 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 30 Sep 2020 15:13:02 +0100 Subject: [PATCH 19/78] Merge Master into HDDS-1880-Decom (additional fixes) * Fixing style issues and ReplicationManager failing tests * Fixed failing tests related to TestIncrementalContainerReportHandler and TestSCMNodeMetrics Closes #1461 --- .../scm/container/ReplicationManager.java | 62 ++++++++------- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 4 - .../hdds/scm/node/states/NodeStateMap.java | 2 +- .../hdds/scm/pipeline/PipelineProvider.java | 1 - ...ocationProtocolServerSideTranslatorPB.java | 40 +++++----- .../scm/container/TestReplicationManager.java | 3 +- .../container/TestUnknownContainerReport.java | 1 - .../hdds/scm/node/TestSCMNodeManager.java | 1 - .../ozone/scm/node/TestSCMNodeMetrics.java | 77 ++++++++++++++++--- .../apache/hadoop/ozone/om/OzoneManager.java | 15 +++- 10 files changed, 136 insertions(+), 70 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java index d8bd6a6b8ba1..d9b0378dbe3e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; @@ -280,10 +279,9 @@ private void processContainer(ContainerID id) { * the replicas are not in OPEN state, send CLOSE_CONTAINER command. */ if (state == LifeCycleState.OPEN) { - // TODO - fix this - //if (!isContainerHealthy(container, replicas)) { - // eventPublisher.fireEvent(SCMEvents.CLOSE_CONTAINER, id); - // } + if (!isOpenContainerHealthy(container, replicas)) { + eventPublisher.fireEvent(SCMEvents.CLOSE_CONTAINER, id); + } return; } @@ -325,13 +323,16 @@ private void processContainer(ContainerID id) { ContainerReplicaCount replicaSet = getContainerReplicaCount(container, replicas); + ContainerPlacementStatus placementStatus = getPlacementStatus( + replicas, container.getReplicationFactor().getNumber()); /* * Check if the container is under replicated and take appropriate * action. */ - if (!replicaSet.isSufficientlyReplicated()) { - handleUnderReplicatedContainer(container, replicaSet); + if (!replicaSet.isSufficientlyReplicated() + || !placementStatus.isPolicySatisfied()) { + handleUnderReplicatedContainer(container, replicaSet, placementStatus); return; } @@ -544,26 +545,21 @@ private void forceCloseContainer(final ContainerInfo container, * current replica count and inflight adds and deletes */ private void handleUnderReplicatedContainer(final ContainerInfo container, - final ContainerReplicaCount replicaSet) { + final ContainerReplicaCount replicaSet, + final ContainerPlacementStatus placementStatus) { LOG.debug("Handling under-replicated container: {}", - container.getContainerID()); Set replicas = replicaSet.getReplica(); try { - if (replicaSet.isSufficientlyReplicated()) { + if (replicaSet.isSufficientlyReplicated() + && placementStatus.isPolicySatisfied()) { LOG.info("The container {} with replicas {} is sufficiently "+ - "replicated", container.getContainerID(), replicaSet); - return; - } - int repDelta = replicaSet.additionalReplicaNeeded(); - if (repDelta <= 0) { - LOG.info("The container {} with {} is not sufficiently " + - "replicated but no further replicas will be scheduled until "+ - "in-flight operations complete", + "replicated and is not mis-replicated", container.getContainerID(), replicaSet); return; } + int repDelta = replicaSet.additionalReplicaNeeded(); final ContainerID id = container.containerID(); final List deletionInFlight = inflightDeletion .getOrDefault(id, Collections.emptyList()) @@ -597,10 +593,10 @@ private void handleUnderReplicatedContainer(final ContainerInfo container, List targetReplicas = new ArrayList<>(source); // Then add any pending additions targetReplicas.addAll(replicationInFlight); - final ContainerPlacementStatus placementStatus = + final ContainerPlacementStatus inFlightplacementStatus = containerPlacement.validateContainerPlacement( targetReplicas, replicationFactor); - final int misRepDelta = placementStatus.misReplicationCount(); + final int misRepDelta = inFlightplacementStatus.misReplicationCount(); final int replicasNeeded = repDelta < misRepDelta ? misRepDelta : repDelta; if (replicasNeeded <= 0) { @@ -689,13 +685,13 @@ private void handleOverReplicatedContainer(final ContainerInfo container, .putIfAbsent(r.getOriginDatanodeId(), r)); eligibleReplicas.removeAll(uniqueReplicas.values()); - // Replica which are maintenance or decommissioned are not eligible to - // be removed, as they do not count toward over-replication and they - // also many not be available - eligibleReplicas.removeIf(r -> - r.getDatanodeDetails().getPersistedOpState() != - HddsProtos.NodeOperationalState.IN_SERVICE); } + // Replica which are maintenance or decommissioned are not eligible to + // be removed, as they do not count toward over-replication and they + // also many not be available + eligibleReplicas.removeIf(r -> + r.getDatanodeDetails().getPersistedOpState() != + HddsProtos.NodeOperationalState.IN_SERVICE); final List unhealthyReplicas = eligibleReplicas .stream() @@ -960,6 +956,20 @@ public static boolean compareState(final LifeCycleState containerState, } } + /** + * An open container is healthy if all its replicas are in the same state as + * the container. + * @param container The container to check + * @param replicas The replicas belonging to the container + * @return True if the container is healthy, false otherwise + */ + private boolean isOpenContainerHealthy( + ContainerInfo container, Set replicas) { + LifeCycleState state = container.getState(); + return replicas.stream() + .allMatch(r -> ReplicationManager.compareState(state, r.getState())); + } + @Override public void getMetrics(MetricsCollector collector, boolean all) { collector.addRecord(ReplicationManager.class.getSimpleName()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java index 14bafe9694e1..5ea4d911a9e1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeMetrics.java @@ -35,10 +35,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.util.StringUtils; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE; - /** * This class maintains Node related metrics. */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java index 3cf232af392b..d44f6b798af7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java @@ -75,7 +75,7 @@ public void addNode(DatanodeDetails datanodeDetails, NodeStatus nodeStatus) throw new NodeAlreadyExistsException("Node UUID: " + id); } nodeMap.put(id, new DatanodeInfo(datanodeDetails, nodeStatus)); - nodeToContainer.put(id, Collections.emptySet()); + nodeToContainer.put(id, new HashSet<>()); } finally { lock.writeLock().unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java index 6b704f33255e..8df976ca383a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java @@ -25,7 +25,6 @@ import java.util.stream.Collectors; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.scm.exceptions.SCMException; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index b2646dcf36ac..f4ee6d133200 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -277,27 +277,27 @@ public ScmContainerLocationResponse processRequest( .setGetSafeModeRuleStatusesResponse(getSafeModeRuleStatues( request.getGetSafeModeRuleStatusesRequest())) .build(); - case DecommissionNodes: - return ScmContainerLocationResponse.newBuilder() - .setCmdType(request.getCmdType()) - .setStatus(Status.OK) - .setDecommissionNodesResponse(decommissionNodes( - request.getDecommissionNodesRequest())) - .build(); - case RecommissionNodes: - return ScmContainerLocationResponse.newBuilder() - .setCmdType(request.getCmdType()) - .setStatus(Status.OK) - .setRecommissionNodesResponse(recommissionNodes( - request.getRecommissionNodesRequest())) - .build(); - case StartMaintenanceNodes: - return ScmContainerLocationResponse.newBuilder() - .setCmdType(request.getCmdType()) - .setStatus(Status.OK) - .setStartMaintenanceNodesResponse(startMaintenanceNodes( - request.getStartMaintenanceNodesRequest())) + case DecommissionNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setDecommissionNodesResponse(decommissionNodes( + request.getDecommissionNodesRequest())) + .build(); + case RecommissionNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setRecommissionNodesResponse(recommissionNodes( + request.getRecommissionNodesRequest())) .build(); + case StartMaintenanceNodes: + return ScmContainerLocationResponse.newBuilder() + .setCmdType(request.getCmdType()) + .setStatus(Status.OK) + .setStartMaintenanceNodesResponse(startMaintenanceNodes( + request.getStartMaintenanceNodesRequest())) + .build(); default: throw new IllegalArgumentException( "Unknown command type: " + request.getCmdType()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java index 81aa3057e94f..887850f92480 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java @@ -89,7 +89,8 @@ public class TestReplicationManager { private SCMNodeManager scmNodeManager; @Before - public void setup() throws IOException, InterruptedException, NodeNotFoundException { + public void setup() + throws IOException, InterruptedException, NodeNotFoundException { conf = new OzoneConfiguration(); containerManager = Mockito.mock(ContainerManager.class); nodeManager = new SimpleMockNodeManager(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java index be3e2cb6a155..f2e496820562 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestUnknownContainerReport.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index e3e1cefc55de..31cd6e2dfb66 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java index 7576e8babd45..fd652837755a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/scm/node/TestSCMNodeMetrics.java @@ -172,17 +172,72 @@ public void testNodeCountAndInfoMetricsReported() throws Exception { MetricsRecordBuilder metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME); - assertGauge("HealthyNodes", 1, metricsSource); - assertGauge("StaleNodes", 0, metricsSource); - assertGauge("DeadNodes", 0, metricsSource); - assertGauge("DecommissioningNodes", 0, metricsSource); - assertGauge("DecommissionedNodes", 0, metricsSource); - assertGauge("DiskCapacity", 100L, metricsSource); - assertGauge("DiskUsed", 10L, metricsSource); - assertGauge("DiskRemaining", 90L, metricsSource); - assertGauge("SSDCapacity", 0L, metricsSource); - assertGauge("SSDUsed", 0L, metricsSource); - assertGauge("SSDRemaining", 0L, metricsSource); + assertGauge("InServiceHealthyNodes", 1, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InServiceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InServiceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissioningDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("EnteringMaintenanceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceHealthyNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceStaleNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("InMaintenanceDeadNodes", 0, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DiskCapacity", 100L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DiskUsed", 10L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DiskRemaining", 90L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("SSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("SSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("SSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("MaintenanceSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedDiskRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDCapacity", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDUsed", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); + assertGauge("DecommissionedSSDRemaining", 0L, + getMetrics(SCMNodeMetrics.class.getSimpleName())); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 99dc2028968d..fb7a23f8c67a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -47,7 +47,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import com.google.common.base.Optional; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; @@ -141,6 +140,13 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.snapshot.OzoneManagerSnapshotProvider; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserVolumeInfo; import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB; import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager; @@ -168,16 +174,18 @@ import org.apache.hadoop.util.KMSUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.Time; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; import com.google.protobuf.ProtocolMessageEnum; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; - import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients; @@ -219,8 +227,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; - -import org.apache.hadoop.util.Time; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService; import org.apache.ratis.proto.RaftProtos.RaftPeerRole; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.ExitUtils; From f43a370169f2d8cc2b8635ac9e026278157b16db Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Mon, 5 Oct 2020 14:42:41 +0100 Subject: [PATCH 20/78] HDDS-4300. Removed unneeded class DatanodeAdminNodeDetails (#1465) --- .../scm/node/DatanodeAdminMonitorImpl.java | 105 +++++++------- .../scm/node/DatanodeAdminNodeDetails.java | 137 ------------------ .../scm/node/TestDatanodeAdminMonitor.java | 43 +++--- .../node/TestDatanodeAdminNodeDetails.java | 81 ----------- 4 files changed, 67 insertions(+), 299 deletions(-) delete mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java delete mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index f9d1a3216de5..0bbd13d6fa86 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -64,9 +64,9 @@ public class DatanodeAdminMonitorImpl implements DatanodeAdminMonitor { private EventPublisher eventQueue; private NodeManager nodeManager; private ReplicationManager replicationManager; - private Queue pendingNodes = new ArrayDeque(); - private Queue cancelledNodes = new ArrayDeque(); - private Set trackedNodes = new HashSet<>(); + private Queue pendingNodes = new ArrayDeque(); + private Queue cancelledNodes = new ArrayDeque(); + private Set trackedNodes = new HashSet<>(); private static final Logger LOG = LoggerFactory.getLogger(DatanodeAdminMonitorImpl.class); @@ -93,10 +93,8 @@ public DatanodeAdminMonitorImpl( */ @Override public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { - DatanodeAdminNodeDetails nodeDetails = - new DatanodeAdminNodeDetails(dn, endInHours); - cancelledNodes.remove(nodeDetails); - pendingNodes.add(nodeDetails); + cancelledNodes.remove(dn); + pendingNodes.add(dn); } /** @@ -108,9 +106,8 @@ public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { */ @Override public synchronized void stopMonitoring(DatanodeDetails dn) { - DatanodeAdminNodeDetails nodeDetails = new DatanodeAdminNodeDetails(dn, 0); - pendingNodes.remove(nodeDetails); - cancelledNodes.add(nodeDetails); + pendingNodes.remove(dn); + cancelledNodes.add(dn); } /** @@ -155,20 +152,19 @@ public int getTrackedNodeCount() { } @VisibleForTesting - public Set getTrackedNodes() { + public Set getTrackedNodes() { return trackedNodes; } private void processCancelledNodes() { while (!cancelledNodes.isEmpty()) { - DatanodeAdminNodeDetails dn = cancelledNodes.poll(); + DatanodeDetails dn = cancelledNodes.poll(); try { stopTrackingNode(dn); putNodeBackInService(dn); - LOG.info("Recommissioned node {}", dn.getDatanodeDetails()); + LOG.info("Recommissioned node {}", dn); } catch (NodeNotFoundException e) { - LOG.warn("Failed processing the cancel admin request for {}", - dn.getDatanodeDetails(), e); + LOG.warn("Failed processing the cancel admin request for {}", dn, e); } } } @@ -180,11 +176,11 @@ private void processPendingNodes() { } private void processTransitioningNodes() { - Iterator iterator = trackedNodes.iterator(); + Iterator iterator = trackedNodes.iterator(); while (iterator.hasNext()) { - DatanodeAdminNodeDetails dn = iterator.next(); + DatanodeDetails dn = iterator.next(); try { - NodeStatus status = getNodeStatus(dn.getDatanodeDetails()); + NodeStatus status = getNodeStatus(dn); if (!shouldContinueWorkflow(dn, status)) { abortWorkflow(dn); @@ -193,7 +189,7 @@ private void processTransitioningNodes() { } if (status.isMaintenance()) { - if (dn.shouldMaintenanceEnd()) { + if (status.operationalStateExpired()) { completeMaintenance(dn); iterator.remove(); continue; @@ -205,12 +201,12 @@ private void processTransitioningNodes() { // Ensure the DN has received and persisted the current maint // state. && status.getOperationalState() - == dn.getDatanodeDetails().getPersistedOpState() + == dn.getPersistedOpState() && checkContainersReplicatedOnNode(dn)) { // CheckContainersReplicatedOnNode may take a short time to run // so after it completes, re-get the nodestatus to check the health // and ensure the state is still good to continue - status = getNodeStatus(dn.getDatanodeDetails()); + status = getNodeStatus(dn); if (status.isDead()) { LOG.warn("Datanode {} is dead and the admin workflow cannot " + "continue. The node will be put back to IN_SERVICE and " + @@ -228,7 +224,7 @@ && checkContainersReplicatedOnNode(dn)) { } catch (NodeNotFoundException e) { LOG.error("An unexpected error occurred processing datanode {}. " + - "Aborting the admin workflow", dn.getDatanodeDetails(), e); + "Aborting the admin workflow", dn, e); abortWorkflow(dn); iterator.remove(); } @@ -244,43 +240,43 @@ && checkContainersReplicatedOnNode(dn)) { * @param nodeStatus The current NodeStatus for the datanode * @return True if admin can continue, false otherwise */ - private boolean shouldContinueWorkflow(DatanodeAdminNodeDetails dn, + private boolean shouldContinueWorkflow(DatanodeDetails dn, NodeStatus nodeStatus) { if (!nodeStatus.isDecommission() && !nodeStatus.isMaintenance()) { LOG.warn("Datanode {} has an operational state of {} when it should " + "be undergoing decommission or maintenance. Aborting admin for " + - "this node.", - dn.getDatanodeDetails(), nodeStatus.getOperationalState()); + "this node.", dn, nodeStatus.getOperationalState()); return false; } if (nodeStatus.isDead() && !nodeStatus.isInMaintenance()) { LOG.error("Datanode {} is dead but is not IN_MAINTENANCE. Aborting the " + - "admin workflow for this node", dn.getDatanodeDetails()); + "admin workflow for this node", dn); return false; } return true; } - private boolean checkPipelinesClosedOnNode(DatanodeAdminNodeDetails dn) { - DatanodeDetails dnd = dn.getDatanodeDetails(); - Set pipelines = nodeManager.getPipelines(dnd); + private boolean checkPipelinesClosedOnNode(DatanodeDetails dn) + throws NodeNotFoundException { + Set pipelines = nodeManager.getPipelines(dn); + NodeStatus status = nodeManager.getNodeStatus(dn); if (pipelines == null || pipelines.size() == 0 - || dn.shouldMaintenanceEnd()) { + || status.operationalStateExpired()) { return true; } else { LOG.info("Waiting for pipelines to close for {}. There are {} " + - "pipelines", dnd, pipelines.size()); + "pipelines", dn, pipelines.size()); return false; } } - private boolean checkContainersReplicatedOnNode(DatanodeAdminNodeDetails dn) + private boolean checkContainersReplicatedOnNode(DatanodeDetails dn) throws NodeNotFoundException { int sufficientlyReplicated = 0; int underReplicated = 0; int unhealthy = 0; Set containers = - nodeManager.getContainers(dn.getDatanodeDetails()); + nodeManager.getContainers(dn); for (ContainerID cid : containers) { try { ContainerReplicaCount replicaSet = @@ -295,46 +291,40 @@ private boolean checkContainersReplicatedOnNode(DatanodeAdminNodeDetails dn) } } catch (ContainerNotFoundException e) { LOG.warn("ContainerID {} present in node list for {} but not found " + - "in containerManager", cid, dn.getDatanodeDetails()); + "in containerManager", cid, dn); } } - dn.setSufficientlyReplicatedContainers(sufficientlyReplicated); - dn.setUnderReplicatedContainers(underReplicated); - dn.setUnHealthyContainers(unhealthy); - return underReplicated == 0 && unhealthy == 0; } - private void completeDecommission(DatanodeAdminNodeDetails dn) + private void completeDecommission(DatanodeDetails dn) throws NodeNotFoundException { setNodeOpState(dn, NodeOperationalState.DECOMMISSIONED); LOG.info("Datanode {} has completed the admin workflow. The operational " + - "state has been set to {}", dn.getDatanodeDetails(), + "state has been set to {}", dn, NodeOperationalState.DECOMMISSIONED); } - private void putIntoMaintenance(DatanodeAdminNodeDetails dn) + private void putIntoMaintenance(DatanodeDetails dn) throws NodeNotFoundException { - LOG.info("Datanode {} has entered maintenance", dn.getDatanodeDetails()); + LOG.info("Datanode {} has entered maintenance", dn); setNodeOpState(dn, NodeOperationalState.IN_MAINTENANCE); } - private void completeMaintenance(DatanodeAdminNodeDetails dn) + private void completeMaintenance(DatanodeDetails dn) throws NodeNotFoundException { // The end state of Maintenance is to put the node back IN_SERVICE, whether // it is dead or not. - LOG.info("Datanode {} has ended maintenance automatically", - dn.getDatanodeDetails()); + LOG.info("Datanode {} has ended maintenance automatically", dn); putNodeBackInService(dn); } - private void startTrackingNode(DatanodeAdminNodeDetails dn) { - eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, - dn.getDatanodeDetails()); + private void startTrackingNode(DatanodeDetails dn) { + eventQueue.fireEvent(SCMEvents.START_ADMIN_ON_NODE, dn); trackedNodes.add(dn); } - private void stopTrackingNode(DatanodeAdminNodeDetails dn) { + private void stopTrackingNode(DatanodeDetails dn) { trackedNodes.remove(dn); } @@ -345,24 +335,29 @@ private void stopTrackingNode(DatanodeAdminNodeDetails dn) { * * @param dn The datanode for which to abort tracking */ - private void abortWorkflow(DatanodeAdminNodeDetails dn) { + private void abortWorkflow(DatanodeDetails dn) { try { putNodeBackInService(dn); } catch (NodeNotFoundException e) { LOG.error("Unable to set the node OperationalState for {} while " + - "aborting the datanode admin workflow", dn.getDatanodeDetails()); + "aborting the datanode admin workflow", dn); } } - private void putNodeBackInService(DatanodeAdminNodeDetails dn) + private void putNodeBackInService(DatanodeDetails dn) throws NodeNotFoundException { setNodeOpState(dn, NodeOperationalState.IN_SERVICE); } - private void setNodeOpState(DatanodeAdminNodeDetails dn, + private void setNodeOpState(DatanodeDetails dn, HddsProtos.NodeOperationalState state) throws NodeNotFoundException { - nodeManager.setNodeOperationalState(dn.getDatanodeDetails(), state, - dn.getMaintenanceEnd() / 1000); + long expiry = 0; + if ((state == NodeOperationalState.IN_MAINTENANCE) + || (state == NodeOperationalState.ENTERING_MAINTENANCE)) { + NodeStatus status = nodeManager.getNodeStatus(dn); + expiry = status.getOpStateExpiryEpochSeconds(); + } + nodeManager.setNodeOperationalState(dn, state, expiry); } private NodeStatus getNodeStatus(DatanodeDetails dnd) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java deleted file mode 100644 index 9c8a905bb996..000000000000 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminNodeDetails.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdds.scm.node; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is used by the DatanodeAdminMonitor to track the state and - * details for Datanode decommission and maintenance. It provides a wrapper - * around a DatanodeDetails object adding some additional states and helper - * methods related to the admin workflow. - */ -public class DatanodeAdminNodeDetails { - private DatanodeDetails datanodeDetails; - private long maintenanceEndTime; - private long enteredStateAt = 0; - private int unHealthyContainers = 0; - private int underReplicatedContainers = 0; - private int sufficientlyReplicatedContainers = 0; - - private static final Logger LOG = - LoggerFactory.getLogger(DatanodeAdminNodeDetails.class); - - - /** - * Create a new object given the DatanodeDetails and the maintenance endtime. - * @param dn The datanode going through the admin workflow - * @param maintenanceEnd The number of hours from 'now', when maintenance - * should end automatically. Passing zero indicates - * indicates maintenance will never end automatically. - */ - DatanodeAdminNodeDetails(DatanodeDetails dn, long maintenanceEnd) { - datanodeDetails = dn; - setMaintenanceEnd(maintenanceEnd); - enteredStateAt = System.currentTimeMillis(); - } - - public boolean shouldMaintenanceEnd() { - if (0 == maintenanceEndTime) { - return false; - } - return System.currentTimeMillis() >= maintenanceEndTime; - } - - public DatanodeDetails getDatanodeDetails() { - return datanodeDetails; - } - - public void setUnHealthyContainers(int val) { - this.unHealthyContainers = val; - } - - public void setUnderReplicatedContainers(int val) { - this.underReplicatedContainers = val; - } - - public void setSufficientlyReplicatedContainers(int val) { - this.sufficientlyReplicatedContainers = val; - } - - public int getUnHealthyContainers() { - return unHealthyContainers; - } - - public int getUnderReplicatedContainers() { - return underReplicatedContainers; - } - - public int getSufficientlyReplicatedContainers() { - return sufficientlyReplicatedContainers; - } - - /** - * Set the number of hours after which maintenance should end. Passing zero - * indicates maintenance will never end automatically. It is possible to pass - * a negative number of hours can be passed for testing purposes. - * @param hoursFromNow The number of hours from now when maintenance should - * end, or zero for it to never end. - */ - @VisibleForTesting - public void setMaintenanceEnd(long hoursFromNow) { - if (0 == hoursFromNow) { - maintenanceEndTime = 0; - return; - } - // Convert hours to ms - long msFromNow = hoursFromNow * 60L * 60L * 1000L; - maintenanceEndTime = System.currentTimeMillis() + msFromNow; - } - - /** - * Returns the maintenance end time as milli seconds from the epoch. - * @return The maintenance end time, or zero if no end time is set. - */ - public long getMaintenanceEnd() { - return maintenanceEndTime; - } - - /** - * Matches only on the DatanodeDetails field, which compares only the UUID - * of the node to determine of they are the same object or not. - * - * @param o The object to compare this with - * @return True if the object match, otherwise false - * - */ - @Override - public boolean equals(Object o) { - return o instanceof DatanodeAdminNodeDetails && - datanodeDetails.equals( - ((DatanodeAdminNodeDetails) o).getDatanodeDetails()); - } - - @Override - public int hashCode() { - return datanodeDetails.hashCode(); - } - -} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index 0592f94a6105..e0f23ae0909f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -181,7 +181,7 @@ public void testDecommissionNodeWaitsForContainersToReplicate() // REPLICATE_CONTAINERS as there are no pipelines to close. monitor.startMonitoring(dn1, 0); monitor.run(); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertEquals(1, monitor.getTrackedNodeCount()); assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, nodeManager.getNodeStatus(dn1).getOperationalState()); @@ -191,9 +191,6 @@ public void testDecommissionNodeWaitsForContainersToReplicate() monitor.run(); assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, nodeManager.getNodeStatus(dn1).getOperationalState()); - assertEquals(0, node.getSufficientlyReplicatedContainers()); - assertEquals(0, node.getUnHealthyContainers()); - assertEquals(3, node.getUnderReplicatedContainers()); // Now change the replicationManager mock to return 3 CLOSED replicas // and the node should complete the REPLICATE_CONTAINERS step, moving to @@ -207,10 +204,7 @@ public void testDecommissionNodeWaitsForContainersToReplicate() monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); - assertEquals(3, node.getSufficientlyReplicatedContainers()); - assertEquals(0, node.getUnHealthyContainers()); - assertEquals(0, node.getUnderReplicatedContainers()); - assertEquals(DECOMMISSIONED, + assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONED, nodeManager.getNodeStatus(dn1).getOperationalState()); } @@ -234,10 +228,9 @@ public void testDecommissionAbortedWhenNodeInUnexpectedState() monitor.startMonitoring(dn1, 0); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, nodeManager.getNodeStatus(dn1).getOperationalState()); - assertEquals(3, node.getUnderReplicatedContainers()); // Set the node to dead, and then the workflow should get aborted, setting // the node state back to IN_SERVICE on the next run. @@ -268,10 +261,9 @@ public void testDecommissionAbortedWhenNodeGoesDead() monitor.startMonitoring(dn1, 0); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertEquals(HddsProtos.NodeOperationalState.DECOMMISSIONING, nodeManager.getNodeStatus(dn1).getOperationalState()); - assertEquals(3, node.getUnderReplicatedContainers()); // Set the node to dead, and then the workflow should get aborted, setting // the node state back to IN_SERVICE. @@ -297,8 +289,7 @@ public void testMaintenanceWaitsForMaintenanceToComplete() monitor.startMonitoring(dn1, 1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); - assertEquals(0, node.getUnderReplicatedContainers()); + DatanodeDetails node = getFirstTrackedNode(); assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); // Running the monitor again causes the node to remain in maintenance @@ -308,7 +299,8 @@ public void testMaintenanceWaitsForMaintenanceToComplete() // Set the maintenance end time to a time in the past and then the node // should complete the workflow and transition to IN_SERVICE - node.setMaintenanceEnd(-1); + nodeManager.setNodeOperationalState(node, + HddsProtos.NodeOperationalState.IN_MAINTENANCE, -1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); assertEquals(IN_SERVICE, @@ -327,13 +319,14 @@ public void testMaintenanceEndsClosingPipelines() // Add the node to the monitor monitor.startMonitoring(dn1, 1); monitor.run(); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertEquals(1, monitor.getTrackedNodeCount()); assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance()); // Set the maintenance end time to the past and the node should complete // the workflow and return to IN_SERVICE - node.setMaintenanceEnd(-1); + nodeManager.setNodeOperationalState(node, + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, -1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); assertEquals(IN_SERVICE, @@ -361,11 +354,11 @@ public void testMaintenanceEndsWhileReplicatingContainers() monitor.startMonitoring(dn1, 1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertTrue(nodeManager.getNodeStatus(dn1).isEnteringMaintenance()); - assertEquals(3, node.getUnderReplicatedContainers()); - node.setMaintenanceEnd(-1); + nodeManager.setNodeOperationalState(node, + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, -1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); assertEquals(IN_SERVICE, @@ -385,9 +378,8 @@ public void testDeadMaintenanceNodeDoesNotAbortWorkflow() monitor.startMonitoring(dn1, 1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); - assertEquals(0, node.getUnderReplicatedContainers()); // Set the node dead and ensure the workflow does not end NodeStatus status = nodeManager.getNodeStatus(dn1); @@ -413,9 +405,8 @@ public void testCancelledNodesMovedToInService() monitor.startMonitoring(dn1, 1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); - DatanodeAdminNodeDetails node = getFirstTrackedNode(); + DatanodeDetails node = getFirstTrackedNode(); assertTrue(nodeManager.getNodeStatus(dn1).isInMaintenance()); - assertEquals(0, node.getUnderReplicatedContainers()); // Now cancel the node and run the monitor, the node should be IN_SERVICE monitor.stopMonitoring(dn1); @@ -489,9 +480,9 @@ private ContainerReplica generateReplica(ContainerID containerID, * the monitor. * @return DatanodeAdminNodeDetails for the first tracked node found. */ - private DatanodeAdminNodeDetails getFirstTrackedNode() { + private DatanodeDetails getFirstTrackedNode() { return - monitor.getTrackedNodes().toArray(new DatanodeAdminNodeDetails[0])[0]; + monitor.getTrackedNodes().toArray(new DatanodeDetails[0])[0]; } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java deleted file mode 100644 index 7a813a9bc1d4..000000000000 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminNodeDetails.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdds.scm.node; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static junit.framework.TestCase.*; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; - -/** - * Tests to validate the DatanodeAdminNodeDetails class. - */ -public class TestDatanodeAdminNodeDetails { - - private OzoneConfiguration conf; - - @Before - public void setup() { - conf = new OzoneConfiguration(); - } - - @After - public void teardown() { - } - - @Test - public void testEqualityBasedOnDatanodeDetails() { - DatanodeDetails dn1 = MockDatanodeDetails.randomDatanodeDetails(); - DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails(); - DatanodeAdminNodeDetails details1 = - new DatanodeAdminNodeDetails(dn1, 0); - DatanodeAdminNodeDetails details2 = - new DatanodeAdminNodeDetails(dn2, 0); - - assertNotEquals(details1, details2); - assertEquals(details1, - new DatanodeAdminNodeDetails(dn1, 0)); - assertNotEquals(details1, dn1); - } - - - - @Test - public void testMaintenanceEnd() { - DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); - // End in zero hours - should never end. - DatanodeAdminNodeDetails details = new DatanodeAdminNodeDetails(dn, 0); - assertFalse(details.shouldMaintenanceEnd()); - - // End 1 hour - maintenance should not end yet. - details.setMaintenanceEnd(1); - assertFalse(details.shouldMaintenanceEnd()); - - // End 1 hour ago - maintenance should end. - details.setMaintenanceEnd(-1); - assertTrue(details.shouldMaintenanceEnd()); - } - -} From f74d73ada0db79a3a14df89d9d6474ef39465424 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 14 Oct 2020 09:03:58 +0100 Subject: [PATCH 21/78] HDDS-4322. Add integration tests for Decommission and resolve issues detected by the tests. (#1484) --- .../hadoop/hdds/protocol/DatanodeDetails.java | 3 + .../scm/node/DatanodeAdminMonitorImpl.java | 3 + .../hadoop/hdds/scm/node/NewNodeHandler.java | 22 ++ .../scm/node/NodeDecommissionManager.java | 16 + .../hdds/scm/node/NodeStateManager.java | 25 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 25 -- .../scm/server/StorageContainerManager.java | 7 +- .../org/apache/hadoop/ozone/TestDataUtil.java | 11 +- .../node/TestDecommissionAndMaintenance.java | 359 ++++++++++++++++-- 9 files changed, 401 insertions(+), 70 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index c60eb1c0080a..c7d5a286c065 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -476,6 +476,9 @@ public Builder setDatanodeDetails(DatanodeDetails details) { this.setupTime = details.getSetupTime(); this.revision = details.getRevision(); this.buildDate = details.getBuildDate(); + this.persistedOpState = details.getPersistedOpState(); + this.persistedOpStateExpiryEpochSec = + details.getPersistedOpStateExpiryEpochSec(); return this; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index 0bbd13d6fa86..cf01af55c518 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -294,6 +294,9 @@ private boolean checkContainersReplicatedOnNode(DatanodeDetails dn) "in containerManager", cid, dn); } } + LOG.info("{} has {} sufficientlyReplicated, {} underReplicated and {} " + + "unhealthy containers", + dn, sufficientlyReplicated, underReplicated, unhealthy); return underReplicated == 0 && unhealthy == 0; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java index a40a63a1dc7e..42369fdac80c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java @@ -20,9 +20,13 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Handles New Node event. @@ -30,11 +34,16 @@ public class NewNodeHandler implements EventHandler { private final PipelineManager pipelineManager; + private final NodeDecommissionManager decommissionManager; private final ConfigurationSource conf; + private static final Logger LOG = + LoggerFactory.getLogger(NewNodeHandler.class); public NewNodeHandler(PipelineManager pipelineManager, + NodeDecommissionManager decommissionManager, ConfigurationSource conf) { this.pipelineManager = pipelineManager; + this.decommissionManager = decommissionManager; this.conf = conf; } @@ -42,5 +51,18 @@ public NewNodeHandler(PipelineManager pipelineManager, public void onMessage(DatanodeDetails datanodeDetails, EventPublisher publisher) { pipelineManager.triggerPipelineCreation(); + HddsProtos.NodeOperationalState opState + = datanodeDetails.getPersistedOpState(); + if (datanodeDetails.getPersistedOpState() + != HddsProtos.NodeOperationalState.IN_SERVICE) { + try { + decommissionManager.continueAdminForNode(datanodeDetails); + } catch (NodeNotFoundException e) { + // Should not happen, as the node has just registered to call this event + // handler. + LOG.warn("NodeNotFound when adding the node to the decommissionManager", + e); + } + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index 3258fefa8fd7..d7432bc4d32c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -233,6 +233,22 @@ public synchronized void decommissionNodes(List nodes) } } + /** + * If a SCM is restarted, then upon re-registration the datanode will already + * be in DECOMMISSIONING or ENTERING_MAINTENANCE state. In that case, it + * needs to be added back into the monitor to track its progress. + * @param dn Datanode to add back to tracking. + * @throws NodeNotFoundException + */ + public synchronized void continueAdminForNode(DatanodeDetails dn) + throws NodeNotFoundException { + NodeOperationalState opState = getNodeStatus(dn).getOperationalState(); + if (opState == NodeOperationalState.DECOMMISSIONING + || opState == NodeOperationalState.ENTERING_MAINTENANCE) { + monitor.startMonitoring(dn, 0); + } + } + public synchronized void startDecommission(DatanodeDetails dn) throws NodeNotFoundException, InvalidNodeStateException { NodeStatus nodeStatus = getNodeStatus(dn); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index 47ce56eec0de..d7af7f5da377 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@ -239,11 +239,32 @@ private void initializeStateMachines() { */ public void addNode(DatanodeDetails datanodeDetails) throws NodeAlreadyExistsException { - nodeStateMap.addNode(datanodeDetails, new NodeStatus( - NodeOperationalState.IN_SERVICE, nodeHealthSM.getInitialState())); + NodeStatus newNodeStatus = newNodeStatus(datanodeDetails); + nodeStateMap.addNode(datanodeDetails, newNodeStatus); eventPublisher.fireEvent(SCMEvents.NEW_NODE, datanodeDetails); } + /** + * When a node registers with SCM, the operational state stored on the + * datanode is the source of truth. Therefore, if the datanode reports + * anything other than IN_SERVICE on registration, the state in SCM should be + * updated to reflect the datanode state. + * @param dn DatanodeDetails reported by the datanode + */ + private NodeStatus newNodeStatus(DatanodeDetails dn) { + HddsProtos.NodeOperationalState dnOpState = dn.getPersistedOpState(); + if (dnOpState != NodeOperationalState.IN_SERVICE) { + LOG.info("Updating nodeOperationalState on registration as the " + + "datanode has a persisted state of {} and expiry of {}", + dnOpState, dn.getPersistedOpStateExpiryEpochSec()); + return new NodeStatus(dnOpState, nodeHealthSM.getInitialState(), + dn.getPersistedOpStateExpiryEpochSec()); + } else { + return new NodeStatus( + NodeOperationalState.IN_SERVICE, nodeHealthSM.getInitialState()); + } + } + /** * Adds a pipeline in the node2PipelineMap. * @param pipeline - Pipeline to be added diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 825593d614e5..323d79a64871 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; @@ -350,7 +349,6 @@ public RegisteredCommand register( datanodeDetails.toString()); } } - registerInitialDatanodeOpState(datanodeDetails); return RegisteredCommand.newBuilder().setErrorCode(ErrorCode.success) .setDatanode(datanodeDetails) @@ -358,29 +356,6 @@ public RegisteredCommand register( .build(); } - /** - * When a node registers with SCM, the operational state stored on the - * datanode is the source of truth. Therefore, if the datanode reports - * anything other than IN_SERVICE on registration, the state in SCM should be - * updated to reflect the datanode state. - * @param dn - */ - private void registerInitialDatanodeOpState(DatanodeDetails dn) { - try { - HddsProtos.NodeOperationalState dnOpState = dn.getPersistedOpState(); - if (dnOpState != NodeOperationalState.IN_SERVICE) { - LOG.info("Updating nodeOperationalState on registration as the " + - "datanode has a persisted state of {} and expiry of {}", - dnOpState, dn.getPersistedOpStateExpiryEpochSec()); - setNodeOperationalState(dn, dnOpState, - dn.getPersistedOpStateExpiryEpochSec()); - } - } catch (NodeNotFoundException e) { - LOG.error("Unable to find the node when setting the operational state", - e); - } - } - /** * Add an entry to the dnsToUuidMap, which maps hostname / IP to the DNs * running on that host. As each address can have many DNs running on it, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 0319845a370e..e26d52138eb3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -296,7 +296,8 @@ public StorageContainerManager(OzoneConfiguration conf, CommandStatusReportHandler cmdStatusReportHandler = new CommandStatusReportHandler(); - NewNodeHandler newNodeHandler = new NewNodeHandler(pipelineManager, conf); + NewNodeHandler newNodeHandler = new NewNodeHandler(pipelineManager, + scmDecommissionManager, conf); StaleNodeHandler staleNodeHandler = new StaleNodeHandler(scmNodeManager, pipelineManager, conf); DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager, @@ -339,8 +340,6 @@ public StorageContainerManager(OzoneConfiguration conf, blockProtocolServer = new SCMBlockProtocolServer(conf, this); clientProtocolServer = new SCMClientProtocolServer(conf, this); httpServer = new StorageContainerManagerHttpServer(conf); - scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, - containerManager, eventQueue, replicationManager); eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager); eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler); @@ -456,6 +455,8 @@ private void initializeSystemManagers(OzoneConfiguration conf, scmSafeModeManager = new SCMSafeModeManager(conf, containerManager.getContainers(), pipelineManager, eventQueue); } + scmDecommissionManager = new NodeDecommissionManager(conf, scmNodeManager, + containerManager, eventQueue, replicationManager); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java index 115336e2ad82..4cdefb8e8ed8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java @@ -73,9 +73,16 @@ public static OzoneBucket createVolumeAndBucket(MiniOzoneCluster cluster, public static void createKey(OzoneBucket bucket, String keyName, String content) throws IOException { + createKey(bucket, keyName, ReplicationFactor.ONE, + ReplicationType.STAND_ALONE, content); + } + + public static void createKey(OzoneBucket bucket, String keyName, + ReplicationFactor repFactor, ReplicationType repType, String content) + throws IOException { try (OutputStream stream = bucket - .createKey(keyName, content.length(), ReplicationType.STAND_ALONE, - ReplicationFactor.ONE, new HashMap<>())) { + .createKey(keyName, content.length(), repType, repFactor, + new HashMap<>())) { stream.write(content.getBytes()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index f39af441e3c2..8f02878f0ba8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -17,28 +17,59 @@ */ package org.apache.hadoop.ozone.scm.node; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Duration; import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static java.util.concurrent.TimeUnit.SECONDS; import static junit.framework.TestCase.assertEquals; -import static org.apache.hadoop.hdds.HddsConfigKeys.*; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; +import static org.junit.Assert.fail; /** * Test from the scmclient for decommission and maintenance. @@ -48,8 +79,15 @@ public class TestDecommissionAndMaintenance { private static final Logger LOG = LoggerFactory.getLogger(TestDecommissionAndMaintenance.class); - private static int numOfDatanodes = 5; + private static int numOfDatanodes = 6; + private static String bucketName = "bucket1"; + private static String volName = "vol1"; + private OzoneBucket bucket; private MiniOzoneCluster cluster; + private NodeManager nm; + private ContainerManager cm; + private PipelineManager pm; + private StorageContainerManager scm; private ContainerOperationClient scmClient; @@ -67,70 +105,315 @@ public void setUp() throws Exception { conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); + conf.setTimeDuration(OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, + 1, SECONDS); + + ReplicationManagerConfiguration replicationConf = + conf.getObject(ReplicationManagerConfiguration.class); + replicationConf.setInterval(Duration.ofSeconds(1)); + conf.setFromObject(replicationConf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(numOfDatanodes) .build(); cluster.waitForClusterToBeReady(); + setManagers(); + + bucket = TestDataUtil.createVolumeAndBucket(cluster, volName, bucketName); scmClient = new ContainerOperationClient(conf); } @After - public void tearDown() throws Exception { + public void tearDown() { if (cluster != null) { cluster.shutdown(); } } @Test - @Ignore // HDDS-3345 - public void testNodeCanBeDecommMaintAndRecommissioned() - throws IOException { - NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager(); + // Decommissioning a node with open pipelines should close the pipelines + // and hence the open containers and then the containers should be replicated + // by the replication manager. + public void testNodeWithOpenPipelineCanBeDecommissioned() + throws Exception { + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + // Locate any container and find its open pipeline + final ContainerInfo container = + scmClient.listContainer(0, 1).get(0); + Pipeline pipeline = pm.getPipeline(container.getPipelineID()); + assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState()); + + // Ensure all 3 replicas of the container have been reported via ICR + GenericTestUtils.waitFor( + () -> getContainerReplicas(container).size() == 3, + 200, 30000); + Set replicas = getContainerReplicas(container); - List dns = nm.getAllNodes(); - scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + final DatanodeDetails toDecommission = getOneDNHostingReplica(replicas); + scmClient.decommissionNodes(Arrays.asList( + getDNHostAndPort(toDecommission))); - // Ensure one node is decommissioning + // Ensure one node transitioned to DECOMMISSIONING List decomNodes = nm.getNodes( - HddsProtos.NodeOperationalState.DECOMMISSIONING, - HddsProtos.NodeState.HEALTHY); + DECOMMISSIONING, + HEALTHY); assertEquals(1, decomNodes.size()); + waitForDnToReachOpState(toDecommission, DECOMMISSIONED); - scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + // Should now be 4 replicas online as the DN is still alive but + // in the DECOMMISSIONED state. + Set newReplicas = + cm.getContainerReplicas(container.containerID()); + assertEquals(4, newReplicas.size()); - // Ensure zero nodes are now decommissioning - decomNodes = nm.getNodes( - HddsProtos.NodeOperationalState.DECOMMISSIONING, - HddsProtos.NodeState.HEALTHY); - assertEquals(0, decomNodes.size()); + // Stop the decommissioned DN + cluster.shutdownHddsDatanode(toDecommission); + waitForDnToReachHealthState(toDecommission, DEAD); - scmClient.startMaintenanceNodes(Arrays.asList( - getDNHostAndPort(dns.get(0))), 10); + // Now the decommissioned node is dead, we should have + // 3 replicas for the tracked container. + newReplicas = cm.getContainerReplicas(container.containerID()); + assertEquals(3, newReplicas.size()); + } - // None are decommissioning - decomNodes = nm.getNodes( - HddsProtos.NodeOperationalState.DECOMMISSIONING, - HddsProtos.NodeState.HEALTHY); - assertEquals(0, decomNodes.size()); + @Test + // After a SCM restart, it will have forgotten all the Operational states. + // However the state will have been persisted on the DNs. Therefore on initial + // registration, the DN operationalState is the source of truth and SCM should + // be updated to reflect that. + public void testDecommissionedStateReinstatedAfterSCMRestart() + throws Exception { + // Decommission any node and wait for it to be DECOMMISSIONED + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + DatanodeDetails dn = nm.getAllNodes().get(0); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + waitForDnToReachOpState(dn, DECOMMISSIONED); - // One is in Maintenance - decomNodes = nm.getNodes( - HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, - HddsProtos.NodeState.HEALTHY); - assertEquals(1, decomNodes.size()); + cluster.restartStorageContainerManager(true); + setManagers(); + DatanodeDetails newDn = nm.getNodeByUuid(dn.getUuid().toString()); - scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + // On initial registration, the DN should report its operational state + // and if it is decommissioned, that should be updated in the NodeStatus + waitForDnToReachOpState(newDn, DECOMMISSIONED); + // Also confirm the datanodeDetails correctly reflect the operational + // state. + waitForDnToReachPersistedOpState(newDn, DECOMMISSIONED); + } + + @Test + // If a node has not yet completed decommission and SCM is restarted, then + // when it re-registers it should re-enter the decommission workflow and + // complete decommissioning. + public void testDecommissioningNodesCompleteDecommissionOnSCMRestart() + throws Exception { + // First stop the replicationManager so nodes marked for decommission cannot + // make any progress. THe node will be stuck DECOMMISSIONING + scm.getReplicationManager().stop(); + // Generate some data and then pick a DN to decommission which is hosting a + // container. This ensures it will not decommission immediately due to + // having no containers. + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + final ContainerInfo container = + scmClient.listContainer(0, 1).get(0); + GenericTestUtils.waitFor( + () -> getContainerReplicas(container).size() == 3, + 200, 30000); + final DatanodeDetails dn + = getOneDNHostingReplica(getContainerReplicas(container)); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); - // None are in maintenance - decomNodes = nm.getNodes( - HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, - HddsProtos.NodeState.HEALTHY); - assertEquals(0, decomNodes.size()); + // Wait for the state to be persisted on the DN so it can report it on + // restart of SCM. + waitForDnToReachPersistedOpState(dn, DECOMMISSIONING); + cluster.restartStorageContainerManager(true); + setManagers(); + + // After the SCM restart, the DN should report as DECOMMISSIONING, then + // it should re-enter the decommission workflow and move to DECOMMISSIONED + DatanodeDetails newDn = nm.getNodeByUuid(dn.getUuid().toString()); + waitForDnToReachOpState(newDn, DECOMMISSIONED); + waitForDnToReachPersistedOpState(newDn, DECOMMISSIONED); } + @Test + // If a node was decommissioned, and then stopped so it is dead. Then it is + // recommissioned in SCM and restarted, the SCM state should be taken as the + // source of truth and the node will go to the IN_SERVICE state and the state + // should be updated on the DN. + public void testStoppedDecommissionedNodeTakesSCMStateOnRestart() + throws Exception { + // Decommission node and wait for it to be DECOMMISSIONED + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + DatanodeDetails dn = nm.getAllNodes().get(0); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + waitForDnToReachOpState(dn, DECOMMISSIONED); + waitForDnToReachPersistedOpState(dn, DECOMMISSIONED); + + int dnIndex = cluster.getHddsDatanodeIndex(dn); + cluster.shutdownHddsDatanode(dnIndex); + waitForDnToReachHealthState(dn, DEAD); + + // Datanode is shutdown and dead. Now recommission it in SCM + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + + // Now restart it and ensure it remains IN_SERVICE + cluster.restartHddsDatanode(dnIndex, true); + DatanodeDetails newDn = nm.getNodeByUuid(dn.getUuid().toString()); + + // As this is not an initial registration since SCM was started, the DN + // should report its operational state and if it differs from what SCM + // has, then the SCM state should be used and the DN state updated. + waitForDnToReachHealthState(newDn, HEALTHY); + waitForDnToReachOpState(newDn, IN_SERVICE); + waitForDnToReachPersistedOpState(dn, IN_SERVICE); + } + + @Test + // A node which is decommissioning or decommissioned can be move back to + // IN_SERVICE. + public void testDecommissionedNodeCanBeRecommissioned() throws Exception { + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + DatanodeDetails dn = nm.getAllNodes().get(0); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + + GenericTestUtils.waitFor( + () -> !dn.getPersistedOpState() + .equals(IN_SERVICE), + 200, 30000); + + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + waitForDnToReachOpState(dn, IN_SERVICE); + waitForDnToReachPersistedOpState(dn, IN_SERVICE); + } + + /** + * Sets the instance variables to the values for the current MiniCluster. + */ + private void setManagers() { + scm = cluster.getStorageContainerManager(); + nm = scm.getScmNodeManager(); + cm = scm.getContainerManager(); + pm = scm.getPipelineManager(); + } + + /** + * Generates some data on the cluster so the cluster has some containers. + * @param keyCount The number of keys to create + * @param keyPrefix The prefix to use for the key name. + * @param repFactor The replication Factor for the keys + * @param repType The replication Type for the keys + * @throws IOException + */ + private void generateData(int keyCount, String keyPrefix, + ReplicationFactor repFactor, ReplicationType repType) throws IOException { + for (int i=0; i getContainerReplicas(ContainerInfo c) { + Set replicas = null; + try { + replicas = cm.getContainerReplicas(c.containerID()); + } catch (ContainerNotFoundException e) { + fail("Unexpected ContainerNotFoundException"); + } + return replicas; + } + + /** + * Select any DN hosting a replica from the Replica Set. + * @param replicas The set of ContainerReplica + * @return Any datanode associated one of the replicas + */ + private DatanodeDetails getOneDNHostingReplica( + Set replicas) { + // Now Decommission a host with one of the replicas + Iterator iter = replicas.iterator(); + ContainerReplica c = iter.next(); + return c.getDatanodeDetails(); + } + + /** + * Given a Datanode, return a string consisting of the hostname and one of its + * ports in the for host:post. + * @param dn Datanode for which to retrieve the host:post string + * @return host:port for the given DN. + */ private String getDNHostAndPort(DatanodeDetails dn) { return dn.getHostName()+":"+dn.getPorts().get(0).getValue(); } + /** + * Wait for the given datanode to reach the given operational state. + * @param dn Datanode for which to check the state + * @param state The state to wait for. + * @throws TimeoutException + * @throws InterruptedException + */ + private void waitForDnToReachOpState(DatanodeDetails dn, + HddsProtos.NodeOperationalState state) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor( + () -> getNodeStatus(dn).getOperationalState().equals(state), + 200, 30000); + } + + /** + * Wait for the given datanode to reach the given Health state. + * @param dn Datanode for which to check the state + * @param state The state to wait for. + * @throws TimeoutException + * @throws InterruptedException + */ + private void waitForDnToReachHealthState(DatanodeDetails dn, + HddsProtos.NodeState state) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor( + () -> getNodeStatus(dn).getHealth().equals(state), + 200, 30000); + } + + /** + * Wait for the given datanode to reach the given persisted state. + * @param dn Datanode for which to check the state + * @param state The state to wait for. + * @throws TimeoutException + * @throws InterruptedException + */ + private void waitForDnToReachPersistedOpState(DatanodeDetails dn, + HddsProtos.NodeOperationalState state) + throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor( + () -> dn.getPersistedOpState().equals(state), + 200, 30000); + } + } From 39f5dc1b5f221090da29358a764c575bd8aa19b9 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 14 Oct 2020 10:43:48 +0100 Subject: [PATCH 22/78] Removed unused variable detected by findbugs --- .../java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java index 42369fdac80c..f0f9b7215833 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java @@ -51,8 +51,6 @@ public NewNodeHandler(PipelineManager pipelineManager, public void onMessage(DatanodeDetails datanodeDetails, EventPublisher publisher) { pipelineManager.triggerPipelineCreation(); - HddsProtos.NodeOperationalState opState - = datanodeDetails.getPersistedOpState(); if (datanodeDetails.getPersistedOpState() != HddsProtos.NodeOperationalState.IN_SERVICE) { try { From d327b7bc266d389e8520988ae2b1d94c2e573b23 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Mon, 19 Oct 2020 17:19:40 +0100 Subject: [PATCH 23/78] HDDS-4324. DatanodeAdminMonitor no longers needs maintenance end time to be passed (#1500) --- .../hdds/scm/node/DatanodeAdminMonitor.java | 2 +- .../scm/node/DatanodeAdminMonitorImpl.java | 5 +--- .../scm/node/NodeDecommissionManager.java | 6 ++--- .../scm/node/TestDatanodeAdminMonitor.java | 24 +++++++++---------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java index e78eeffd84ad..903b8167aa60 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -25,7 +25,7 @@ */ public interface DatanodeAdminMonitor extends Runnable { - void startMonitoring(DatanodeDetails dn, int endInHours); + void startMonitoring(DatanodeDetails dn); void stopMonitoring(DatanodeDetails dn); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index cf01af55c518..ee90cba8a1b6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -87,12 +87,9 @@ public DatanodeAdminMonitorImpl( * queued and added to the workflow after a defined interval. * * @param dn The datanode to move into an admin state - * @param endInHours For nodes going into maintenance, the number of hours - * from now for maintenance to automatically end. Ignored - * for decommissioning nodes. */ @Override - public synchronized void startMonitoring(DatanodeDetails dn, int endInHours) { + public synchronized void startMonitoring(DatanodeDetails dn) { cancelledNodes.remove(dn); pendingNodes.add(dn); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index d7432bc4d32c..cc81473b7847 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -245,7 +245,7 @@ public synchronized void continueAdminForNode(DatanodeDetails dn) NodeOperationalState opState = getNodeStatus(dn).getOperationalState(); if (opState == NodeOperationalState.DECOMMISSIONING || opState == NodeOperationalState.ENTERING_MAINTENANCE) { - monitor.startMonitoring(dn, 0); + monitor.startMonitoring(dn); } } @@ -257,7 +257,7 @@ public synchronized void startDecommission(DatanodeDetails dn) LOG.info("Starting Decommission for node {}", dn); nodeManager.setNodeOperationalState( dn, NodeOperationalState.DECOMMISSIONING); - monitor.startMonitoring(dn, 0); + monitor.startMonitoring(dn); } else if (nodeStatus.isDecommission()) { LOG.info("Start Decommission called on node {} in state {}. Nothing to "+ "do.", dn, opState); @@ -339,7 +339,7 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) if (opState == NodeOperationalState.IN_SERVICE) { nodeManager.setNodeOperationalState( dn, NodeOperationalState.ENTERING_MAINTENANCE, maintenanceEnd); - monitor.startMonitoring(dn, endInHours); + monitor.startMonitoring(dn); LOG.info("Starting Maintenance for node {}", dn); } else if (nodeStatus.isMaintenance()) { LOG.info("Starting Maintenance called on node {} with state {}. "+ diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java index e0f23ae0909f..33fe35f50642 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDatanodeAdminMonitor.java @@ -85,14 +85,14 @@ public void teardown() { @Test public void testNodeCanBeQueuedAndCancelled() { DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); - monitor.startMonitoring(dn, 0); + monitor.startMonitoring(dn); assertEquals(1, monitor.getPendingCount()); monitor.stopMonitoring(dn); assertEquals(0, monitor.getPendingCount()); assertEquals(1, monitor.getCancelledCount()); - monitor.startMonitoring(dn, 0); + monitor.startMonitoring(dn); assertEquals(1, monitor.getPendingCount()); assertEquals(0, monitor.getCancelledCount()); } @@ -112,7 +112,7 @@ public void testClosePipelinesEventFiredWhenAdminStarted() // Ensure the node has some pipelines nodeManager.setPipelines(dn1, 2); // Add the node to the monitor - monitor.startMonitoring(dn1, 0); + monitor.startMonitoring(dn1); monitor.run(); // Ensure a StartAdmin event was fired eventQueue.processAll(20000); @@ -152,7 +152,7 @@ public void testDecommissionNodeTransitionsToCompleteWhenNoContainers() // Add the node to the monitor. By default we have zero pipelines and // zero containers in the test setup, so the node should immediately // transition to COMPLETED state - monitor.startMonitoring(dn1, 0); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(0, monitor.getTrackedNodeCount()); NodeStatus newStatus = nodeManager.getNodeStatus(dn1); @@ -179,7 +179,7 @@ public void testDecommissionNodeWaitsForContainersToReplicate() // Run the monitor for the first time and the node will transition to // REPLICATE_CONTAINERS as there are no pipelines to close. - monitor.startMonitoring(dn1, 0); + monitor.startMonitoring(dn1); monitor.run(); DatanodeDetails node = getFirstTrackedNode(); assertEquals(1, monitor.getTrackedNodeCount()); @@ -225,7 +225,7 @@ public void testDecommissionAbortedWhenNodeInUnexpectedState() // Add the node to the monitor, it should have 3 under-replicated containers // after the first run - monitor.startMonitoring(dn1, 0); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); @@ -258,7 +258,7 @@ public void testDecommissionAbortedWhenNodeGoesDead() // Add the node to the monitor, it should have 3 under-replicated containers // after the first run - monitor.startMonitoring(dn1, 0); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); @@ -286,7 +286,7 @@ public void testMaintenanceWaitsForMaintenanceToComplete() // Add the node to the monitor, it should transiting to // IN_MAINTENANCE as there are no containers to replicate. - monitor.startMonitoring(dn1, 1); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); @@ -317,7 +317,7 @@ public void testMaintenanceEndsClosingPipelines() // Ensure the node has some pipelines nodeManager.setPipelines(dn1, 2); // Add the node to the monitor - monitor.startMonitoring(dn1, 1); + monitor.startMonitoring(dn1); monitor.run(); DatanodeDetails node = getFirstTrackedNode(); assertEquals(1, monitor.getTrackedNodeCount()); @@ -351,7 +351,7 @@ public void testMaintenanceEndsWhileReplicatingContainers() // Add the node to the monitor, it should transiting to // REPLICATE_CONTAINERS as the containers are under-replicated for // maintenance. - monitor.startMonitoring(dn1, 1); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); @@ -375,7 +375,7 @@ public void testDeadMaintenanceNodeDoesNotAbortWorkflow() // Add the node to the monitor, it should transiting to // AWAIT_MAINTENANCE_END as there are no under-replicated containers. - monitor.startMonitoring(dn1, 1); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); @@ -402,7 +402,7 @@ public void testCancelledNodesMovedToInService() // Add the node to the monitor, it should transiting to // AWAIT_MAINTENANCE_END as there are no under-replicated containers. - monitor.startMonitoring(dn1, 1); + monitor.startMonitoring(dn1); monitor.run(); assertEquals(1, monitor.getTrackedNodeCount()); DatanodeDetails node = getFirstTrackedNode(); From 42d53bce288db8d617fe4d3c82eb4c0f35fba135 Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 21 Oct 2020 15:04:22 +0100 Subject: [PATCH 24/78] HDDS-4323. Add integration tests for putting nodes into maintenance and fix any issues uncovered in the tests (#1501) --- .../hdds/scm/node/DatanodeAdminMonitor.java | 3 + .../scm/node/DatanodeAdminMonitorImpl.java | 17 +- .../scm/node/NodeDecommissionManager.java | 12 +- .../scm/server/StorageContainerManager.java | 7 + .../node/TestDecommissionAndMaintenance.java | 300 +++++++++++++++++- 5 files changed, 319 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java index 903b8167aa60..34665477516a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitor.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import java.util.Set; + /** * Interface used by the DatanodeAdminMonitor, which can be used to * decommission or recommission nodes and take them in and out of maintenance. @@ -27,5 +29,6 @@ public interface DatanodeAdminMonitor extends Runnable { void startMonitoring(DatanodeDetails dn); void stopMonitoring(DatanodeDetails dn); + Set getTrackedNodes(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java index ee90cba8a1b6..247a3071c247 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorImpl.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdds.scm.node; -import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -107,6 +105,16 @@ public synchronized void stopMonitoring(DatanodeDetails dn) { cancelledNodes.add(dn); } + /** + * Get the set of nodes which are currently tracked in the decommissioned + * and maintenance workflow. + * @return An unmodifiable set of the tracked nodes. + */ + @Override + public synchronized Set getTrackedNodes() { + return Collections.unmodifiableSet(trackedNodes); + } + /** * Run an iteration of the monitor. This is the main run loop, and performs * the following checks: @@ -148,11 +156,6 @@ public int getTrackedNodeCount() { return trackedNodes.size(); } - @VisibleForTesting - public Set getTrackedNodes() { - return trackedNodes; - } - private void processCancelledNodes() { while (!cancelledNodes.isEmpty()) { DatanodeDetails dn = cancelledNodes.poll(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index cc81473b7847..30cae106dc16 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -244,7 +244,8 @@ public synchronized void continueAdminForNode(DatanodeDetails dn) throws NodeNotFoundException { NodeOperationalState opState = getNodeStatus(dn).getOperationalState(); if (opState == NodeOperationalState.DECOMMISSIONING - || opState == NodeOperationalState.ENTERING_MAINTENANCE) { + || opState == NodeOperationalState.ENTERING_MAINTENANCE + || opState == NodeOperationalState.IN_MAINTENANCE) { monitor.startMonitoring(dn); } } @@ -351,6 +352,15 @@ public synchronized void startMaintenance(DatanodeDetails dn, int endInHours) } } + /** + * Stops the decommission monitor from running when SCM is shutdown. + */ + public void stop() { + if (executor != null) { + executor.shutdown(); + } + } + private NodeStatus getNodeStatus(DatanodeDetails dn) throws NodeNotFoundException { return nodeManager.getNodeStatus(dn); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index e26d52138eb3..2e9f576273d7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -844,6 +844,13 @@ public void stop() { LOG.error("Replication manager service stop failed.", ex); } + try { + LOG.info("Stopping the Datanode Admin Monitor."); + scmDecommissionManager.stop(); + } catch (Exception ex) { + LOG.error("The Datanode Admin Monitor failed to stop", ex); + } + try { LOG.info("Stopping Lease Manager of the command watchers"); commandWatcherLeaseManager.shutdown(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index 8f02878f0ba8..42b4cd8d90f3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ContainerReplicaCount; import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; @@ -46,15 +47,18 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertTrue; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; @@ -62,6 +66,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; @@ -138,16 +143,11 @@ public void testNodeWithOpenPipelineCanBeDecommissioned() throws Exception { // Generate some data on the empty cluster to create some containers generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + // Locate any container and find its open pipeline - final ContainerInfo container = - scmClient.listContainer(0, 1).get(0); + final ContainerInfo container = waitForAndReturnContainer(); Pipeline pipeline = pm.getPipeline(container.getPipelineID()); assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState()); - - // Ensure all 3 replicas of the container have been reported via ICR - GenericTestUtils.waitFor( - () -> getContainerReplicas(container).size() == 3, - 200, 30000); Set replicas = getContainerReplicas(container); final DatanodeDetails toDecommission = getOneDNHostingReplica(replicas); @@ -215,11 +215,7 @@ public void testDecommissioningNodesCompleteDecommissionOnSCMRestart() // container. This ensures it will not decommission immediately due to // having no containers. generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); - final ContainerInfo container = - scmClient.listContainer(0, 1).get(0); - GenericTestUtils.waitFor( - () -> getContainerReplicas(container).size() == 3, - 200, 30000); + final ContainerInfo container = waitForAndReturnContainer(); final DatanodeDetails dn = getOneDNHostingReplica(getContainerReplicas(container)); scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); @@ -246,6 +242,7 @@ public void testStoppedDecommissionedNodeTakesSCMStateOnRestart() throws Exception { // Decommission node and wait for it to be DECOMMISSIONED generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + DatanodeDetails dn = nm.getAllNodes().get(0); scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dn))); waitForDnToReachOpState(dn, DECOMMISSIONED); @@ -288,6 +285,270 @@ public void testDecommissionedNodeCanBeRecommissioned() throws Exception { waitForDnToReachPersistedOpState(dn, IN_SERVICE); } + @Test + // When putting a single node into maintenance, its pipelines should be closed + // but no new replicas should be create and the node should transition into + // maintenance + public void testSingleNodeWithOpenPipelineCanGotoMaintenance() + throws Exception { + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + + // Locate any container and find its open pipeline + final ContainerInfo container = waitForAndReturnContainer(); + Pipeline pipeline = pm.getPipeline(container.getPipelineID()); + assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState()); + Set replicas = getContainerReplicas(container); + + final DatanodeDetails dn = getOneDNHostingReplica(replicas); + scmClient.startMaintenanceNodes(Arrays.asList( + getDNHostAndPort(dn)), 0); + + waitForDnToReachOpState(dn, IN_MAINTENANCE); + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + + // Should still be 3 replicas online as no replication should happen for + // maintenance + Set newReplicas = + cm.getContainerReplicas(container.containerID()); + assertEquals(3, newReplicas.size()); + + // Stop the maintenance DN + cluster.shutdownHddsDatanode(dn); + waitForDnToReachHealthState(dn, DEAD); + + // Now the maintenance node is dead, we should still have + // 3 replicas as we don't purge the replicas for a dead maintenance node + newReplicas = cm.getContainerReplicas(container.containerID()); + assertEquals(3, newReplicas.size()); + + // Restart the DN and it should keep the IN_MAINTENANCE state + cluster.restartHddsDatanode(dn, true); + DatanodeDetails newDN = nm.getNodeByUuid(dn.getUuid().toString()); + waitForDnToReachHealthState(newDN, HEALTHY); + waitForDnToReachPersistedOpState(newDN, IN_MAINTENANCE); + } + + @Test + // After a node enters maintenance and is stopped, it can be recommissioned in + // SCM. Then when it is restarted, it should go back to IN_SERVICE and have + // that persisted on the DN. + public void testStoppedMaintenanceNodeTakesScmStateOnRestart() + throws Exception { + // Put a node into maintenance and wait for it to complete + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + DatanodeDetails dn = nm.getAllNodes().get(0); + scmClient.startMaintenanceNodes(Arrays.asList(getDNHostAndPort(dn)), 0); + waitForDnToReachOpState(dn, IN_MAINTENANCE); + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + + int dnIndex = cluster.getHddsDatanodeIndex(dn); + cluster.shutdownHddsDatanode(dnIndex); + waitForDnToReachHealthState(dn, DEAD); + + // Datanode is shutdown and dead. Now recommission it in SCM + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dn))); + + // Now restart it and ensure it remains IN_SERVICE + cluster.restartHddsDatanode(dnIndex, true); + DatanodeDetails newDn = nm.getNodeByUuid(dn.getUuid().toString()); + + // As this is not an initial registration since SCM was started, the DN + // should report its operational state and if it differs from what SCM + // has, then the SCM state should be used and the DN state updated. + waitForDnToReachHealthState(newDn, HEALTHY); + waitForDnToReachOpState(newDn, IN_SERVICE); + waitForDnToReachPersistedOpState(dn, IN_SERVICE); + } + + @Test + // By default a node can enter maintenance if there are two replicas left + // available when the maintenance nodes are stopped. Therefore putting all + // nodes hosting a replica to maintenance should cause new replicas to get + // created before the nodes can enter maintenance. When the maintenance nodes + // return, the excess replicas should be removed. + public void testContainerIsReplicatedWhenAllNodesGotoMaintenance() + throws Exception { + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + // Locate any container and find its open pipeline + final ContainerInfo container = waitForAndReturnContainer(); + Set replicas = getContainerReplicas(container); + + List forMaintenance = new ArrayList<>(); + replicas.forEach(r ->forMaintenance.add(r.getDatanodeDetails())); + + scmClient.startMaintenanceNodes(forMaintenance.stream() + .map(d -> getDNHostAndPort(d)) + .collect(Collectors.toList()), 0); + + // Ensure all 3 DNs go to maintenance + for(DatanodeDetails dn : forMaintenance) { + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + } + + // There should now be 5 replicas of the container we are tracking + Set newReplicas = + cm.getContainerReplicas(container.containerID()); + assertEquals(5, newReplicas.size()); + + scmClient.recommissionNodes(forMaintenance.stream() + .map(d -> getDNHostAndPort(d)) + .collect(Collectors.toList())); + + // Ensure all 3 DNs go to maintenance + for(DatanodeDetails dn : forMaintenance) { + waitForDnToReachOpState(dn, IN_SERVICE); + } + + GenericTestUtils.waitFor(() -> getContainerReplicas(container).size() == 3, + 200, 30000); + } + + @Test + // If SCM is restarted when a node is ENTERING_MAINTENANCE, then when the node + // re-registers, it should continue to enter maintenance. + public void testEnteringMaintenanceNodeCompletesAfterSCMRestart() + throws Exception { + // Stop Replication Manager to sure no containers are replicated + scm.getReplicationManager().stop(); + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + // Locate any container and find its open pipeline + final ContainerInfo container = waitForAndReturnContainer(); + Set replicas = getContainerReplicas(container); + + List forMaintenance = new ArrayList<>(); + replicas.forEach(r ->forMaintenance.add(r.getDatanodeDetails())); + + scmClient.startMaintenanceNodes(forMaintenance.stream() + .map(d -> getDNHostAndPort(d)) + .collect(Collectors.toList()), 0); + + // Ensure all 3 DNs go to maintenance + for(DatanodeDetails dn : forMaintenance) { + waitForDnToReachOpState(dn, IN_MAINTENANCE); + } + + cluster.restartStorageContainerManager(true); + + List newDns = new ArrayList<>(); + for(DatanodeDetails dn : forMaintenance) { + newDns.add(nm.getNodeByUuid(dn.getUuid().toString())); + } + + // Ensure all 3 DNs go to maintenance + for(DatanodeDetails dn : forMaintenance) { + waitForDnToReachOpState(dn, IN_MAINTENANCE); + } + + // There should now be 5 replicas of the container we are tracking + Set newReplicas = + cm.getContainerReplicas(container.containerID()); + assertEquals(5, newReplicas.size()); + } + + @Test + // For a node which is online the maintenance should end automatically when + // maintenance expires and the node should go back into service. + // If the node is dead when maintenance expires, its replicas will be purge + // and new replicas created. + public void testMaintenanceEndsAutomaticallyAtTimeout() + throws Exception { + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + ContainerInfo container = waitForAndReturnContainer(); + DatanodeDetails dn = + getOneDNHostingReplica(getContainerReplicas(container)); + + scmClient.startMaintenanceNodes(Arrays.asList(getDNHostAndPort(dn)), 0); + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + + long newEndTime = System.currentTimeMillis() / 1000 + 5; + // Update the maintenance end time via NM manually. As the current + // decommission interface only allows us to specify hours from now as the + // end time, that is not really suitable for a test like this. + nm.setNodeOperationalState(dn, IN_MAINTENANCE, newEndTime); + waitForDnToReachOpState(dn, IN_SERVICE); + waitForDnToReachPersistedOpState(dn, IN_SERVICE); + + // Put the node back into maintenance and then stop it and wait for it to + // go dead + scmClient.startMaintenanceNodes(Arrays.asList(getDNHostAndPort(dn)), 0); + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + cluster.shutdownHddsDatanode(dn); + waitForDnToReachHealthState(dn, DEAD); + + newEndTime = System.currentTimeMillis() / 1000 + 5; + nm.setNodeOperationalState(dn, IN_MAINTENANCE, newEndTime); + waitForDnToReachOpState(dn, IN_SERVICE); + // Ensure there are 3 replicas not including the dead node, indicating a new + // replica was created + GenericTestUtils.waitFor(() -> getContainerReplicas(container) + .stream() + .filter(r -> !r.getDatanodeDetails().equals(dn)) + .count() == 3, + 200, 30000); + } + + @Test + // If is SCM is Restarted when a maintenance node is dead, then we lose all + // the replicas associated with it, as the dead node cannot report them back + // in. If that happens, SCM has no choice except to replicate the containers. + public void testSCMHandlesRestartForMaintenanceNode() + throws Exception { + // Generate some data on the empty cluster to create some containers + generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); + ContainerInfo container = waitForAndReturnContainer(); + DatanodeDetails dn = + getOneDNHostingReplica(getContainerReplicas(container)); + + scmClient.startMaintenanceNodes(Arrays.asList(getDNHostAndPort(dn)), 0); + waitForDnToReachPersistedOpState(dn, IN_MAINTENANCE); + + cluster.restartStorageContainerManager(true); + setManagers(); + + // Ensure there are 3 replicas with one in maintenance indicating no new + // replicas were created + final ContainerInfo newContainer = cm.getContainer(container.containerID()); + GenericTestUtils.waitFor(() -> + getContainerReplicas(newContainer).size() == 3, 200, 30000); + + ContainerReplicaCount counts = + scm.getReplicationManager().getContainerReplicaCount(newContainer); + assertEquals(1, counts.getMaintenanceCount()); + assertTrue(counts.isSufficientlyReplicated()); + + // The node should be added back to the decommission monitor to ensure + // maintenance end time is correctly tracked. + GenericTestUtils.waitFor(() -> scm.getScmDecommissionManager().getMonitor() + .getTrackedNodes().size() == 1, 200, 30000); + + // Now let the node go dead and repeat the test. This time ensure a new + // replica is created. + cluster.shutdownHddsDatanode(dn); + waitForDnToReachHealthState(dn, DEAD); + + cluster.restartStorageContainerManager(false); + setManagers(); + + GenericTestUtils.waitFor(() + -> nm.getNodeCount(IN_SERVICE, null) == 5, 200, 30000); + + // Ensure there are 3 replicas not including the dead node, indicating a new + // replica was created + final ContainerInfo nextContainer + = cm.getContainer(container.containerID()); + GenericTestUtils.waitFor(() -> + getContainerReplicas(nextContainer).size() == 3, 200, 30000); + // There should be no IN_MAINTENANCE node: + assertEquals(0, nm.getNodeCount(IN_MAINTENANCE, null)); + counts = scm.getReplicationManager().getContainerReplicaCount(newContainer); + assertEquals(0, counts.getMaintenanceCount()); + assertTrue(counts.isSufficientlyReplicated()); + } + /** * Sets the instance variables to the values for the current MiniCluster. */ @@ -416,4 +677,19 @@ private void waitForDnToReachPersistedOpState(DatanodeDetails dn, 200, 30000); } + /** + * Get any container present in the cluster and wait to ensure 3 replicas + * have been reported before returning the container. + * @return A single container present on the cluster + * @throws Exception + */ + private ContainerInfo waitForAndReturnContainer() throws Exception { + final ContainerInfo container = cm.getContainers().get(0); + // Ensure all 3 replicas of the container have been reported via ICR + GenericTestUtils.waitFor( + () -> getContainerReplicas(container).size() == 3, + 200, 30000); + return container; + } + } From f64476c06e6268fbec683f6104ff39876adbb90c Mon Sep 17 00:00:00 2001 From: Stephen O'Donnell Date: Wed, 21 Oct 2020 17:10:30 +0100 Subject: [PATCH 25/78] HDDS-4340. Add Operational State to the datanode list command (#1491) --- hadoop-hdds/tools/pom.xml | 6 + .../scm/cli/datanode/ListInfoSubcommand.java | 46 +++++-- .../cli/datanode/TestListInfoSubcommand.java | 119 ++++++++++++++++++ 3 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index dfda5a668fdd..9da6f93b0bb4 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -78,6 +78,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.xerial sqlite-jdbc + + org.mockito + mockito-all + ${mockito1-hadoop.version} + test + diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 2ce9c8730ad5..38ad390cb85d 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -60,30 +60,34 @@ public void execute(ScmClient scmClient) throws IOException { if (Strings.isNullOrEmpty(ipaddress) && Strings.isNullOrEmpty(uuid)) { getAllNodes(scmClient).forEach(this::printDatanodeInfo); } else { - Stream allNodes = getAllNodes(scmClient).stream(); + Stream allNodes = getAllNodes(scmClient).stream(); if (!Strings.isNullOrEmpty(ipaddress)) { - allNodes = allNodes.filter(p -> p.getIpAddress() + allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress() .compareToIgnoreCase(ipaddress) == 0); } if (!Strings.isNullOrEmpty(uuid)) { - allNodes = allNodes.filter(p -> p.getUuid().toString().equals(uuid)); + allNodes = allNodes.filter(p -> + p.getDatanodeDetails().toString().equals(uuid)); } allNodes.forEach(this::printDatanodeInfo); } } - private List getAllNodes(ScmClient scmClient) + private List getAllNodes(ScmClient scmClient) throws IOException { List nodes = scmClient.queryNode(null, HddsProtos.NodeState.HEALTHY, HddsProtos.QueryScope.CLUSTER, ""); return nodes.stream() - .map(p -> DatanodeDetails.getFromProtoBuf(p.getNodeID())) + .map(p -> new DatanodeWithAttributes( + DatanodeDetails.getFromProtoBuf(p.getNodeID()), + p.getNodeOperationalStates(0), p.getNodeStates(0))) .collect(Collectors.toList()); } - private void printDatanodeInfo(DatanodeDetails datanode) { + private void printDatanodeInfo(DatanodeWithAttributes dna) { StringBuilder pipelineListInfo = new StringBuilder(); + DatanodeDetails datanode = dna.getDatanodeDetails(); int relatedPipelineNum = 0; if (!pipelines.isEmpty()) { List relatedPipelines = pipelines.stream().filter( @@ -108,6 +112,34 @@ private void printDatanodeInfo(DatanodeDetails datanode) { System.out.println("Datanode: " + datanode.getUuid().toString() + " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + "/" + relatedPipelineNum + - " pipelines) \n" + "Related pipelines: \n" + pipelineListInfo); + " pipelines)"); + System.out.println("Operational State: " + dna.getOpState()); + System.out.println("Related pipelines: \n" + pipelineListInfo); + } + + private static class DatanodeWithAttributes { + private DatanodeDetails datanodeDetails; + private HddsProtos.NodeOperationalState operationalState; + private HddsProtos.NodeState healthState; + + DatanodeWithAttributes(DatanodeDetails dn, + HddsProtos.NodeOperationalState opState, + HddsProtos.NodeState healthState) { + this.datanodeDetails = dn; + this.operationalState = opState; + this.healthState = healthState; + } + + public DatanodeDetails getDatanodeDetails() { + return datanodeDetails; + } + + public HddsProtos.NodeOperationalState getOpState() { + return operationalState; + } + + public HddsProtos.NodeState getHealthState() { + return healthState; + } } } diff --git a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java new file mode 100644 index 000000000000..45d4d7b908ca --- /dev/null +++ b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.scm.cli.datanode; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.client.ScmClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.mockito.Mockito; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.any; + +/** + * Unit tests to validate the the TestListInfoSubCommand class includes the + * correct output when executed against a mock client. + */ +public class TestListInfoSubcommand { + + private ListInfoSubcommand cmd; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + + @Before + public void setup() { + cmd = new ListInfoSubcommand(); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @After + public void tearDown() { + System.setOut(originalOut); + System.setErr(originalErr); + } + + @Test + public void testDataNodeOperationalStateIncludedInOutput() throws Exception { + ScmClient scmClient = mock(ScmClient.class); + Mockito.when(scmClient.queryNode(any(HddsProtos.NodeOperationalState.class), + any(HddsProtos.NodeState.class), any(HddsProtos.QueryScope.class), + Mockito.anyString())) + .thenAnswer(invocation -> getNodeDetails()); + Mockito.when(scmClient.listPipelines()) + .thenReturn(new ArrayList<>()); + + cmd.execute(scmClient); + + // The output should contain a string like: + // + // Operational State: + // + Pattern p = Pattern.compile( + "^Operational State:\\s+IN_SERVICE$", Pattern.MULTILINE); + Matcher m = p.matcher(outContent.toString()); + assertTrue(m.find()); + // Should also have a node with the state DECOMMISSIONING + p = Pattern.compile( + "^Operational State:\\s+DECOMMISSIONING$", Pattern.MULTILINE); + m = p.matcher(outContent.toString()); + assertTrue(m.find()); + } + + private List getNodeDetails() { + List nodes = new ArrayList<>(); + + for (int i=0; i<2; i++) { + HddsProtos.DatanodeDetailsProto.Builder dnd = + HddsProtos.DatanodeDetailsProto.newBuilder(); + dnd.setHostName("host" + i); + dnd.setIpAddress("1.2.3." + i+1); + dnd.setNetworkLocation("/default"); + dnd.setNetworkName("host" + i); + dnd.addPorts(HddsProtos.Port.newBuilder() + .setName("ratis").setValue(5678).build()); + dnd.setUuid(UUID.randomUUID().toString()); + + HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); + if (i == 0) { + builder.addNodeOperationalStates( + HddsProtos.NodeOperationalState.IN_SERVICE); + } else { + builder.addNodeOperationalStates( + HddsProtos.NodeOperationalState.DECOMMISSIONING); + } + builder.addNodeStates(HddsProtos.NodeState.HEALTHY); + builder.setNodeID(dnd.build()); + nodes.add(builder.build()); + } + return nodes; + } +} \ No newline at end of file From 2c411920b951b9efc571cf5e970a4af889ce67cb Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 12 Nov 2020 15:23:45 +0000 Subject: [PATCH 26/78] Trigger build From 536a6e7de7a5bd53605f6935ae848667432af22b Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Mon, 30 Nov 2020 15:00:18 +0100 Subject: [PATCH 27/78] HDDS-4461. Reuse compiled binaries in acceptance test (#1588) --- .github/workflows/post-commit.yml | 90 +++++++++++-------------------- 1 file changed, 32 insertions(+), 58 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 1aaf27c6da23..44d14c04b7e4 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -19,8 +19,7 @@ on: schedule: - cron: 30 0,12 * * * jobs: - build: - name: compile + compile: runs-on: ubuntu-18.04 strategy: matrix: @@ -49,7 +48,14 @@ jobs: with: java-version: ${{ matrix.java }} - name: Run a full build - run: hadoop-ozone/dev-support/checks/build.sh + run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist + - name: Store binaries for tests + uses: actions/upload-artifact@v2 + if: matrix.java == '8' + with: + name: ozone-bin + path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz + retention-days: 1 bats: runs-on: ubuntu-18.04 steps: @@ -167,6 +173,7 @@ jobs: path: target/findbugs continue-on-error: true acceptance: + needs: compile runs-on: ubuntu-18.04 strategy: matrix: @@ -178,37 +185,20 @@ jobs: steps: - name: Checkout project uses: actions/checkout@v2 - - name: Cache for maven dependencies - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: maven-repo- - - name: Cache for npm dependencies - uses: actions/cache@v2 with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Checkout to /mnt/ozone + path: ozone + - name: Move Ozone to /mnt run: | sudo chmod 777 /mnt - git clone 'https://github.com/${{ github.repository }}.git' /mnt/ozone - cd /mnt/ozone - if [[ '${{ github.event_name }}' == 'pull_request' ]]; then - git fetch --verbose origin '${{ github.ref }}' - else - git fetch --verbose origin '${{ github.sha }}' - fi - git checkout FETCH_HEAD - git reset --hard - - name: Run a full build + mv ozone /mnt/ + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries run: | - cd /mnt/ozone - hadoop-ozone/dev-support/checks/build.sh -Pcoverage + mkdir -p /mnt/ozone/hadoop-ozone/dist/target + tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target - name: Install robotframework run: sudo pip install robotframework - name: Execute tests @@ -304,37 +294,25 @@ jobs: path: target/coverage continue-on-error: true kubernetes: + needs: compile runs-on: ubuntu-18.04 steps: - name: Checkout project uses: actions/checkout@v2 - - name: Cache for maven dependencies - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: maven-repo- - - name: Cache for npm dependencies - uses: actions/cache@v2 with: - path: | - ~/.pnpm-store - **/node_modules - key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm- - - name: Checkout to /mnt/ozone + path: ozone + - name: Move Ozone to /mnt run: | sudo chmod 777 /mnt - git clone 'https://github.com/${{ github.repository }}.git' /mnt/ozone - cd /mnt/ozone - if [[ '${{ github.event_name }}' == 'pull_request' ]]; then - git fetch --verbose origin '${{ github.ref }}' - else - git fetch --verbose origin '${{ github.sha }}' - fi - git checkout FETCH_HEAD - git reset --hard + mv ozone /mnt/ + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir -p /mnt/ozone/hadoop-ozone/dist/target + tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target - name: Install robotframework run: sudo pip install robotframework - name: Install k3s @@ -350,10 +328,6 @@ jobs: wget https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz -O - | tar -zx chmod +x flekszible sudo mv flekszible /usr/bin/flekszible - - name: Run a full build - run: | - cd /mnt/ozone - hadoop-ozone/dev-support/checks/build.sh -Pcoverage - name: Execute tests run: | cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws From 21c387d9cedcda31658170ffbd6a8a91067ccf92 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 30 Nov 2020 15:33:17 +0000 Subject: [PATCH 28/78] Address intermittent test failures --- .../node/TestDecommissionAndMaintenance.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index 42b4cd8d90f3..a1a2dfaefc20 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -66,6 +66,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONED; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD; @@ -154,12 +155,12 @@ public void testNodeWithOpenPipelineCanBeDecommissioned() scmClient.decommissionNodes(Arrays.asList( getDNHostAndPort(toDecommission))); + waitForDnToReachOpState(toDecommission, DECOMMISSIONED); // Ensure one node transitioned to DECOMMISSIONING List decomNodes = nm.getNodes( - DECOMMISSIONING, + DECOMMISSIONED, HEALTHY); assertEquals(1, decomNodes.size()); - waitForDnToReachOpState(toDecommission, DECOMMISSIONED); // Should now be 4 replicas online as the DN is still alive but // in the DECOMMISSIONED state. @@ -210,7 +211,7 @@ public void testDecommissioningNodesCompleteDecommissionOnSCMRestart() throws Exception { // First stop the replicationManager so nodes marked for decommission cannot // make any progress. THe node will be stuck DECOMMISSIONING - scm.getReplicationManager().stop(); + stopReplicationManager(); // Generate some data and then pick a DN to decommission which is hosting a // container. This ensures it will not decommission immediately due to // having no containers. @@ -411,7 +412,7 @@ public void testContainerIsReplicatedWhenAllNodesGotoMaintenance() public void testEnteringMaintenanceNodeCompletesAfterSCMRestart() throws Exception { // Stop Replication Manager to sure no containers are replicated - scm.getReplicationManager().stop(); + stopReplicationManager(); // Generate some data on the empty cluster to create some containers generateData(20, "key", ReplicationFactor.THREE, ReplicationType.RATIS); // Locate any container and find its open pipeline @@ -425,12 +426,12 @@ public void testEnteringMaintenanceNodeCompletesAfterSCMRestart() .map(d -> getDNHostAndPort(d)) .collect(Collectors.toList()), 0); - // Ensure all 3 DNs go to maintenance + // Ensure all 3 DNs go to entering_maintenance for(DatanodeDetails dn : forMaintenance) { - waitForDnToReachOpState(dn, IN_MAINTENANCE); + waitForDnToReachPersistedOpState(dn, ENTERING_MAINTENANCE); } - cluster.restartStorageContainerManager(true); + setManagers(); List newDns = new ArrayList<>(); for(DatanodeDetails dn : forMaintenance) { @@ -438,7 +439,7 @@ public void testEnteringMaintenanceNodeCompletesAfterSCMRestart() } // Ensure all 3 DNs go to maintenance - for(DatanodeDetails dn : forMaintenance) { + for(DatanodeDetails dn : newDns) { waitForDnToReachOpState(dn, IN_MAINTENANCE); } @@ -692,4 +693,16 @@ private ContainerInfo waitForAndReturnContainer() throws Exception { return container; } + /** + * Wait for the ReplicationManager thread to start, and when it does, stop + * it. + * @throws Exception + */ + private void stopReplicationManager() throws Exception { + GenericTestUtils.waitFor( + () -> scm.getReplicationManager().isRunning(), + 200, 30000); + scm.getReplicationManager().stop(); + } + } From 9c6f805c57ceaf42ed08341d9d575d3043119a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= Date: Mon, 30 Nov 2020 20:16:24 +0100 Subject: [PATCH 29/78] HDDS-4429. Create unit test for SimpleContainerDownloader. (#1551) --- .../SimpleContainerDownloader.java | 80 ++++++----- .../TestSimpleContainerDownloader.java | 127 +++++++++++++++++- 2 files changed, 170 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java index c1445bde58ca..5d8a86bc930a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SimpleContainerDownloader.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.container.replication; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.security.cert.X509Certificate; @@ -26,7 +25,6 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.function.Function; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -35,6 +33,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import com.google.common.annotations.VisibleForTesting; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +53,10 @@ public class SimpleContainerDownloader implements ContainerDownloader { private final SecurityConfig securityConfig; private final X509Certificate caCert; - public SimpleContainerDownloader(ConfigurationSource conf, - X509Certificate caCert) { + public SimpleContainerDownloader( + ConfigurationSource conf, + X509Certificate caCert + ) { String workDirString = conf.get(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR); @@ -71,38 +72,31 @@ public SimpleContainerDownloader(ConfigurationSource conf, } @Override - public CompletableFuture getContainerDataFromReplicas(long containerId, - List sourceDatanodes) { + public CompletableFuture getContainerDataFromReplicas( + long containerId, + List sourceDatanodes + ) { CompletableFuture result = null; - //There is a chance for the download is successful but import is failed, - //due to data corruption. We need a random selected datanode to have a - //chance to succeed next time. - final ArrayList shuffledDatanodes = - new ArrayList<>(sourceDatanodes); - Collections.shuffle(shuffledDatanodes); + final List shuffledDatanodes = + shuffleDatanodes(sourceDatanodes); for (DatanodeDetails datanode : shuffledDatanodes) { try { if (result == null) { result = downloadContainer(containerId, datanode); } else { - result = result.thenApply(CompletableFuture::completedFuture) - .exceptionally(t -> { - LOG.error("Error on replicating container: " + containerId, t); - try { - GrpcReplicationClient grpcReplicationClient = - new GrpcReplicationClient(datanode.getIpAddress(), - datanode.getPort(Name.STANDALONE).getValue(), - workingDirectory, securityConfig, caCert); - return grpcReplicationClient.download(containerId); - } catch (IOException e) { - LOG.error("Error on replicating container: " + containerId, - t); - return null; - } - }).thenCompose(Function.identity()); + result = result.exceptionally(t -> { + LOG.error("Error on replicating container: " + containerId, t); + try { + return downloadContainer(containerId, datanode).join(); + } catch (Exception e) { + LOG.error("Error on replicating container: " + containerId, + e); + return null; + } + }); } } catch (Exception ex) { LOG.error(String.format( @@ -114,18 +108,42 @@ public CompletableFuture getContainerDataFromReplicas(long containerId, } + //There is a chance for the download is successful but import is failed, + //due to data corruption. We need a random selected datanode to have a + //chance to succeed next time. + @NotNull + protected List shuffleDatanodes( + List sourceDatanodes + ) { + + final ArrayList shuffledDatanodes = + new ArrayList<>(sourceDatanodes); + + Collections.shuffle(shuffledDatanodes); + + return shuffledDatanodes; + } + @VisibleForTesting protected CompletableFuture downloadContainer( long containerId, DatanodeDetails datanode ) throws Exception { CompletableFuture result; - try (GrpcReplicationClient grpcReplicationClient = + GrpcReplicationClient grpcReplicationClient = new GrpcReplicationClient(datanode.getIpAddress(), datanode.getPort(Name.STANDALONE).getValue(), - workingDirectory, securityConfig, caCert)) { - result = grpcReplicationClient.download(containerId); - } + workingDirectory, securityConfig, caCert); + result = grpcReplicationClient.download(containerId) + .thenApply(r -> { + try { + grpcReplicationClient.close(); + } catch (Exception e) { + LOG.error("Couldn't close Grpc replication client", e); + } + return r; + }); + return result; } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java index 3f0f5a9faaeb..f29b1579198a 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSimpleContainerDownloader.java @@ -21,10 +21,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; @@ -32,17 +35,74 @@ import org.junit.Assert; import org.junit.Test; -/** - * Test container downloader. +/* + * Test SimpleContainerDownloader. */ public class TestSimpleContainerDownloader { private static final String SUCCESS_PATH = "downloaded"; + @Test + public void testGetContainerDataFromReplicasHappyPath() throws Exception { + + //GIVEN + List datanodes = createDatanodes(); + + SimpleContainerDownloader downloader = + createDownloaderWithPredefinedFailures(true); + + //WHEN + final Path result = + downloader.getContainerDataFromReplicas(1L, datanodes) + .get(1L, TimeUnit.SECONDS); + + //THEN + Assert.assertEquals(datanodes.get(0).getUuidString(), result.toString()); + } + + @Test + public void testGetContainerDataFromReplicasDirectFailure() + throws Exception { + + //GIVEN + List datanodes = createDatanodes(); + + SimpleContainerDownloader downloader = + createDownloaderWithPredefinedFailures(true, datanodes.get(0)); + + //WHEN + final Path result = + downloader.getContainerDataFromReplicas(1L, datanodes) + .get(1L, TimeUnit.SECONDS); + + //THEN + //first datanode is failed, second worked + Assert.assertEquals(datanodes.get(1).getUuidString(), result.toString()); + } + + @Test + public void testGetContainerDataFromReplicasAsyncFailure() throws Exception { + + //GIVEN + List datanodes = createDatanodes(); + + SimpleContainerDownloader downloader = + createDownloaderWithPredefinedFailures(false, datanodes.get(0)); + + //WHEN + final Path result = + downloader.getContainerDataFromReplicas(1L, datanodes) + .get(1L, TimeUnit.SECONDS); + + //THEN + //first datanode is failed, second worked + Assert.assertEquals(datanodes.get(1).getUuidString(), result.toString()); + } + /** * Test if different datanode is used for each download attempt. */ - @Test(timeout = 1000L) + @Test(timeout = 10_000L) public void testRandomSelection() throws ExecutionException, InterruptedException { @@ -62,24 +122,79 @@ protected CompletableFuture downloadContainer( } }; - //WHEN executed, THEN at least once the second datanode should be returned. + //WHEN executed, THEN at least once the second datanode should be + //returned. for (int i = 0; i < 10000; i++) { - Path path = downloader.getContainerDataFromReplicas(1L, datanodes).get(); + Path path = + downloader.getContainerDataFromReplicas(1L, datanodes).get(); if (path.toString().equals(datanodes.get(1).getUuidString())) { return; } } - //there is 1/2^10_000 chance for false positive, which is practically 0. + //there is 1/3^10_000 chance for false positive, which is practically 0. Assert.fail( "Datanodes are selected 10000 times but second datanode was never " + "used."); } + /** + * Creates downloader which fails with datanodes in the arguments. + * + * @param directException if false the exception will be wrapped in the + * returning future. + */ + private SimpleContainerDownloader createDownloaderWithPredefinedFailures( + boolean directException, + DatanodeDetails... failedDatanodes + ) { + + ConfigurationSource conf = new OzoneConfiguration(); + + final List datanodes = + Arrays.asList(failedDatanodes); + + return new SimpleContainerDownloader(conf, null) { + + //for retry testing we use predictable list of datanodes. + @Override + protected List shuffleDatanodes( + List sourceDatanodes + ) { + //turn off randomization + return sourceDatanodes; + } + + @Override + protected CompletableFuture downloadContainer( + long containerId, + DatanodeDetails datanode + ) throws Exception { + + if (datanodes.contains(datanode)) { + if (directException) { + throw new RuntimeException("Unavailable datanode"); + } else { + return CompletableFuture.supplyAsync(() -> { + throw new RuntimeException("Unavailable datanode"); + }); + } + } else { + + //path includes the dn id to make it possible to assert. + return CompletableFuture.completedFuture( + Paths.get(datanode.getUuidString())); + } + + } + }; + } + private List createDatanodes() { List datanodes = new ArrayList<>(); datanodes.add(MockDatanodeDetails.randomDatanodeDetails()); datanodes.add(MockDatanodeDetails.randomDatanodeDetails()); + datanodes.add(MockDatanodeDetails.randomDatanodeDetails()); return datanodes; } } \ No newline at end of file From b0d54d41b51504359373be541b6a337ce9aa2c52 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 30 Nov 2020 19:53:59 -0800 Subject: [PATCH 30/78] HDDS-4484. Use RaftServerImpl isLeader instead of periodic leader update logic in OM and isLeaderReady for read/write requests (#1638) --- .../src/main/resources/ozone-default.xml | 11 -- .../apache/hadoop/ozone/om/OMConfigKeys.java | 7 - .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +- .../hadoop/ozone/TestMiniOzoneHACluster.java | 2 +- .../recon/TestReconWithOzoneManagerHA.java | 2 +- .../hadoop/ozone/om/KeyDeletingService.java | 2 +- .../apache/hadoop/ozone/om/OzoneManager.java | 20 +-- .../om/ratis/OzoneManagerRatisServer.java | 169 +++--------------- .../om/ratis/OzoneManagerStateMachine.java | 1 - ...ManagerProtocolServerSideTranslatorPB.java | 57 +++--- 10 files changed, 75 insertions(+), 198 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index d8402f7b9df6..d2421b1a0e40 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1643,17 +1643,6 @@ - - ozone.om.ratis.server.role.check.interval - 15s - OZONE, OM, RATIS, MANAGEMENT - The interval between OM leader performing a role - check on its ratis server. Ratis server informs OM if it - loses the leader role. The scheduled check is an secondary - check to ensure that the leader role is updated periodically - . - - ozone.om.ratis.snapshot.dir diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index f16679a681eb..a5f2d573f22f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -168,13 +168,6 @@ private OMConfigKeys() { OZONE_OM_RATIS_SERVER_FAILURE_TIMEOUT_DURATION_DEFAULT = TimeDuration.valueOf(120, TimeUnit.SECONDS); - // OM Leader server role check interval - public static final String OZONE_OM_RATIS_SERVER_ROLE_CHECK_INTERVAL_KEY - = "ozone.om.ratis.server.role.check.interval"; - public static final TimeDuration - OZONE_OM_RATIS_SERVER_ROLE_CHECK_INTERVAL_DEFAULT - = TimeDuration.valueOf(15, TimeUnit.SECONDS); - // OM SnapshotProvider configurations public static final String OZONE_OM_RATIS_SNAPSHOT_DIR = "ozone.om.ratis.snapshot.dir"; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index 6953594747bc..d16618a373d9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -171,7 +171,7 @@ public List getOzoneManagersList() { public OzoneManager getOMLeader() { OzoneManager res = null; for (OzoneManager ozoneManager : this.ozoneManagers) { - if (ozoneManager.isLeader()) { + if (ozoneManager.isLeaderReady()) { if (res != null) { // Found more than one leader // Return null, expect the caller to retry in a while diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneHACluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneHACluster.java index 96121afed966..051eb94d582e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneHACluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneHACluster.java @@ -107,6 +107,6 @@ public void testGetOMLeader() throws InterruptedException, TimeoutException { Assert.assertNotNull("Timed out waiting OM leader election to finish: " + "no leader or more than one leader.", ozoneManager); Assert.assertTrue("Should have gotten the leader!", - ozoneManager.get().isLeader()); + ozoneManager.get().isLeaderReady()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index be146fc9d49f..2ff79de07500 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -103,7 +103,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception { Assert.assertNotNull("Timed out waiting OM leader election to finish: " + "no leader or more than one leader.", ozoneManager); Assert.assertTrue("Should have gotten the leader!", - ozoneManager.get().isLeader()); + ozoneManager.get().isLeaderReady()); OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java index 466a55f18300..418d7a88274b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java @@ -129,7 +129,7 @@ private boolean shouldRun() { // OzoneManager can be null for testing return true; } - return ozoneManager.isLeader(); + return ozoneManager.isLeaderReady(); } private boolean isRatisEnabled() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 6229beffa265..cc5af0f11edd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -234,6 +234,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK; +import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.LEADER_AND_READY; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService; import org.apache.hadoop.util.Time; @@ -3502,21 +3503,16 @@ public String getComponent() { public long getMaxUserVolumeCount() { return maxUserVolumeCount; } - /** - * Checks the Leader status of OM Ratis Server. - * Note that this status has a small window of error. It should not be used - * to determine the absolute leader status. - * If it is the leader, the role status is cached till Ratis server - * notifies of leader change. If it is not leader, the role information is - * retrieved through by submitting a GroupInfoRequest to Ratis server. - *

- * If ratis is not enabled, then it always returns true. + * Return true, if the current OM node is leader and in ready state to + * process the requests. * - * @return Return true if this node is the leader, false otherwsie. + * If ratis is not enabled, then it always returns true. + * @return */ - public boolean isLeader() { - return isRatisEnabled ? omRatisServer.isLeader() : true; + public boolean isLeaderReady() { + return isRatisEnabled ? + omRatisServer.checkLeaderStatus() == LEADER_AND_READY : true; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 467764b2f6a0..0f9e20b8d9fa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -26,13 +26,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -59,11 +54,7 @@ import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.grpc.GrpcConfigKeys; import org.apache.ratis.netty.NettyConfigKeys; -import org.apache.ratis.proto.RaftProtos.RaftPeerRole; -import org.apache.ratis.proto.RaftProtos.RoleInfoProto; import org.apache.ratis.protocol.ClientId; -import org.apache.ratis.protocol.GroupInfoReply; -import org.apache.ratis.protocol.GroupInfoRequest; import org.apache.ratis.protocol.exceptions.LeaderNotReadyException; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.apache.ratis.protocol.exceptions.StateMachineException; @@ -78,8 +69,9 @@ import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.impl.RaftServerImpl; +import org.apache.ratis.server.impl.RaftServerProxy; import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.StringUtils; @@ -106,20 +98,6 @@ public final class OzoneManagerRatisServer { private final OzoneManager ozoneManager; private final OzoneManagerStateMachine omStateMachine; - private final ClientId clientId = ClientId.randomId(); - - private final ScheduledExecutorService scheduledRoleChecker; - private long roleCheckInitialDelayMs = 1000; // 1 second default - private long roleCheckIntervalMs; - private ReentrantReadWriteLock roleCheckLock = new ReentrantReadWriteLock(); - private Optional cachedPeerRole = Optional.empty(); - private Optional cachedLeaderPeerId = Optional.empty(); - - private static final AtomicLong CALL_ID_COUNTER = new AtomicLong(); - - private static long nextCallId() { - return CALL_ID_COUNTER.getAndIncrement() & Long.MAX_VALUE; - } /** * Submit request to Ratis server. @@ -301,20 +279,6 @@ private OzoneManagerRatisServer(ConfigurationSource conf, .setProperties(serverProperties) .setStateMachine(omStateMachine) .build(); - - // Run a scheduler to check and update the server role on the leader - // periodically - this.scheduledRoleChecker = Executors.newSingleThreadScheduledExecutor(); - this.scheduledRoleChecker.scheduleWithFixedDelay(new Runnable() { - @Override - public void run() { - // Run this check only on the leader OM - if (cachedPeerRole.isPresent() && - cachedPeerRole.get() == RaftPeerRole.LEADER) { - updateServerRole(); - } - } - }, roleCheckInitialDelayMs, roleCheckIntervalMs, TimeUnit.MILLISECONDS); } /** @@ -556,19 +520,6 @@ private RaftProperties newRaftProperties(ConfigurationSource conf) { RaftServerConfigKeys.Rpc.setSlownessTimeout(properties, nodeFailureTimeout); - TimeUnit roleCheckIntervalUnit = - OMConfigKeys.OZONE_OM_RATIS_SERVER_ROLE_CHECK_INTERVAL_DEFAULT - .getUnit(); - long roleCheckIntervalDuration = conf.getTimeDuration( - OMConfigKeys.OZONE_OM_RATIS_SERVER_ROLE_CHECK_INTERVAL_KEY, - OMConfigKeys.OZONE_OM_RATIS_SERVER_ROLE_CHECK_INTERVAL_DEFAULT - .getDuration(), nodeFailureTimeoutUnit); - this.roleCheckIntervalMs = TimeDuration.valueOf( - roleCheckIntervalDuration, roleCheckIntervalUnit) - .toLong(TimeUnit.MILLISECONDS); - this.roleCheckInitialDelayMs = leaderElectionMinTimeout - .toLong(TimeUnit.MILLISECONDS); - // Set auto trigger snapshot. We don't need to configure auto trigger // threshold in OM, as last applied index is flushed during double buffer // flush automatically. (But added this property internally, so that this @@ -591,107 +542,39 @@ private RaftProperties newRaftProperties(ConfigurationSource conf) { } /** - * Check the cached leader status. - * @return true if cached role is Leader, false otherwise. - */ - private boolean checkCachedPeerRoleIsLeader() { - this.roleCheckLock.readLock().lock(); - try { - if (cachedPeerRole.isPresent() && - cachedPeerRole.get() == RaftPeerRole.LEADER) { - return true; - } - return false; - } finally { - this.roleCheckLock.readLock().unlock(); - } - } - - /** - * Check if the current OM node is the leader node. - * @return true if Leader, false otherwise. - */ - public boolean isLeader() { - if (checkCachedPeerRoleIsLeader()) { - return true; - } - - // Get the server role from ratis server and update the cached values. - updateServerRole(); - - // After updating the server role, check and return if leader or not. - return checkCachedPeerRoleIsLeader(); - } - - /** - * Get the suggested leader peer id. - * @return RaftPeerId of the suggested leader node. + * Defines RaftServer Status. */ - public Optional getCachedLeaderPeerId() { - this.roleCheckLock.readLock().lock(); - try { - return cachedLeaderPeerId; - } finally { - this.roleCheckLock.readLock().unlock(); - } + public enum RaftServerStatus { + NOT_LEADER, + LEADER_AND_NOT_READY, + LEADER_AND_READY; } /** - * Get the gorup info (peer role and leader peer id) from Ratis server and - * update the OM server role. + * Check Leader status and return the state of the RaftServer. + * + * @return RaftServerStatus. */ - public void updateServerRole() { + public RaftServerStatus checkLeaderStatus() { + Preconditions.checkState(server instanceof RaftServerProxy); + RaftServerImpl serverImpl; try { - GroupInfoReply groupInfo = getGroupInfo(); - RoleInfoProto roleInfoProto = groupInfo.getRoleInfoProto(); - RaftPeerRole thisNodeRole = roleInfoProto.getRole(); - - if (thisNodeRole.equals(RaftPeerRole.LEADER)) { - setServerRole(thisNodeRole, raftPeerId); - - } else if (thisNodeRole.equals(RaftPeerRole.FOLLOWER)) { - ByteString leaderNodeId = roleInfoProto.getFollowerInfo() - .getLeaderInfo().getId().getId(); - // There may be a chance, here we get leaderNodeId as null. For - // example, in 3 node OM Ratis, if 2 OM nodes are down, there will - // be no leader. - RaftPeerId leaderPeerId = null; - if (leaderNodeId != null && !leaderNodeId.isEmpty()) { - leaderPeerId = RaftPeerId.valueOf(leaderNodeId); + serverImpl = ((RaftServerProxy) server).getImpl(raftGroupId); + if (serverImpl != null) { + if (!serverImpl.isLeader()) { + return RaftServerStatus.NOT_LEADER; + } else if (serverImpl.isLeaderReady()) { + return RaftServerStatus.LEADER_AND_READY; + } else { + return RaftServerStatus.LEADER_AND_NOT_READY; } - - setServerRole(thisNodeRole, leaderPeerId); - - } else { - setServerRole(thisNodeRole, null); - } - } catch (IOException e) { - LOG.error("Failed to retrieve RaftPeerRole. Setting cached role to " + - "{} and resetting leader info.", RaftPeerRole.UNRECOGNIZED, e); - setServerRole(null, null); - } - } - - /** - * Set the current server role and the leader peer id. - */ - private void setServerRole(RaftPeerRole currentRole, - RaftPeerId leaderPeerId) { - this.roleCheckLock.writeLock().lock(); - try { - this.cachedPeerRole = Optional.ofNullable(currentRole); - this.cachedLeaderPeerId = Optional.ofNullable(leaderPeerId); - } finally { - this.roleCheckLock.writeLock().unlock(); + } catch (IOException ioe) { + // In this case we return not a leader. + LOG.error("Fail to get RaftServer impl and therefore it's not clear " + + "whether it's leader. ", ioe); } - } - - private GroupInfoReply getGroupInfo() throws IOException { - GroupInfoRequest groupInfoRequest = new GroupInfoRequest(clientId, - raftPeerId, raftGroupId, nextCallId()); - GroupInfoReply groupInfo = server.getGroupInfo(groupInfoRequest); - return groupInfo; + return RaftServerStatus.NOT_LEADER; } public int getServerPort() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index aaf94e9b8c56..183fe461cbcd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -380,7 +380,6 @@ public CompletableFuture notifyInstallSnapshotFromLeader( @Override public void notifyNotLeader(Collection pendingEntries) throws IOException { - omRatisServer.updateServerRole(); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 57b8fac6d0d1..c118307fd28d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.protocolPB; import java.io.IOException; -import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; @@ -26,10 +25,12 @@ import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -44,6 +45,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.LEADER_AND_READY; +import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.NOT_LEADER; + /** * This class is the server-side translator that forwards requests received on * {@link OzoneManagerProtocolPB} @@ -121,13 +125,14 @@ public OMResponse submitRequest(RpcController controller, private OMResponse processRequest(OMRequest request) throws ServiceException { - + RaftServerStatus raftServerStatus; if (isRatisEnabled) { // Check if the request is a read only request if (OmUtils.isReadOnly(request)) { return submitReadRequestToOM(request); } else { - if (omRatisServer.isLeader()) { + raftServerStatus = omRatisServer.checkLeaderStatus(); + if (raftServerStatus == LEADER_AND_READY) { try { OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(request); @@ -138,12 +143,7 @@ private OMResponse processRequest(OMRequest request) throws } return submitRequestToRatis(request); } else { - // throw not leader exception. This is being done, so to avoid - // unnecessary execution of preExecute on follower OM's. This - // will be helpful in the case like where we we reduce the - // chance of allocate blocks on follower OM's. Right now our - // leader status is updated every 1 second. - throw createNotLeaderException(); + throw createLeaderErrorException(raftServerStatus); } } } else { @@ -185,25 +185,22 @@ private OMResponse submitRequestToRatis(OMRequest request) private OMResponse submitReadRequestToOM(OMRequest request) throws ServiceException { // Check if this OM is the leader. - if (omRatisServer.isLeader()) { + RaftServerStatus raftServerStatus = omRatisServer.checkLeaderStatus(); + if (raftServerStatus == LEADER_AND_READY) { return handler.handleReadRequest(request); } else { - throw createNotLeaderException(); + throw createLeaderErrorException(raftServerStatus); } } private ServiceException createNotLeaderException() { RaftPeerId raftPeerId = omRatisServer.getRaftPeerId(); - Optional leaderRaftPeerId = omRatisServer - .getCachedLeaderPeerId(); - OMNotLeaderException notLeaderException; - if (leaderRaftPeerId.isPresent()) { - notLeaderException = new OMNotLeaderException( - raftPeerId, leaderRaftPeerId.get()); - } else { - notLeaderException = new OMNotLeaderException(raftPeerId); - } + // TODO: Set suggest leaderID. Right now, client is not using suggest + // leaderID. Need to fix this. + + OMNotLeaderException notLeaderException = + new OMNotLeaderException(raftPeerId); if (LOG.isDebugEnabled()) { LOG.debug(notLeaderException.getMessage()); @@ -212,6 +209,26 @@ private ServiceException createNotLeaderException() { return new ServiceException(notLeaderException); } + private ServiceException createLeaderErrorException( + RaftServerStatus raftServerStatus) { + if (raftServerStatus == NOT_LEADER) { + return createNotLeaderException(); + } else { + return createLeaderNotReadyException(); + } + } + + + private ServiceException createLeaderNotReadyException() { + RaftPeerId raftPeerId = omRatisServer.getRaftPeerId(); + + OMLeaderNotReadyException leaderNotReadyException = + new OMLeaderNotReadyException(raftPeerId.toString() + " is Leader " + + "but not ready to process request"); + + return new ServiceException(leaderNotReadyException); + } + /** * Submits request directly to OM. */ From e970a46311e59100df9c0c8f358d5d480e48b834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=2C=20M=C3=A1rton?= Date: Tue, 1 Dec 2020 09:18:27 +0100 Subject: [PATCH 31/78] HDDS-4424. Update README with information how to report security issues (#1548) --- README.md | 1 + SECURITY.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 SECURITY.md diff --git a/README.md b/README.md index b4a40bfcd863..e2d55aeda7e5 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Ozone is a top level project under the [Apache Software Foundation](https://apac * Chat: You can find the #ozone channel on the official ASF slack. Invite link is [here](http://s.apache.org/slack-invite). * There are Open [Weekly calls](https://cwiki.apache.org/confluence/display/HADOOP/Ozone+Community+Calls) where you can ask anything about Ozone. * Past meeting notes are also available from the wiki. + * Reporting security issues: Please consult with [SECURITY.md](./SECURITY.md) about reporting security vulerabilities and issues. ## Download diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..8d8a42b48d36 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,23 @@ +# Security Policy + +## Supported Versions + +The first stable release of Apache Ozone is 1.0, the previous alpha and beta releases are not supported by the community. + +| Version | Supported | +| ------------- | ------------------ | +| 0.3.0 (alpha) | :x: | +| 0.4.0 (alpha) | :x: | +| 0.4.1 (alpha) | :x: | +| 0.5.0 (beta) | :x: | +| 1.0 | :white_check_mark: | + +## Reporting a Vulnerability + +To report any found security issues or vulnerabilities, please send a mail to security@ozone.apache.org, so that they may be investigated and fixed before the vulnerabilities is published. + +This email address is a private mailing list for discussion of potential security vulnerabilities issues. + +This mailing list is **NOT** for end-user questions and discussion on security. Please use the dev@ozone.apache.org list for such issues. + +In order to post to the list, it is **NOT** necessary to first subscribe to it. From 02ba287dd75d199fca0c305d28607d606bdd3e60 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Tue, 1 Dec 2020 03:55:26 -0500 Subject: [PATCH 32/78] HDDS-4528. Upgrade slf4j to 1.7.30 (#1639) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 05c34d55bd41..e61ad50d971c 100644 --- a/pom.xml +++ b/pom.xml @@ -138,7 +138,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 4.4.13 - 1.7.25 + 1.7.30 1.2.17 2.13.3 3.4.2 From 20202f62dc08aac3253e2d82e918a6d8f6cbd28e Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 1 Dec 2020 11:19:22 +0000 Subject: [PATCH 33/78] Trigger CI From c296f780a7ecd246beb9ecb6458132c9dcf6b5d2 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 1 Dec 2020 12:56:41 +0000 Subject: [PATCH 34/78] Trigger CI From fc8b4ec08503520c9f7b0f0e27af6696c5f551ec Mon Sep 17 00:00:00 2001 From: micah zhao Date: Tue, 1 Dec 2020 21:04:21 +0800 Subject: [PATCH 35/78] HDDS-4485. [DOC] add the authentication rules of the Ozone Ranger. (#1603) * Modification of the operating introduction of ozone ranger. * fix style * fix style * Update hadoop-hdds/docs/content/security/SecurityWithRanger.md Co-authored-by: Vivek Ratnavel Subramanian * Update hadoop-hdds/docs/content/security/SecurityWithRanger.md Co-authored-by: Vivek Ratnavel Subramanian * fix review issues. * trigger new CI check Co-authored-by: Vivek Ratnavel Subramanian --- .../content/security/SecurityWithRanger.md | 21 +++++++++++++++++-- .../content/security/SecurityWithRanger.zh.md | 18 +++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/docs/content/security/SecurityWithRanger.md b/hadoop-hdds/docs/content/security/SecurityWithRanger.md index 7daaf8199e18..ee86a115930b 100644 --- a/hadoop-hdds/docs/content/security/SecurityWithRanger.md +++ b/hadoop-hdds/docs/content/security/SecurityWithRanger.md @@ -27,8 +27,9 @@ icon: user Apache Ranger™ is a framework to enable, monitor and manage comprehensive data -security across the Hadoop platform. Any version of Apache Ranger which is greater -than 1.20 is aware of Ozone, and can manage an Ozone cluster. +security across the Hadoop platform. Apache Ranger has supported Ozone authentication +since version 2.0. However, due to some bugs in 2.0, Apache Ranger +2.1 and later versions are recommended. To use Apache Ranger, you must have Apache Ranger installed in your Hadoop @@ -44,3 +45,19 @@ Property|Value --------|------------------------------------------------------------ ozone.acl.enabled | true ozone.acl.authorizer.class| org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer + +The Ranger permissions corresponding to the Ozone operations are as follows: + +| operation&permission | Volume permission | Bucket permission | Key permission | +| :--- | :--- | :--- | :--- | +| Create volume | CREATE | | | +| List volume | LIST | | | +| Get volume Info | READ | | | +| Delete volume | DELETE | | | +| Create bucket | READ | CREATE | | +| List bucket | LIST, READ | | | +| Get bucket info | READ | READ | | +| Delete bucket | READ | DELETE | | +| List key | READ | LIST, READ | | +| Write key | READ | READ | CREATE, WRITE | +| Read key | READ | READ | READ | diff --git a/hadoop-hdds/docs/content/security/SecurityWithRanger.zh.md b/hadoop-hdds/docs/content/security/SecurityWithRanger.zh.md index 4d40a17d0188..e7ff33e9268f 100644 --- a/hadoop-hdds/docs/content/security/SecurityWithRanger.zh.md +++ b/hadoop-hdds/docs/content/security/SecurityWithRanger.zh.md @@ -26,7 +26,7 @@ icon: user --> -Apache Ranger™ 是一个用于管理和监控 Hadoop 平台复杂数据权限的框架。版本大于 1.20 的 Apache Ranger 都可以用于管理 Ozone 集群。 +Apache Ranger™ 是一个用于管理和监控 Hadoop 平台复杂数据权限的框架。Apache Ranger 从2.0版本开始支持Ozone鉴权。但由于在2.0中存在一些bug，因此我们更推荐使用Apache Ranger 2.1及以后版本。 你需要先在你的 Hadoop 集群上安装 Apache Ranger，安装指南可以参考 [Apache Ranger 官网](https://ranger.apache.org/index.html). @@ -36,3 +36,19 @@ Apache Ranger™ 是一个用于管理和监控 Hadoop 平台复杂数据权限 --------|------------------------------------------------------------ ozone.acl.enabled | true ozone.acl.authorizer.class| org.apache.ranger.authorization.ozone.authorizer.RangerOzoneAuthorizer + +Ozone各类操作对应Ranger权限如下： + +| operation&permission | Volume permission | Bucket permission | Key permission | +| :--- | :--- | :--- | :--- | +| Create volume | CREATE | | | +| List volume | LIST | | | +| Get volume Info | READ | | | +| Delete volume | DELETE | | | +| Create bucket | READ | CREATE | | +| List bucket | LIST, READ | | | +| Get bucket info | READ | READ | | +| Delete bucket | READ | DELETE | | +| List key | READ | LIST, READ | | +| Write key | READ | READ | CREATE, WRITE | +| Read key | READ | READ | READ | \ No newline at end of file From cc5eebe0139dbf448382ed64cfc2b237c8655337 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 1 Dec 2020 14:48:43 +0000 Subject: [PATCH 36/78] Trigger CI From 90944636e6ca143dace266d066fbf00a7a56f263 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Tue, 1 Dec 2020 16:57:21 +0100 Subject: [PATCH 37/78] HDDS-4473. Reduce number of sortDatanodes RPC calls (#1610) --- .../hadoop/ozone/om/KeyManagerImpl.java | 98 ++++++++++----- .../hadoop/ozone/om/TestKeyManagerUnit.java | 119 ++++++++++++++---- 2 files changed, 162 insertions(+), 55 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 72ebfd2ea70e..f13f9d278381 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -24,6 +24,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -54,6 +55,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.utils.BackgroundService; @@ -109,6 +111,7 @@ import com.google.common.base.Strings; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; + import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; @@ -699,7 +702,7 @@ public OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress) refreshPipeline(value); if (args.getSortDatanodes()) { - sortDatanodeInPipeline(value, clientAddress); + sortDatanodes(clientAddress, value); } return value; } @@ -1827,7 +1830,7 @@ private OzoneFileStatus getOzoneFileStatus(String volumeName, // Please refer this jira for more details. refreshPipeline(fileKeyInfo); if (sortDatanodes) { - sortDatanodeInPipeline(fileKeyInfo, clientAddress); + sortDatanodes(clientAddress, fileKeyInfo); } return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); } @@ -2212,9 +2215,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, refreshPipeline(keyInfoList); if (args.getSortDatanodes()) { - for (OzoneFileStatus fileStatus : fileStatusList) { - sortDatanodeInPipeline(fileStatus.getKeyInfo(), clientAddress); - } + sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0])); } return fileStatusList; @@ -2309,38 +2310,67 @@ private FileEncryptionInfo getFileEncryptionInfo(OmBucketInfo bucketInfo) return encInfo; } - private void sortDatanodeInPipeline(OmKeyInfo keyInfo, String clientMachine) { - if (keyInfo != null && clientMachine != null && !clientMachine.isEmpty()) { - for (OmKeyLocationInfoGroup key : keyInfo.getKeyLocationVersions()) { - key.getLocationList().forEach(k -> { - List nodes = k.getPipeline().getNodes(); - if (nodes == null || nodes.isEmpty()) { - LOG.warn("Datanodes for pipeline {} is empty", - k.getPipeline().getId().toString()); - return; - } - List nodeList = new ArrayList<>(); - nodes.stream().forEach(node -> - nodeList.add(node.getUuidString())); - try { - List sortedNodes = scmClient.getBlockClient() - .sortDatanodes(nodeList, clientMachine); - k.getPipeline().setNodesInOrder(sortedNodes); - if (LOG.isDebugEnabled()) { - LOG.debug("Sort datanodes {} for client {}, return {}", nodes, - clientMachine, sortedNodes); + @VisibleForTesting + void sortDatanodes(String clientMachine, OmKeyInfo... keyInfos) { + if (keyInfos != null && clientMachine != null && !clientMachine.isEmpty()) { + Map, List> sortedPipelines = new HashMap<>(); + for (OmKeyInfo keyInfo : keyInfos) { + OmKeyLocationInfoGroup key = keyInfo.getLatestVersionLocations(); + if (key == null) { + LOG.warn("No location for key {}", keyInfo); + continue; + } + for (OmKeyLocationInfo k : key.getLocationList()) { + Pipeline pipeline = k.getPipeline(); + List nodes = pipeline.getNodes(); + List uuidList = toNodeUuid(nodes); + Set uuidSet = new HashSet<>(uuidList); + List sortedNodes = sortedPipelines.get(uuidSet); + if (sortedNodes == null) { + if (nodes.isEmpty()) { + LOG.warn("No datanodes in pipeline {}", pipeline.getId()); + continue; } - } catch (IOException e) { - LOG.warn("Unable to sort datanodes based on distance to " + - "client, volume=" + keyInfo.getVolumeName() + - ", bucket=" + keyInfo.getBucketName() + - ", key=" + keyInfo.getKeyName() + - ", client=" + clientMachine + - ", datanodes=" + nodes.toString() + - ", exception=" + e.getMessage()); + sortedNodes = sortDatanodes(clientMachine, nodes, keyInfo, + uuidList); + if (sortedNodes != null) { + sortedPipelines.put(uuidSet, sortedNodes); + } + } else if (LOG.isDebugEnabled()) { + LOG.debug("Found sorted datanodes for pipeline {} and client {} " + + "in cache", pipeline.getId(), clientMachine); } - }); + pipeline.setNodesInOrder(sortedNodes); + } + } + } + } + + private List sortDatanodes(String clientMachine, + List nodes, OmKeyInfo keyInfo, List nodeList) { + List sortedNodes = null; + try { + sortedNodes = scmClient.getBlockClient() + .sortDatanodes(nodeList, clientMachine); + if (LOG.isDebugEnabled()) { + LOG.debug("Sorted datanodes {} for client {}, result: {}", nodes, + clientMachine, sortedNodes); } + } catch (IOException e) { + LOG.warn("Unable to sort datanodes based on distance to client, " + + " volume={}, bucket={}, key={}, client={}, datanodes={}, " + + " exception={}", + keyInfo.getVolumeName(), keyInfo.getBucketName(), + keyInfo.getKeyName(), clientMachine, nodeList, e.getMessage()); + } + return sortedNodes; + } + + private static List toNodeUuid(Collection nodes) { + List nodeSet = new ArrayList<>(nodes.size()); + for (DatanodeDetails node : nodes) { + nodeSet.add(node.getUuidString()); } + return nodeSet; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 6046ac9a5caf..a947b3550ced 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -76,8 +76,12 @@ import org.junit.Test; import org.mockito.Mockito; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Unit test key manager. @@ -91,6 +95,7 @@ public class TestKeyManagerUnit { private Instant startDate; private File testDir; + private ScmBlockLocationProtocol blockClient; @Before public void setup() throws IOException { @@ -100,14 +105,10 @@ public void setup() throws IOException { testDir.toString()); metadataManager = new OmMetadataManagerImpl(configuration); containerClient = Mockito.mock(StorageContainerLocationProtocol.class); + blockClient = Mockito.mock(ScmBlockLocationProtocol.class); keyManager = new KeyManagerImpl( - Mockito.mock(ScmBlockLocationProtocol.class), - containerClient, - metadataManager, - configuration, - "omtest", - Mockito.mock(OzoneBlockTokenSecretManager.class) - ); + blockClient, containerClient, metadataManager, configuration, + "omtest", Mockito.mock(OzoneBlockTokenSecretManager.class)); startDate = Instant.now(); } @@ -372,10 +373,10 @@ public void testLookupFileWithDnFailure() throws IOException { List cps = new ArrayList<>(); ContainerInfo ci = Mockito.mock(ContainerInfo.class); - Mockito.when(ci.getContainerID()).thenReturn(1L); + when(ci.getContainerID()).thenReturn(1L); cps.add(new ContainerWithPipeline(ci, pipelineTwo)); - Mockito.when(containerClient.getContainerWithPipelineBatch(containerIDs)) + when(containerClient.getContainerWithPipelineBatch(containerIDs)) .thenReturn(cps); final OmVolumeArgs volumeArgs = OmVolumeArgs.newBuilder() @@ -443,6 +444,7 @@ public void listStatus() throws Exception { String volume = "vol"; String bucket = "bucket"; String keyPrefix = "key"; + String client = "client.host"; TestOMRequestUtils.addVolumeToDB(volume, OzoneConsts.OZONE, metadataManager); @@ -450,17 +452,12 @@ public void listStatus() throws Exception { TestOMRequestUtils.addBucketToDB(volume, bucket, metadataManager); final Pipeline pipeline = MockPipeline.createPipeline(3); - - OmKeyInfo.Builder keyInfoBuilder = new OmKeyInfo.Builder() - .setVolumeName(volume) - .setBucketName(bucket) - .setCreationTime(Time.now()) - .setOmKeyLocationInfos(singletonList( - new OmKeyLocationInfoGroup(0, new ArrayList<>()))) - .setReplicationFactor(ReplicationFactor.THREE) - .setReplicationType(ReplicationType.RATIS); + final List nodes = pipeline.getNodes().stream() + .map(DatanodeDetails::getUuidString) + .collect(toList()); List containerIDs = new ArrayList<>(); + List containersWithPipeline = new ArrayList<>(); for (long i = 1; i <= 10; i++) { final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() .setBlockID(new BlockID(i, 1L)) @@ -469,9 +466,21 @@ public void listStatus() throws Exception { .setLength(256000) .build(); + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(i) + .build(); + containersWithPipeline.add( + new ContainerWithPipeline(containerInfo, pipeline)); containerIDs.add(i); - OmKeyInfo keyInfo = keyInfoBuilder + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setVolumeName(volume) + .setBucketName(bucket) + .setCreationTime(Time.now()) + .setOmKeyLocationInfos(singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>()))) + .setReplicationFactor(ReplicationFactor.THREE) + .setReplicationType(ReplicationType.RATIS) .setKeyName(keyPrefix + i) .setObjectID(i) .setUpdateID(i) @@ -480,15 +489,83 @@ public void listStatus() throws Exception { TestOMRequestUtils.addKeyToOM(metadataManager, keyInfo); } + when(containerClient.getContainerWithPipelineBatch(containerIDs)) + .thenReturn(containersWithPipeline); + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() .setVolumeName(volume) .setBucketName(bucket) - .setKeyName(""); + .setKeyName("") + .setSortDatanodesInPipeline(true); List fileStatusList = - keyManager.listStatus(builder.build(), false, null, Long.MAX_VALUE); + keyManager.listStatus(builder.build(), false, + null, Long.MAX_VALUE, client); Assert.assertEquals(10, fileStatusList.size()); verify(containerClient).getContainerWithPipelineBatch(containerIDs); + verify(blockClient).sortDatanodes(nodes, client); + } + + @Test + public void sortDatanodes() throws Exception { + // GIVEN + String client = "anyhost"; + int pipelineCount = 3; + int keysPerPipeline = 5; + OmKeyInfo[] keyInfos = new OmKeyInfo[pipelineCount * keysPerPipeline]; + List> expectedSortDatanodesInvocations = new ArrayList<>(); + Map> expectedSortedNodes = new HashMap<>(); + int ki = 0; + for (int p = 0; p < pipelineCount; p++) { + final Pipeline pipeline = MockPipeline.createPipeline(3); + final List nodes = pipeline.getNodes().stream() + .map(DatanodeDetails::getUuidString) + .collect(toList()); + expectedSortDatanodesInvocations.add(nodes); + final List sortedNodes = pipeline.getNodes().stream() + .sorted(comparing(DatanodeDetails::getUuidString)) + .collect(toList()); + expectedSortedNodes.put(pipeline, sortedNodes); + + when(blockClient.sortDatanodes(nodes, client)) + .thenReturn(sortedNodes); + + for (int i = 1; i <= keysPerPipeline; i++) { + OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(i, 1L)) + .setPipeline(pipeline) + .setOffset(0) + .setLength(256000) + .build(); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setOmKeyLocationInfos(Arrays.asList( + new OmKeyLocationInfoGroup(0, emptyList()), + new OmKeyLocationInfoGroup(1, singletonList(keyLocationInfo)))) + .build(); + keyInfos[ki++] = keyInfo; + } + } + + // WHEN + keyManager.sortDatanodes(client, keyInfos); + + // THEN + // verify all key info locations got updated + for (OmKeyInfo keyInfo : keyInfos) { + OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); + Assert.assertNotNull(locations); + for (OmKeyLocationInfo locationInfo : locations.getLocationList()) { + Pipeline pipeline = locationInfo.getPipeline(); + List expectedOrder = expectedSortedNodes.get(pipeline); + Assert.assertEquals(expectedOrder, pipeline.getNodesInOrder()); + } + } + + // expect one invocation per pipeline + for (List nodes : expectedSortDatanodesInvocations) { + verify(blockClient).sortDatanodes(nodes, client); + } } } From d811ed74698ed1ad785c4a3a3d50373786546f16 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 1 Dec 2020 17:06:27 +0000 Subject: [PATCH 38/78] Trigger build From fd015cf6e915d3048ae864d60cca97dfebbe24e8 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Tue, 1 Dec 2020 17:26:30 -0500 Subject: [PATCH 39/78] HDDS-4529. Upgrade snakeyaml to 1.26 (#1640) --- .../hadoop/ozone/container/common/impl/ContainerDataYaml.java | 4 +--- pom.xml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java index 74cbbc0af846..757d7e856d63 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.container.common.impl; -import java.beans.IntrospectionException; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -201,8 +200,7 @@ public static Yaml getYamlForContainerType(ContainerType containerType) */ private static class ContainerDataRepresenter extends Representer { @Override - protected Set getProperties(Class type) - throws IntrospectionException { + protected Set getProperties(Class type) { Set set = super.getProperties(type); Set filtered = new TreeSet(); diff --git a/pom.xml b/pom.xml index e61ad50d971c..a595fbe61acc 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.0.5 1.5.4 - 1.16 + 1.26 ${basedir}/target/classes From f1e46cb25f815ff857ecb4e4fc9ab44395a65943 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Wed, 2 Dec 2020 12:02:59 +0100 Subject: [PATCH 40/78] HDDS-4320. Let Ozone input streams implement CanUnbuffer (#1523) --- .../hdds/scm/storage/BlockInputStream.java | 111 +++++++--- .../hdds/scm/storage/ChunkInputStream.java | 67 ++++-- .../scm/storage/DummyBlockInputStream.java | 31 +-- .../DummyBlockInputStreamWithRetry.java | 4 +- .../scm/storage/DummyChunkInputStream.java | 24 ++- .../scm/storage/TestBlockInputStream.java | 112 +++++++++- .../scm/storage/TestChunkInputStream.java | 62 +++++- .../ozone/client/io/KeyInputStream.java | 11 +- .../ozone/client/io/OzoneInputStream.java | 11 +- .../AbstractContractUnbufferTest.java | 159 ++++++++++++++ .../contract/ITestOzoneContractUnbuffer.java | 47 +++++ .../ITestRootedOzoneContractUnbuffer.java | 48 +++++ .../ozone/client/rpc/TestKeyInputStream.java | 199 ++++++++++++++---- .../hadoop/ozone/container/TestHelper.java | 70 +++++- .../src/test/resources/contract/ozone.xml | 5 + .../hadoop/ozone/om/TestChunkStreams.java | 4 +- .../fs/ozone/CapableOzoneFSInputStream.java | 3 +- .../hadoop/fs/ozone/OzoneFSInputStream.java | 10 +- .../fs/ozone/OzoneStreamCapabilities.java | 9 + 19 files changed, 851 insertions(+), 136 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/ITestRootedOzoneContractUnbuffer.java diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 3578bd6ea1e2..8cb97c1d2070 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.function.Function; +import org.apache.hadoop.fs.CanUnbuffer; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; @@ -33,7 +34,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.XceiverClientFactory; -import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -50,7 +50,8 @@ * This class encapsulates all state management for iterating * through the sequence of chunks through {@link ChunkInputStream}. */ -public class BlockInputStream extends InputStream implements Seekable { +public class BlockInputStream extends InputStream + implements Seekable, CanUnbuffer { private static final Logger LOG = LoggerFactory.getLogger(BlockInputStream.class); @@ -95,25 +96,25 @@ public class BlockInputStream extends InputStream implements Seekable { // can be reset if a new position is seeked. private int chunkIndexOfPrevPosition; - private Function refreshPipelineFunction; + private final Function refreshPipelineFunction; public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline, Token token, boolean verifyChecksum, - XceiverClientFactory xceiverClientFctry, + XceiverClientFactory xceiverClientFactory, Function refreshPipelineFunction) { this.blockID = blockId; this.length = blockLen; this.pipeline = pipeline; this.token = token; this.verifyChecksum = verifyChecksum; - this.xceiverClientFactory = xceiverClientFctry; + this.xceiverClientFactory = xceiverClientFactory; this.refreshPipelineFunction = refreshPipelineFunction; } public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline, Token token, boolean verifyChecksum, - XceiverClientManager xceiverClientFactory + XceiverClientFactory xceiverClientFactory ) { this(blockId, blockLen, pipeline, token, verifyChecksum, xceiverClientFactory, null); @@ -129,22 +130,12 @@ public synchronized void initialize() throws IOException { return; } - List chunks = null; + List chunks; try { chunks = getChunkInfos(); } catch (ContainerNotFoundException ioEx) { - LOG.error("Unable to read block information from pipeline."); - if (refreshPipelineFunction != null) { - LOG.debug("Re-fetching pipeline for block {}", blockID); - Pipeline newPipeline = refreshPipelineFunction.apply(blockID); - if (newPipeline == null || newPipeline.equals(pipeline)) { - throw ioEx; - } else { - LOG.debug("New pipeline got for block {}", blockID); - this.pipeline = newPipeline; - chunks = getChunkInfos(); - } - } + refreshPipeline(ioEx); + chunks = getChunkInfos(); } if (chunks != null && !chunks.isEmpty()) { @@ -171,6 +162,24 @@ public synchronized void initialize() throws IOException { } } + private void refreshPipeline(IOException cause) throws IOException { + LOG.info("Unable to read information for block {} from pipeline {}: {}", + blockID, pipeline.getId(), cause.getMessage()); + if (refreshPipelineFunction != null) { + LOG.debug("Re-fetching pipeline for block {}", blockID); + Pipeline newPipeline = refreshPipelineFunction.apply(blockID); + if (newPipeline == null || newPipeline.equals(pipeline)) { + LOG.warn("No new pipeline for block {}", blockID); + throw cause; + } else { + LOG.debug("New pipeline got for block {}", blockID); + this.pipeline = newPipeline; + } + } else { + throw cause; + } + } + /** * Send RPC call to get the block info from the container. * @return List of chunks in this block. @@ -182,7 +191,7 @@ protected List getChunkInfos() throws IOException { pipeline = Pipeline.newBuilder(pipeline) .setType(HddsProtos.ReplicationType.STAND_ALONE).build(); } - xceiverClient = xceiverClientFactory.acquireClientForReadData(pipeline); + acquireClient(); boolean success = false; List chunks; try { @@ -207,17 +216,25 @@ protected List getChunkInfos() throws IOException { return chunks; } + protected void acquireClient() throws IOException { + xceiverClient = xceiverClientFactory.acquireClientForReadData(pipeline); + } + /** * Append another ChunkInputStream to the end of the list. Note that the * ChunkInputStream is only created here. The chunk will be read from the * Datanode only when a read operation is performed on for that chunk. */ protected synchronized void addStream(ChunkInfo chunkInfo) { - chunkStreams.add(new ChunkInputStream(chunkInfo, blockID, - xceiverClient, verifyChecksum, token)); + chunkStreams.add(createChunkInputStream(chunkInfo)); } - public synchronized long getRemaining() throws IOException { + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + return new ChunkInputStream(chunkInfo, blockID, + xceiverClientFactory, () -> pipeline, verifyChecksum, token); + } + + public synchronized long getRemaining() { return length - getPos(); } @@ -266,7 +283,13 @@ public synchronized int read(byte[] b, int off, int len) throws IOException { // Get the current chunkStream and read data from it ChunkInputStream current = chunkStreams.get(chunkIndex); int numBytesToRead = Math.min(len, (int)current.getRemaining()); - int numBytesRead = current.read(b, off, numBytesToRead); + int numBytesRead; + try { + numBytesRead = current.read(b, off, numBytesToRead); + } catch (IOException e) { + handleReadError(e); + continue; + } if (numBytesRead != numBytesToRead) { // This implies that there is either data loss or corruption in the @@ -356,7 +379,7 @@ public synchronized void seek(long pos) throws IOException { } @Override - public synchronized long getPos() throws IOException { + public synchronized long getPos() { if (length == 0) { return 0; } @@ -376,9 +399,13 @@ public boolean seekToNewSource(long targetPos) throws IOException { @Override public synchronized void close() { + releaseClient(); + xceiverClientFactory = null; + } + + private void releaseClient() { if (xceiverClientFactory != null && xceiverClient != null) { xceiverClientFactory.releaseClient(xceiverClient, false); - xceiverClientFactory = null; xceiverClient = null; } } @@ -393,7 +420,7 @@ public synchronized void resetPosition() { * @throws IOException if stream is closed */ protected synchronized void checkOpen() throws IOException { - if (xceiverClient == null) { + if (xceiverClientFactory == null) { throw new IOException("BlockInputStream has been closed."); } } @@ -416,8 +443,32 @@ synchronized long getBlockPosition() { return blockPosition; } - @VisibleForTesting - synchronized List getChunkStreams() { - return chunkStreams; + @Override + public void unbuffer() { + storePosition(); + releaseClient(); + + final List inputStreams = this.chunkStreams; + if (inputStreams != null) { + for (ChunkInputStream is : inputStreams) { + is.unbuffer(); + } + } + } + + private synchronized void storePosition() { + blockPosition = getPos(); + } + + private void handleReadError(IOException cause) throws IOException { + releaseClient(); + final List inputStreams = this.chunkStreams; + if (inputStreams != null) { + for (ChunkInputStream is : inputStreams) { + is.releaseClient(); + } + } + + refreshPipeline(cause); } } diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java index cfb3a21f62be..9c034538c572 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java @@ -20,14 +20,17 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.CanUnbuffer; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.common.ChecksumData; import org.apache.hadoop.ozone.common.OzoneChecksumException; @@ -40,18 +43,22 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.util.List; +import java.util.function.Supplier; /** * An {@link InputStream} called from BlockInputStream to read a chunk from the * container. Each chunk may contain multiple underlying {@link ByteBuffer} * instances. */ -public class ChunkInputStream extends InputStream implements Seekable { +public class ChunkInputStream extends InputStream + implements Seekable, CanUnbuffer { private ChunkInfo chunkInfo; private final long length; private final BlockID blockID; + private final XceiverClientFactory xceiverClientFactory; private XceiverClientSpi xceiverClient; + private final Supplier pipelineSupplier; private boolean verifyChecksum; private boolean allocated = false; // Buffer to store the chunk data read from the DN container @@ -69,9 +76,8 @@ public class ChunkInputStream extends InputStream implements Seekable { // Position of the ChunkInputStream is maintained by this variable (if a // seek is performed. This position is w.r.t to the chunk only and not the - // block or key. This variable is set only if either the buffers are not - // yet allocated or the if the allocated buffers do not cover the seeked - // position. Once the chunk is read, this variable is reset. + // block or key. This variable is also set before attempting a read to enable + // retry. Once the chunk is read, this variable is reset. private long chunkPosition = -1; private final Token token; @@ -79,17 +85,19 @@ public class ChunkInputStream extends InputStream implements Seekable { private static final int EOF = -1; ChunkInputStream(ChunkInfo chunkInfo, BlockID blockId, - XceiverClientSpi xceiverClient, boolean verifyChecksum, - Token token) { + XceiverClientFactory xceiverClientFactory, + Supplier pipelineSupplier, + boolean verifyChecksum, Token token) { this.chunkInfo = chunkInfo; this.length = chunkInfo.getLen(); this.blockID = blockId; - this.xceiverClient = xceiverClient; + this.xceiverClientFactory = xceiverClientFactory; + this.pipelineSupplier = pipelineSupplier; this.verifyChecksum = verifyChecksum; this.token = token; } - public synchronized long getRemaining() throws IOException { + public synchronized long getRemaining() { return length - getPos(); } @@ -98,7 +106,7 @@ public synchronized long getRemaining() throws IOException { */ @Override public synchronized int read() throws IOException { - checkOpen(); + acquireClient(); int available = prepareRead(1); int dataout = EOF; @@ -143,7 +151,7 @@ public synchronized int read(byte[] b, int off, int len) throws IOException { if (len == 0) { return 0; } - checkOpen(); + acquireClient(); int total = 0; while (len > 0) { int available = prepareRead(len); @@ -196,7 +204,7 @@ public synchronized void seek(long pos) throws IOException { } @Override - public synchronized long getPos() throws IOException { + public synchronized long getPos() { if (chunkPosition >= 0) { return chunkPosition; } @@ -219,19 +227,23 @@ public boolean seekToNewSource(long targetPos) throws IOException { @Override public synchronized void close() { - if (xceiverClient != null) { + releaseClient(); + } + + protected synchronized void releaseClient() { + if (xceiverClientFactory != null && xceiverClient != null) { + xceiverClientFactory.releaseClient(xceiverClient, false); xceiverClient = null; } } /** - * Checks if the stream is open. If not, throw an exception. - * - * @throws IOException if stream is closed + * Acquire new client if previous one was released. */ - protected synchronized void checkOpen() throws IOException { - if (xceiverClient == null) { - throw new IOException("BlockInputStream has been closed."); + protected synchronized void acquireClient() throws IOException { + if (xceiverClientFactory != null && xceiverClient == null) { + xceiverClient = xceiverClientFactory.acquireClientForReadData( + pipelineSupplier.get()); } } @@ -292,6 +304,11 @@ private synchronized void readChunkFromContainer(int len) throws IOException { startByteIndex = bufferOffset + bufferLength; } + // bufferOffset and bufferLength are updated below, but if read fails + // and is retried, we need the previous position. Position is reset after + // successful read in adjustBufferPosition() + storePosition(); + if (verifyChecksum) { // Update the bufferOffset and bufferLength as per the checksum // boundary requirement. @@ -437,7 +454,8 @@ private void adjustBufferPosition(long bufferPosition) { /** * Check if the buffers have been allocated data and false otherwise. */ - private boolean buffersAllocated() { + @VisibleForTesting + protected boolean buffersAllocated() { return buffers != null && !buffers.isEmpty(); } @@ -538,6 +556,10 @@ void resetPosition() { this.chunkPosition = -1; } + private void storePosition() { + chunkPosition = getPos(); + } + String getChunkName() { return chunkInfo.getChunkName(); } @@ -550,4 +572,11 @@ protected long getLength() { protected long getChunkPosition() { return chunkPosition; } + + @Override + public synchronized void unbuffer() { + storePosition(); + releaseBuffers(); + releaseClient(); + } } diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java index 5db722acaa41..1c7968b13446 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStream.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier; import org.apache.hadoop.security.token.Token; @@ -34,9 +34,9 @@ */ class DummyBlockInputStream extends BlockInputStream { - private List chunks; + private final List chunks; - private Map chunkDataMap; + private final Map chunkDataMap; @SuppressWarnings("parameternumber") DummyBlockInputStream( @@ -45,23 +45,7 @@ class DummyBlockInputStream extends BlockInputStream { Pipeline pipeline, Token token, boolean verifyChecksum, - XceiverClientManager xceiverClientManager, - List chunkList, - Map chunkMap) { - super(blockId, blockLen, pipeline, token, verifyChecksum, - xceiverClientManager); - this.chunks = chunkList; - this.chunkDataMap = chunkMap; - } - - @SuppressWarnings("parameternumber") - DummyBlockInputStream( - BlockID blockId, - long blockLen, - Pipeline pipeline, - Token token, - boolean verifyChecksum, - XceiverClientManager xceiverClientManager, + XceiverClientFactory xceiverClientManager, Function refreshFunction, List chunkList, Map chunks) { @@ -78,11 +62,10 @@ protected List getChunkInfos() throws IOException { } @Override - protected void addStream(ChunkInfo chunkInfo) { - TestChunkInputStream testChunkInputStream = new TestChunkInputStream(); - getChunkStreams().add(new DummyChunkInputStream(testChunkInputStream, + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + return new DummyChunkInputStream( chunkInfo, null, null, false, - chunkDataMap.get(chunkInfo.getChunkName()).clone())); + chunkDataMap.get(chunkInfo.getChunkName()).clone(), null); } @Override diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java index 1686ed45b859..51ba2c6eddce 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyBlockInputStreamWithRetry.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; @@ -49,7 +49,7 @@ final class DummyBlockInputStreamWithRetry Pipeline pipeline, Token token, boolean verifyChecksum, - XceiverClientManager xceiverClientManager, + XceiverClientFactory xceiverClientManager, List chunkList, Map chunkMap, AtomicBoolean isRerfreshed) { diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyChunkInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyChunkInputStream.java index e654d119f75d..15f7edab6fcf 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyChunkInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/DummyChunkInputStream.java @@ -22,8 +22,9 @@ import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; /** @@ -31,18 +32,18 @@ */ public class DummyChunkInputStream extends ChunkInputStream { - private byte[] chunkData; + private final byte[] chunkData; // Stores the read chunk data in each readChunk call - private List readByteBuffers = new ArrayList<>(); + private final List readByteBuffers = new ArrayList<>(); - public DummyChunkInputStream(TestChunkInputStream testChunkInputStream, - ChunkInfo chunkInfo, + public DummyChunkInputStream(ChunkInfo chunkInfo, BlockID blockId, - XceiverClientSpi xceiverClient, + XceiverClientFactory xceiverClientFactory, boolean verifyChecksum, - byte[] data) { - super(chunkInfo, blockId, xceiverClient, verifyChecksum, null); + byte[] data, Pipeline pipeline) { + super(chunkInfo, blockId, xceiverClientFactory, () -> pipeline, + verifyChecksum, null); this.chunkData = data; } @@ -56,10 +57,15 @@ protected ByteString readChunk(ChunkInfo readChunkInfo) { } @Override - protected void checkOpen() { + protected void acquireClient() { // No action needed } + @Override + protected void releaseClient() { + // no-op + } + public List getReadByteBuffers() { return readByteBuffers; } diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java index 3f5e12a8e16b..e1ede9771970 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java @@ -23,25 +23,42 @@ import org.apache.hadoop.hdds.client.ContainerBlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.common.Checksum; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; import java.io.EOFException; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_NOT_FOUND; import static org.apache.hadoop.hdds.scm.storage.TestChunkInputStream.generateRandomData; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Tests for {@link BlockInputStream}'s functionality. */ +@RunWith(MockitoJUnitRunner.class) public class TestBlockInputStream { private static final int CHUNK_SIZE = 100; @@ -52,7 +69,9 @@ public class TestBlockInputStream { private int blockSize; private List chunks; private Map chunkDataMap; - private AtomicBoolean isRefreshed = new AtomicBoolean(); + + @Mock + private Function refreshPipeline; @Before public void setup() throws Exception { @@ -61,7 +80,7 @@ public void setup() throws Exception { createChunkList(5); blockStream = new DummyBlockInputStream(blockID, blockSize, null, null, - false, null, chunks, chunkDataMap); + false, null, refreshPipeline, chunks, chunkDataMap); } /** @@ -199,9 +218,11 @@ public void testSeekAndRead() throws Exception { @Test public void testRefreshPipelineFunction() throws Exception { BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); + AtomicBoolean isRefreshed = new AtomicBoolean(); createChunkList(5); BlockInputStream blockInputStreamWithRetry = - new DummyBlockInputStreamWithRetry(blockID, blockSize, null, null, + new DummyBlockInputStreamWithRetry(blockID, blockSize, + MockPipeline.createSingleNodePipeline(), null, false, null, chunks, chunkDataMap, isRefreshed); Assert.assertFalse(isRefreshed.get()); @@ -210,4 +231,89 @@ public void testRefreshPipelineFunction() throws Exception { blockInputStreamWithRetry.read(b, 0, 200); Assert.assertTrue(isRefreshed.get()); } + + @Test + public void testRefreshOnReadFailure() throws Exception { + // GIVEN + BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + Pipeline newPipeline = MockPipeline.createSingleNodePipeline(); + + final int len = 200; + final ChunkInputStream stream = mock(ChunkInputStream.class); + when(stream.read(any(), anyInt(), anyInt())) + .thenThrow(new StorageContainerException("test", CONTAINER_NOT_FOUND)) + .thenReturn(len); + when(stream.getRemaining()) + .thenReturn((long) len); + + when(refreshPipeline.apply(blockID)) + .thenReturn(newPipeline); + + BlockInputStream subject = new DummyBlockInputStream(blockID, blockSize, + pipeline, null, false, null, refreshPipeline, chunks, null) { + @Override + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + return stream; + } + }; + subject.initialize(); + + // WHEN + byte[] b = new byte[len]; + int bytesRead = subject.read(b, 0, len); + + // THEN + Assert.assertEquals(len, bytesRead); + verify(refreshPipeline).apply(blockID); + } + + @Test + public void testRefreshOnReadFailureAfterUnbuffer() throws Exception { + // GIVEN + BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + Pipeline newPipeline = MockPipeline.createSingleNodePipeline(); + XceiverClientFactory clientFactory = mock(XceiverClientFactory.class); + XceiverClientSpi client = mock(XceiverClientSpi.class); + when(clientFactory.acquireClientForReadData(pipeline)) + .thenReturn(client); + + final int len = 200; + final ChunkInputStream stream = mock(ChunkInputStream.class); + when(stream.read(any(), anyInt(), anyInt())) + .thenThrow(new StorageContainerException("test", CONTAINER_NOT_FOUND)) + .thenReturn(len); + when(stream.getRemaining()) + .thenReturn((long) len); + + when(refreshPipeline.apply(blockID)) + .thenReturn(newPipeline); + + BlockInputStream subject = new BlockInputStream(blockID, blockSize, + pipeline, null, false, clientFactory, refreshPipeline) { + @Override + protected List getChunkInfos() throws IOException { + acquireClient(); + return chunks; + } + + @Override + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + return stream; + } + }; + subject.initialize(); + subject.unbuffer(); + + // WHEN + byte[] b = new byte[len]; + int bytesRead = subject.read(b, 0, len); + + // THEN + Assert.assertEquals(len, bytesRead); + verify(refreshPipeline).apply(blockID); + verify(clientFactory).acquireClientForReadData(pipeline); + verify(clientFactory).releaseClient(client, false); + } } diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java index eea8e1f14ef4..cb110b182d85 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java @@ -20,16 +20,26 @@ import java.io.EOFException; import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.scm.XceiverClientFactory; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.pipeline.MockPipeline; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + /** * Tests for {@link ChunkInputStream}'s functionality. */ @@ -59,8 +69,8 @@ public void setup() throws Exception { chunkData, 0, CHUNK_SIZE).getProtoBufMessage()) .build(); - chunkStream = - new DummyChunkInputStream(this, chunkInfo, null, null, true, chunkData); + chunkStream = new DummyChunkInputStream(chunkInfo, null, null, true, + chunkData, null); } static byte[] generateRandomData(int length) { @@ -174,4 +184,50 @@ public void testSeekAndRead() throws Exception { chunkStream.read(b2, 0, 20); matchWithInputData(b2, 70, 20); } -} \ No newline at end of file + + @Test + public void testUnbuffer() throws Exception { + byte[] b1 = new byte[20]; + chunkStream.read(b1, 0, 20); + matchWithInputData(b1, 0, 20); + + chunkStream.unbuffer(); + + Assert.assertFalse(chunkStream.buffersAllocated()); + + // Next read should start from the position of the last read + 1 i.e. 20 + byte[] b2 = new byte[20]; + chunkStream.read(b2, 0, 20); + matchWithInputData(b2, 20, 20); + } + + @Test + public void connectsToNewPipeline() throws Exception { + // GIVEN + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + Pipeline newPipeline = MockPipeline.createSingleNodePipeline(); + XceiverClientFactory clientFactory = mock(XceiverClientFactory.class); + XceiverClientSpi client = mock(XceiverClientSpi.class); + when(clientFactory.acquireClientForReadData(pipeline)) + .thenReturn(client); + + AtomicReference pipelineRef = new AtomicReference<>(pipeline); + + ChunkInputStream subject = new ChunkInputStream(chunkInfo, null, + clientFactory, pipelineRef::get, false, null) { + @Override + protected ByteString readChunk(ChunkInfo readChunkInfo) { + return ByteString.copyFrom(chunkData); + } + }; + + // WHEN + subject.unbuffer(); + pipelineRef.set(newPipeline); + int b = subject.read(); + + // THEN + Assert.assertNotEquals(-1, b); + verify(clientFactory).acquireClientForReadData(newPipeline); + } +} diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java index f8f6cd3fea7e..a837feafbf29 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java @@ -26,6 +26,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hadoop.fs.CanUnbuffer; import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.hdds.client.BlockID; @@ -43,7 +44,8 @@ /** * Maintaining a list of BlockInputStream. Read based on offset. */ -public class KeyInputStream extends InputStream implements Seekable { +public class KeyInputStream extends InputStream + implements Seekable, CanUnbuffer { private static final Logger LOG = LoggerFactory.getLogger(KeyInputStream.class); @@ -333,4 +335,11 @@ public long skip(long n) throws IOException { seek(getPos() + toSkip); return toSkip; } + + @Override + public void unbuffer() { + for (BlockInputStream is : blockStreams) { + is.unbuffer(); + } + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java index 14b28665bdb1..f01975c94d8e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.client.io; +import org.apache.hadoop.fs.CanUnbuffer; + import java.io.IOException; import java.io.InputStream; @@ -24,7 +26,7 @@ * OzoneInputStream is used to read data from Ozone. * It uses {@link KeyInputStream} for reading the data. */ -public class OzoneInputStream extends InputStream { +public class OzoneInputStream extends InputStream implements CanUnbuffer { private final InputStream inputStream; @@ -65,4 +67,11 @@ public long skip(long n) throws IOException { public InputStream getInputStream() { return inputStream; } + + @Override + public void unbuffer() { + if (inputStream instanceof CanUnbuffer) { + ((CanUnbuffer) inputStream).unbuffer(); + } + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java new file mode 100644 index 000000000000..809e6d1b90b0 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; + +/** + * Contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}. + * Note: this is from Hadoop 3.3, can be removed after dependency upgrade. + */ +public abstract class AbstractContractUnbufferTest + extends AbstractFSContractTestBase { + + private Path file; + private byte[] fileBytes; + + private static final String SUPPORTS_UNBUFFER = "supports-unbuffer"; + + @Override + public void setup() throws Exception { + super.setup(); + skipIfUnsupported(SUPPORTS_UNBUFFER); + file = path("unbufferFile"); + fileBytes = dataset(TEST_FILE_LEN, 0, 255); + createFile(getFileSystem(), file, true, fileBytes); + } + + @Test + public void testUnbufferAfterRead() throws IOException { + describe("unbuffer a file after a single read"); + try (FSDataInputStream stream = getFileSystem().open(file)) { + validateFullFileContents(stream); + unbuffer(stream); + } + } + + @Test + public void testUnbufferBeforeRead() throws IOException { + describe("unbuffer a file before a read"); + try (FSDataInputStream stream = getFileSystem().open(file)) { + unbuffer(stream); + validateFullFileContents(stream); + } + } + + @Test + public void testUnbufferEmptyFile() throws IOException { + Path emptyFile = path("emptyUnbufferFile"); + getFileSystem().create(emptyFile, true).close(); + describe("unbuffer an empty file"); + try (FSDataInputStream stream = getFileSystem().open(emptyFile)) { + unbuffer(stream); + } + } + + @Test + public void testUnbufferOnClosedFile() throws IOException { + describe("unbuffer a file after it is closed"); + FSDataInputStream stream = null; + try { + stream = getFileSystem().open(file); + validateFullFileContents(stream); + } finally { + if (stream != null) { + stream.close(); + } + } + if (stream != null) { + unbuffer(stream); + } + } + + @Test + public void testMultipleUnbuffers() throws IOException { + describe("unbuffer a file multiple times"); + try (FSDataInputStream stream = getFileSystem().open(file)) { + unbuffer(stream); + unbuffer(stream); + validateFullFileContents(stream); + unbuffer(stream); + unbuffer(stream); + } + } + + @Test + public void testUnbufferMultipleReads() throws IOException { + describe("unbuffer a file multiple times"); + try (FSDataInputStream stream = getFileSystem().open(file)) { + unbuffer(stream); + validateFileContents(stream, TEST_FILE_LEN / 8, 0); + unbuffer(stream); + validateFileContents(stream, TEST_FILE_LEN / 8, TEST_FILE_LEN / 8); + validateFileContents(stream, TEST_FILE_LEN / 4, TEST_FILE_LEN / 4); + unbuffer(stream); + validateFileContents(stream, TEST_FILE_LEN / 2, TEST_FILE_LEN / 2); + unbuffer(stream); + assertEquals("stream should be at end of file", TEST_FILE_LEN, + stream.getPos()); + } + } + + private void unbuffer(FSDataInputStream stream) throws IOException { + long pos = stream.getPos(); + stream.unbuffer(); + assertEquals("unbuffer unexpectedly changed the stream position", pos, + stream.getPos()); + } + + protected void validateFullFileContents(FSDataInputStream stream) + throws IOException { + validateFileContents(stream, TEST_FILE_LEN, 0); + } + + protected void validateFileContents(FSDataInputStream stream, int length, + int startIndex) + throws IOException { + byte[] streamData = new byte[length]; + assertEquals("failed to read expected number of bytes from " + + "stream. This may be transient", + length, stream.read(streamData)); + byte[] validateFileBytes; + if (startIndex == 0 && length == fileBytes.length) { + validateFileBytes = fileBytes; + } else { + validateFileBytes = Arrays.copyOfRange(fileBytes, startIndex, + startIndex + length); + } + assertArrayEquals("invalid file contents", validateFileBytes, streamData); + } + + protected Path getFile() { + return file; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java new file mode 100644 index 000000000000..e40b22e55aaf --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUnbuffer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Ozone contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}. + */ +public class ITestOzoneContractUnbuffer extends AbstractContractUnbufferTest { + + @BeforeClass + public static void createCluster() throws IOException { + OzoneContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + OzoneContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new OzoneContract(conf); + } +} \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/ITestRootedOzoneContractUnbuffer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/ITestRootedOzoneContractUnbuffer.java new file mode 100644 index 000000000000..e081e8d5b80a --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/ITestRootedOzoneContractUnbuffer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.ozone.contract.rooted; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Ozone contract tests for {@link org.apache.hadoop.fs.CanUnbuffer#unbuffer}. + */ +public class ITestRootedOzoneContractUnbuffer + extends AbstractContractUnbufferTest { + + @BeforeClass + public static void createCluster() throws IOException { + RootedOzoneContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() { + RootedOzoneContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RootedOzoneContract(conf); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java index 4dbb0b60cf0e..cc2eb94b5819 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java @@ -18,20 +18,30 @@ package org.apache.hadoop.ozone.client.rpc; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientMetrics; +import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -45,21 +55,39 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; +import static org.apache.hadoop.ozone.container.TestHelper.countReplicas; +import static org.apache.hadoop.ozone.container.TestHelper.waitForReplicaCount; + +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Tests {@link KeyInputStream}. */ @RunWith(Parameterized.class) public class TestKeyInputStream { + + private static final Logger LOG = + LoggerFactory.getLogger(TestKeyInputStream.class); + + private static final int TIMEOUT = 300_000; + private static MiniOzoneCluster cluster; private static OzoneConfiguration conf = new OzoneConfiguration(); private static OzoneClient client; @@ -105,12 +133,19 @@ public void init() throws Exception { conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS); + conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, TimeUnit.SECONDS); conf.setQuietMode(false); conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 64, StorageUnit.MB); conf.set(ScmConfigKeys.OZONE_SCM_CHUNK_LAYOUT_KEY, chunkLayout.name()); + + ReplicationManagerConfiguration repConf = + conf.getObject(ReplicationManagerConfiguration.class); + repConf.setInterval(Duration.ofSeconds(1)); + conf.setFromObject(repConf); + cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) + .setNumDatanodes(4) .setTotalPipelineNumLimit(5) .setBlockSize(blockSize) .setChunkSize(chunkSize) @@ -130,7 +165,7 @@ public void init() throws Exception { } @Rule - public Timeout timeout = new Timeout(300_000); + public Timeout timeout = new Timeout(TIMEOUT); /** * Shutdown MiniDFSCluster. @@ -156,12 +191,7 @@ public void testSeekRandomly() throws Exception { // write data of more than 2 blocks. int dataLength = (2 * blockSize) + (chunkSize); - Random rd = new Random(); - byte[] inputData = new byte[dataLength]; - rd.nextBytes(inputData); - key.write(inputData); - key.close(); - + byte[] inputData = writeRandomBytes(key, dataLength); KeyInputStream keyInputStream = (KeyInputStream) objectStore .getVolume(volumeName).getBucket(bucketName).readKey(keyName) @@ -304,40 +334,23 @@ public void testReadChunk() throws Exception { OzoneOutputStream key = TestHelper.createKey(keyName, ReplicationType.RATIS, 0, objectStore, volumeName, bucketName); - // write data spanning multiple chunks + // write data spanning multiple blocks/chunks int dataLength = 2 * blockSize + (blockSize / 2); - byte[] originData = new byte[dataLength]; - Random r = new Random(); - r.nextBytes(originData); - key.write(originData); - key.close(); + byte[] data = writeRandomBytes(key, dataLength); // read chunk data - KeyInputStream keyInputStream = (KeyInputStream) objectStore + try (KeyInputStream keyInputStream = (KeyInputStream) objectStore .getVolume(volumeName).getBucket(bucketName).readKey(keyName) - .getInputStream(); - - int[] bufferSizeList = {chunkSize / 4, chunkSize / 2, chunkSize - 1, - chunkSize, chunkSize + 1, blockSize - 1, blockSize, blockSize + 1, - blockSize * 2}; - for (int bufferSize : bufferSizeList) { - byte[] data = new byte[bufferSize]; - int totalRead = 0; - while (totalRead < dataLength) { - int numBytesRead = keyInputStream.read(data); - if (numBytesRead == -1 || numBytesRead == 0) { - break; - } - byte[] tmp1 = - Arrays.copyOfRange(originData, totalRead, totalRead + numBytesRead); - byte[] tmp2 = - Arrays.copyOfRange(data, 0, numBytesRead); - Assert.assertArrayEquals(tmp1, tmp2); - totalRead += numBytesRead; + .getInputStream()) { + + int[] bufferSizeList = {chunkSize / 4, chunkSize / 2, chunkSize - 1, + chunkSize, chunkSize + 1, blockSize - 1, blockSize, blockSize + 1, + blockSize * 2}; + for (int bufferSize : bufferSizeList) { + assertReadFully(data, keyInputStream, bufferSize, 0); + keyInputStream.seek(0); } - keyInputStream.seek(0); } - keyInputStream.close(); } @Test @@ -397,4 +410,118 @@ public void testSkip() throws Exception { Assert.assertEquals(inputData[chunkSize + 50 + i], readData[i]); } } + + @Test + public void readAfterReplication() throws Exception { + testReadAfterReplication(false); + } + + @Test + public void readAfterReplicationWithUnbuffering() throws Exception { + testReadAfterReplication(true); + } + + private void testReadAfterReplication(boolean doUnbuffer) throws Exception { + Assume.assumeTrue(cluster.getHddsDatanodes().size() > 3); + + int dataLength = 2 * chunkSize; + String keyName = getKeyName(); + OzoneOutputStream key = TestHelper.createKey(keyName, + ReplicationType.RATIS, dataLength, objectStore, volumeName, bucketName); + + byte[] data = writeRandomBytes(key, dataLength); + + OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setType(HddsProtos.ReplicationType.RATIS) + .setFactor(HddsProtos.ReplicationFactor.THREE) + .build(); + OmKeyInfo keyInfo = cluster.getOzoneManager().lookupKey(keyArgs); + + OmKeyLocationInfoGroup locations = keyInfo.getLatestVersionLocations(); + Assert.assertNotNull(locations); + List locationInfoList = locations.getLocationList(); + Assert.assertEquals(1, locationInfoList.size()); + OmKeyLocationInfo loc = locationInfoList.get(0); + long containerID = loc.getContainerID(); + Assert.assertEquals(3, countReplicas(containerID, cluster)); + + TestHelper.waitForContainerClose(cluster, containerID); + + List pipelineNodes = loc.getPipeline().getNodes(); + + // read chunk data + try (KeyInputStream keyInputStream = (KeyInputStream) objectStore + .getVolume(volumeName).getBucket(bucketName) + .readKey(keyName).getInputStream()) { + + int b = keyInputStream.read(); + Assert.assertNotEquals(-1, b); + + if (doUnbuffer) { + keyInputStream.unbuffer(); + } + + // stop one node, wait for container to be replicated to another one + cluster.shutdownHddsDatanode(pipelineNodes.get(0)); + waitForNodeToBecomeDead(pipelineNodes.get(0)); + waitForReplicaCount(containerID, 2, cluster); + waitForReplicaCount(containerID, 3, cluster); + + // avoid polluting the logs + cluster.getStorageContainerManager().getReplicationManager().stop(); + + // stop original pipeline's remaining nodes + cluster.shutdownHddsDatanode(pipelineNodes.get(1)); + cluster.shutdownHddsDatanode(pipelineNodes.get(2)); + + // now only the new node has the container + waitForReplicaCount(containerID, 1, cluster); + + // check that we can still read it + assertReadFully(data, keyInputStream, dataLength - 1, 1); + } + } + + private static void waitForNodeToBecomeDead( + DatanodeDetails datanode) throws TimeoutException, InterruptedException { + StorageContainerManager scm = cluster.getStorageContainerManager(); + NodeManager nodeManager = scm.getScmNodeManager(); + GenericTestUtils.waitFor(() -> + HddsProtos.NodeState.DEAD == nodeManager.getNodeState(datanode), + 100, 30000); + LOG.info("Node {} is {}", datanode.getUuidString(), + nodeManager.getNodeState(datanode)); + } + + private byte[] writeRandomBytes(OutputStream key, int size) + throws IOException { + byte[] data = new byte[size]; + Random r = new Random(); + r.nextBytes(data); + key.write(data); + key.close(); + return data; + } + + private static void assertReadFully(byte[] data, InputStream in, + int bufferSize, int totalRead) throws IOException { + + byte[] buffer = new byte[bufferSize]; + while (totalRead < data.length) { + int numBytesRead = in.read(buffer); + if (numBytesRead == -1 || numBytesRead == 0) { + break; + } + byte[] tmp1 = + Arrays.copyOfRange(data, totalRead, totalRead + numBytesRead); + byte[] tmp2 = + Arrays.copyOfRange(buffer, 0, numBytesRead); + Assert.assertArrayEquals(tmp1, tmp2); + totalRead += numBytesRead; + } + Assert.assertEquals(data.length, totalRead); + } + } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java index 12ffce6c2411..7888e39519e5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestHelper.java @@ -27,7 +27,9 @@ import org.apache.hadoop.hdds.ratis.RatisHelper; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException; @@ -51,12 +53,19 @@ import org.apache.ratis.server.impl.RaftServerProxy; import org.apache.ratis.statemachine.StateMachine; import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.stream.Collectors.toList; /** * Helpers for container tests. */ public final class TestHelper { + private static final Logger LOG = + LoggerFactory.getLogger(TestHelper.class); + /** * Never constructed. */ @@ -95,6 +104,19 @@ public static boolean isContainerPresent(MiniOzoneCluster cluster, return false; } + public static int countReplicas(long containerID, + Set datanodes) { + int count = 0; + for (HddsDatanodeService datanodeService : datanodes) { + Container container = datanodeService.getDatanodeStateMachine() + .getContainer().getContainerSet().getContainer(containerID); + if (container != null) { + count++; + } + } + return count; + } + public static OzoneOutputStream createKey(String keyName, ReplicationType type, long size, ObjectStore objectStore, String volumeName, String bucketName) throws Exception { @@ -317,12 +339,52 @@ public static StateMachine getStateMachine(HddsDatanodeService dn, } public static HddsDatanodeService getDatanodeService(OmKeyLocationInfo info, - MiniOzoneCluster cluster) - throws IOException { + MiniOzoneCluster cluster) + throws IOException { DatanodeDetails dnDetails = info.getPipeline(). - getFirstNode(); + getFirstNode(); return cluster.getHddsDatanodes().get(cluster. - getHddsDatanodeIndex(dnDetails)); + getHddsDatanodeIndex(dnDetails)); + } + + public static Set getDatanodeServices( + MiniOzoneCluster cluster, Pipeline pipeline) { + Set services = new HashSet<>(); + Set pipelineNodes = pipeline.getNodeSet(); + for (HddsDatanodeService service : cluster.getHddsDatanodes()) { + if (pipelineNodes.contains(service.getDatanodeDetails())) { + services.add(service); + } + } + Assert.assertEquals(pipelineNodes.size(), services.size()); + return services; + } + + public static int countReplicas(long containerID, MiniOzoneCluster cluster) { + ContainerManager containerManager = cluster.getStorageContainerManager() + .getContainerManager(); + try { + Set replicas = containerManager + .getContainerReplicas(ContainerID.valueof(containerID)); + LOG.info("Container {} has {} replicas on {}", containerID, + replicas.size(), + replicas.stream() + .map(ContainerReplica::getDatanodeDetails) + .map(DatanodeDetails::getUuidString) + .sorted() + .collect(toList()) + ); + return replicas.size(); + } catch (ContainerNotFoundException e) { + LOG.warn("Container {} not found", containerID); + return 0; + } + } + + public static void waitForReplicaCount(long containerID, int count, + MiniOzoneCluster cluster) throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> countReplicas(containerID, cluster) == count, + 1000, 30_000); } } diff --git a/hadoop-ozone/integration-test/src/test/resources/contract/ozone.xml b/hadoop-ozone/integration-test/src/test/resources/contract/ozone.xml index 11da52c60ba3..c94054969d19 100644 --- a/hadoop-ozone/integration-test/src/test/resources/contract/ozone.xml +++ b/hadoop-ozone/integration-test/src/test/resources/contract/ozone.xml @@ -111,6 +111,11 @@ true + + fs.contract.supports-unbuffer + true + + fs.contract.supports-unix-permissions false diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java index 78e1c4456ee9..cbde28b02a0f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestChunkStreams.java @@ -58,7 +58,7 @@ public synchronized void seek(long pos) throws IOException { } @Override - public synchronized long getPos() throws IOException { + public synchronized long getPos() { return pos; } @@ -114,7 +114,7 @@ public synchronized void seek(long pos) throws IOException { } @Override - public synchronized long getPos() throws IOException { + public synchronized long getPos() { return pos; } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSInputStream.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSInputStream.java index cef6a5856605..2e8a469050b3 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSInputStream.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSInputStream.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,6 +34,7 @@ final class CapableOzoneFSInputStream extends OzoneFSInputStream public boolean hasCapability(String capability) { switch (StringUtils.toLowerCase(capability)) { case OzoneStreamCapabilities.READBYTEBUFFER: + case OzoneStreamCapabilities.UNBUFFER: return true; default: return false; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java index 313ae6ffcb49..35bd0d5829cb 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.nio.ReadOnlyBufferException; +import org.apache.hadoop.fs.CanUnbuffer; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.fs.ByteBufferReadable; @@ -39,7 +40,7 @@ @InterfaceAudience.Private @InterfaceStability.Evolving public class OzoneFSInputStream extends FSInputStream - implements ByteBufferReadable { + implements ByteBufferReadable, CanUnbuffer { private final InputStream inputStream; private final Statistics statistics; @@ -123,4 +124,11 @@ public int read(ByteBuffer buf) throws IOException { return bytesRead; } + + @Override + public void unbuffer() { + if (inputStream instanceof CanUnbuffer) { + ((CanUnbuffer) inputStream).unbuffer(); + } + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java index db90cd9db5d7..5dd69a45009f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.fs.ozone; +import org.apache.hadoop.fs.CanUnbuffer; + import java.nio.ByteBuffer; /** @@ -35,4 +37,11 @@ private OzoneStreamCapabilities() { * TODO: If Hadoop dependency is upgraded, this string can be removed. */ static final String READBYTEBUFFER = "in:readbytebuffer"; + + /** + * Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}. + * + * TODO: If Hadoop dependency is upgraded, this string can be removed. + */ + static final String UNBUFFER = "in:unbuffer"; } From 062ad25bdc99cf577a344c8fd0965c12b14ef839 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 2 Dec 2020 14:51:07 +0000 Subject: [PATCH 41/78] Trigger CI From 7b6eafda0e2215b2c719ab5cb29898469f126a5c Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Thu, 3 Dec 2020 16:19:01 +0530 Subject: [PATCH 42/78] HDDS-4356. SCM is flooded with useless "Deleting blocks" messages. (#1604) --- .../hdds/scm/block/BlockManagerImpl.java | 6 ++-- .../hdds/scm/block/DeletedBlockLogImpl.java | 28 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java index 014c76c6e64c..fae21b4fed73 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java @@ -297,8 +297,10 @@ public void deleteBlocks(List keyBlocksInfoList) // TODO: track the block size info so that we can reclaim the container // TODO: used space when the block is deleted. for (BlockGroup bg : keyBlocksInfoList) { - LOG.info("Deleting blocks {}", - StringUtils.join(",", bg.getBlockIDList())); + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting blocks {}", + StringUtils.join(",", bg.getBlockIDList())); + } for (BlockID block : bg.getBlockIDList()) { long containerID = block.getContainerID(); if (containerBlocks.containsKey(containerID)) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java index aa554808f499..f13505a22185 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java @@ -129,9 +129,12 @@ public void incrementCount(List txIDs) throws IOException { DeletedBlocksTransaction block = scmMetadataStore.getDeletedBlocksTXTable().get(txID); if (block == null) { - // Should we make this an error ? How can we not find the deleted - // TXID? - LOG.warn("Deleted TXID {} not found.", txID); + if (LOG.isDebugEnabled()) { + // This can occur due to race condition between retry and old + // service task where old task removes the transaction and the new + // task is resending + LOG.debug("Deleted TXID {} not found.", txID); + } continue; } DeletedBlocksTransaction.Builder builder = block.toBuilder(); @@ -196,9 +199,12 @@ public void commitTransactions( transactionResult.getContainerID()); if (dnsWithCommittedTxn == null) { // Mostly likely it's a retried delete command response. - LOG.debug("Transaction txId={} commit by dnId={} for containerID={}" - + " failed. Corresponding entry not found.", txID, dnID, - containerId); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Transaction txId={} commit by dnId={} for containerID={}" + + " failed. Corresponding entry not found.", txID, dnID, + containerId); + } continue; } @@ -218,12 +224,16 @@ public void commitTransactions( .collect(Collectors.toList()); if (dnsWithCommittedTxn.containsAll(containerDns)) { transactionToDNsCommitMap.remove(txID); - LOG.debug("Purging txId={} from block deletion log", txID); + if (LOG.isDebugEnabled()) { + LOG.debug("Purging txId={} from block deletion log", txID); + } scmMetadataStore.getDeletedBlocksTXTable().delete(txID); } } - LOG.debug("Datanode txId={} containerId={} committed by dnId={}", - txID, containerId, dnID); + if (LOG.isDebugEnabled()) { + LOG.debug("Datanode txId={} containerId={} committed by dnId={}", + txID, containerId, dnID); + } } catch (IOException e) { LOG.warn("Could not commit delete block transaction: " + transactionResult.getTxID(), e); From 43f03eb90200e30a29baf004c72c35eb8d1f4605 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 2 Dec 2020 21:29:36 +0000 Subject: [PATCH 43/78] Address compile issues after merging master into HDDS-1880-Decom --- .../ozone/client/rpc/TestKeyInputStream.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java index cc2eb94b5819..8217e3baeb09 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientMetrics; import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration; import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -59,6 +59,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.ozone.container.TestHelper.countReplicas; import static org.apache.hadoop.ozone.container.TestHelper.waitForReplicaCount; +import static org.junit.Assert.fail; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -486,13 +487,23 @@ private void testReadAfterReplication(boolean doUnbuffer) throws Exception { private static void waitForNodeToBecomeDead( DatanodeDetails datanode) throws TimeoutException, InterruptedException { - StorageContainerManager scm = cluster.getStorageContainerManager(); - NodeManager nodeManager = scm.getScmNodeManager(); GenericTestUtils.waitFor(() -> - HddsProtos.NodeState.DEAD == nodeManager.getNodeState(datanode), + HddsProtos.NodeState.DEAD == getNodeHealth(datanode), 100, 30000); LOG.info("Node {} is {}", datanode.getUuidString(), - nodeManager.getNodeState(datanode)); + getNodeHealth(datanode)); + } + + private static HddsProtos.NodeState getNodeHealth(DatanodeDetails dn) { + HddsProtos.NodeState health = null; + try { + NodeManager nodeManager = + cluster.getStorageContainerManager().getScmNodeManager(); + health = nodeManager.getNodeStatus(dn).getHealth(); + } catch (NodeNotFoundException e) { + fail("Unexpected NodeNotFound exception"); + } + return health; } private byte[] writeRandomBytes(OutputStream key, int size) From e6007b11d19801ef53099643237780ff2661f10d Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Fri, 4 Dec 2020 13:10:48 +0800 Subject: [PATCH 44/78] HDDS-4543. Failed to list keys when there are two bucket names with same prefix. (#1653) --- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 3 ++- .../hadoop/ozone/om/TestOmMetadataManager.java | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 27a36623b564..15b193d82f59 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -765,7 +765,8 @@ public List listKeys(String volumeName, String bucketName, skipStartKey = true; } else { // This allows us to seek directly to the first key with the right prefix. - seekKey = getOzoneKey(volumeName, bucketName, keyPrefix); + seekKey = getOzoneKey(volumeName, bucketName, + StringUtil.isNotBlank(keyPrefix) ? keyPrefix : OM_KEY_PREFIX); } String seekPrefix; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 7c2d258e9a00..5cc048b12a12 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -324,19 +324,21 @@ public void testListKeys() throws Exception { String volumeNameB = "volumeB"; String ozoneBucket = "ozoneBucket"; String hadoopBucket = "hadoopBucket"; - + String ozoneTestBucket = "ozoneBucket-Test"; // Create volumes and buckets. TestOMRequestUtils.addVolumeToDB(volumeNameA, omMetadataManager); TestOMRequestUtils.addVolumeToDB(volumeNameB, omMetadataManager); addBucketsToCache(volumeNameA, ozoneBucket); addBucketsToCache(volumeNameB, hadoopBucket); - + addBucketsToCache(volumeNameA, ozoneTestBucket); String prefixKeyA = "key-a"; String prefixKeyB = "key-b"; + String prefixKeyC = "key-c"; TreeSet keysASet = new TreeSet<>(); TreeSet keysBSet = new TreeSet<>(); + TreeSet keysCSet = new TreeSet<>(); for (int i=1; i<= 100; i++) { if (i % 2 == 0) { keysASet.add( @@ -348,7 +350,8 @@ public void testListKeys() throws Exception { addKeysToOM(volumeNameA, hadoopBucket, prefixKeyB + i, i); } } - + keysCSet.add(prefixKeyC + 1); + addKeysToOM(volumeNameA, ozoneTestBucket, prefixKeyC + 0, 0); TreeSet keysAVolumeBSet = new TreeSet<>(); TreeSet keysBVolumeBSet = new TreeSet<>(); @@ -442,6 +445,14 @@ public void testListKeys() throws Exception { Assert.assertEquals(omKeyInfoList.size(), 0); + // List all keys with empty prefix + omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket, + null, null, 100); + Assert.assertEquals(50, omKeyInfoList.size()); + for (OmKeyInfo omKeyInfo : omKeyInfoList) { + Assert.assertTrue(omKeyInfo.getKeyName().startsWith( + prefixKeyA)); + } } @Test From 5eae96cbde3f572fc749e851950391c755c1f298 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Fri, 4 Dec 2020 00:12:05 -0500 Subject: [PATCH 45/78] HDDS-4547. Upgrade aws-java-sdk to 1.11.901 (#1654) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a595fbe61acc..d8a76fd1ad7d 100644 --- a/pom.xml +++ b/pom.xml @@ -224,7 +224,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 3.1.0 8.29 1200 - 1.11.615 + 1.11.901 2.3.4 1.10.0 -# Hadoop Ozone/HDDS docs +# Apache Ozone/HDDS docs This subproject contains the inline documentation for Ozone/HDDS components. diff --git a/hadoop-hdds/docs/content/_index.md b/hadoop-hdds/docs/content/_index.md index 7890f6f16580..be0b30303e97 100644 --- a/hadoop-hdds/docs/content/_index.md +++ b/hadoop-hdds/docs/content/_index.md @@ -21,7 +21,7 @@ weight: -10 limitations under the License. --> -# Apache Hadoop Ozone +# Apache Ozone {{

}} diff --git a/hadoop-hdds/docs/content/_index.zh.md b/hadoop-hdds/docs/content/_index.zh.md index 689490be11ad..69fdb3f3f141 100644 --- a/hadoop-hdds/docs/content/_index.zh.md +++ b/hadoop-hdds/docs/content/_index.zh.md @@ -20,7 +20,7 @@ weight: -10 limitations under the License. --> -# Apache Hadoop Ozone +# Apache Ozone {{
}} diff --git a/hadoop-hdds/docs/pom.xml b/hadoop-hdds/docs/pom.xml index 404b6c2a253b..3a6aea0aa6c0 100644 --- a/hadoop-hdds/docs/pom.xml +++ b/hadoop-hdds/docs/pom.xml @@ -24,8 +24,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> hadoop-hdds-docs 1.1.0-SNAPSHOT - Apache Hadoop HDDS/Ozone Documentation - Apache Hadoop HDDS/Ozone Documentation + Apache Ozone/HDDS Documentation + Apache Ozone/HDDS Documentation jar diff --git a/hadoop-hdds/docs/static/ozone-logo-monochrome.svg b/hadoop-hdds/docs/static/ozone-logo-monochrome.svg index cd046a00b7cf..89cc166afe79 100644 --- a/hadoop-hdds/docs/static/ozone-logo-monochrome.svg +++ b/hadoop-hdds/docs/static/ozone-logo-monochrome.svg @@ -28,7 +28,7 @@ sodipodi:docname="ozone_bolt.svg" inkscape:version="0.92.4 (5da689c313, 2019-01-14)"> Apache Hadoop Ozone Logo + id="title39">Apache Ozone Logo @@ -37,7 +37,7 @@ image/svg+xml - Apache Hadoop Ozone Logo + Apache Ozone Logo diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/header.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/header.html index a4e24c951330..8f475b61f8d1 100644 --- a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/header.html +++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/header.html @@ -21,9 +21,9 @@ - + - Documentation for Apache Hadoop Ozone + Documentation for Apache Ozone diff --git a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/navbar.html b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/navbar.html index f942e4a1d25d..d4c9f2e994de 100644 --- a/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/navbar.html +++ b/hadoop-hdds/docs/themes/ozonedoc/layouts/partials/navbar.html @@ -27,9 +27,9 @@ - Hadoop Ozone + Apache Ozone