HDDS-7775. EC: Exception encountered while deleting UNHEALTHY replica in Datanode

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java

@@ -408,14 +408,15 @@ public void sendCloseContainerEvent(ContainerID containerID) {
    * @param container Container to be deleted
    * @param replicaIndex Index of the container replica to be deleted
    * @param datanode  The datanode on which the replica should be deleted
+   * @param force true to force delete a container that is open or not empty
    * @throws NotLeaderException when this SCM is not the leader
    */
   public void sendDeleteCommand(final ContainerInfo container, int replicaIndex,
-      final DatanodeDetails datanode) throws NotLeaderException {
+      final DatanodeDetails datanode, boolean force) throws NotLeaderException {
     LOG.debug("Sending delete command for container {} and index {} on {}",
         container, replicaIndex, datanode);
     final DeleteContainerCommand deleteCommand =
-        new DeleteContainerCommand(container.containerID(), false);
+        new DeleteContainerCommand(container.containerID(), force);
     deleteCommand.setReplicaIndex(replicaIndex);
     sendDatanodeCommand(deleteCommand, container, datanode);
   }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java

@@ -121,7 +121,7 @@ private void sendDeleteCommand(ContainerInfo containerInfo,
     LOG.debug("Trying to delete UNHEALTHY replica [{}]", replica);
     try {
       replicationManager.sendDeleteCommand(containerInfo,
-          replica.getReplicaIndex(), replica.getDatanodeDetails());
+          replica.getReplicaIndex(), replica.getDatanodeDetails(), true);
     } catch (NotLeaderException e) {
       LOG.warn("Failed to delete UNHEALTHY replica [{}]", replica, e);
     }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java

@@ -87,7 +87,8 @@ public boolean handle(ContainerCheckRequest request) {
         .forEach(rp -> {
           try {
             replicationManager.sendDeleteCommand(
-                containerInfo, rp.getReplicaIndex(), rp.getDatanodeDetails());
+                containerInfo, rp.getReplicaIndex(), rp.getDatanodeDetails(),
+                false);
           } catch (NotLeaderException e) {
             LOG.warn("Failed to delete empty replica with index {} for " +
                     "container {} on datanode {}", rp.getReplicaIndex(),

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java

@@ -112,7 +112,7 @@ private void deleteContainerReplicas(final ContainerInfo containerInfo,
 
       try {
         replicationManager.sendDeleteCommand(containerInfo,
-            rp.getReplicaIndex(), rp.getDatanodeDetails());
+            rp.getReplicaIndex(), rp.getDatanodeDetails(), false);
       } catch (NotLeaderException e) {
         LOG.warn("Failed to delete empty replica with index {} for container" +
                 " {} on datanode {}",

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java

@@ -143,7 +143,7 @@ public void testRatisContainerReturnsFalse() {
         ArgumentCaptor.forClass(Integer.class);
     Mockito.verify(replicationManager, Mockito.times(2))
         .sendDeleteCommand(Mockito.eq(container), Mockito.anyInt(), Mockito.any(
-            DatanodeDetails.class));
+            DatanodeDetails.class), Mockito.eq(true));
     // replica index that delete was sent for should either be 2 or 5
     replicaIndexCaptor.getAllValues()
         .forEach(index -> Assert.assertTrue(index == 2 || index == 5));

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java

@@ -233,6 +233,6 @@ private void verifyDeleteCommandCount(ContainerInfo containerInfo,
 
     Mockito.verify(replicationManager, Mockito.times(times))
         .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(),
-            Mockito.any(DatanodeDetails.class));
+            Mockito.any(DatanodeDetails.class), Mockito.eq(false));
   }
 }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java

@@ -215,7 +215,7 @@ private void assertAndVerify(ContainerCheckRequest request,
     Assertions.assertEquals(assertion, emptyContainerHandler.handle(request));
     Mockito.verify(replicationManager, Mockito.times(times))
         .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(),
-            Mockito.any(DatanodeDetails.class));
+            Mockito.any(DatanodeDetails.class), Mockito.eq(false));
     Assertions.assertEquals(numEmptyExpected, request.getReport().getStat(
         ReplicationManagerReport.HealthState.EMPTY));