HDDS-6447. Refine SCM handling of unhealthy container replicas.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java

@@ -209,8 +209,10 @@ public void onMessage(final ContainerReportFromDatanode reportFromDatanode,
   }
 
   /**
-   * Processes the ContainerReport, unknown container reported
-   * that will be deleted by SCM.
+   * Processes the ContainerReport.
+   * Any unknown container reported by DN and not present in SCM
+   * containerSet will either be logged as an error or deleted based on
+   * unknownContainerHandleAction.
    *
    * @param datanodeDetails Datanode from which this report was received
    * @param container ContainerInfo representing the container

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaCount.java

@@ -21,7 +21,7 @@
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.container.ContainerReplica;
 
-import java.util.Set;
+import java.util.List;
 
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
 
@@ -32,7 +32,7 @@
 public interface ContainerReplicaCount {
   ContainerInfo getContainer();
 
-  Set<ContainerReplica> getReplicas();
+  List<ContainerReplica> getReplicas();
 
   boolean isSufficientlyReplicated();
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerReplicaCount.java

@@ -25,6 +25,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -74,14 +75,20 @@ public class ECContainerReplicaCount implements ContainerReplicaCount {
   private final Map<Integer, Integer> healthyIndexes = new HashMap<>();
   private final Map<Integer, Integer> decommissionIndexes = new HashMap<>();
   private final Map<Integer, Integer> maintenanceIndexes = new HashMap<>();
-  private final Set<ContainerReplica> replicas;
+  private final List<ContainerReplica> replicas;
 
   public ECContainerReplicaCount(ContainerInfo containerInfo,
       Set<ContainerReplica> replicas,
       List<ContainerReplicaOp> replicaPendingOps,
       int remainingMaintenanceRedundancy) {
     this.containerInfo = containerInfo;
-    this.replicas = replicas;
+    // Iterate replicas in deterministic order to avoid potential data loss
+    // on delete.
+    // See https://issues.apache.org/jira/browse/HDDS-4589.
+    // N.B., sort replicas by (containerID, datanodeDetails).
+    this.replicas = replicas.stream()
+        .sorted(Comparator.comparingLong(ContainerReplica::hashCode))
+        .collect(Collectors.toList());
     this.repConfig = (ECReplicationConfig)containerInfo.getReplicationConfig();
     this.pendingAdd = new ArrayList<>();
     this.pendingDelete = new ArrayList<>();
@@ -158,7 +165,7 @@ public ContainerInfo getContainer() {
   }
 
   @Override
-  public Set<ContainerReplica> getReplicas() {
+  public List<ContainerReplica> getReplicas() {
     return replicas;
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+
+import java.util.Set;
+
+/**
+ * When HDDS-6447 was done to improve the LegacyReplicationManager, work on
+ * the new replication manager had already started. When this class was added,
+ * the LegacyReplicationManager needed separate handling for healthy and
+ * unhealthy container replicas, but the new replication manager did not yet
+ * have this functionality. This class is used by the
+ * LegacyReplicationManager to allow {@link RatisContainerReplicaCount} to
+ * function for both use cases. When the new replication manager is finished
+ * and LegacyReplicationManager is removed, this class should be deleted and
+ * all necessary functionality consolidated to
+ * {@link RatisContainerReplicaCount}
+ */
+public class LegacyRatisContainerReplicaCount extends
+    RatisContainerReplicaCount {
+  public LegacyRatisContainerReplicaCount(ContainerInfo container,
+                                    Set<ContainerReplica> replicas,
+                                    int inFlightAdd,
+                                    int inFlightDelete, int replicationFactor,
+                                    int minHealthyForMaintenance) {
+    super(container, replicas, inFlightAdd, inFlightDelete, replicationFactor,
+        minHealthyForMaintenance);
+  }
+
+  @Override
+  protected int healthyReplicaCountAdapter() {
+    return 0;
+  }
+}