From 8b67b65ec35ad79ffedb546296e73718e6d97780 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 18 Apr 2023 11:23:32 +0200 Subject: [PATCH 1/2] Move PlacementPolicy back to hdds-server-scm --- .../src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java | 0 .../org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename hadoop-hdds/{common => server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java (100%) rename hadoop-hdds/{common => server-scm}/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java (100%) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicyValidateProxy.java From 8446db2a72c092db24055194e02050f562e15465 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 18 Apr 2023 11:30:38 +0200 Subject: [PATCH 2/2] Eliminate generic type of PlacementPolicy --- .../org/apache/hadoop/hdds/scm/PlacementPolicy.java | 11 ++++++----- .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 2 +- .../algorithms/SCMContainerPlacementRandom.java | 3 +-- .../replication/ECMisReplicationHandler.java | 2 +- .../container/replication/MisReplicationHandler.java | 4 ++-- .../replication/RatisMisReplicationHandler.java | 2 +- .../hdds/scm/server/StorageContainerManager.java | 5 ++--- .../algorithms/TestContainerPlacementFactory.java | 2 +- .../replication/TestECMisReplicationHandler.java | 5 ++++- .../replication/TestMisReplicationHandler.java | 4 ++-- .../ozone/recon/fsck/TestContainerHealthTask.java | 2 +- 11 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java index 248e7d715b22..a792e2cea6b7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; import java.io.IOException; import java.util.Collections; @@ -29,7 +30,7 @@ * A PlacementPolicy support choosing datanodes to build * pipelines or containers with specified constraints. */ -public interface PlacementPolicy { +public interface PlacementPolicy { default List chooseDatanodes( List excludedNodes, @@ -75,8 +76,8 @@ ContainerPlacementStatus validateContainerPlacement( * @param replicas: Map of replicas with value signifying if * replica can be copied */ - Set replicasToCopyToFixMisreplication( - Map replicas); + Set replicasToCopyToFixMisreplication( + Map replicas); /** * Given a set of replicas of a container which are overreplicated, @@ -85,6 +86,6 @@ Set replicasToCopyToFixMisreplication( * @param expectedCountPerUniqueReplica: Replication factor of each * unique replica */ - Set replicasToRemoveToFixOverreplication( - Set replicas, int expectedCountPerUniqueReplica); + Set replicasToRemoveToFixOverreplication( + Set replicas, int expectedCountPerUniqueReplica); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java index 8d02d7fc3b80..e679ec72a150 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java @@ -56,7 +56,7 @@ * functions which are common to placement policies. */ public abstract class SCMCommonPlacementPolicy implements - PlacementPolicy { + PlacementPolicy { @VisibleForTesting static final Logger LOG = LoggerFactory.getLogger(SCMCommonPlacementPolicy.class); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java index 2aa11211015c..cdfd57d1d09b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java @@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.node.NodeManager; @@ -41,7 +40,7 @@ * can be practically used. */ public final class SCMContainerPlacementRandom extends SCMCommonPlacementPolicy - implements PlacementPolicy { + implements PlacementPolicy { @VisibleForTesting public static final Logger LOG = LoggerFactory.getLogger(SCMContainerPlacementRandom.class); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java index 3c6c3a27f167..aeb096ce8e05 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java @@ -34,7 +34,7 @@ */ public class ECMisReplicationHandler extends MisReplicationHandler { public ECMisReplicationHandler( - PlacementPolicy containerPlacement, + PlacementPolicy containerPlacement, ConfigurationSource conf, ReplicationManager replicationManager, boolean push) { super(containerPlacement, conf, replicationManager, push); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java index 4ab454955651..6977e15b7557 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java @@ -53,13 +53,13 @@ public abstract class MisReplicationHandler implements public static final Logger LOG = LoggerFactory.getLogger(MisReplicationHandler.class); - private final PlacementPolicy containerPlacement; + private final PlacementPolicy containerPlacement; private final long currentContainerSize; private final ReplicationManager replicationManager; private boolean push; public MisReplicationHandler( - final PlacementPolicy containerPlacement, + final PlacementPolicy containerPlacement, final ConfigurationSource conf, ReplicationManager replicationManager, final boolean push) { this.containerPlacement = containerPlacement; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java index 7e9f0f48d1e2..e6341d675873 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java @@ -36,7 +36,7 @@ public class RatisMisReplicationHandler extends MisReplicationHandler { public RatisMisReplicationHandler( - PlacementPolicy containerPlacement, + PlacementPolicy containerPlacement, ConfigurationSource conf, ReplicationManager replicationManager, boolean push) { super(containerPlacement, conf, replicationManager, push); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index e5faff4b6907..b1c83e390ecd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; import org.apache.hadoop.hdds.scm.PlacementPolicyValidateProxy; -import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.balancer.MoveManager; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaPendingOps; import org.apache.hadoop.hdds.scm.container.replication.DatanodeCommandCountUpdatedHandler; @@ -275,8 +274,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private final OzoneConfiguration configuration; private SCMContainerMetrics scmContainerMetrics; private SCMContainerPlacementMetrics placementMetrics; - private PlacementPolicy containerPlacementPolicy; - private PlacementPolicy ecContainerPlacementPolicy; + private PlacementPolicy containerPlacementPolicy; + private PlacementPolicy ecContainerPlacementPolicy; private PlacementPolicyValidateProxy placementPolicyValidateProxy; private MetricsSystem ms; private final Map ratisMetricsMap = diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java index 31016f017c04..695422b46c1b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java @@ -184,7 +184,7 @@ public void testECPolicy() throws IOException { * A dummy container placement implementation for test. */ public static class DummyImpl implements - PlacementPolicy { + PlacementPolicy { @Override public List chooseDatanodes( List usedNodes, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java index a0d8f3220738..aeee7917837d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicy; @@ -193,10 +194,12 @@ public void commandsForFewerThanRequiredNodes() throws IOException { Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4), Pair.of(IN_SERVICE, 5)); PlacementPolicy placementPolicy = Mockito.mock(PlacementPolicy.class); + List targetDatanodes = singletonList( + availableReplicas.iterator().next().getDatanodeDetails()); Mockito.when(placementPolicy.chooseDatanodes( any(), any(), any(), Mockito.anyInt(), Mockito.anyLong(), Mockito.anyLong())) - .thenReturn(singletonList(availableReplicas.iterator().next())); + .thenReturn(targetDatanodes); assertThrows(InsufficientDatanodesException.class, () -> testMisReplication(availableReplicas, Collections.emptyList(), 0, 2, 1)); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java index fac35fd395b0..40fce5562d90 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java @@ -99,8 +99,8 @@ protected ReplicationManager getReplicationManager() { return replicationManager; } - static PlacementPolicy mockPlacementPolicy() { - PlacementPolicy placementPolicy = Mockito.mock(PlacementPolicy.class); + static PlacementPolicy mockPlacementPolicy() { + PlacementPolicy placementPolicy = Mockito.mock(PlacementPolicy.class); ContainerPlacementStatus mockedContainerPlacementStatus = Mockito.mock(ContainerPlacementStatus.class); Mockito.when(mockedContainerPlacementStatus.isPolicySatisfied()) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java index 33a2a33811b7..77cd4baccee7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java @@ -345,7 +345,7 @@ private ContainerInfo getMockDeletedContainer(int containerID) { * to validateContainerPlacement, then it will return an invalid placement. */ private static class MockPlacementPolicy implements - PlacementPolicy { + PlacementPolicy { private UUID misRepWhenDnPresent = null;