diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java index 77142848b692..b7befbc720a2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.health.ClosedWithMismatchedReplicasHandler; import org.apache.hadoop.hdds.scm.container.replication.health.ClosingContainerHandler; +import org.apache.hadoop.hdds.scm.container.replication.health.DeletingContainerHandler; import org.apache.hadoop.hdds.scm.container.replication.health.ECReplicationCheckHandler; import org.apache.hadoop.hdds.scm.container.replication.health.HealthCheck; import org.apache.hadoop.hdds.scm.container.replication.health.OpenContainerHandler; @@ -51,6 +52,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.DeleteContainerCommand; @@ -220,6 +222,7 @@ public ReplicationManager(final ConfigurationSource conf, .addNext(new ClosingContainerHandler(this)) .addNext(new QuasiClosedContainerHandler(this)) .addNext(new ClosedWithMismatchedReplicasHandler(this)) + .addNext(new DeletingContainerHandler(this)) .addNext(ecReplicationCheckHandler) .addNext(ratisReplicationCheckHandler); start(); @@ -403,6 +406,23 @@ public void sendDeleteCommand(final ContainerInfo container, int replicaIndex, } } + /** + * update container state. + * + * @param containerID Container to be updated + * @param event the event to update the container + */ + public void updateContainerState(ContainerID containerID, + HddsProtos.LifeCycleEvent event) { + try { + containerManager.updateContainerState(containerID, event); + } catch (IOException | InvalidStateTransitionException | + TimeoutException e) { + LOG.error("Failed to update the state of container {}, update Event {}", + containerID, event, e); + } + } + /** * Add an under replicated container back to the queue if it was unable to diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java new file mode 100644 index 000000000000..a31e5ecb161f --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class used in Replication Manager to handle the + * replicas of containers in DELETING State. + */ +public class DeletingContainerHandler extends AbstractCheck { + private final ReplicationManager replicationManager; + + public static final Logger LOG = + LoggerFactory.getLogger(DeletingContainerHandler.class); + + public DeletingContainerHandler(ReplicationManager replicationManager) { + this.replicationManager = replicationManager; + } + + /** + * If the number of replicas of the container is 0, change the state + * of the container to Deleted, otherwise resend delete command if needed. + * @param request ContainerCheckRequest object representing the container + * @return false if the specified container is not in DELETING state, + * otherwise true. + */ + @Override + public boolean handle(ContainerCheckRequest request) { + ContainerInfo containerInfo = request.getContainerInfo(); + ContainerID cID = containerInfo.containerID(); + HddsProtos.LifeCycleState containerState = containerInfo.getState(); + + if (containerState == HddsProtos.LifeCycleState.DELETED) { + return true; + } + + if (containerState != HddsProtos.LifeCycleState.DELETING) { + return false; + } + + if (request.getContainerReplicas().size() == 0) { + replicationManager.updateContainerState( + cID, HddsProtos.LifeCycleEvent.CLEANUP); + return true; + } + + Set pendingDelete = request.getPendingOps().stream() + .filter(o -> o.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) + .map(o -> o.getTarget()).collect(Collectors.toSet()); + //resend deleteCommand if needed + request.getContainerReplicas().stream() + .filter(r -> !pendingDelete.contains(r.getDatanodeDetails())) + .forEach(rp -> { + try { + replicationManager.sendDeleteCommand( + containerInfo, rp.getReplicaIndex(), rp.getDatanodeDetails()); + } catch (NotLeaderException e) { + LOG.warn("Failed to delete empty replica with index {} for " + + "container {} on datanode {}", rp.getReplicaIndex(), + cID, rp.getDatanodeDetails().getUuidString(), e); + } + }); + return true; + } +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java index 47ac50ce3a02..19c8d0a93aaa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java @@ -20,20 +20,16 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; -import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Set; -import java.util.concurrent.TimeoutException; /** * This handler deletes a container if it's closed and empty (0 key count) @@ -44,12 +40,9 @@ public class EmptyContainerHandler extends AbstractCheck { LoggerFactory.getLogger(EmptyContainerHandler.class); private final ReplicationManager replicationManager; - private final ContainerManager containerManager; - public EmptyContainerHandler(ReplicationManager replicationManager, - ContainerManager containerManager) { + public EmptyContainerHandler(ReplicationManager replicationManager) { this.replicationManager = replicationManager; - this.containerManager = containerManager; } /** @@ -72,14 +65,8 @@ public boolean handle(ContainerCheckRequest request) { deleteContainerReplicas(containerInfo, replicas); // Update the container's state - try { - containerManager.updateContainerState(containerInfo.containerID(), - HddsProtos.LifeCycleEvent.DELETE); - } catch (IOException | InvalidStateTransitionException | - TimeoutException e) { - LOG.error("Failed to delete empty container {}", - request.getContainerInfo(), e); - } + replicationManager.updateContainerState( + containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE); return true; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java new file mode 100644 index 000000000000..e2a147ffd2f6 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.replication.health; + +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; +import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; +import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager; +import org.apache.hadoop.hdds.scm.container.replication.ReplicationTestUtil; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.DELETING; + +/** + * Tests for {@link DeletingContainerHandler}. + */ +public class TestDeletingContainerHandler { + private ReplicationManager replicationManager; + private DeletingContainerHandler deletingContainerHandler; + private ECReplicationConfig ecReplicationConfig; + private RatisReplicationConfig ratisReplicationConfig; + + + @BeforeEach + public void setup() throws IOException { + + ecReplicationConfig = new ECReplicationConfig(3, 2); + ratisReplicationConfig = RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE); + replicationManager = Mockito.mock(ReplicationManager.class); + + Mockito.doNothing().when(replicationManager) + .updateContainerState(Mockito.any(ContainerID.class), + Mockito.any(HddsProtos.LifeCycleEvent.class)); + + deletingContainerHandler = + new DeletingContainerHandler(replicationManager); + } + + /** + * If a container is not in Deleting state, it should not be handled by + * DeletingContainerHandler. It should return false so the request can be + * passed to the next handler in the chain. + */ + @Test + public void testNonDeletingContainerReturnsFalse() { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, CLOSED); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSING, 1, 2, 3, 4, 5); + + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertFalse(deletingContainerHandler.handle(request)); + } + + @Test + public void testNonDeletingRatisContainerReturnsFalse() { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, CLOSED); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSING, 0, 0, 0); + + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertFalse(deletingContainerHandler.handle(request)); + } + + /** + * If a container is in Deleting state and no replica exists, + * change the state of the container to DELETED. + */ + @Test + public void testCleanupIfNoReplicaExist() { + //ratis container + cleanupIfNoReplicaExist(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.THREE), 1); + + //ec container + //since updateContainerState is called once when testing + //ratis container, so here should be 1+1 = 2 times + cleanupIfNoReplicaExist(ecReplicationConfig, 2); + } + + + private void cleanupIfNoReplicaExist( + ReplicationConfig replicationConfig, int times) { + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + replicationConfig, 1, DELETING); + + Set containerReplicas = new HashSet<>(); + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(Collections.EMPTY_LIST) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertTrue(deletingContainerHandler.handle(request)); + Mockito.verify(replicationManager, Mockito.times(times)) + .updateContainerState(Mockito.any(ContainerID.class), + Mockito.any(HddsProtos.LifeCycleEvent.class)); + } + + /** + * If a container is in Deleting state , some replicas exist and + * for each replica there is a pending delete, then do nothing. + */ + @Test + public void testNoNeedResendDeleteCommand() throws NotLeaderException { + //ratis container + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, DELETING); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0, 0); + List pendingOps = new ArrayList<>(); + containerReplicas.forEach(r -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + r.getDatanodeDetails(), r.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 0); + + //EC container + containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, DELETING); + containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4, 5); + pendingOps.clear(); + containerReplicas.forEach(r -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + r.getDatanodeDetails(), r.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 0); + + } + + /** + * If a container is in Deleting state , some replicas exist and + * for some replica there is no pending delete, then resending delete + * command. + */ + @Test + public void testResendDeleteCommand() throws NotLeaderException { + //ratis container + ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( + ratisReplicationConfig, 1, DELETING); + Set containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 0, 0, 0); + List pendingOps = new ArrayList<>(); + containerReplicas.stream().limit(2).forEach(replica -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + replica.getDatanodeDetails(), replica.getReplicaIndex()))); + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 1); + + //EC container + containerInfo = ReplicationTestUtil.createContainerInfo( + ecReplicationConfig, 1, DELETING); + containerReplicas = ReplicationTestUtil + .createReplicas(containerInfo.containerID(), + ContainerReplicaProto.State.CLOSED, 1, 2, 3, 4, 5); + pendingOps.clear(); + containerReplicas.stream().limit(3).forEach(replica -> pendingOps.add( + ContainerReplicaOp.create(ContainerReplicaOp.PendingOpType.DELETE, + replica.getDatanodeDetails(), replica.getReplicaIndex()))); + //since one delete command is end when testing ratis container, so + //here should be 1+2 = 3 times + verifyDeleteCommandCount(containerInfo, containerReplicas, pendingOps, 3); + + } + + private void verifyDeleteCommandCount(ContainerInfo containerInfo, + Set containerReplicas, + List pendingOps, + int times) throws NotLeaderException { + ContainerCheckRequest request = new ContainerCheckRequest.Builder() + .setPendingOps(pendingOps) + .setReport(new ReplicationManagerReport()) + .setContainerInfo(containerInfo) + .setContainerReplicas(containerReplicas) + .build(); + + Assert.assertTrue(deletingContainerHandler.handle(request)); + + Mockito.verify(replicationManager, Mockito.times(times)) + .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(), + Mockito.any(DatanodeDetails.class)); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java index ac746900f544..bac90b4c390e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; import org.apache.hadoop.hdds.scm.container.replication.ContainerCheckRequest; @@ -50,7 +49,6 @@ */ public class TestEmptyContainerHandler { private ReplicationManager replicationManager; - private ContainerManager containerManager; private EmptyContainerHandler emptyContainerHandler; private ECReplicationConfig ecReplicationConfig; private RatisReplicationConfig ratisReplicationConfig; @@ -62,10 +60,8 @@ public void setup() ratisReplicationConfig = RatisReplicationConfig.getInstance( HddsProtos.ReplicationFactor.THREE); replicationManager = Mockito.mock(ReplicationManager.class); - containerManager = Mockito.mock(ContainerManager.class); - emptyContainerHandler = - new EmptyContainerHandler(replicationManager, containerManager); + new EmptyContainerHandler(replicationManager); } /** @@ -74,7 +70,7 @@ public void setup() */ @Test public void testEmptyAndClosedECContainerReturnsTrue() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -96,7 +92,7 @@ public void testEmptyAndClosedECContainerReturnsTrue() @Test public void testEmptyAndClosedRatisContainerReturnsTrue() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -122,7 +118,7 @@ public void testEmptyAndClosedRatisContainerReturnsTrue() */ @Test public void testEmptyAndNonClosedECContainerReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 0L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -150,7 +146,7 @@ public void testEmptyAndNonClosedECContainerReturnsFalse() */ @Test public void testNonEmptyRatisContainerReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { long keyCount = 5L; long bytesUsed = 123L; ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( @@ -177,7 +173,7 @@ public void testNonEmptyRatisContainerReturnsFalse() */ @Test public void testEmptyECContainerWithNonEmptyReplicaReturnsFalse() - throws InvalidStateTransitionException, IOException, TimeoutException { + throws IOException { ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo( ecReplicationConfig, 1, CLOSED, 0L, 0L); Set containerReplicas = ReplicationTestUtil @@ -215,7 +211,7 @@ public void testEmptyECContainerWithNonEmptyReplicaReturnsFalse() */ private void assertAndVerify(ContainerCheckRequest request, boolean assertion, int times, long numEmptyExpected) - throws IOException, InvalidStateTransitionException, TimeoutException { + throws IOException { Assertions.assertEquals(assertion, emptyContainerHandler.handle(request)); Mockito.verify(replicationManager, Mockito.times(times)) .sendDeleteCommand(Mockito.any(ContainerInfo.class), Mockito.anyInt(), @@ -224,7 +220,7 @@ private void assertAndVerify(ContainerCheckRequest request, ReplicationManagerReport.HealthState.EMPTY)); if (times > 0) { - Mockito.verify(containerManager, Mockito.times(1)) + Mockito.verify(replicationManager, Mockito.times(1)) .updateContainerState(Mockito.any(ContainerID.class), Mockito.any(HddsProtos.LifeCycleEvent.class)); }