diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java index 840e8acf7c4..7c563e13dab 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; +import org.apache.hadoop.ozone.recon.api.types.ContainerStateCounts; import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.scm.ReconContainerManager; @@ -52,7 +53,6 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.VOLUME_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; - /** * Endpoint to fetch current state of ozone cluster. */ @@ -91,25 +91,37 @@ public class ClusterStateEndpoint { */ @GET public Response getClusterState() { + ContainerStateCounts containerStateCounts = new ContainerStateCounts(); List datanodeDetails = nodeManager.getAllNodes(); - int containers = this.containerManager.getContainers().size(); + int pipelines = this.pipelineManager.getPipelines().size(); + List missingContainers = containerHealthSchemaManager .getUnhealthyContainers( ContainerSchemaDefinition.UnHealthyContainerStates.MISSING, 0, MISSING_CONTAINER_COUNT_LIMIT); - int totalMissingContainerCount = missingContainers.size() == - MISSING_CONTAINER_COUNT_LIMIT ? - MISSING_CONTAINER_COUNT_LIMIT : missingContainers.size(); - int openContainersCount = this.containerManager.getContainerStateCount( - HddsProtos.LifeCycleState.OPEN); + + containerStateCounts.setMissingContainerCount( + missingContainers.size() == MISSING_CONTAINER_COUNT_LIMIT ? + MISSING_CONTAINER_COUNT_LIMIT : missingContainers.size()); + + containerStateCounts.setOpenContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.OPEN)); + + containerStateCounts.setDeletedContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.DELETED)); + int healthyDatanodes = nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) + nodeManager.getNodeCount(NodeStatus.inServiceHealthyReadOnly()); + SCMNodeStat stats = nodeManager.getStats(); DatanodeStorageReport storageReport = new DatanodeStorageReport(stats.getCapacity().get(), stats.getScmUsed().get(), stats.getRemaining().get()); + ClusterStateResponse.Builder builder = ClusterStateResponse.newBuilder(); GlobalStats volumeRecord = globalStatsDao.findById( TableCountTask.getRowKeyFromTable(VOLUME_TABLE)); @@ -156,14 +168,19 @@ public Response getClusterState() { builder.setDeletedKeys(deletedKeys); builder.setDeletedDirs(deletedDirs); + // Subtract deleted containers from total containers. + containerStateCounts.setTotalContainerCount( + this.containerManager.getContainers().size() - + containerStateCounts.getDeletedContainersCount()); ClusterStateResponse response = builder .setStorageReport(storageReport) .setPipelines(pipelines) - .setContainers(containers) - .setMissingContainers(totalMissingContainerCount) + .setContainers(containerStateCounts.getTotalContainerCount()) + .setMissingContainers(containerStateCounts.getMissingContainerCount()) .setTotalDatanodes(datanodeDetails.size()) .setHealthyDatanodes(healthyDatanodes) - .setOpenContainers(openContainersCount) + .setOpenContainers(containerStateCounts.getOpenContainersCount()) + .setDeletedContainers(containerStateCounts.getDeletedContainersCount()) .build(); return Response.ok(response).build(); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java index 118cedc2de7..e7296d71000 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java @@ -66,6 +66,12 @@ public final class ClusterStateResponse { @JsonProperty("openContainers") private int openContainers; + /** + * Total count of deleted containers in the cluster. + */ + @JsonProperty("deletedContainers") + private int deletedContainers; + /** * Total count of volumes in the cluster. */ @@ -118,6 +124,7 @@ private ClusterStateResponse(Builder b) { this.openContainers = b.openContainers; this.deletedKeys = b.deletedKeys; this.deletedDirs = b.deletedDirs; + this.deletedContainers = b.deletedContainers; } /** @@ -132,6 +139,7 @@ public static final class Builder { private int containers; private int missingContainers; private int openContainers; + private int deletedContainers; private long volumes; private long buckets; private long keys; @@ -143,6 +151,7 @@ public Builder() { this.containers = 0; this.missingContainers = 0; this.openContainers = 0; + this.deletedContainers = 0; this.volumes = 0; this.buckets = 0; this.keys = 0; @@ -188,6 +197,11 @@ public Builder setOpenContainers(int openContainers) { return this; } + public Builder setDeletedContainers(int deletedContainers) { + this.deletedContainers = deletedContainers; + return this; + } + public Builder setVolumes(long volumes) { this.volumes = volumes; return this; @@ -250,6 +264,10 @@ public int getOpenContainers() { return openContainers; } + public int getDeletedContainers() { + return deletedContainers; + } + public long getBuckets() { return buckets; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java new file mode 100644 index 00000000000..d4db87f9dfe --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api.types; + +/** + * Represents statistics related to containers in the Ozone cluster. + */ +public class ContainerStateCounts { + + private int totalContainerCount; + private int missingContainerCount; + private int openContainersCount; + private int deletedContainersCount; + + public int getTotalContainerCount() { + return totalContainerCount; + } + + public void setTotalContainerCount(int totalContainerCount) { + this.totalContainerCount = totalContainerCount; + } + + public int getMissingContainerCount() { + return missingContainerCount; + } + + public void setMissingContainerCount(int missingContainerCount) { + this.missingContainerCount = missingContainerCount; + } + + public int getOpenContainersCount() { + return openContainersCount; + } + + public void setOpenContainersCount(int openContainersCount) { + this.openContainersCount = openContainersCount; + } + + public int getDeletedContainersCount() { + return deletedContainersCount; + } + + public void setDeletedContainersCount(int deletedContainersCount) { + this.deletedContainersCount = deletedContainersCount; + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java similarity index 82% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java index 04eb53bb6e1..c61bb139012 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java @@ -17,22 +17,19 @@ */ package org.apache.hadoop.ozone.recon.api; + import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos - .ExtendedDatanodeDetailsProto; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ExtendedDatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.LayoutVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageTypeProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.LayoutVersionProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; @@ -55,36 +52,33 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.ozone.test.GenericTestUtils; -import org.apache.ozone.test.LambdaTestUtils; import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; +import java.net.HttpURLConnection; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.Response; -import java.net.HttpURLConnection; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; - /** - * Test for Recon API endpoints. + * Test for clusterStateEndpoint for checking deletedContainers count. */ -public class TestTotalOpenContainerCount extends AbstractReconSqlDBTest { +public class TestContainerStateCounts extends AbstractReconSqlDBTest { private NodeEndpoint nodeEndpoint; private ClusterStateEndpoint clusterStateEndpoint; private ReconOMMetadataManager reconOMMetadataManager; @@ -94,7 +88,7 @@ public class TestTotalOpenContainerCount extends AbstractReconSqlDBTest { private DatanodeDetails datanodeDetails; private DatanodeDetails datanodeDetails2; private ContainerReportsProto containerReportsProto; - private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; + private HddsProtos.ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; private Pipeline pipeline, pipeline2; private static final String HOST1 = "host1.datanode"; private static final String HOST2 = "host2.datanode"; @@ -132,18 +126,20 @@ private void initializeInjector() throws Exception { when(mockScmServiceProvider.getPipeline( pipeline2.getId().getProtobuf())).thenReturn(pipeline2); - // Open 5 containers on pipeline 1 + // Initialize 5 DELETED containers associated with pipeline 1 + // which is associated with DataNode1 containerIDs = new LinkedList<>(); cpw = new LinkedList<>(); for (long i = 1L; i <= 5L; ++i) { ContainerInfo containerInfo = new ContainerInfo.Builder() .setContainerID(i) - .setReplicationConfig( - RatisReplicationConfig.getInstance(ReplicationFactor.ONE)) - .setState(LifeCycleState.OPEN) + .setReplicationConfig(RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setState(LifeCycleState.DELETED) .setOwner("test") .setPipelineID(pipeline.getId()) .build(); + ContainerWithPipeline containerWithPipeline = new ContainerWithPipeline(containerInfo, pipeline); when(mockScmServiceProvider.getContainerWithPipeline(i)) @@ -152,13 +148,18 @@ private void initializeInjector() throws Exception { cpw.add(containerWithPipeline); } - // Open 5 containers on pipeline 2 + + // Initialize 2 CLOSED and 3 OPEN containers associated with pipeline 2 + // which is associated with DataNode2 for (long i = 6L; i <= 10L; ++i) { + LifeCycleState lifeCycleState = (i == 6L || i == 7L) ? + LifeCycleState.CLOSED : LifeCycleState.OPEN; ContainerInfo containerInfo = new ContainerInfo.Builder() .setContainerID(i) .setReplicationConfig( - RatisReplicationConfig.getInstance(ReplicationFactor.ONE)) - .setState(LifeCycleState.OPEN) + RatisReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setState(lifeCycleState) .setOwner("test") .setPipelineID(pipeline2.getId()) .build(); @@ -170,6 +171,7 @@ private void initializeInjector() throws Exception { cpw.add(containerWithPipeline); } + when(mockScmServiceProvider .getExistContainerWithPipelinesInBatch(containerIDs)) .thenReturn(cpw); @@ -213,43 +215,51 @@ private void initializeInjector() throws Exception { @BeforeEach public void setUp() throws Exception { - // The following setup runs only once + // Check if the setup has already been done if (!isSetupDone) { + // Initialize the injector if setup has not been done initializeInjector(); + // Mark the setup as done isSetupDone = true; } + // Get UUIDs for datanodes String datanodeId = datanodeDetails.getUuid().toString(); String datanodeId2 = datanodeDetails2.getUuid().toString(); // initialize container report builder = ContainerReportsProto.newBuilder(); - for (long i = 1L; i <= 10L; i++) { - if (i >= 1L && i < 6L) { - builder.addReports( - ContainerReplicaProto.newBuilder() - .setContainerID(i) - .setState(ContainerReplicaProto.State.OPEN) - .setOriginNodeId(datanodeId) - .build() - ); + builder = ContainerReportsProto.newBuilder(); + + // Generate container reports with different states + for (long i = 1L; i < 11L; i++) { + ContainerReplicaProto.State state; + if (i < 5L) { + state = ContainerReplicaProto.State.DELETED; + } else if (i < 8L) { + state = ContainerReplicaProto.State.CLOSED; } else { - builder.addReports( - ContainerReplicaProto.newBuilder() - .setContainerID(i) - .setState(ContainerReplicaProto.State.OPEN) - .setOriginNodeId(datanodeId2) - .build() - ); + state = ContainerReplicaProto.State.OPEN; } + + builder.addReports( + ContainerReplicaProto.newBuilder() + .setContainerID(i) + .setState(state) + .setOriginNodeId(i < 5L ? datanodeId : datanodeId2) + .build() + ); } + // Build container reports containerReportsProto = builder.build(); + // Build UUID object for pipeline UUID pipelineUuid = UUID.fromString(pipelineId); HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder() .setMostSigBits(pipelineUuid.getMostSignificantBits()) .setLeastSigBits(pipelineUuid.getLeastSignificantBits()) .build(); + // Build pipeline report for pipeline 1 PipelineReport pipelineReport = PipelineReport.newBuilder() .setPipelineID( PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128) @@ -292,6 +302,8 @@ public void setUp() throws Exception { .setMostSigBits(pipelineUuid2.getMostSignificantBits()) .setLeastSigBits(pipelineUuid2.getLeastSignificantBits()) .build(); + + // Build pipeline report for pipeline 2 PipelineReport pipelineReport2 = PipelineReport.newBuilder() .setPipelineID( PipelineID.newBuilder().setId(pipelineId2).setUuid128(uuid128) @@ -307,7 +319,7 @@ public void setUp() throws Exception { .setUuid(datanodeId2) .setIpAddress(IP2) .build(); - ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = + HddsProtos.ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = ExtendedDatanodeDetailsProto.newBuilder() .setDatanodeDetails(datanodeDetailsProto2) .setVersion("0.6.0") @@ -349,48 +361,30 @@ public void setUp() throws Exception { } @Test - public void testOpenContainerCount() throws Exception { + public void testDeletedContainerCount() throws Exception { - waitAndCheckConditionAfterHeartbeat(() -> { - Response response1 = clusterStateEndpoint.getClusterState(); - ClusterStateResponse clusterStateResponse1 = - (ClusterStateResponse) response1.getEntity(); - return (clusterStateResponse1.getContainers() == 10); - }); + // Total Available Containers = Total - Deleted Containers = 10 - 5 = 5 + // Total Deleted Containers = 5 + // Total Open Containers = 3 + // Total Closed Containers = 2 - Response response = clusterStateEndpoint.getClusterState(); - response = nodeEndpoint.getDatanodes(); + Response response = nodeEndpoint.getDatanodes(); DatanodesResponse datanodesResponse = (DatanodesResponse) response.getEntity(); Assertions.assertEquals(2, datanodesResponse.getTotalCount()); - AtomicInteger expectedCount = new AtomicInteger(); Response response1 = clusterStateEndpoint.getClusterState(); ClusterStateResponse clusterStateResponse1 = (ClusterStateResponse) response1.getEntity(); - // Get the total count of Open containers across all DataNodes - datanodesResponse.getDatanodes().forEach(datanodeMetadata -> { - expectedCount.set( - expectedCount.get() + - datanodeMetadata.getOpenContainers()); - }); - - Assertions.assertEquals(expectedCount.intValue(), - clusterStateResponse1.getOpenContainers()); - } - private void waitAndCheckConditionAfterHeartbeat(Callable check) - throws Exception { - // if container report is processed first, and pipeline does not exist - // then container is not added until the next container report is processed - SCMHeartbeatRequestProto heartbeatRequestProto = - SCMHeartbeatRequestProto.newBuilder() - .setContainerReport(containerReportsProto) - .setDatanodeDetails(extendedDatanodeDetailsProto - .getDatanodeDetails()) - .setDataNodeLayoutVersion(defaultLayoutVersionProto()) - .build(); - reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); - LambdaTestUtils.await(30000, 1000, check); + // Test for total pipelines + Assertions.assertEquals(2, clusterStateResponse1.getPipelines()); + // Test for total containers + Assertions.assertEquals(5, clusterStateResponse1.getContainers()); + // Test for total deleted containers + Assertions.assertEquals(5, clusterStateResponse1.getDeletedContainers()); + // Test for OPEN containers + Assertions.assertEquals(3, clusterStateResponse1.getOpenContainers()); } + }