From 2ee951930c4290b9ae2d135ae6b2e067d4ec2649 Mon Sep 17 00:00:00 2001 From: JiaChi Wang Date: Tue, 18 Apr 2023 10:34:47 +0800 Subject: [PATCH] add transitional container count in recon --- .../ozone/recon/api/ClusterStateEndpoint.java | 22 ++++++ .../recon/api/types/ClusterStateResponse.java | 71 +++++++++++++++++++ .../recon/api/types/ContainerStateCounts.java | 36 ++++++++++ .../src/views/overview/overview.tsx | 24 +++++-- .../recon/api/TestContainerStateCounts.java | 25 ++++++- 5 files changed, 173 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java index 7c563e13dabe..f17a3bd261d2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ClusterStateEndpoint.java @@ -109,6 +109,22 @@ public Response getClusterState() { this.containerManager.getContainerStateCount( HddsProtos.LifeCycleState.OPEN)); + containerStateCounts.setClosingContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.CLOSING)); + + containerStateCounts.setQuasiClosedContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.QUASI_CLOSED)); + + containerStateCounts.setClosedContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.CLOSED)); + + containerStateCounts.setDeletingContainersCount( + this.containerManager.getContainerStateCount( + HddsProtos.LifeCycleState.DELETING)); + containerStateCounts.setDeletedContainersCount( this.containerManager.getContainerStateCount( HddsProtos.LifeCycleState.DELETED)); @@ -180,6 +196,12 @@ public Response getClusterState() { .setTotalDatanodes(datanodeDetails.size()) .setHealthyDatanodes(healthyDatanodes) .setOpenContainers(containerStateCounts.getOpenContainersCount()) + .setClosingContainers(containerStateCounts.getClosingContainersCount()) + .setQuasiClosedContainers( + containerStateCounts.getQuasiClosedContainersCount()) + .setClosedContainers(containerStateCounts.getClosedContainersCount()) + .setDeletingContainers( + containerStateCounts.getDeletingContainersCount()) .setDeletedContainers(containerStateCounts.getDeletedContainersCount()) .build(); return Response.ok(response).build(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java index e7296d71000d..a89012c582ca 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ClusterStateResponse.java @@ -66,6 +66,30 @@ public final class ClusterStateResponse { @JsonProperty("openContainers") private int openContainers; + /** + * Total count of closing containers in the cluster. + */ + @JsonProperty("closingContainers") + private int closingContainers; + + /** + * Total count of quasi closed containers in the cluster. + */ + @JsonProperty("quasiClosedContainers") + private int quasiClosedContainers; + + /** + * Total count of closed containers. + */ + @JsonProperty("closedContainers") + private int closedContainers; + + /** + * Total count of deleting containers. + */ + @JsonProperty("deletingContainers") + private int deletingContainers; + /** * Total count of deleted containers in the cluster. */ @@ -124,6 +148,10 @@ private ClusterStateResponse(Builder b) { this.openContainers = b.openContainers; this.deletedKeys = b.deletedKeys; this.deletedDirs = b.deletedDirs; + this.closingContainers = b.closingContainers; + this.quasiClosedContainers = b.quasiClosedContainers; + this.closedContainers = b.closedContainers; + this.deletingContainers = b.deletingContainers; this.deletedContainers = b.deletedContainers; } @@ -139,6 +167,10 @@ public static final class Builder { private int containers; private int missingContainers; private int openContainers; + private int closingContainers; + private int quasiClosedContainers; + private int closedContainers; + private int deletingContainers; private int deletedContainers; private long volumes; private long buckets; @@ -151,6 +183,10 @@ public Builder() { this.containers = 0; this.missingContainers = 0; this.openContainers = 0; + this.closingContainers = 0; + this.quasiClosedContainers = 0; + this.closedContainers = 0; + this.deletingContainers = 0; this.deletedContainers = 0; this.volumes = 0; this.buckets = 0; @@ -197,6 +233,25 @@ public Builder setOpenContainers(int openContainers) { return this; } + public Builder setClosingContainers(int closingContainers) { + this.closingContainers = closingContainers; + return this; + } + + public Builder setQuasiClosedContainers(int quasiClosedContainers) { + this.quasiClosedContainers = quasiClosedContainers; + return this; + } + public Builder setClosedContainers(int closedContainers) { + this.closedContainers = closedContainers; + return this; + } + + public Builder setDeletingContainers(int deletingContainers) { + this.deletingContainers = deletingContainers; + return this; + } + public Builder setDeletedContainers(int deletedContainers) { this.deletedContainers = deletedContainers; return this; @@ -264,6 +319,22 @@ public int getOpenContainers() { return openContainers; } + public int getClosingContainers() { + return closingContainers; + } + + public int getQuasiClosedContainers() { + return quasiClosedContainers; + } + + public int getClosedContainers() { + return closedContainers; + } + + public int getDeletingContainers() { + return deletingContainers; + } + public int getDeletedContainers() { return deletedContainers; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java index d4db87f9dfe4..d1fd74172f00 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerStateCounts.java @@ -26,6 +26,10 @@ public class ContainerStateCounts { private int totalContainerCount; private int missingContainerCount; private int openContainersCount; + private int closingContainersCount; + private int quasiClosedContainersCount; + private int closedContainersCount; + private int deletingContainersCount; private int deletedContainersCount; public int getTotalContainerCount() { @@ -52,6 +56,38 @@ public void setOpenContainersCount(int openContainersCount) { this.openContainersCount = openContainersCount; } + public int getClosingContainersCount() { + return closingContainersCount; + } + + public void setClosingContainersCount(int closingContainersCount) { + this.closingContainersCount = closingContainersCount; + } + + public int getQuasiClosedContainersCount() { + return quasiClosedContainersCount; + } + + public void setQuasiClosedContainersCount(int quasiClosedContainersCount) { + this.quasiClosedContainersCount = quasiClosedContainersCount; + } + + public int getClosedContainersCount() { + return closedContainersCount; + } + + public void setClosedContainersCount(int closedContainersCount) { + this.closedContainersCount = closedContainersCount; + } + + public int getDeletingContainersCount() { + return deletingContainersCount; + } + + public void setDeletingContainersCount(int deletingContainersCount) { + this.deletingContainersCount = deletingContainersCount; + } + public int getDeletedContainersCount() { return deletedContainersCount; } diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx index cc007ae79cc8..e5fc68852cee 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/overview/overview.tsx @@ -41,6 +41,7 @@ interface IClusterStateResponse { buckets: number; keys: number; openContainers: number; + deletingContainers: number; } interface IOverviewState { @@ -58,6 +59,10 @@ interface IOverviewState { lastUpdatedOMDBFull: number; omStatus: string; openContainers: number; + closingContainers: number, + quasiClosedContainers: number, + closedContainers: number, + deletingContainers: number; } export class Overview extends React.Component, IOverviewState> { @@ -84,7 +89,11 @@ export class Overview extends React.Component, IOverviewS lastUpdatedOMDBDelta: 0, lastUpdatedOMDBFull: 0, omStatus: '', - openContainers: 0 + openContainers: 0, + closingContainers: 0, + quasiClosedContainers:0, + closedContainers:0, + deletingContainers: 0 }; this.autoReload = new AutoReloadHelper(this._loadData); } @@ -115,6 +124,10 @@ export class Overview extends React.Component, IOverviewS keys: clusterState.keys, missingContainersCount, openContainers: clusterState.openContainers, + closingContainers: clusterState.closingContainers, + quasiClosedContainers: clusterState.quasiClosedContainers, + closedContainers: clusterState.closedContainers, + deletingContainers: clusterState.deletingContainers, lastRefreshed: Number(moment()), lastUpdatedOMDBDelta: omDBDeltaObject && omDBDeltaObject.lastUpdatedTimestamp, lastUpdatedOMDBFull: omDBFullObject && omDBFullObject.lastUpdatedTimestamp @@ -158,7 +171,8 @@ export class Overview extends React.Component, IOverviewS render() { const {loading, datanodes, pipelines, storageReport, containers, volumes, buckets, - keys, missingContainersCount, lastRefreshed, lastUpdatedOMDBDelta, lastUpdatedOMDBFull, omStatus, openContainers } = this.state; + keys, missingContainersCount, lastRefreshed, lastUpdatedOMDBDelta, lastUpdatedOMDBFull, omStatus, + openContainers, closingContainers, quasiClosedContainers, closedContainers,deletingContainers } = this.state; const datanodesElement = ( @@ -168,6 +182,8 @@ export class Overview extends React.Component, IOverviewS const containersTooltip = missingContainersCount === 1 ? 'container is missing' : 'containers are missing'; const containersLink = missingContainersCount > 0 ? '/MissingContainers' : '/Containers'; const duLink = '/DiskUsage'; + const transitionalContainers = closingContainers + quasiClosedContainers + closedContainers + deletingContainers; + const containersElementTitle = missingContainersCount > 0 ? "Containers" : "Containers(Open/Transition)" const containersElement = missingContainersCount > 0 ? ( 1000 ? `1000+ Containers are missing. For more information, go to the Containers page.` : `${missingContainersCount} ${containersTooltip}`}> @@ -179,7 +195,7 @@ export class Overview extends React.Component, IOverviewS
{containers.toString()} - ({openContainers}) + ({openContainers}/{transitionalContainers})
const clusterCapacity = `${size(storageReport.capacity - storageReport.remaining)}/${size(storageReport.capacity)}`; @@ -212,7 +228,7 @@ export class Overview extends React.Component, IOverviewS 0} linkToUrl={containersLink}/> diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java index 78db1b5ab87d..b01d484bf4d4 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerStateCounts.java @@ -69,6 +69,9 @@ public class TestContainerStateCounts extends AbstractReconSqlDBTest { private static final int NUM_OPEN_CONTAINERS = 3; private static final int NUM_DELETED_CONTAINERS = 4; private static final int NUM_CLOSED_CONTAINERS = 3; + private static final int NUM_QUASI_CLOSED_CONTAINERS = 5; + private static final int NUM_CLOSING_CONTAINERS = 6; + private static final int NUM_DELETING_CONTAINERS = 7; @BeforeEach @@ -117,6 +120,12 @@ public void testGetContainerCounts() throws Exception { HddsProtos.LifeCycleState.DELETED); putContainerInfos(NUM_CLOSED_CONTAINERS, HddsProtos.LifeCycleState.CLOSED); + putContainerInfos(NUM_CLOSING_CONTAINERS, + HddsProtos.LifeCycleState.CLOSING); + putContainerInfos(NUM_QUASI_CLOSED_CONTAINERS, + HddsProtos.LifeCycleState.QUASI_CLOSED); + putContainerInfos(NUM_DELETING_CONTAINERS, + HddsProtos.LifeCycleState.DELETING); // Get the cluster state using the ClusterStateEndpoint Response response1 = clusterStateEndpoint.getClusterState(); @@ -124,8 +133,14 @@ public void testGetContainerCounts() throws Exception { (ClusterStateResponse) response1.getEntity(); // Calculate expected counts - int expectedTotalContainers = NUM_OPEN_CONTAINERS + NUM_CLOSED_CONTAINERS; + int expectedTotalContainers = NUM_OPEN_CONTAINERS + NUM_CLOSED_CONTAINERS + + NUM_CLOSING_CONTAINERS + NUM_QUASI_CLOSED_CONTAINERS + + NUM_DELETING_CONTAINERS; int expectedOpenContainers = NUM_OPEN_CONTAINERS; + int expectedClosingContainers = NUM_CLOSING_CONTAINERS; + int expectedQuasiClosedContainers = NUM_QUASI_CLOSED_CONTAINERS; + int expectedClosedContainers = NUM_CLOSED_CONTAINERS; + int expectedDeletingContainers = NUM_DELETING_CONTAINERS; int expectedDeletedContainers = NUM_DELETED_CONTAINERS; // Verify counts using assertions @@ -133,6 +148,14 @@ public void testGetContainerCounts() throws Exception { clusterStateResponse1.getContainers()); Assertions.assertEquals(expectedOpenContainers, clusterStateResponse1.getOpenContainers()); + Assertions.assertEquals(expectedClosingContainers, + clusterStateResponse1.getClosingContainers()); + Assertions.assertEquals(expectedQuasiClosedContainers, + clusterStateResponse1.getQuasiClosedContainers()); + Assertions.assertEquals(expectedClosedContainers, + clusterStateResponse1.getClosedContainers()); + Assertions.assertEquals(expectedDeletingContainers, + clusterStateResponse1.getDeletingContainers()); Assertions.assertEquals(expectedDeletedContainers, clusterStateResponse1.getDeletedContainers()); }