HDDS-13093. Add metrics for the cumulative state of volumes

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java

@@ -84,6 +84,7 @@ public class MutableVolumeSet implements VolumeSet {
   private final StorageVolumeFactory volumeFactory;
   private final StorageVolume.VolumeType volumeType;
   private int maxVolumeFailuresTolerated;
+  private VolumeHealthMetrics volumeHealthMetrics;
 
   public MutableVolumeSet(String dnUuid, ConfigurationSource conf,
       StateContext context, StorageVolume.VolumeType volumeType,
@@ -124,6 +125,8 @@ public MutableVolumeSet(String dnUuid, String clusterID,
     }
 
     initializeVolumeSet();
+
+    this.volumeHealthMetrics = VolumeHealthMetrics.create(volumeType, this);
   }
 
   public void setFailedVolumeListener(CheckedRunnable<IOException> runnable) {
@@ -415,6 +418,10 @@ public void shutdown() {
       }
     }
     volumeMap.clear();
+
+    if (volumeHealthMetrics != null) {
+      volumeHealthMetrics.unregister();
+    }
   }
 
   @Override

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeHealthMetrics.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.volume;
+
+import java.util.List;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * This class is used to track Volume Health metrics for all volumes on a datanode.
+ */
+@Metrics(about = "Ozone Volume Health Metrics",
+    context = OzoneConsts.OZONE)
+public class VolumeHealthMetrics implements MetricsSource {
+
+  private static final String SOURCE_BASENAME =
+      VolumeHealthMetrics.class.getSimpleName();
+
+  private static final MetricsInfo TOTAL_VOLUMES =
+      Interns.info("TotalVolumes", "Total number of volumes");
+  private static final MetricsInfo HEALTHY_VOLUMES =
+      Interns.info("NumHealthyVolumes", "Number of healthy volumes");
+  private static final MetricsInfo FAILED_VOLUMES =
+      Interns.info("NumFailedVolumes", "Number of failed volumes");
+
+  private final MetricsRegistry registry;
+  private final String metricsSourceName;
+  private final VolumeSet volumeSet;
+
+  /**
+   * Constructor for VolumeHealthMetrics.
+   *
+   * @param volumeType Type of volumes (DATA_VOLUME, META_VOLUME, DB_VOLUME)
+   * @param volumeSet The volume set to track metrics for
+   */
+  VolumeHealthMetrics(StorageVolume.VolumeType volumeType, VolumeSet volumeSet) {
+    this.volumeSet = volumeSet;
+
+    metricsSourceName = SOURCE_BASENAME + '-' + volumeType.name();
+    registry = new MetricsRegistry(metricsSourceName);
+  }
+
+  /**
+   * Creates and registers a new VolumeHealthMetrics instance.
+   *
+   * @param volumeType Type of volumes (DATA_VOLUME, META_VOLUME, DB_VOLUME)
+   * @param volumeSet  The volume set to track metrics for
+   * @return The registered VolumeHealthMetrics instance
+   */
+  public static VolumeHealthMetrics create(StorageVolume.VolumeType volumeType,
+      VolumeSet volumeSet) {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    VolumeHealthMetrics metrics = new VolumeHealthMetrics(volumeType, volumeSet);
+    return ms.register(metrics.metricsSourceName, "Volume Health Statistics", metrics);
+  }
+
+  public void unregister() {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    ms.unregisterSource(metricsSourceName);
+  }
+
+  @Override
+  public void getMetrics(MetricsCollector collector, boolean all) {
+    MetricsRecordBuilder builder = collector.addRecord(metricsSourceName);
+    registry.snapshot(builder, all);
+
+    // Get the list of volumes
+    List<StorageVolume> volumes = volumeSet.getVolumesList();
+
+    int totalVolumes = volumes.size();
+    int healthyVolumes = 0;
+    int failedVolumes = 0;
+    for (StorageVolume volume : volumes) {
+      if (volume.getStorageState() == StorageVolume.VolumeState.NORMAL) {
+        healthyVolumes++;
+      } else if (volume.getStorageState() == StorageVolume.VolumeState.FAILED) {
+        // we don't use getFailedVolumesList() because it is a private method and
+        // to maintain a consistent count of currently active volumes
+        failedVolumes++;
+      }
+    }
+
+    builder
+        .addGauge(TOTAL_VOLUMES, totalVolumes)
+        .addGauge(HEALTHY_VOLUMES, healthyVolumes)
+        .addGauge(FAILED_VOLUMES, failedVolumes);
+  }
+}

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeHealthMetrics.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.volume;
+
+import static org.apache.ozone.test.MetricsAsserts.assertGauge;
+import static org.apache.ozone.test.MetricsAsserts.getMetrics;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test for VolumeHealthMetrics.
+ */
+class TestVolumeHealthMetrics {
+
+  private MetricsSystem metrics;
+  private VolumeSet mockVolumeSet;
+  private List<StorageVolume> volumes;
+  private VolumeHealthMetrics volumeHealthMetrics;
+
+  @BeforeEach
+  void setUp() {
+    metrics = DefaultMetricsSystem.instance();
+    metrics.init("test");
+
+    mockVolumeSet = mock(VolumeSet.class);
+    volumes = new ArrayList<>();
+
+    volumeHealthMetrics = VolumeHealthMetrics.create(StorageVolume.VolumeType.DATA_VOLUME, mockVolumeSet);
+  }
+
+  @AfterEach
+  void tearDown() {
+    if (volumeHealthMetrics != null) {
+      volumeHealthMetrics.unregister();
+    }
+
+    metrics.stop();
+    metrics.shutdown();
+  }
+
+  private StorageVolume createMockVolume(StorageVolume.VolumeState state) {
+    StorageVolume volume = mock(StorageVolume.class);
+    when(volume.getStorageState()).thenReturn(state);
+    return volume;
+  }
+
+  /**
+   * Test metrics with empty volume list.
+   */
+  @Test
+  void testEmptyVolumeList() {
+    when(mockVolumeSet.getVolumesList()).thenReturn(new ArrayList<>());
+
+    MetricsRecordBuilder metricsRecords = getMetrics(volumeHealthMetrics);
+
+    assertGauge("TotalVolumes", 0, metricsRecords);
+    assertGauge("NumHealthyVolumes", 0, metricsRecords);
+    assertGauge("NumFailedVolumes", 0, metricsRecords);
+  }
+
+  /**
+   * Test that metrics are correctly initialized and reported.
+   */
+  @Test
+  void testBasicMetrics() {
+    StorageVolume normalVolume1 = createMockVolume(StorageVolume.VolumeState.NORMAL);
+    StorageVolume normalVolume2 = createMockVolume(StorageVolume.VolumeState.NORMAL);
+    StorageVolume failedVolume = createMockVolume(StorageVolume.VolumeState.FAILED);
+
+    volumes.add(normalVolume1);
+    volumes.add(normalVolume2);
+    volumes.add(failedVolume);
+
+    when(mockVolumeSet.getVolumesList()).thenReturn(volumes);
+
+    MetricsRecordBuilder metricsRecords = getMetrics(volumeHealthMetrics);
+
+    assertGauge("TotalVolumes", 3, metricsRecords);
+    assertGauge("NumHealthyVolumes", 2, metricsRecords);
+    assertGauge("NumFailedVolumes", 1, metricsRecords);
+  }
+
+  /**
+   * Test that metrics are updated when volume states change.
+   */
+  @Test
+  void testMetricsUpdateOnStateChange() {
+    StorageVolume volume1 = createMockVolume(StorageVolume.VolumeState.NORMAL);
+    StorageVolume volume2 = createMockVolume(StorageVolume.VolumeState.NORMAL);
+
+    volumes.add(volume1);
+    volumes.add(volume2);
+
+    when(mockVolumeSet.getVolumesList()).thenReturn(volumes);
+
+    // Verify initial metrics
+    MetricsRecordBuilder initialMetrics = getMetrics(volumeHealthMetrics);
+    assertGauge("TotalVolumes", 2, initialMetrics);
+    assertGauge("NumHealthyVolumes", 2, initialMetrics);
+    assertGauge("NumFailedVolumes", 0, initialMetrics);
+
+    when(volume1.getStorageState()).thenReturn(StorageVolume.VolumeState.FAILED);
+
+    // Verify updated metrics
+    MetricsRecordBuilder updatedMetrics = getMetrics(volumeHealthMetrics);
+    assertGauge("TotalVolumes", 2, updatedMetrics);
+    assertGauge("NumHealthyVolumes", 1, updatedMetrics);
+    assertGauge("NumFailedVolumes", 1, updatedMetrics);
+
+    when(volume1.getStorageState()).thenReturn(StorageVolume.VolumeState.NORMAL);
+
+    // Verify metrics are back to initial state
+    MetricsRecordBuilder finalMetrics = getMetrics(volumeHealthMetrics);
+    assertGauge("TotalVolumes", 2, finalMetrics);
+    assertGauge("NumHealthyVolumes", 2, finalMetrics);
+    assertGauge("NumFailedVolumes", 0, finalMetrics);
+  }
+
+  /**
+   * Test metrics with volumes in states other than NORMAL or FAILED.
+   */
+  @Test
+  void testOtherVolumeStates() {
+    StorageVolume normalVolume = createMockVolume(StorageVolume.VolumeState.NORMAL);
+    StorageVolume failedVolume = createMockVolume(StorageVolume.VolumeState.FAILED);
+    StorageVolume nonExistentVolume = createMockVolume(StorageVolume.VolumeState.NON_EXISTENT);
+    StorageVolume inconsistentVolume = createMockVolume(StorageVolume.VolumeState.INCONSISTENT);
+
+    volumes.add(normalVolume);
+    volumes.add(failedVolume);
+    volumes.add(nonExistentVolume);
+    volumes.add(inconsistentVolume);
+
+    when(mockVolumeSet.getVolumesList()).thenReturn(volumes);
+
+    MetricsRecordBuilder metricsRecords = getMetrics(volumeHealthMetrics);
+
+    // Verify metrics values - only NORMAL and FAILED state volumes should be counted
+    assertGauge("TotalVolumes", 4, metricsRecords);
+    assertGauge("NumHealthyVolumes", 1, metricsRecords);
+    assertGauge("NumFailedVolumes", 1, metricsRecords);
+  }
+}