HADOOP-17471. ABFS to collect IOStatistics

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java

@@ -316,6 +316,30 @@ public final class StoreStatisticNames {
   public static final String ACTION_HTTP_GET_REQUEST
       = "action_http_get_request";
 
+  /**
+   * An HTTP DELETE request was made: {@value}.
+   */
+  public static final String ACTION_HTTP_DELETE_REQUEST
+      = "action_http_delete_request";
+
+  /**
+   * An HTTP PUT request was made: {@value}.
+   */
+  public static final String ACTION_HTTP_PUT_REQUEST
+      = "action_http_put_request";
+
+  /**
+   * An HTTP PATCH request was made: {@value}.
+   */
+  public static final String ACTION_HTTP_PATCH_REQUEST
+      = "action_http_patch_request";
+
+  /**
+   * An HTTP POST request was made: {@value}.
+   */
+  public static final String ACTION_HTTP_POST_REQUEST
+      = "action_http_post_request";
+
   /**
    * An HTTP HEAD request was made: {@value}.
    */

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java

@@ -19,24 +19,23 @@
 package org.apache.hadoop.fs.azurebfs;
 
 import java.net.URI;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 
 import org.apache.hadoop.fs.azurebfs.services.AbfsCounters;
-import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStoreBuilder;
 import org.apache.hadoop.metrics2.MetricStringBuilder;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.MetricsInfo;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.MetricsTag;
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 import org.apache.hadoop.metrics2.lib.MutableMetric;
 
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
 
 /**
  * Instrumentation of Abfs counters.
@@ -62,6 +61,8 @@ public class AbfsCountersImpl implements AbfsCounters {
   private final MetricsRegistry registry =
       new MetricsRegistry("abfsMetrics").setContext(CONTEXT);
 
+  private final IOStatisticsStore ioStatisticsStore;
+
   private static final AbfsStatistic[] STATISTIC_LIST = {
       CALL_CREATE,
       CALL_OPEN,
@@ -85,7 +86,17 @@ public class AbfsCountersImpl implements AbfsCounters {
       BYTES_SENT,
       BYTES_RECEIVED,
       READ_THROTTLES,
-      WRITE_THROTTLES
+      WRITE_THROTTLES,
+      SERVER_UNAVAILABLE
+  };
+
+  private static final AbfsStatistic[] DURATION_TRACKER_LIST = {
+      HTTP_HEAD_REQUEST,
+      HTTP_GET_REQUEST,
+      HTTP_DELETE_REQUEST,
+      HTTP_PUT_REQUEST,
+      HTTP_PATCH_REQUEST,
+      HTTP_POST_REQUEST
   };
 
   public AbfsCountersImpl(URI uri) {
@@ -95,9 +106,17 @@ public AbfsCountersImpl(URI uri) {
         fileSystemInstanceId.toString());
     registry.tag(METRIC_BUCKET, "Hostname from the FS URL", uri.getHost());
 
+    IOStatisticsStoreBuilder ioStatisticsStoreBuilder = iostatisticsStore();
+    // Declaring the counters.
     for (AbfsStatistic stats : STATISTIC_LIST) {
+      ioStatisticsStoreBuilder.withCounters(stats.getStatName());
       createCounter(stats);
     }
+    // Declaring the DurationTrackers.
+    for (AbfsStatistic durationStats : DURATION_TRACKER_LIST) {
+      ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName());
+    }
+    ioStatisticsStore = ioStatisticsStoreBuilder.build();
   }
 
   /**
@@ -149,6 +168,7 @@ private MutableCounterLong createCounter(AbfsStatistic stats) {
    */
   @Override
   public void incrementCounter(AbfsStatistic statistic, long value) {
+    ioStatisticsStore.incrementCounter(statistic.getStatName(), value);
     MutableCounterLong counter = lookupCounter(statistic.getStatName());
     if (counter != null) {
       counter.incr(value);
@@ -189,98 +209,35 @@ public String formString(String prefix, String separator, String suffix,
   /**
    * {@inheritDoc}
    *
-   * Creating a map of all the counters for testing.
+   * Map of all the counters for testing.
    *
-   * @return a map of the metrics.
+   * @return a map of the IOStatistics counters.
    */
   @VisibleForTesting
   @Override
   public Map<String, Long> toMap() {
-    MetricsToMap metricBuilder = new MetricsToMap(null);
-    registry.snapshot(metricBuilder, true);
-    return metricBuilder.getMap();
+    return ioStatisticsStore.counters();
   }
 
-  protected static class MetricsToMap extends MetricsRecordBuilder {
-    private final MetricsCollector parent;
-    private final Map<String, Long> map =
-        new HashMap<>();
-
-    MetricsToMap(MetricsCollector parent) {
-      this.parent = parent;
-    }
-
-    @Override
-    public MetricsRecordBuilder tag(MetricsInfo info, String value) {
-      return this;
-    }
-
-    @Override
-    public MetricsRecordBuilder add(MetricsTag tag) {
-      return this;
-    }
-
-    @Override
-    public MetricsRecordBuilder add(AbstractMetric metric) {
-      return this;
-    }
-
-    @Override
-    public MetricsRecordBuilder setContext(String value) {
-      return this;
-    }
-
-    @Override
-    public MetricsRecordBuilder addCounter(MetricsInfo info, int value) {
-      return tuple(info, value);
-    }
-
-    @Override
-    public MetricsRecordBuilder addCounter(MetricsInfo info, long value) {
-      return tuple(info, value);
-    }
-
-    @Override
-    public MetricsRecordBuilder addGauge(MetricsInfo info, int value) {
-      return tuple(info, value);
-    }
-
-    @Override
-    public MetricsRecordBuilder addGauge(MetricsInfo info, long value) {
-      return tuple(info, value);
-    }
-
-    public MetricsToMap tuple(MetricsInfo info, long value) {
-      return tuple(info.name(), value);
-    }
-
-    public MetricsToMap tuple(String name, long value) {
-      map.put(name, value);
-      return this;
-    }
-
-    @Override
-    public MetricsRecordBuilder addGauge(MetricsInfo info, float value) {
-      return tuple(info, (long) value);
-    }
-
-    @Override
-    public MetricsRecordBuilder addGauge(MetricsInfo info, double value) {
-      return tuple(info, (long) value);
-    }
-
-    @Override
-    public MetricsCollector parent() {
-      return parent;
-    }
+  /**
+   * Returning the instance of IOStatisticsStore used to collect the metrics
+   * in AbfsCounters.
+   *
+   * @return instance of IOStatistics.
+   */
+  @Override
+  public IOStatistics getIOStatistics() {
+    return ioStatisticsStore;
+  }
 
-    /**
-     * Get the map.
-     *
-     * @return the map of metrics.
-     */
-    public Map<String, Long> getMap() {
-      return map;
-    }
+  /**
+   * Tracks the duration of a statistic.
+   *
+   * @param key name of the statistic.
+   * @return DurationTracker for that statistic.
+   */
+  @Override
+  public DurationTracker trackDuration(String key) {
+    return ioStatisticsStore.trackDuration(key);
   }
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java

@@ -18,7 +18,12 @@
 
 package org.apache.hadoop.fs.azurebfs;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
 
 /**
  * Statistic which are collected in Abfs.
@@ -73,11 +78,45 @@ public enum AbfsStatistic {
   READ_THROTTLES("read_throttles",
       "Total number of times a read operation is throttled."),
   WRITE_THROTTLES("write_throttles",
-      "Total number of times a write operation is throttled.");
+      "Total number of times a write operation is throttled."),
+  SERVER_UNAVAILABLE("server_unavailable",
+      "Total number of times HTTP 503 status code is received in response."),
+
+  // HTTP Duration Trackers
+  HTTP_HEAD_REQUEST(StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST,
+      "Time taken to complete a HEAD request",
+      AbfsHttpConstants.HTTP_METHOD_HEAD),
+  HTTP_GET_REQUEST(StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+      "Time taken to complete a GET request",
+      AbfsHttpConstants.HTTP_METHOD_GET),
+  HTTP_DELETE_REQUEST(StoreStatisticNames.ACTION_HTTP_DELETE_REQUEST,
+      "Time taken to complete a DELETE request",
+      AbfsHttpConstants.HTTP_METHOD_DELETE),
+  HTTP_PUT_REQUEST(StoreStatisticNames.ACTION_HTTP_PUT_REQUEST,
+      "Time taken to complete a PUT request",
+      AbfsHttpConstants.HTTP_METHOD_PUT),
+  HTTP_PATCH_REQUEST(StoreStatisticNames.ACTION_HTTP_PATCH_REQUEST,
+      "Time taken to complete a PATCH request",
+      AbfsHttpConstants.HTTP_METHOD_PATCH),
+  HTTP_POST_REQUEST(StoreStatisticNames.ACTION_HTTP_POST_REQUEST,
+      "Time taken to complete a POST request",
+      AbfsHttpConstants.HTTP_METHOD_POST);
 
   private String statName;
   private String statDescription;
 
+  //For http call stats only.
+  private String httpCall;
+  private static final Map<String, String> HTTP_CALL_TO_NAME_MAP = new HashMap<>();
+
+  static {
+    for (AbfsStatistic statistic : values()) {
+      if (statistic.getHttpCall() != null) {
+        HTTP_CALL_TO_NAME_MAP.put(statistic.getHttpCall(), statistic.getStatName());
+      }
+    }
+  }
+
   /**
    * Constructor of AbfsStatistic to set statistic name and description.
    *
@@ -89,6 +128,19 @@ public enum AbfsStatistic {
     this.statDescription = statDescription;
   }
 
+  /**
+   * Constructor for AbfsStatistic for HTTP durationTrackers.
+   *
+   * @param statName        Name of the statistic.
+   * @param statDescription Description of the statistic.
+   * @param httpCall        HTTP call associated with the stat name.
+   */
+  AbfsStatistic(String statName, String statDescription, String httpCall) {
+    this.statName = statName;
+    this.statDescription = statDescription;
+    this.httpCall = httpCall;
+  }
+
   /**
    * Getter for statistic name.
    *
@@ -106,4 +158,23 @@ public String getStatName() {
   public String getStatDescription() {
     return statDescription;
   }
+
+  /**
+   * Getter for http call for HTTP duration trackers.
+   *
+   * @return http call of a statistic.
+   */
+  public String getHttpCall() {
+    return httpCall;
+  }
+
+  /**
+   * Get the statistic name using the http call name.
+   *
+   * @param httpCall The HTTP call used to get the statistic name.
+   * @return Statistic name.
+   */
+  public static String getStatNameFromHttpCall(String httpCall) {
+    return HTTP_CALL_TO_NAME_MAP.get(httpCall);
+  }
 }