YARN-10846. Add dispatcher metrics to NM.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -3070,6 +3070,10 @@ public static boolean isAclEnabled(Configuration conf) {
       + "amrmproxy.ha.enable";
   public static final boolean DEFAULT_AMRM_PROXY_HA_ENABLED = false;
 
+  // Enable NM Dispatcher Metric default False.
+  public static final String NM_DISPATCHER_METRIC_ENABLED = NM_PREFIX + "dispatcher.metric.enable";
+  public static final boolean DEFAULT_NM_DISPATCHER_METRIC_ENABLED = false;
+
   /**
    * Default platform-agnostic CLASSPATH for YARN applications. A
    * comma-separated list of CLASSPATH entries. The parameter expansion marker

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/GenericEventTypeMetrics.java

@@ -57,10 +57,8 @@ public GenericEventTypeMetrics(MetricsInfo info, MetricsSystem ms,
 
     //Initialize enum
     for (final T type : enums) {
-      String eventCountMetricsName =
-          type.toString() + "_" + "event_count";
-      String processingTimeMetricsName =
-          type.toString() + "_" + "processing_time";
+      String eventCountMetricsName = type + "_" + "event_count";
+      String processingTimeMetricsName = type + "_" + "processing_time";
       eventCountMetrics.put(type, this.registry.
           newGauge(eventCountMetricsName, eventCountMetricsName, 0L));
       processingTimeMetrics.put(type, this.registry.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -5065,6 +5065,16 @@
     </description>
   </property>
 
+  <property>
+    <name>yarn.nodemanager.dispatcher.metric.enable</name>
+    <value>false</value>
+    <description>
+      Yarn NodeManager enables Dispatcher Metric.
+      if true, will enable dispatcher metric; if false, will not enable dispatcher metric;
+      Default is false.
+    </description>
+  </property>
+
   <property>
     <name>yarn.router.interceptor.user-thread-pool.minimum-pool-size</name>
     <value>5</value>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/GenericEventTypeMetricsManager.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+public final class GenericEventTypeMetricsManager {
+
+  private GenericEventTypeMetricsManager() {
+      // nothing to do
+  }
+
+  // Construct a GenericEventTypeMetrics for dispatcher
+  @SuppressWarnings("unchecked")
+  public static <T extends Enum<T>> GenericEventTypeMetrics
+      create(String dispatcherName, Class<T> eventTypeClass) {
+    return new GenericEventTypeMetrics.EventTypeMetricsBuilder<T>()
+        .setMs(DefaultMetricsSystem.instance())
+        .setInfo(info("GenericEventTypeMetrics for " + eventTypeClass.getName(),
+            "Metrics for " + dispatcherName))
+        .setEnumClass(eventTypeClass)
+        .setEnums(eventTypeClass.getEnumConstants())
+        .build().registerMetrics();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java

@@ -48,6 +48,7 @@
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
 import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
 import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
@@ -144,8 +145,10 @@ public int getExitCode() {
   private AtomicBoolean isStopping = new AtomicBoolean(false);
   private boolean rmWorkPreservingRestartEnabled;
   private boolean shouldExitOnShutdownEvent = false;
+  private boolean nmDispatherMetricEnabled;
 
   private NMLogAggregationStatusTracker nmLogAggregationStatusTracker;
+
   /**
    * Default Container State transition listener.
    */
@@ -366,6 +369,10 @@ protected void serviceInit(Configuration conf) throws Exception {
             .RM_WORK_PRESERVING_RECOVERY_ENABLED,
         YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);
 
+    nmDispatherMetricEnabled = conf.getBoolean(
+        YarnConfiguration.NM_DISPATCHER_METRIC_ENABLED,
+        YarnConfiguration.DEFAULT_NM_DISPATCHER_METRIC_ENABLED);
+
     try {
       initAndStartRecoveryStore(conf);
     } catch (IOException e) {
@@ -1006,8 +1013,17 @@ ContainerManagerImpl getContainerManager() {
   /**
    * Unit test friendly.
    */
+  @SuppressWarnings("unchecked")
   protected AsyncDispatcher createNMDispatcher() {
-    return new AsyncDispatcher("NM Event dispatcher");
+    dispatcher = new AsyncDispatcher("NM Event dispatcher");
+    if (nmDispatherMetricEnabled) {
+      GenericEventTypeMetrics<ContainerManagerEventType> eventTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(),
+          ContainerManagerEventType.class);
+      dispatcher.addMetrics(eventTypeMetrics, eventTypeMetrics.getEnumClass());
+      LOG.info("NM Event dispatcher Metric Initialization Completed.");
+    }
+    return dispatcher;
   }
 
   //For testing
@@ -1052,4 +1068,10 @@ private NMLogAggregationStatusTracker createNMLogAggregationStatusTracker(
       Context ctxt) {
     return new NMLogAggregationStatusTracker(ctxt);
   }
+
+  @VisibleForTesting
+  @Private
+  public AsyncDispatcher getDispatcher() {
+    return dispatcher;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java

@@ -24,7 +24,9 @@
 import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse;
 import org.apache.hadoop.yarn.api.records.LocalizationStatus;
+import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerTokenUpdatedEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.RecoveryIterator;
@@ -105,6 +107,7 @@
 import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
 import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
+import org.apache.hadoop.yarn.server.nodemanager.GenericEventTypeMetricsManager;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrUpdateContainersEvent;
 import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
@@ -120,6 +123,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
+
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent;
@@ -217,7 +221,7 @@ private enum ReInitOp {
   protected final NodeStatusUpdater nodeStatusUpdater;
 
   protected LocalDirsHandlerService dirsHandler;
-  protected final AsyncDispatcher dispatcher;
+  protected AsyncDispatcher dispatcher;
 
   private final DeletionService deletionService;
   private LogHandler logHandler;
@@ -233,6 +237,7 @@ private enum ReInitOp {
   // NM metrics publisher is set only if the timeline service v.2 is enabled
   private NMTimelinePublisher nmMetricsPublisher;
   private boolean timelineServiceV2Enabled;
+  private boolean nmDispatherMetricEnabled;
 
   public ContainerManagerImpl(Context context, ContainerExecutor exec,
       DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
@@ -242,7 +247,7 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec,
     this.dirsHandler = dirsHandler;
 
     // ContainerManager level dispatcher.
-    dispatcher = new AsyncDispatcher("NM ContainerManager dispatcher");
+    dispatcher = createContainerManagerDispatcher();
     this.deletionService = deletionContext;
     this.metrics = metrics;
 
@@ -324,10 +329,66 @@ public void serviceInit(Configuration conf) throws Exception {
             YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS) +
         SHUTDOWN_CLEANUP_SLOP_MS;
 
+    nmDispatherMetricEnabled = conf.getBoolean(
+        YarnConfiguration.NM_DISPATCHER_METRIC_ENABLED,
+        YarnConfiguration.DEFAULT_NM_DISPATCHER_METRIC_ENABLED);
+
     super.serviceInit(conf);
     recover();
   }
 
+  @SuppressWarnings("unchecked")
+  protected AsyncDispatcher createContainerManagerDispatcher() {
+    dispatcher = new AsyncDispatcher("NM ContainerManager dispatcher");
+
+    if (nmDispatherMetricEnabled) {
+      GenericEventTypeMetrics<ContainerEventType> containerEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(), ContainerEventType.class);
+      dispatcher.addMetrics(containerEventTypeMetrics,
+          containerEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<LocalizationEventType> localizationEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(), LocalizationEventType.class);
+      dispatcher.addMetrics(localizationEventTypeMetrics,
+          localizationEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<ApplicationEventType> applicationEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(), ApplicationEventType.class);
+      dispatcher.addMetrics(applicationEventTypeMetrics,
+          applicationEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<ContainersLauncherEventType> containersLauncherEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(
+              dispatcher.getName(), ContainersLauncherEventType.class);
+      dispatcher.addMetrics(containersLauncherEventTypeMetrics,
+          containersLauncherEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<ContainerSchedulerEventType> containerSchedulerEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(
+              dispatcher.getName(), ContainerSchedulerEventType.class);
+      dispatcher.addMetrics(containerSchedulerEventTypeMetrics,
+          containerSchedulerEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<ContainersMonitorEventType> containersMonitorEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(
+              dispatcher.getName(), ContainersMonitorEventType.class);
+      dispatcher.addMetrics(containersMonitorEventTypeMetrics,
+          containersMonitorEventTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<AuxServicesEventType> auxServicesEventTypeTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(), AuxServicesEventType.class);
+      dispatcher.addMetrics(auxServicesEventTypeTypeMetrics,
+          auxServicesEventTypeTypeMetrics.getEnumClass());
+
+      GenericEventTypeMetrics<LocalizerEventType> localizerEventTypeMetrics =
+          GenericEventTypeMetricsManager.create(dispatcher.getName(), LocalizerEventType.class);
+      dispatcher.addMetrics(localizerEventTypeMetrics, localizerEventTypeMetrics.getEnumClass());
+      LOG.info("NM ContainerManager dispatcher Metric Initialization Completed.");
+    }
+
+    return dispatcher;
+  }
+
   protected void createAMRMProxyService(Configuration conf) {
     this.amrmProxyEnabled =
         conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java

@@ -17,11 +17,24 @@
  */
 package org.apache.hadoop.yarn.server.nodemanager.metrics;
 
+import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
 import static org.apache.hadoop.test.MetricsAsserts.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.util.Records;
 
 import org.junit.After;
@@ -37,6 +50,7 @@ public class TestNodeManagerMetrics {
   @Before
   public void setup() {
     DefaultMetricsSystem.initialize("NodeManager");
+    DefaultMetricsSystem.setMiniClusterMode(true);
     metrics = NodeManagerMetrics.create();
   }
 
@@ -140,4 +154,84 @@ public static void checkMetrics(int launched, int completed, int failed,
     assertGauge("NodeGpuUtilization", nodeGpuUtilization, rb);
     assertGauge("ApplicationsRunning", applicationsRunning, rb);
   }
+
+  private enum TestEnum {
+    TestEventType
+  }
+
+  private static class TestHandler implements EventHandler<Event> {
+
+    private long sleepTime = 1500;
+
+    TestHandler() {
+    }
+
+    TestHandler(long sleepTime) {
+      this.sleepTime = sleepTime;
+    }
+
+    @Override
+    public void handle(Event event) {
+      try {
+        // As long as 10000 events queued
+        Thread.sleep(this.sleepTime);
+      } catch (InterruptedException e) {
+      }
+    }
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testNMDispatcherMetricsHistogram() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+
+    NodeManager nm = new NodeManager();
+    nm.init(conf);
+    AsyncDispatcher dispatcher = nm.getDispatcher();
+
+    MetricsInfo metricsInfo = info(
+        "GenericEventTypeMetrics for " + TestEnum.class.getName(),
+        "Metrics for " + dispatcher.getName());
+
+    GenericEventTypeMetrics<TestEnum> genericEventTypeMetrics =
+        new GenericEventTypeMetrics.EventTypeMetricsBuilder()
+        .setMs(DefaultMetricsSystem.instance())
+        .setInfo(metricsInfo)
+        .setEnumClass(TestEnum.class)
+        .setEnums(TestEnum.class.getEnumConstants())
+        .build().registerMetrics();
+
+    dispatcher.addMetrics(genericEventTypeMetrics, genericEventTypeMetrics.getEnumClass());
+    dispatcher.init(conf);
+
+    // Register handler
+    dispatcher.register(TestEnum.class, new TestHandler());
+    dispatcher.start();
+
+    for (int i = 0; i < 3; ++i) {
+      Event event = mock(Event.class);
+      when(event.getType()).thenReturn(TestEnum.TestEventType);
+      dispatcher.getEventHandler().handle(event);
+    }
+
+    // Check event type count.
+    GenericTestUtils.waitFor(() -> genericEventTypeMetrics.
+        get(TestEnum.TestEventType) == 3, 1000, 10000);
+
+    String testEventTypeCountExpect =
+        Long.toString(genericEventTypeMetrics.get(TestEnum.TestEventType));
+    Assert.assertNotNull(testEventTypeCountExpect);
+    String testEventTypeCountMetric =
+        genericEventTypeMetrics.getRegistry().get("TestEventType_event_count").toString();
+    Assert.assertNotNull(testEventTypeCountMetric);
+    Assert.assertEquals(testEventTypeCountExpect, testEventTypeCountMetric);
+
+    String testEventTypeProcessingTimeExpect =
+        Long.toString(genericEventTypeMetrics.getTotalProcessingTime(TestEnum.TestEventType));
+    Assert.assertNotNull(testEventTypeProcessingTimeExpect);
+    String testEventTypeProcessingTimeMetric =
+        genericEventTypeMetrics.getRegistry().get("TestEventType_processing_time").toString();
+    Assert.assertNotNull(testEventTypeProcessingTimeMetric);
+    Assert.assertEquals(testEventTypeProcessingTimeExpect, testEventTypeProcessingTimeMetric);
+  }
 }