feat: Add grain_type tag to app-request timeout/cancel counters

src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using System.Diagnostics.Metrics;
 
 namespace Orleans.Runtime;
@@ -29,13 +30,13 @@ internal void OnAppRequestsEnd(long durationMilliseconds)
             _appRequestsLatencyHistogramAggregator.Record(durationMilliseconds);
     }
 
-    internal void OnAppRequestsTimedOut()
+    internal void OnAppRequestsTimedOut(string grainType)
     {
-        _timedOutRequestsCounter.Add(1);
+        _timedOutRequestsCounter.Add(1, new KeyValuePair<string, object?>("grain_type", grainType));
     }
 
-    internal void OnAppRequestsCanceled()
+    internal void OnAppRequestsCanceled(string grainType)
     {
-        _canceledRequestsCounter.Add(1);
+        _canceledRequestsCounter.Add(1, new KeyValuePair<string, object?>("grain_type", grainType));
     }
 }

src/Orleans.Core/Runtime/CallbackData.cs

@@ -83,6 +83,12 @@ private long GetResponseTimeoutStopwatchTicks()
 
         private TimeSpan GetResponseTimeout() => (Message.BodyObject as IInvokable)?.GetDefaultResponseTimeout() ?? shared.ResponseTimeout;
 
+        private string GetTargetGrainType()
+        {
+            var type = Message.TargetGrain.Type;
+            return type.IsDefault ? "unknown" : type.ToString()!;
+        }
+
         private void OnCancellation()
         {
             // If waiting for acknowledgement is enabled, simply signal to the remote grain that cancellation
@@ -104,7 +110,7 @@ private void OnCancellation()
             SignalCancellation();
             shared.Unregister(Message);
             _applicationRequestInstruments.OnAppRequestsEnd((long)stopwatch.Elapsed.TotalMilliseconds);
-            _applicationRequestInstruments.OnAppRequestsTimedOut();
+            _applicationRequestInstruments.OnAppRequestsCanceled(GetTargetGrainType());
             OrleansCallBackDataEvent.Instance.OnCanceled(Message);
             context.Complete(Response.FromException(new OperationCanceledException(_cancellationTokenRegistration.Token)));
             _cancellationTokenRegistration.Dispose();
@@ -126,7 +132,7 @@ public void OnTimeout()
             this.shared.Unregister(this.Message);
             _cancellationTokenRegistration.Dispose();
             _applicationRequestInstruments.OnAppRequestsEnd((long)this.stopwatch.Elapsed.TotalMilliseconds);
-            _applicationRequestInstruments.OnAppRequestsTimedOut();
+            _applicationRequestInstruments.OnAppRequestsTimedOut(GetTargetGrainType());
 
             OrleansCallBackDataEvent.Instance.OnTimeout(this.Message);
 

test/Orleans.Runtime.Tests/Diagnostics/ApplicationRequestInstrumentsTests.cs

@@ -0,0 +1,34 @@
+using System.Diagnostics.Metrics;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.Metrics.Testing;
+using Orleans.Runtime;
+using Xunit;
+
+namespace Tester.Diagnostics;
+
+public class ApplicationRequestInstrumentsTests
+{
+    [Fact, TestCategory("BVT")]
+    public void TimedOutAndCanceledCounters_CarryGrainTypeTag()
+    {
+        var services = new ServiceCollection();
+        services.AddMetrics();
+
+        using var serviceProvider = services.BuildServiceProvider();
+        var meterFactory = serviceProvider.GetRequiredService<IMeterFactory>();
+        var instruments = new ApplicationRequestInstruments(new OrleansInstruments(meterFactory));
+        using var timedOutCollector = new MetricCollector<long>(meterFactory, "Microsoft.Orleans", InstrumentNames.APP_REQUESTS_TIMED_OUT);
+        using var canceledCollector = new MetricCollector<long>(meterFactory, "Microsoft.Orleans", InstrumentNames.APP_REQUESTS_CANCELED);
+
+        instruments.OnAppRequestsTimedOut("mygrain");
+        instruments.OnAppRequestsCanceled("othergrain");
+
+        var timedOut = Assert.Single(timedOutCollector.GetMeasurementSnapshot());
+        Assert.Equal(1, timedOut.Value);
+        Assert.Equal("mygrain", Assert.Contains("grain_type", timedOut.Tags));
+
+        var canceled = Assert.Single(canceledCollector.GetMeasurementSnapshot());
+        Assert.Equal(1, canceled.Value);
+        Assert.Equal("othergrain", Assert.Contains("grain_type", canceled.Tags));
+    }
+}