From 1b59e8bef850f6ea61a8e010e1954fd5a765e119 Mon Sep 17 00:00:00 2001 From: Lennart Erpenbeck Date: Thu, 4 Jun 2026 12:12:45 +0200 Subject: [PATCH] feat(metrics): add grain_type tag to app-request timeout/cancel counters Tag orleans-app-requests-timedout and orleans-app-requests-canceled with the target grain implementation type (msg.TargetGrain.Type), so timeouts and cancellations can be broken down by grain type. Default grain types are reported as "unknown". Also fix a mis-wiring in CallbackData.OnCancellation(): it incremented the timed-out counter instead of the canceled counter, so cancellations were counted as timeouts and orleans-app-requests-canceled never emitted. --- .../Metrics/ApplicationRequestInstruments.cs | 9 ++--- src/Orleans.Core/Runtime/CallbackData.cs | 10 ++++-- .../ApplicationRequestInstrumentsTests.cs | 34 +++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 test/Orleans.Runtime.Tests/Diagnostics/ApplicationRequestInstrumentsTests.cs diff --git a/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs index c45ce5ea9b7..94a9216c4a7 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Diagnostics.Metrics; namespace Orleans.Runtime; @@ -29,13 +30,13 @@ internal void OnAppRequestsEnd(long durationMilliseconds) _appRequestsLatencyHistogramAggregator.Record(durationMilliseconds); } - internal void OnAppRequestsTimedOut() + internal void OnAppRequestsTimedOut(string grainType) { - _timedOutRequestsCounter.Add(1); + _timedOutRequestsCounter.Add(1, new KeyValuePair("grain_type", grainType)); } - internal void OnAppRequestsCanceled() + internal void OnAppRequestsCanceled(string grainType) { - _canceledRequestsCounter.Add(1); + _canceledRequestsCounter.Add(1, new KeyValuePair("grain_type", grainType)); } } diff --git a/src/Orleans.Core/Runtime/CallbackData.cs b/src/Orleans.Core/Runtime/CallbackData.cs index 98536d69ce7..40453bf800c 100644 --- a/src/Orleans.Core/Runtime/CallbackData.cs +++ b/src/Orleans.Core/Runtime/CallbackData.cs @@ -83,6 +83,12 @@ private long GetResponseTimeoutStopwatchTicks() private TimeSpan GetResponseTimeout() => (Message.BodyObject as IInvokable)?.GetDefaultResponseTimeout() ?? shared.ResponseTimeout; + private string GetTargetGrainType() + { + var type = Message.TargetGrain.Type; + return type.IsDefault ? "unknown" : type.ToString()!; + } + private void OnCancellation() { // If waiting for acknowledgement is enabled, simply signal to the remote grain that cancellation @@ -104,7 +110,7 @@ private void OnCancellation() SignalCancellation(); shared.Unregister(Message); _applicationRequestInstruments.OnAppRequestsEnd((long)stopwatch.Elapsed.TotalMilliseconds); - _applicationRequestInstruments.OnAppRequestsTimedOut(); + _applicationRequestInstruments.OnAppRequestsCanceled(GetTargetGrainType()); OrleansCallBackDataEvent.Instance.OnCanceled(Message); context.Complete(Response.FromException(new OperationCanceledException(_cancellationTokenRegistration.Token))); _cancellationTokenRegistration.Dispose(); @@ -126,7 +132,7 @@ public void OnTimeout() this.shared.Unregister(this.Message); _cancellationTokenRegistration.Dispose(); _applicationRequestInstruments.OnAppRequestsEnd((long)this.stopwatch.Elapsed.TotalMilliseconds); - _applicationRequestInstruments.OnAppRequestsTimedOut(); + _applicationRequestInstruments.OnAppRequestsTimedOut(GetTargetGrainType()); OrleansCallBackDataEvent.Instance.OnTimeout(this.Message); diff --git a/test/Orleans.Runtime.Tests/Diagnostics/ApplicationRequestInstrumentsTests.cs b/test/Orleans.Runtime.Tests/Diagnostics/ApplicationRequestInstrumentsTests.cs new file mode 100644 index 00000000000..6c87be03354 --- /dev/null +++ b/test/Orleans.Runtime.Tests/Diagnostics/ApplicationRequestInstrumentsTests.cs @@ -0,0 +1,34 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.Metrics.Testing; +using Orleans.Runtime; +using Xunit; + +namespace Tester.Diagnostics; + +public class ApplicationRequestInstrumentsTests +{ + [Fact, TestCategory("BVT")] + public void TimedOutAndCanceledCounters_CarryGrainTypeTag() + { + var services = new ServiceCollection(); + services.AddMetrics(); + + using var serviceProvider = services.BuildServiceProvider(); + var meterFactory = serviceProvider.GetRequiredService(); + var instruments = new ApplicationRequestInstruments(new OrleansInstruments(meterFactory)); + using var timedOutCollector = new MetricCollector(meterFactory, "Microsoft.Orleans", InstrumentNames.APP_REQUESTS_TIMED_OUT); + using var canceledCollector = new MetricCollector(meterFactory, "Microsoft.Orleans", InstrumentNames.APP_REQUESTS_CANCELED); + + instruments.OnAppRequestsTimedOut("mygrain"); + instruments.OnAppRequestsCanceled("othergrain"); + + var timedOut = Assert.Single(timedOutCollector.GetMeasurementSnapshot()); + Assert.Equal(1, timedOut.Value); + Assert.Equal("mygrain", Assert.Contains("grain_type", timedOut.Tags)); + + var canceled = Assert.Single(canceledCollector.GetMeasurementSnapshot()); + Assert.Equal(1, canceled.Value); + Assert.Equal("othergrain", Assert.Contains("grain_type", canceled.Tags)); + } +}