From ad57e1c8fcc2a6fc9a609ccd689dadf0806e2335 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Tue, 31 Mar 2026 12:16:43 -0700 Subject: [PATCH 01/12] feat(membership): detect local GC pauses during direct probes --- .../MembershipService/SiloHealthMonitor.cs | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index ae69ed79e3c..c7b1f95111f 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -212,7 +212,7 @@ private async Task Run() if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout).ConfigureAwait(false); } else { @@ -321,12 +321,14 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// Probes the remote silo. /// /// A token to cancel and fail the probe attempt. + /// The timeout used for this probe, for GC pause evaluation. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); + var gcPauseBefore = GC.GetTotalPauseDuration(); var roundTripTimer = TimeProviderValueStopwatch.StartNew(_timeProvider); ProbeResult probeResult; Exception? failureException; @@ -361,12 +363,23 @@ private async Task ProbeDirectly(CancellationToken cancellation) } else { - _messagingInstruments?.OnPingReplyMissed(TargetSiloAddress); - - var failedProbes = ++_failedProbes; - LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes); + // Check if a GC pause consumed a significant portion of the probe timeout. + // If so, the local silo may have been unable to process the response in time, + // so we treat this as an inconclusive result rather than a failure. + var gcPauseDuring = GC.GetTotalPauseDuration() - gcPauseBefore; + if (gcPauseDuring > probeTimeout.Multiply(0.25)) + { + LogWarningProbeFailureDuringGcPause(_log, id, TargetSiloAddress, roundTripTime, gcPauseDuring, _failedProbes); + probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); + } + else + { + _messagingInstruments?.OnPingReplyMissed(TargetSiloAddress); + var failedProbes = ++_failedProbes; + LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes); - probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed); + probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed); + } } return probeResult; @@ -536,5 +549,11 @@ public enum ProbeResultStatus Message = "Exception monitoring silo {SiloAddress}" )] private static partial void LogErrorExceptionMonitoringSilo(ILogger logger, Exception exception, SiloAddress siloAddress); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a GC pause of {GcPauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probes remains at {FailedProbeCount}." + )] + private static partial void LogWarningProbeFailureDuringGcPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan gcPauseDuration, int failedProbeCount); } } From fa09fef538f4af7205edc0ae74bec10e8a19d988 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 16 Aug 2026 02:58:03 -0700 Subject: [PATCH 02/12] fix(membership): constrain GC pause suppression Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MembershipService/SiloHealthMonitor.cs | 13 ++-- .../Membership/SiloHealthMonitorTests.cs | 69 +++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index c7b1f95111f..ba15351e7f8 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -30,6 +30,7 @@ internal partial class SiloHealthMonitor : ITestAccessor, IHealthCheckable, IDis private readonly ILocalSiloDetails _localSiloDetails; private readonly TimeProvider _timeProvider; private readonly CancellationTokenSource _stoppingCancellation = new(); + private Func _getTotalPauseDuration = GC.GetTotalPauseDuration; #if NET9_0_OR_GREATER private readonly Lock _lockObj = new(); #else @@ -97,6 +98,7 @@ internal interface ITestAccessor int MissedProbes { get; } int ProbeTimeoutSampleCount { get; } TimeSpan CurrentProbeTimeout { get; } + Func GetTotalPauseDuration { set; } } /// @@ -112,6 +114,7 @@ internal interface ITestAccessor int ITestAccessor.MissedProbes => _failedProbes; int ITestAccessor.ProbeTimeoutSampleCount => _failureDetector?.SampleCount ?? 0; TimeSpan ITestAccessor.CurrentProbeTimeout => _failureDetector?.GetTimeout() ?? _clusterMembershipOptions.CurrentValue.ProbeTimeout; + Func ITestAccessor.GetTotalPauseDuration { set => _getTotalPauseDuration = value; } /// /// Start the monitor. @@ -328,10 +331,11 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); - var gcPauseBefore = GC.GetTotalPauseDuration(); + var gcPauseBefore = _getTotalPauseDuration(); var roundTripTimer = TimeProviderValueStopwatch.StartNew(_timeProvider); ProbeResult probeResult; Exception? failureException; + var probeTimedOut = false; try { await _prober.Probe(TargetSiloAddress, id, cancellation).WaitAsync(cancellation); @@ -339,6 +343,7 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti } catch (OperationCanceledException exception) { + probeTimedOut = cancellation.IsCancellationRequested; failureException = new OperationCanceledException( $"The ping attempt was cancelled after {roundTripTimer.GetElapsedTime(out _)}. Ping #{id}", exception); @@ -366,8 +371,8 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti // Check if a GC pause consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - var gcPauseDuring = GC.GetTotalPauseDuration() - gcPauseBefore; - if (gcPauseDuring > probeTimeout.Multiply(0.25)) + var gcPauseDuring = _getTotalPauseDuration() - gcPauseBefore; + if (probeTimedOut && gcPauseDuring > probeTimeout.Multiply(0.25)) { LogWarningProbeFailureDuringGcPause(_log, id, TargetSiloAddress, roundTripTime, gcPauseDuring, _failedProbes); probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); @@ -552,7 +557,7 @@ public enum ProbeResultStatus [LoggerMessage( Level = LogLevel.Warning, - Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a GC pause of {GcPauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probes remains at {FailedProbeCount}." + Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a GC pause of {GcPauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." )] private static partial void LogWarningProbeFailureDuringGcPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan gcPauseDuration, int failedProbeCount); } diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index ec08323dedd..e82fc13017a 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -223,10 +223,79 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() await Shutdown(); } + [Fact] + public async Task SiloHealthMonitor_TimeoutDuringGcPause_IsInconclusive() + { + var timeProvider = new Microsoft.Extensions.Time.Testing.FakeTimeProvider( + new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero)); + var options = new ClusterMembershipOptions + { + ProbeTimeout = TimeSpan.FromSeconds(5), + MaxProbeTimeout = TimeSpan.FromSeconds(45), + EnableIndirectProbes = false, + }; + var optionsMonitor = Substitute.For>(); + optionsMonitor.CurrentValue.Returns(options); + var prober = Substitute.For(); + var probeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingProbe = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + prober.Probe(default!, default, default).ReturnsForAnyArgs(call => + { + probeEntered.TrySetResult(call.ArgAt(2)); + return pendingProbe.Task; + }); + var probeResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timer = new DilationProbeTimer(); + var monitor = new SiloHealthMonitor( + _targetSilo, + (_, result) => + { + probeResult.TrySetResult(result); + return Task.CompletedTask; + }, + optionsMonitor, + _loggerFactory, + prober, + new DelegateAsyncTimerFactory((_, _) => timer), + _localSiloHealthMonitor, + _membershipService, + _localSiloDetails, + timeProvider); + var totalPauseDuration = TimeSpan.Zero; + ((ITestAccessor)monitor).GetTotalPauseDuration = () => totalPauseDuration; + + try + { + monitor.Start(); + var firstTick = await timer.Ticks.ReadAsync(); + firstTick.Completion.SetResult(true); + var cancellationToken = await probeEntered.Task; + + totalPauseDuration = TimeSpan.FromSeconds(2); + timeProvider.Advance(options.ProbeTimeout); + + Assert.True(cancellationToken.IsCancellationRequested); + var result = await probeResult.Task; + Assert.Equal(ProbeResultStatus.Unknown, result.Status); + Assert.Equal(0, result.FailedProbeCount); + Assert.Equal(0, ((ITestAccessor)monitor).MissedProbes); + + var nextTick = await timer.Ticks.ReadAsync(); + nextTick.Completion.SetResult(false); + } + finally + { + await monitor.StopAsync(CancellationToken.None); + } + } + [Fact] public async Task SiloHealthMonitor_FailedProbe_Exception() { _clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2); + var pauseSample = 0; + ((ITestAccessor)_monitor).GetTotalPauseDuration = () => + Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout; _prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope")); _prober.ProbeIndirectly(default!, default!, default, default).ThrowsAsyncForAnyArgs(new InvalidOperationException("No")); From 00e411888dc82517a4d8f2c539fe52bf38fb4842 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 16 Aug 2026 03:05:17 -0700 Subject: [PATCH 03/12] fix(membership): include GC pause threshold boundary Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs | 2 +- test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index ba15351e7f8..f01648104e7 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -372,7 +372,7 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. var gcPauseDuring = _getTotalPauseDuration() - gcPauseBefore; - if (probeTimedOut && gcPauseDuring > probeTimeout.Multiply(0.25)) + if (probeTimedOut && gcPauseDuring >= probeTimeout.Multiply(0.25)) { LogWarningProbeFailureDuringGcPause(_log, id, TargetSiloAddress, roundTripTime, gcPauseDuring, _failedProbes); probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index e82fc13017a..02cb600238d 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -271,7 +271,7 @@ public async Task SiloHealthMonitor_TimeoutDuringGcPause_IsInconclusive() firstTick.Completion.SetResult(true); var cancellationToken = await probeEntered.Task; - totalPauseDuration = TimeSpan.FromSeconds(2); + totalPauseDuration = options.ProbeTimeout.Multiply(0.25); timeProvider.Advance(options.ProbeTimeout); Assert.True(cancellationToken.IsCancellationRequested); From bc8528052e199af2b622d397b6b44e7c59278d89 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 12:21:16 -0700 Subject: [PATCH 04/12] fix(membership): sample GC pauses when arming probe timeout --- src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index f01648104e7..12f573e5898 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -211,11 +211,12 @@ private async Task Run() var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); probeStartTimestamp = _timeProvider.GetTimestamp(); using var cancellation = new CancellationTokenSource(timeout, _timeProvider); + var gcPauseBefore = isDirectProbe ? _getTotalPauseDuration() : TimeSpan.Zero; if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token, timeout).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout, gcPauseBefore).ConfigureAwait(false); } else { @@ -325,13 +326,13 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// /// A token to cancel and fail the probe attempt. /// The timeout used for this probe, for GC pause evaluation. + /// The cumulative GC pause duration when the probe timeout was armed. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, TimeSpan gcPauseBefore) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); - var gcPauseBefore = _getTotalPauseDuration(); var roundTripTimer = TimeProviderValueStopwatch.StartNew(_timeProvider); ProbeResult probeResult; Exception? failureException; From ab321153b1d37634e5d5d682892b74eabc9a308f Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 12:56:33 -0700 Subject: [PATCH 05/12] refactor(membership): expose GC pauses via health monitor --- .../MembershipService/LocalSiloHealthMonitor.cs | 8 ++++++++ .../MembershipService/SiloHealthMonitor.cs | 9 ++++----- .../Membership/SiloHealthMonitorTests.cs | 6 +++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index f92c78624d7..9a3337e9abb 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -53,6 +53,11 @@ public ImmutableArray Complaints internal interface ILocalSiloHealthMonitor { + /// + /// Gets the total duration of garbage collection pauses since process start. + /// + TimeSpan TotalGarbageCollectionPauseDuration { get; } + /// /// Returns the aggregate local health status over the provided interval. /// @@ -172,6 +177,9 @@ public LocalSiloHealthMonitor( /// public ImmutableArray Complaints { get; private set; } = []; + /// + public TimeSpan TotalGarbageCollectionPauseDuration => GC.GetTotalPauseDuration(); + /// public LocalSiloHealthStatus GetLocalHealthStatus( long startTimestamp, diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 12f573e5898..f006f8b5aee 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -30,7 +30,6 @@ internal partial class SiloHealthMonitor : ITestAccessor, IHealthCheckable, IDis private readonly ILocalSiloDetails _localSiloDetails; private readonly TimeProvider _timeProvider; private readonly CancellationTokenSource _stoppingCancellation = new(); - private Func _getTotalPauseDuration = GC.GetTotalPauseDuration; #if NET9_0_OR_GREATER private readonly Lock _lockObj = new(); #else @@ -98,7 +97,6 @@ internal interface ITestAccessor int MissedProbes { get; } int ProbeTimeoutSampleCount { get; } TimeSpan CurrentProbeTimeout { get; } - Func GetTotalPauseDuration { set; } } /// @@ -114,7 +112,6 @@ internal interface ITestAccessor int ITestAccessor.MissedProbes => _failedProbes; int ITestAccessor.ProbeTimeoutSampleCount => _failureDetector?.SampleCount ?? 0; TimeSpan ITestAccessor.CurrentProbeTimeout => _failureDetector?.GetTimeout() ?? _clusterMembershipOptions.CurrentValue.ProbeTimeout; - Func ITestAccessor.GetTotalPauseDuration { set => _getTotalPauseDuration = value; } /// /// Start the monitor. @@ -211,7 +208,9 @@ private async Task Run() var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); probeStartTimestamp = _timeProvider.GetTimestamp(); using var cancellation = new CancellationTokenSource(timeout, _timeProvider); - var gcPauseBefore = isDirectProbe ? _getTotalPauseDuration() : TimeSpan.Zero; + var gcPauseBefore = isDirectProbe + ? _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration + : TimeSpan.Zero; if (isDirectProbe) { @@ -372,7 +371,7 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti // Check if a GC pause consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - var gcPauseDuring = _getTotalPauseDuration() - gcPauseBefore; + var gcPauseDuring = _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration - gcPauseBefore; if (probeTimedOut && gcPauseDuring >= probeTimeout.Multiply(0.25)) { LogWarningProbeFailureDuringGcPause(_log, id, TargetSiloAddress, roundTripTime, gcPauseDuring, _failedProbes); diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index 02cb600238d..05ad1071899 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -262,7 +262,7 @@ public async Task SiloHealthMonitor_TimeoutDuringGcPause_IsInconclusive() _localSiloDetails, timeProvider); var totalPauseDuration = TimeSpan.Zero; - ((ITestAccessor)monitor).GetTotalPauseDuration = () => totalPauseDuration; + _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration.Returns(_ => totalPauseDuration); try { @@ -294,8 +294,8 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() { _clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2); var pauseSample = 0; - ((ITestAccessor)_monitor).GetTotalPauseDuration = () => - Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout; + _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration.Returns(_ => + Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout); _prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope")); _prober.ProbeIndirectly(default!, default!, default, default).ThrowsAsyncForAnyArgs(new InvalidOperationException("No")); From 6c1d7b21f565951d0a0cdaa19b4b93cc9023b2cb Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 13:19:39 -0700 Subject: [PATCH 06/12] refactor(membership): generalize local pause metric --- .../LocalSiloHealthMonitor.cs | 6 ++--- .../MembershipService/SiloHealthMonitor.cs | 24 +++++++++---------- .../Membership/SiloHealthMonitorTests.cs | 6 ++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index 9a3337e9abb..21644f79ddb 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -54,9 +54,9 @@ public ImmutableArray Complaints internal interface ILocalSiloHealthMonitor { /// - /// Gets the total duration of garbage collection pauses since process start. + /// Gets the total duration of local runtime pauses since process start. /// - TimeSpan TotalGarbageCollectionPauseDuration { get; } + TimeSpan TotalPauseDuration { get; } /// /// Returns the aggregate local health status over the provided interval. @@ -178,7 +178,7 @@ public LocalSiloHealthMonitor( public ImmutableArray Complaints { get; private set; } = []; /// - public TimeSpan TotalGarbageCollectionPauseDuration => GC.GetTotalPauseDuration(); + public TimeSpan TotalPauseDuration => GC.GetTotalPauseDuration(); /// public LocalSiloHealthStatus GetLocalHealthStatus( diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index f006f8b5aee..04f0e8a4b29 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -208,14 +208,14 @@ private async Task Run() var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); probeStartTimestamp = _timeProvider.GetTimestamp(); using var cancellation = new CancellationTokenSource(timeout, _timeProvider); - var gcPauseBefore = isDirectProbe - ? _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration + var pauseDurationBefore = isDirectProbe + ? _localSiloHealthMonitor.TotalPauseDuration : TimeSpan.Zero; if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token, timeout, gcPauseBefore).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout, pauseDurationBefore).ConfigureAwait(false); } else { @@ -324,10 +324,10 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// Probes the remote silo. /// /// A token to cancel and fail the probe attempt. - /// The timeout used for this probe, for GC pause evaluation. - /// The cumulative GC pause duration when the probe timeout was armed. + /// The timeout used for this probe, for local pause evaluation. + /// The cumulative local pause duration when the probe timeout was armed. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, TimeSpan gcPauseBefore) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, TimeSpan pauseDurationBefore) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); @@ -368,13 +368,13 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti } else { - // Check if a GC pause consumed a significant portion of the probe timeout. + // Check if a local pause consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - var gcPauseDuring = _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration - gcPauseBefore; - if (probeTimedOut && gcPauseDuring >= probeTimeout.Multiply(0.25)) + var pauseDurationDuring = _localSiloHealthMonitor.TotalPauseDuration - pauseDurationBefore; + if (probeTimedOut && pauseDurationDuring >= probeTimeout.Multiply(0.25)) { - LogWarningProbeFailureDuringGcPause(_log, id, TargetSiloAddress, roundTripTime, gcPauseDuring, _failedProbes); + LogWarningProbeFailureDuringLocalPause(_log, id, TargetSiloAddress, roundTripTime, pauseDurationDuring, _failedProbes); probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); } else @@ -557,8 +557,8 @@ public enum ProbeResultStatus [LoggerMessage( Level = LogLevel.Warning, - Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a GC pause of {GcPauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." + Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a local pause of {PauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." )] - private static partial void LogWarningProbeFailureDuringGcPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan gcPauseDuration, int failedProbeCount); + private static partial void LogWarningProbeFailureDuringLocalPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan pauseDuration, int failedProbeCount); } } diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index 05ad1071899..0156f40a4cb 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -224,7 +224,7 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() } [Fact] - public async Task SiloHealthMonitor_TimeoutDuringGcPause_IsInconclusive() + public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() { var timeProvider = new Microsoft.Extensions.Time.Testing.FakeTimeProvider( new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero)); @@ -262,7 +262,7 @@ public async Task SiloHealthMonitor_TimeoutDuringGcPause_IsInconclusive() _localSiloDetails, timeProvider); var totalPauseDuration = TimeSpan.Zero; - _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration.Returns(_ => totalPauseDuration); + _localSiloHealthMonitor.TotalPauseDuration.Returns(_ => totalPauseDuration); try { @@ -294,7 +294,7 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() { _clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2); var pauseSample = 0; - _localSiloHealthMonitor.TotalGarbageCollectionPauseDuration.Returns(_ => + _localSiloHealthMonitor.TotalPauseDuration.Returns(_ => Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout); _prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope")); From 3463b116d9c66bdf9777eda4f79b14a2fe3a86a8 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 21 Aug 2026 14:57:39 -0700 Subject: [PATCH 07/12] feat(membership): aggregate local pause durations --- .../LocalSiloHealthEventHistory.cs | 189 ++++++++++++++++++ .../LocalSiloHealthMonitor.cs | 109 +++++++++- .../MembershipService/SiloHealthMonitor.cs | 15 +- src/Orleans.Runtime/Silo/Watchdog.cs | 9 - .../LocalSiloHealthEventHistoryTests.cs | 107 +++++++++- .../Membership/SiloHealthMonitorTests.cs | 15 +- .../Diagnostics/WatchdogTests.cs | 18 -- 7 files changed, 416 insertions(+), 46 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs index b2a76373bd1..8a2d9cfd9bf 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; using System.Runtime.InteropServices; namespace Orleans.Runtime.MembershipService; @@ -13,6 +14,12 @@ internal sealed class LocalSiloHealthEventHistory private readonly Bucket[] _buckets; private readonly Dictionary _candidates = []; private readonly List _selected = []; + private readonly List _pauseEvents = []; + private readonly SortedDictionary _activePauseIntervals = []; + private readonly Dictionary> _pauseIntervalsByKind = []; + private readonly List _allPauseIntervals = []; + private readonly List _pauseDurations = []; + private int _pauseEventHead; private int _count; private int _occupiedBucketCount; @@ -35,6 +42,37 @@ public LocalSiloHealthEventHistory( internal int OccupiedBucketCount => _occupiedBucketCount; + public void BeginPauseCollection(long startTimestamp) + { + if (_activePauseIntervals.TryGetValue(startTimestamp, out var count)) + { + _activePauseIntervals[startTimestamp] = count + 1; + } + else + { + _activePauseIntervals.Add(startTimestamp, 1); + } + } + + public void EndPauseCollection(long startTimestamp, long nowTimestamp) + { + if (!_activePauseIntervals.TryGetValue(startTimestamp, out var count)) + { + throw new InvalidOperationException($"Pause collection interval {startTimestamp} is not active."); + } + + if (count == 1) + { + _activePauseIntervals.Remove(startTimestamp); + } + else + { + _activePauseIntervals[startTimestamp] = count - 1; + } + + RemoveExpiredPauseEvents(nowTimestamp, force: true); + } + public void Add(LocalSiloHealthEvent healthEvent) { RemoveExpired(healthEvent.Timestamp); @@ -106,6 +144,7 @@ public LocalSiloHealthStatus Aggregate( entry.HasIntervalEvent = true; } } + } _selected.EnsureCapacity(_candidates.Count); @@ -148,6 +187,57 @@ public LocalSiloHealthStatus Aggregate( } } + public LocalSiloPauseStatus AggregatePauses( + long startTimestamp, + long endTimestamp, + long nowTimestamp, + LocalSiloHealthCheckCategory categories) + { + RemoveExpired(nowTimestamp); + if (categories == LocalSiloHealthCheckCategory.None) + { + return new(TimeSpan.Zero, []); + } + + try + { + for (var i = _pauseEventHead; i < _pauseEvents.Count; i++) + { + var healthEvent = _pauseEvents[i]; + if ((healthEvent.Category & categories) == 0 + || !TryGetPauseInterval(healthEvent, startTimestamp, endTimestamp, out var interval)) + { + continue; + } + + if (!_pauseIntervalsByKind.TryGetValue(healthEvent.Kind, out var intervals)) + { + intervals = []; + _pauseIntervalsByKind.Add(healthEvent.Kind, intervals); + } + + intervals.Add(interval); + _allPauseIntervals.Add(interval); + } + + foreach (var (kind, intervals) in _pauseIntervalsByKind) + { + _pauseDurations.Add(new(kind, GetUnionDuration(intervals))); + } + + _pauseDurations.Sort(static (left, right) => left.Kind.CompareTo(right.Kind)); + return new( + GetUnionDuration(_allPauseIntervals), + ImmutableArray.CreateRange(_pauseDurations)); + } + finally + { + _pauseIntervalsByKind.Clear(); + _allPauseIntervals.Clear(); + _pauseDurations.Clear(); + } + } + private void AddCore(LocalSiloHealthEvent healthEvent) { var bucketIndex = GetBucketIndex(healthEvent.Timestamp); @@ -173,6 +263,10 @@ private void AddCore(LocalSiloHealthEvent healthEvent) bucket.Events!.Add(healthEvent); _count++; + if (IsPause(healthEvent)) + { + _pauseEvents.Add(healthEvent); + } } private void RemoveExpired(long nowTimestamp) @@ -215,6 +309,31 @@ private void RemoveExpired(long nowTimestamp) Clear(ref bucket); } } + + } + + RemoveExpiredPauseEvents(nowTimestamp, force: false); + } + + private void RemoveExpiredPauseEvents(long nowTimestamp, bool force) + { + var oldestTimestamp = SubtractTimestamp(nowTimestamp, _retentionPeriod); + if (_activePauseIntervals.Count > 0) + { + oldestTimestamp = Math.Min(oldestTimestamp, _activePauseIntervals.First().Key); + } + + while (_pauseEventHead < _pauseEvents.Count + && _pauseEvents[_pauseEventHead].Timestamp < oldestTimestamp) + { + _pauseEventHead++; + } + + if (_pauseEventHead > 0 + && (force || _pauseEventHead >= 1024 && _pauseEventHead >= _pauseEvents.Count / 2)) + { + _pauseEvents.RemoveRange(0, _pauseEventHead); + _pauseEventHead = 0; } } @@ -239,6 +358,68 @@ private bool Overlaps(LocalSiloHealthEvent healthEvent, long startTimestamp, lon && _timeProvider.GetElapsedTime(endTimestamp, healthEvent.Timestamp) <= duration; } + private bool TryGetPauseInterval( + LocalSiloHealthEvent healthEvent, + long startTimestamp, + long endTimestamp, + out PauseInterval interval) + { + if (healthEvent.Duration is not { } duration || duration <= TimeSpan.Zero) + { + interval = default; + return false; + } + + var pauseStartTimestamp = SubtractTimestamp(healthEvent.Timestamp, duration); + var overlapStartTimestamp = Math.Max(startTimestamp, pauseStartTimestamp); + var overlapEndTimestamp = Math.Min(endTimestamp, healthEvent.Timestamp); + if (overlapEndTimestamp <= overlapStartTimestamp) + { + interval = default; + return false; + } + + interval = new(overlapStartTimestamp, overlapEndTimestamp); + return true; + } + + private TimeSpan GetUnionDuration(List intervals) + { + if (intervals.Count == 0) + { + return TimeSpan.Zero; + } + + intervals.Sort(static (left, right) => + { + var startComparison = left.StartTimestamp.CompareTo(right.StartTimestamp); + return startComparison != 0 + ? startComparison + : left.EndTimestamp.CompareTo(right.EndTimestamp); + }); + + var totalTimestampLength = 0L; + var currentStartTimestamp = intervals[0].StartTimestamp; + var currentEndTimestamp = intervals[0].EndTimestamp; + for (var i = 1; i < intervals.Count; i++) + { + var interval = intervals[i]; + if (interval.StartTimestamp <= currentEndTimestamp) + { + currentEndTimestamp = Math.Max(currentEndTimestamp, interval.EndTimestamp); + } + else + { + totalTimestampLength = checked(totalTimestampLength + currentEndTimestamp - currentStartTimestamp); + currentStartTimestamp = interval.StartTimestamp; + currentEndTimestamp = interval.EndTimestamp; + } + } + + totalTimestampLength = checked(totalTimestampLength + currentEndTimestamp - currentStartTimestamp); + return _timeProvider.GetElapsedTime(0, totalTimestampLength); + } + private static bool IsStateful(LocalSiloHealthCheckKind kind) => kind is LocalSiloHealthCheckKind.MembershipStatus or LocalSiloHealthCheckKind.HealthCheckParticipant @@ -246,6 +427,12 @@ or LocalSiloHealthCheckKind.ThreadPoolQueueDelay or LocalSiloHealthCheckKind.ProbeRequests or LocalSiloHealthCheckKind.ProbeResponses; + private static bool IsPause(LocalSiloHealthEvent healthEvent) + => healthEvent.Duration > TimeSpan.Zero + && healthEvent.Kind is LocalSiloHealthCheckKind.GarbageCollectionPause + or LocalSiloHealthCheckKind.RuntimeStall + or LocalSiloHealthCheckKind.ComponentHealthCheckStall; + private static bool IsBetter( LocalSiloHealthEvent candidate, LocalSiloHealthEvent current) @@ -270,6 +457,8 @@ private readonly record struct HealthEventIdentity( LocalSiloHealthCheckKind Kind, string? Source); + private readonly record struct PauseInterval(long StartTimestamp, long EndTimestamp); + private struct AggregateEntry { public LocalSiloHealthEvent PriorState; diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index 21644f79ddb..d1c2607246e 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -51,12 +51,43 @@ public ImmutableArray Complaints => [.. Events.Where(static status => status.Complaint is not null).Select(static status => status.Complaint!)]; } + internal readonly record struct LocalSiloPauseDuration(LocalSiloHealthCheckKind Kind, TimeSpan Duration); + + internal readonly record struct LocalSiloPauseStatus( + TimeSpan TotalPauseDuration, + ImmutableArray Durations) + { + public TimeSpan GetDuration(LocalSiloHealthCheckKind kind) + { + if (!Durations.IsDefaultOrEmpty) + { + foreach (var item in Durations) + { + if (item.Kind == kind) + { + return item.Duration; + } + } + } + + return TimeSpan.Zero; + } + } + internal interface ILocalSiloHealthMonitor { /// - /// Gets the total duration of local runtime pauses since process start. + /// Captures the start of a pause aggregation interval. + /// + /// The timestamp which begins the interval. + long CapturePauseTimestamp(); + + /// + /// Returns pause durations since the provided timestamp, aggregated by kind and as a deduplicated total. /// - TimeSpan TotalPauseDuration { get; } + /// The timestamp returned by . + /// The pause duration summary. + LocalSiloPauseStatus GetPauseStatus(long startTimestamp); /// /// Returns the aggregate local health status over the provided interval. @@ -129,6 +160,7 @@ internal partial class LocalSiloHealthMonitor : private readonly IAsyncTimer _degradationCheckTimer; private readonly ThreadPoolMonitor _threadPoolMonitor; private readonly TimeProvider _timeProvider; + private TimeSpan _cumulativeGCPauseDuration; #if NET9_0_OR_GREATER private readonly Lock _historyLock = new(); private readonly Lock _samplingLock = new(); @@ -166,6 +198,7 @@ public LocalSiloHealthMonitor( _log = log; _clusterMembershipOptions = clusterMembershipOptions.Value; _timeProvider = timeProvider; + _cumulativeGCPauseDuration = GC.GetTotalPauseDuration(); _healthHistory = new(timeProvider, HistoryDuration, MinimumCheckPeriod); _degradationCheckTimer = timerFactory.Create( MinimumCheckPeriod, @@ -178,7 +211,47 @@ public LocalSiloHealthMonitor( public ImmutableArray Complaints { get; private set; } = []; /// - public TimeSpan TotalPauseDuration => GC.GetTotalPauseDuration(); + public long CapturePauseTimestamp() + { + lock (_historyLock) + { + var timestamp = _timeProvider.GetTimestamp(); + RecordGarbageCollectionPause(timestamp, GC.GetTotalPauseDuration()); + _healthHistory.BeginPauseCollection(timestamp); + return timestamp; + } + } + + /// + public LocalSiloPauseStatus GetPauseStatus(long startTimestamp) + { + lock (_historyLock) + { + var cumulativePauseDuration = GC.GetTotalPauseDuration(); + var endTimestamp = _timeProvider.GetTimestamp(); + if (endTimestamp < startTimestamp) + { + throw new ArgumentOutOfRangeException( + nameof(startTimestamp), + startTimestamp, + "The interval start must not follow the current timestamp."); + } + + RecordGarbageCollectionPause(endTimestamp, cumulativePauseDuration); + try + { + return _healthHistory.AggregatePauses( + startTimestamp, + endTimestamp, + endTimestamp, + LocalSiloHealthCheckCategory.Local); + } + finally + { + _healthHistory.EndPauseCollection(startTimestamp, endTimestamp); + } + } + } /// public LocalSiloHealthStatus GetLocalHealthStatus( @@ -209,10 +282,13 @@ public LocalSiloHealthStatus GetLocalHealthStatus( lock (_historyLock) { + var cumulativePauseDuration = GC.GetTotalPauseDuration(); + var nowTimestamp = _timeProvider.GetTimestamp(); + RecordGarbageCollectionPause(nowTimestamp, cumulativePauseDuration); return _healthHistory.Aggregate( startTimestamp, endTimestamp, - _timeProvider.GetTimestamp(), + nowTimestamp, categories, MaxScore); } @@ -358,6 +434,25 @@ private void AddEvents(long timestamp, List events) } } + private void RecordGarbageCollectionPause(long timestamp, TimeSpan cumulativePauseDuration) + { + var pauseDuration = cumulativePauseDuration - _cumulativeGCPauseDuration; + _cumulativeGCPauseDuration = cumulativePauseDuration; + if (pauseDuration <= TimeSpan.Zero) + { + return; + } + + _healthHistory.Add(new( + timestamp, + LocalSiloHealthCheckKind.GarbageCollectionPause, + LocalSiloHealthCheckCategory.Local, + Source: null, + Score: 0, + Complaint: $"The .NET runtime paused for garbage collection for {pauseDuration}.", + pauseDuration)); + } + private void CheckThreadPoolQueueDelay( long timestamp, List events, @@ -573,6 +668,12 @@ private async Task Run() var networkHealthCheckVersion = Volatile.Read(ref _networkHealthCheckVersion); lock (_samplingLock) { + lock (_historyLock) + { + var cumulativePauseDuration = GC.GetTotalPauseDuration(); + RecordGarbageCollectionPause(_timeProvider.GetTimestamp(), cumulativePauseDuration); + } + status = EnsureHealthCheck( _timeProvider.GetUtcNow(), _timeProvider.GetTimestamp(), diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 04f0e8a4b29..48924a7334c 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -207,15 +207,15 @@ private async Task Run() localDegradationScore = GetLocalDegradationScore(previousProbePeriod); var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); probeStartTimestamp = _timeProvider.GetTimestamp(); + var pauseStartTimestamp = isDirectProbe + ? _localSiloHealthMonitor.CapturePauseTimestamp() + : 0; using var cancellation = new CancellationTokenSource(timeout, _timeProvider); - var pauseDurationBefore = isDirectProbe - ? _localSiloHealthMonitor.TotalPauseDuration - : TimeSpan.Zero; if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token, timeout, pauseDurationBefore).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout, pauseStartTimestamp).ConfigureAwait(false); } else { @@ -325,9 +325,9 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// /// A token to cancel and fail the probe attempt. /// The timeout used for this probe, for local pause evaluation. - /// The cumulative local pause duration when the probe timeout was armed. + /// The start of the local pause aggregation interval. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, TimeSpan pauseDurationBefore) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, long pauseStartTimestamp) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); @@ -353,6 +353,7 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti failureException = exception; } var roundTripTime = roundTripTimer.GetElapsedTime(out _); + var pauseStatus = _localSiloHealthMonitor.GetPauseStatus(pauseStartTimestamp); if (failureException is null) { @@ -371,7 +372,7 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti // Check if a local pause consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - var pauseDurationDuring = _localSiloHealthMonitor.TotalPauseDuration - pauseDurationBefore; + var pauseDurationDuring = pauseStatus.TotalPauseDuration; if (probeTimedOut && pauseDurationDuring >= probeTimeout.Multiply(0.25)) { LogWarningProbeFailureDuringLocalPause(_log, id, TargetSiloAddress, roundTripTime, pauseDurationDuring, _failedProbes); diff --git a/src/Orleans.Runtime/Silo/Watchdog.cs b/src/Orleans.Runtime/Silo/Watchdog.cs index 2de14c3d78f..194990564ed 100644 --- a/src/Orleans.Runtime/Silo/Watchdog.cs +++ b/src/Orleans.Runtime/Silo/Watchdog.cs @@ -107,15 +107,6 @@ private void CheckRuntimeHealth() var pauseDurationSinceLastTick = GC.GetTotalPauseDuration() - _cumulativeGCPauseDuration; var timeSinceLastTick = _timeProvider.GetElapsedTime(_platformWatchdogTimestamp); - if (pauseDurationSinceLastTick > TimeSpan.Zero) - { - _healthEventRecorder.RecordHealthEvent( - LocalSiloHealthCheckKind.GarbageCollectionPause, - score: 0, - complaint: $"The .NET runtime paused for garbage collection for {pauseDurationSinceLastTick}.", - duration: pauseDurationSinceLastTick); - } - if (timeSinceLastTick > PlatformWatchdogHeartbeatPeriod.Multiply(2)) { var gc = new[] { GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2) }; diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs index 79191e0ed92..a72a0c42016 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs @@ -173,6 +173,108 @@ public void Aggregate_ClampsWithoutOverflow() Assert.Equal(LocalSiloHealthMonitor.MaxScore, status.Score); } + [Fact] + public void AggregatePauses_ReportsPerKindDurationsAndDeduplicatedTotal() + { + var history = CreateHistory(); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(3)), + LocalSiloHealthCheckKind.GarbageCollectionPause, + duration: TimeSpan.FromSeconds(2))); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(8)), + LocalSiloHealthCheckKind.GarbageCollectionPause, + duration: TimeSpan.FromSeconds(2))); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(7)), + LocalSiloHealthCheckKind.RuntimeStall, + duration: TimeSpan.FromSeconds(5))); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(9)), + LocalSiloHealthCheckKind.ThreadPoolQueueDelay, + duration: TimeSpan.FromSeconds(1))); + + var end = TimestampAt(TimeSpan.FromSeconds(10)); + var status = history.AggregatePauses( + _startTimestamp, + end, + end, + LocalSiloHealthCheckCategory.Local); + + Assert.Equal(TimeSpan.FromSeconds(7), status.TotalPauseDuration); + Assert.Equal(TimeSpan.FromSeconds(4), status.GetDuration(LocalSiloHealthCheckKind.GarbageCollectionPause)); + Assert.Equal(TimeSpan.FromSeconds(5), status.GetDuration(LocalSiloHealthCheckKind.RuntimeStall)); + Assert.Equal(TimeSpan.Zero, status.GetDuration(LocalSiloHealthCheckKind.ThreadPoolQueueDelay)); + Assert.Equal(TimeSpan.Zero, status.GetDuration(LocalSiloHealthCheckKind.ComponentHealthCheckStall)); + } + + [Fact] + public void AggregatePauses_UnionsOverlappingPausesOfSameKind() + { + var history = CreateHistory(); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(4)), + LocalSiloHealthCheckKind.RuntimeStall, + duration: TimeSpan.FromSeconds(3))); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(6)), + LocalSiloHealthCheckKind.RuntimeStall, + duration: TimeSpan.FromSeconds(3))); + + var end = TimestampAt(TimeSpan.FromSeconds(6)); + var status = history.AggregatePauses( + _startTimestamp, + end, + end, + LocalSiloHealthCheckCategory.Local); + + Assert.Equal(TimeSpan.FromSeconds(5), status.TotalPauseDuration); + Assert.Equal(TimeSpan.FromSeconds(5), status.GetDuration(LocalSiloHealthCheckKind.RuntimeStall)); + } + + [Fact] + public void AggregatePauses_ClipsPausesToRequestedInterval() + { + var history = CreateHistory(); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(6)), + LocalSiloHealthCheckKind.ComponentHealthCheckStall, + duration: TimeSpan.FromSeconds(5))); + + var start = TimestampAt(TimeSpan.FromSeconds(3)); + var end = TimestampAt(TimeSpan.FromSeconds(5)); + var status = history.AggregatePauses( + start, + end, + TimestampAt(TimeSpan.FromSeconds(6)), + LocalSiloHealthCheckCategory.Local); + + Assert.Equal(TimeSpan.FromSeconds(2), status.TotalPauseDuration); + Assert.Equal(TimeSpan.FromSeconds(2), status.GetDuration(LocalSiloHealthCheckKind.ComponentHealthCheckStall)); + } + + [Fact] + public void AggregatePauses_RetainsEventsForActiveLongRunningInterval() + { + var history = CreateHistory(); + history.BeginPauseCollection(_startTimestamp); + history.Add(Event( + TimestampAt(TimeSpan.FromSeconds(30)), + LocalSiloHealthCheckKind.GarbageCollectionPause, + duration: TimeSpan.FromSeconds(10))); + + var end = TimestampAt(TimeSpan.FromMinutes(2)); + var status = history.AggregatePauses( + _startTimestamp, + end, + end, + LocalSiloHealthCheckCategory.Local); + history.EndPauseCollection(_startTimestamp, end); + + Assert.Equal(TimeSpan.FromSeconds(10), status.TotalPauseDuration); + Assert.Equal(TimeSpan.FromSeconds(10), status.GetDuration(LocalSiloHealthCheckKind.GarbageCollectionPause)); + } + private LocalSiloHealthEventHistory CreateHistory() => new(_timeProvider, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); @@ -180,7 +282,8 @@ private LocalSiloHealthEvent Event( long timestamp, LocalSiloHealthCheckKind kind, int score = 1, - string? source = null) + string? source = null, + TimeSpan? duration = null) => new( timestamp, kind, @@ -188,7 +291,7 @@ private LocalSiloHealthEvent Event( source, score, Complaint: null, - Duration: null); + duration); private long TimestampAt(TimeSpan elapsed) => _startTimestamp diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index 0156f40a4cb..54988322d0f 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -144,6 +144,8 @@ public async Task SiloHealthMonitor_SuccessfulProbe() Assert.Equal(0, probeResult.FailedProbeCount); Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); + _localSiloHealthMonitor.Received(1).CapturePauseTimestamp(); + _localSiloHealthMonitor.Received(1).GetPauseStatus(default); await Shutdown(); } @@ -261,8 +263,10 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() _membershipService, _localSiloDetails, timeProvider); - var totalPauseDuration = TimeSpan.Zero; - _localSiloHealthMonitor.TotalPauseDuration.Returns(_ => totalPauseDuration); + _localSiloHealthMonitor.CapturePauseTimestamp().Returns(_ => timeProvider.GetTimestamp()); + _localSiloHealthMonitor.GetPauseStatus(default).ReturnsForAnyArgs(new LocalSiloPauseStatus( + options.ProbeTimeout.Multiply(0.25), + [new(LocalSiloHealthCheckKind.GarbageCollectionPause, options.ProbeTimeout.Multiply(0.25))])); try { @@ -271,7 +275,6 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() firstTick.Completion.SetResult(true); var cancellationToken = await probeEntered.Task; - totalPauseDuration = options.ProbeTimeout.Multiply(0.25); timeProvider.Advance(options.ProbeTimeout); Assert.True(cancellationToken.IsCancellationRequested); @@ -293,9 +296,9 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() public async Task SiloHealthMonitor_FailedProbe_Exception() { _clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2); - var pauseSample = 0; - _localSiloHealthMonitor.TotalPauseDuration.Returns(_ => - Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout); + _localSiloHealthMonitor.GetPauseStatus(default).ReturnsForAnyArgs(new LocalSiloPauseStatus( + _clusterMembershipOptions.ProbeTimeout, + [new(LocalSiloHealthCheckKind.RuntimeStall, _clusterMembershipOptions.ProbeTimeout)])); _prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope")); _prober.ProbeIndirectly(default!, default!, default, default).ThrowsAsyncForAnyArgs(new InvalidOperationException("No")); diff --git a/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs b/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs index f1bfb8f4105..f41b2e3ffe8 100644 --- a/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs +++ b/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs @@ -15,24 +15,6 @@ public class WatchdogTests { private static readonly DateTimeOffset Start = new(2026, 1, 1, 12, 0, 0, TimeSpan.Zero); - [Fact, TestCategory("BVT")] - public void CheckRuntimeHealth_RecordsGarbageCollectionPauseEvent() - { - using var fixture = new WatchdogFixture(); - fixture.SetWatchdogTimestamp("_platformWatchdogTimestamp"); - fixture.SetWatchdogTimestamp("_componentWatchdogTimestamp"); - fixture.SetField("_cumulativeGCPauseDuration", GC.GetTotalPauseDuration() - TimeSpan.FromTicks(1)); - - fixture.CheckRuntimeHealth(); - - var healthEvent = Assert.Single(fixture.Events); - Assert.Equal(LocalSiloHealthCheckKind.GarbageCollectionPause, healthEvent.Kind); - Assert.Equal(0, healthEvent.Score); - Assert.Contains("garbage collection", healthEvent.Complaint, StringComparison.OrdinalIgnoreCase); - Assert.True(healthEvent.Duration > TimeSpan.Zero); - Assert.Null(healthEvent.Source); - } - [Fact, TestCategory("BVT")] public void CheckRuntimeHealth_RecordsRuntimeStallEvent() { From 607699bc956f7c9efedca10e462d8dcdb498fcf5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 02:24:38 -0700 Subject: [PATCH 08/12] perf(membership): skip pause aggregation for completed probes --- .../LocalSiloHealthMonitor.cs | 18 ++++++++++++++++++ .../MembershipService/SiloHealthMonitor.cs | 11 +++++++++-- .../Membership/SiloHealthMonitorTests.cs | 7 ++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index d1c2607246e..fbe2e9e702d 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -82,6 +82,12 @@ internal interface ILocalSiloHealthMonitor /// The timestamp which begins the interval. long CapturePauseTimestamp(); + /// + /// Completes pause collection without aggregating the interval. + /// + /// The timestamp returned by . + void EndPauseCollection(long startTimestamp); + /// /// Returns pause durations since the provided timestamp, aggregated by kind and as a deduplicated total. /// @@ -222,6 +228,18 @@ public long CapturePauseTimestamp() } } + /// + public void EndPauseCollection(long startTimestamp) + { + lock (_historyLock) + { + var cumulativePauseDuration = GC.GetTotalPauseDuration(); + var endTimestamp = _timeProvider.GetTimestamp(); + RecordGarbageCollectionPause(endTimestamp, cumulativePauseDuration); + _healthHistory.EndPauseCollection(startTimestamp, endTimestamp); + } + } + /// public LocalSiloPauseStatus GetPauseStatus(long startTimestamp) { diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 48924a7334c..5ed9ddf9b0a 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -353,7 +353,15 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti failureException = exception; } var roundTripTime = roundTripTimer.GetElapsedTime(out _); - var pauseStatus = _localSiloHealthMonitor.GetPauseStatus(pauseStartTimestamp); + var pauseDurationDuring = TimeSpan.Zero; + if (probeTimedOut) + { + pauseDurationDuring = _localSiloHealthMonitor.GetPauseStatus(pauseStartTimestamp).TotalPauseDuration; + } + else + { + _localSiloHealthMonitor.EndPauseCollection(pauseStartTimestamp); + } if (failureException is null) { @@ -372,7 +380,6 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti // Check if a local pause consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - var pauseDurationDuring = pauseStatus.TotalPauseDuration; if (probeTimedOut && pauseDurationDuring >= probeTimeout.Multiply(0.25)) { LogWarningProbeFailureDuringLocalPause(_log, id, TargetSiloAddress, roundTripTime, pauseDurationDuring, _failedProbes); diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index 54988322d0f..b9be0333c38 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -145,7 +145,8 @@ public async Task SiloHealthMonitor_SuccessfulProbe() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); _localSiloHealthMonitor.Received(1).CapturePauseTimestamp(); - _localSiloHealthMonitor.Received(1).GetPauseStatus(default); + _localSiloHealthMonitor.Received(1).EndPauseCollection(default); + _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetPauseStatus(default); await Shutdown(); } @@ -221,6 +222,8 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); + _localSiloHealthMonitor.Received(1).GetPauseStatus(default); + _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().EndPauseCollection(default); await Shutdown(); } @@ -319,6 +322,8 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); + _localSiloHealthMonitor.Received(2).EndPauseCollection(default); + _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetPauseStatus(default); await Shutdown(); } From 06147d97dc56387c874b630e930db6079212a303 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 15:50:23 -0700 Subject: [PATCH 09/12] refactor(membership): use continuous stall detector --- .../LocalSiloHealthEventHistory.cs | 189 ----------- .../LocalSiloHealthMonitor.cs | 239 +++----------- .../MembershipService/SiloHealthMonitor.cs | 38 +-- .../ThreadPoolStallDetector.cs | 311 ++++++++++++++++++ src/Orleans.Runtime/Silo/Watchdog.cs | 9 + .../LocalSiloHealthEventHistoryTests.cs | 107 +----- .../Membership/LocalSiloHealthMonitorTests.cs | 19 +- .../Membership/SiloHealthMonitorTests.cs | 41 ++- .../ThreadPoolStallDetectorTests.cs | 217 ++++++++++++ .../Diagnostics/WatchdogTests.cs | 18 + 10 files changed, 659 insertions(+), 529 deletions(-) create mode 100644 src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs create mode 100644 test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs index 8a2d9cfd9bf..b2a76373bd1 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Linq; using System.Runtime.InteropServices; namespace Orleans.Runtime.MembershipService; @@ -14,12 +13,6 @@ internal sealed class LocalSiloHealthEventHistory private readonly Bucket[] _buckets; private readonly Dictionary _candidates = []; private readonly List _selected = []; - private readonly List _pauseEvents = []; - private readonly SortedDictionary _activePauseIntervals = []; - private readonly Dictionary> _pauseIntervalsByKind = []; - private readonly List _allPauseIntervals = []; - private readonly List _pauseDurations = []; - private int _pauseEventHead; private int _count; private int _occupiedBucketCount; @@ -42,37 +35,6 @@ public LocalSiloHealthEventHistory( internal int OccupiedBucketCount => _occupiedBucketCount; - public void BeginPauseCollection(long startTimestamp) - { - if (_activePauseIntervals.TryGetValue(startTimestamp, out var count)) - { - _activePauseIntervals[startTimestamp] = count + 1; - } - else - { - _activePauseIntervals.Add(startTimestamp, 1); - } - } - - public void EndPauseCollection(long startTimestamp, long nowTimestamp) - { - if (!_activePauseIntervals.TryGetValue(startTimestamp, out var count)) - { - throw new InvalidOperationException($"Pause collection interval {startTimestamp} is not active."); - } - - if (count == 1) - { - _activePauseIntervals.Remove(startTimestamp); - } - else - { - _activePauseIntervals[startTimestamp] = count - 1; - } - - RemoveExpiredPauseEvents(nowTimestamp, force: true); - } - public void Add(LocalSiloHealthEvent healthEvent) { RemoveExpired(healthEvent.Timestamp); @@ -144,7 +106,6 @@ public LocalSiloHealthStatus Aggregate( entry.HasIntervalEvent = true; } } - } _selected.EnsureCapacity(_candidates.Count); @@ -187,57 +148,6 @@ public LocalSiloHealthStatus Aggregate( } } - public LocalSiloPauseStatus AggregatePauses( - long startTimestamp, - long endTimestamp, - long nowTimestamp, - LocalSiloHealthCheckCategory categories) - { - RemoveExpired(nowTimestamp); - if (categories == LocalSiloHealthCheckCategory.None) - { - return new(TimeSpan.Zero, []); - } - - try - { - for (var i = _pauseEventHead; i < _pauseEvents.Count; i++) - { - var healthEvent = _pauseEvents[i]; - if ((healthEvent.Category & categories) == 0 - || !TryGetPauseInterval(healthEvent, startTimestamp, endTimestamp, out var interval)) - { - continue; - } - - if (!_pauseIntervalsByKind.TryGetValue(healthEvent.Kind, out var intervals)) - { - intervals = []; - _pauseIntervalsByKind.Add(healthEvent.Kind, intervals); - } - - intervals.Add(interval); - _allPauseIntervals.Add(interval); - } - - foreach (var (kind, intervals) in _pauseIntervalsByKind) - { - _pauseDurations.Add(new(kind, GetUnionDuration(intervals))); - } - - _pauseDurations.Sort(static (left, right) => left.Kind.CompareTo(right.Kind)); - return new( - GetUnionDuration(_allPauseIntervals), - ImmutableArray.CreateRange(_pauseDurations)); - } - finally - { - _pauseIntervalsByKind.Clear(); - _allPauseIntervals.Clear(); - _pauseDurations.Clear(); - } - } - private void AddCore(LocalSiloHealthEvent healthEvent) { var bucketIndex = GetBucketIndex(healthEvent.Timestamp); @@ -263,10 +173,6 @@ private void AddCore(LocalSiloHealthEvent healthEvent) bucket.Events!.Add(healthEvent); _count++; - if (IsPause(healthEvent)) - { - _pauseEvents.Add(healthEvent); - } } private void RemoveExpired(long nowTimestamp) @@ -309,31 +215,6 @@ private void RemoveExpired(long nowTimestamp) Clear(ref bucket); } } - - } - - RemoveExpiredPauseEvents(nowTimestamp, force: false); - } - - private void RemoveExpiredPauseEvents(long nowTimestamp, bool force) - { - var oldestTimestamp = SubtractTimestamp(nowTimestamp, _retentionPeriod); - if (_activePauseIntervals.Count > 0) - { - oldestTimestamp = Math.Min(oldestTimestamp, _activePauseIntervals.First().Key); - } - - while (_pauseEventHead < _pauseEvents.Count - && _pauseEvents[_pauseEventHead].Timestamp < oldestTimestamp) - { - _pauseEventHead++; - } - - if (_pauseEventHead > 0 - && (force || _pauseEventHead >= 1024 && _pauseEventHead >= _pauseEvents.Count / 2)) - { - _pauseEvents.RemoveRange(0, _pauseEventHead); - _pauseEventHead = 0; } } @@ -358,68 +239,6 @@ private bool Overlaps(LocalSiloHealthEvent healthEvent, long startTimestamp, lon && _timeProvider.GetElapsedTime(endTimestamp, healthEvent.Timestamp) <= duration; } - private bool TryGetPauseInterval( - LocalSiloHealthEvent healthEvent, - long startTimestamp, - long endTimestamp, - out PauseInterval interval) - { - if (healthEvent.Duration is not { } duration || duration <= TimeSpan.Zero) - { - interval = default; - return false; - } - - var pauseStartTimestamp = SubtractTimestamp(healthEvent.Timestamp, duration); - var overlapStartTimestamp = Math.Max(startTimestamp, pauseStartTimestamp); - var overlapEndTimestamp = Math.Min(endTimestamp, healthEvent.Timestamp); - if (overlapEndTimestamp <= overlapStartTimestamp) - { - interval = default; - return false; - } - - interval = new(overlapStartTimestamp, overlapEndTimestamp); - return true; - } - - private TimeSpan GetUnionDuration(List intervals) - { - if (intervals.Count == 0) - { - return TimeSpan.Zero; - } - - intervals.Sort(static (left, right) => - { - var startComparison = left.StartTimestamp.CompareTo(right.StartTimestamp); - return startComparison != 0 - ? startComparison - : left.EndTimestamp.CompareTo(right.EndTimestamp); - }); - - var totalTimestampLength = 0L; - var currentStartTimestamp = intervals[0].StartTimestamp; - var currentEndTimestamp = intervals[0].EndTimestamp; - for (var i = 1; i < intervals.Count; i++) - { - var interval = intervals[i]; - if (interval.StartTimestamp <= currentEndTimestamp) - { - currentEndTimestamp = Math.Max(currentEndTimestamp, interval.EndTimestamp); - } - else - { - totalTimestampLength = checked(totalTimestampLength + currentEndTimestamp - currentStartTimestamp); - currentStartTimestamp = interval.StartTimestamp; - currentEndTimestamp = interval.EndTimestamp; - } - } - - totalTimestampLength = checked(totalTimestampLength + currentEndTimestamp - currentStartTimestamp); - return _timeProvider.GetElapsedTime(0, totalTimestampLength); - } - private static bool IsStateful(LocalSiloHealthCheckKind kind) => kind is LocalSiloHealthCheckKind.MembershipStatus or LocalSiloHealthCheckKind.HealthCheckParticipant @@ -427,12 +246,6 @@ or LocalSiloHealthCheckKind.ThreadPoolQueueDelay or LocalSiloHealthCheckKind.ProbeRequests or LocalSiloHealthCheckKind.ProbeResponses; - private static bool IsPause(LocalSiloHealthEvent healthEvent) - => healthEvent.Duration > TimeSpan.Zero - && healthEvent.Kind is LocalSiloHealthCheckKind.GarbageCollectionPause - or LocalSiloHealthCheckKind.RuntimeStall - or LocalSiloHealthCheckKind.ComponentHealthCheckStall; - private static bool IsBetter( LocalSiloHealthEvent candidate, LocalSiloHealthEvent current) @@ -457,8 +270,6 @@ private readonly record struct HealthEventIdentity( LocalSiloHealthCheckKind Kind, string? Source); - private readonly record struct PauseInterval(long StartTimestamp, long EndTimestamp); - private struct AggregateEntry { public LocalSiloHealthEvent PriorState; diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index fbe2e9e702d..d4a9c02fc9c 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -51,49 +51,25 @@ public ImmutableArray Complaints => [.. Events.Where(static status => status.Complaint is not null).Select(static status => status.Complaint!)]; } - internal readonly record struct LocalSiloPauseDuration(LocalSiloHealthCheckKind Kind, TimeSpan Duration); - - internal readonly record struct LocalSiloPauseStatus( - TimeSpan TotalPauseDuration, - ImmutableArray Durations) - { - public TimeSpan GetDuration(LocalSiloHealthCheckKind kind) - { - if (!Durations.IsDefaultOrEmpty) - { - foreach (var item in Durations) - { - if (item.Kind == kind) - { - return item.Duration; - } - } - } - - return TimeSpan.Zero; - } - } - internal interface ILocalSiloHealthMonitor { /// - /// Captures the start of a pause aggregation interval. - /// - /// The timestamp which begins the interval. - long CapturePauseTimestamp(); - - /// - /// Completes pause collection without aggregating the interval. + /// Returns a timestamp from the stall detector's time source. /// - /// The timestamp returned by . - void EndPauseCollection(long startTimestamp); + /// A timestamp suitable for use with . + long GetTimestamp(); /// - /// Returns pause durations since the provided timestamp, aggregated by kind and as a deduplicated total. + /// Waits for the stall detector to sample past the end of the interval, then returns the detected stall duration. /// - /// The timestamp returned by . - /// The pause duration summary. - LocalSiloPauseStatus GetPauseStatus(long startTimestamp); + /// The start of the interval. + /// The end of the interval. + /// A token which cancels the wait. + /// The detected stall duration. + ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken); /// /// Returns the aggregate local health status over the provided interval. @@ -143,7 +119,7 @@ void RecordHealthEvent( /// Check that no other silo suspects this silo. /// Check for recently received successful ping responses (via ). /// Check for recently received ping requests (via ). - /// Check that the .NET Thread Pool is able to process work items within one second. + /// Check that the .NET Thread Pool executes periodic timer callbacks on schedule. /// Check that local async timers have been firing on-time (within 3 seconds of their due time). /// /// @@ -151,7 +127,8 @@ internal partial class LocalSiloHealthMonitor : ILifecycleParticipant, ILifecycleObserver, ILocalSiloHealthMonitor, - ILocalSiloHealthEventRecorder + ILocalSiloHealthEventRecorder, + IDisposable { internal const int MaxScore = 8; private static readonly TimeSpan HistoryDuration = TimeSpan.FromMinutes(1); @@ -164,9 +141,8 @@ internal partial class LocalSiloHealthMonitor : private readonly ILogger _log; private readonly ClusterMembershipOptions _clusterMembershipOptions; private readonly IAsyncTimer _degradationCheckTimer; - private readonly ThreadPoolMonitor _threadPoolMonitor; + private readonly ThreadPoolStallDetector _stallDetector; private readonly TimeProvider _timeProvider; - private TimeSpan _cumulativeGCPauseDuration; #if NET9_0_OR_GREATER private readonly Lock _historyLock = new(); private readonly Lock _samplingLock = new(); @@ -204,72 +180,36 @@ public LocalSiloHealthMonitor( _log = log; _clusterMembershipOptions = clusterMembershipOptions.Value; _timeProvider = timeProvider; - _cumulativeGCPauseDuration = GC.GetTotalPauseDuration(); _healthHistory = new(timeProvider, HistoryDuration, MinimumCheckPeriod); _degradationCheckTimer = timerFactory.Create( MinimumCheckPeriod, nameof(LocalSiloHealthMonitor), timeProvider); - _threadPoolMonitor = new ThreadPoolMonitor(loggerFactory.CreateLogger(), timeProvider); - } - - /// - public ImmutableArray Complaints { get; private set; } = []; - - /// - public long CapturePauseTimestamp() - { - lock (_historyLock) + var stallRetentionPeriod = _clusterMembershipOptions.MaxProbeTimeout + ThreadPoolStallDetector.DetectionPeriod; + if (stallRetentionPeriod < HistoryDuration) { - var timestamp = _timeProvider.GetTimestamp(); - RecordGarbageCollectionPause(timestamp, GC.GetTotalPauseDuration()); - _healthHistory.BeginPauseCollection(timestamp); - return timestamp; + stallRetentionPeriod = HistoryDuration; } + + _stallDetector = new( + loggerFactory.CreateLogger(), + timeProvider, + ThreadPoolStallDetector.DetectionPeriod, + stallRetentionPeriod); } /// - public void EndPauseCollection(long startTimestamp) - { - lock (_historyLock) - { - var cumulativePauseDuration = GC.GetTotalPauseDuration(); - var endTimestamp = _timeProvider.GetTimestamp(); - RecordGarbageCollectionPause(endTimestamp, cumulativePauseDuration); - _healthHistory.EndPauseCollection(startTimestamp, endTimestamp); - } - } + public ImmutableArray Complaints { get; private set; } = []; /// - public LocalSiloPauseStatus GetPauseStatus(long startTimestamp) - { - lock (_historyLock) - { - var cumulativePauseDuration = GC.GetTotalPauseDuration(); - var endTimestamp = _timeProvider.GetTimestamp(); - if (endTimestamp < startTimestamp) - { - throw new ArgumentOutOfRangeException( - nameof(startTimestamp), - startTimestamp, - "The interval start must not follow the current timestamp."); - } + public long GetTimestamp() => _timeProvider.GetTimestamp(); - RecordGarbageCollectionPause(endTimestamp, cumulativePauseDuration); - try - { - return _healthHistory.AggregatePauses( - startTimestamp, - endTimestamp, - endTimestamp, - LocalSiloHealthCheckCategory.Local); - } - finally - { - _healthHistory.EndPauseCollection(startTimestamp, endTimestamp); - } - } - } + /// + public ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken) + => _stallDetector.GetStallDurationAsync(startTimestamp, endTimestamp, cancellationToken); /// public LocalSiloHealthStatus GetLocalHealthStatus( @@ -300,13 +240,10 @@ public LocalSiloHealthStatus GetLocalHealthStatus( lock (_historyLock) { - var cumulativePauseDuration = GC.GetTotalPauseDuration(); - var nowTimestamp = _timeProvider.GetTimestamp(); - RecordGarbageCollectionPause(nowTimestamp, cumulativePauseDuration); return _healthHistory.Aggregate( startTimestamp, endTimestamp, - nowTimestamp, + _timeProvider.GetTimestamp(), categories, MaxScore); } @@ -367,7 +304,7 @@ private LocalSiloHealthStatus EnsureHealthCheck( var events = new List(_healthCheckParticipants.Count + 1); var complaints = new List(); CheckLocalHealthCheckParticipants(now.UtcDateTime, timestamp, events, complaints); - CheckThreadPoolQueueDelay(timestamp, events, complaints); + CheckThreadPoolStalls(timestamp, events, complaints); AddEvents(timestamp, events); var score = GetScore(events); @@ -452,31 +389,14 @@ private void AddEvents(long timestamp, List events) } } - private void RecordGarbageCollectionPause(long timestamp, TimeSpan cumulativePauseDuration) - { - var pauseDuration = cumulativePauseDuration - _cumulativeGCPauseDuration; - _cumulativeGCPauseDuration = cumulativePauseDuration; - if (pauseDuration <= TimeSpan.Zero) - { - return; - } - - _healthHistory.Add(new( - timestamp, - LocalSiloHealthCheckKind.GarbageCollectionPause, - LocalSiloHealthCheckCategory.Local, - Source: null, - Score: 0, - Complaint: $"The .NET runtime paused for garbage collection for {pauseDuration}.", - pauseDuration)); - } - - private void CheckThreadPoolQueueDelay( + private void CheckThreadPoolStalls( long timestamp, List events, List complaints) { - var delay = _threadPoolMonitor.MeasureQueueDelay(); + var delay = _stallDetector.GetMaximumStallDuration( + SubtractTimestamp(timestamp, MinimumCheckPeriod), + timestamp); var score = (int)delay.TotalSeconds; string? complaint = null; if (score >= 1) @@ -686,12 +606,6 @@ private async Task Run() var networkHealthCheckVersion = Volatile.Read(ref _networkHealthCheckVersion); lock (_samplingLock) { - lock (_historyLock) - { - var cumulativePauseDuration = GC.GetTotalPauseDuration(); - RecordGarbageCollectionPause(_timeProvider.GetTimestamp(), cumulativePauseDuration); - } - status = EnsureHealthCheck( _timeProvider.GetUtcNow(), _timeProvider.GetTimestamp(), @@ -771,74 +685,7 @@ public async Task OnStop(CancellationToken ct) } } - /// - /// Measures queue delay on the .NET . - /// - private class ThreadPoolMonitor - { - private static readonly WaitCallback Callback = state => ((ThreadPoolMonitor)state!).Execute(); -#if NET9_0_OR_GREATER - private readonly Lock _lockObj = new(); -#else - private readonly object _lockObj = new(); -#endif - private readonly ILogger _log; - private readonly TimeProvider _timeProvider; - private bool _scheduled; - private TimeSpan _lastQueueDelay; - private long _queueDelayTimestamp; - - public ThreadPoolMonitor(ILogger log, TimeProvider timeProvider) - { - _log = log; - _timeProvider = timeProvider; - } - - public TimeSpan MeasureQueueDelay() - { - bool shouldSchedule; - TimeSpan delay; - lock (_lockObj) - { - var currentQueueDelay = _scheduled ? _timeProvider.GetElapsedTime(_queueDelayTimestamp) : TimeSpan.Zero; - delay = currentQueueDelay > _lastQueueDelay ? currentQueueDelay : _lastQueueDelay; - - if (!_scheduled) - { - _scheduled = true; - shouldSchedule = true; - _queueDelayTimestamp = _timeProvider.GetTimestamp(); - } - else - { - shouldSchedule = false; - } - } - - if (shouldSchedule) - { - _ = ThreadPool.UnsafeQueueUserWorkItem(Callback, this); - } - - return delay; - } - - private void Execute() - { - try - { - lock (_lockObj) - { - _scheduled = false; - _lastQueueDelay = _timeProvider.GetElapsedTime(_queueDelayTimestamp); - } - } - catch (Exception exception) - { - LocalSiloHealthMonitor.LogThreadPoolDelayMonitorError(_log, exception); - } - } - } + public void Dispose() => _stallDetector.Dispose(); [LoggerMessage( Message = ".NET Thread Pool is exhibiting delays of {ThreadPoolQueueDelaySeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses." @@ -890,12 +737,6 @@ private partial void LogRecordedHealthIssue( )] private partial void LogSelfMonitoringDegraded(int score, int maxScore, string complaints); - [LoggerMessage( - Level = LogLevel.Error, - Message = "Exception monitoring .NET thread pool delay" - )] - private static partial void LogThreadPoolDelayMonitorError(ILogger logger, Exception exception); - [LoggerMessage( Level = LogLevel.Error, Message = "Error while monitoring local silo health" diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 5ed9ddf9b0a..22c75ec54f7 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -206,16 +206,17 @@ private async Task Run() isDirectProbe = !options.EnableIndirectProbes || _failedProbes < options.NumMissedProbesLimit - 1 || otherNodes.Length == 0; localDegradationScore = GetLocalDegradationScore(previousProbePeriod); var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); - probeStartTimestamp = _timeProvider.GetTimestamp(); - var pauseStartTimestamp = isDirectProbe - ? _localSiloHealthMonitor.CapturePauseTimestamp() + var currentProbeStartTimestamp = _timeProvider.GetTimestamp(); + probeStartTimestamp = currentProbeStartTimestamp; + var stallStartTimestamp = isDirectProbe + ? _localSiloHealthMonitor.GetTimestamp() : 0; using var cancellation = new CancellationTokenSource(timeout, _timeProvider); if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token, timeout, pauseStartTimestamp).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout, stallStartTimestamp).ConfigureAwait(false); } else { @@ -324,10 +325,10 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// Probes the remote silo. /// /// A token to cancel and fail the probe attempt. - /// The timeout used for this probe, for local pause evaluation. - /// The start of the local pause aggregation interval. + /// The timeout used for this probe, for local stall evaluation. + /// The start of the local stall collection interval. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, long pauseStartTimestamp) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, long stallStartTimestamp) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); @@ -353,16 +354,15 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti failureException = exception; } var roundTripTime = roundTripTimer.GetElapsedTime(out _); - var pauseDurationDuring = TimeSpan.Zero; + var stallDurationDuring = TimeSpan.Zero; if (probeTimedOut) { - pauseDurationDuring = _localSiloHealthMonitor.GetPauseStatus(pauseStartTimestamp).TotalPauseDuration; + var probeEndTimestamp = _localSiloHealthMonitor.GetTimestamp(); + stallDurationDuring = await _localSiloHealthMonitor.GetStallDurationAsync( + stallStartTimestamp, + probeEndTimestamp, + _stoppingCancellation.Token).ConfigureAwait(false); } - else - { - _localSiloHealthMonitor.EndPauseCollection(pauseStartTimestamp); - } - if (failureException is null) { _messagingInstruments?.OnPingReplyReceived(TargetSiloAddress); @@ -377,12 +377,12 @@ private async Task ProbeDirectly(CancellationToken cancellation, Ti } else { - // Check if a local pause consumed a significant portion of the probe timeout. + // Check if a local stall consumed a significant portion of the probe timeout. // If so, the local silo may have been unable to process the response in time, // so we treat this as an inconclusive result rather than a failure. - if (probeTimedOut && pauseDurationDuring >= probeTimeout.Multiply(0.25)) + if (probeTimedOut && stallDurationDuring >= probeTimeout.Multiply(0.25)) { - LogWarningProbeFailureDuringLocalPause(_log, id, TargetSiloAddress, roundTripTime, pauseDurationDuring, _failedProbes); + LogWarningProbeFailureDuringLocalStall(_log, id, TargetSiloAddress, roundTripTime, stallDurationDuring, _failedProbes); probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); } else @@ -565,8 +565,8 @@ public enum ProbeResultStatus [LoggerMessage( Level = LogLevel.Warning, - Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a local pause of {PauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." + Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but local execution stalled for {StallDuration} during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." )] - private static partial void LogWarningProbeFailureDuringLocalPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan pauseDuration, int failedProbeCount); + private static partial void LogWarningProbeFailureDuringLocalStall(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan stallDuration, int failedProbeCount); } } diff --git a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs new file mode 100644 index 00000000000..322e09d857e --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs @@ -0,0 +1,311 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Orleans.Runtime.MembershipService; + +/// +/// Detects delays in .NET Thread Pool execution using a periodic timer callback. +/// +/// +/// Timer callbacks execute on the Thread Pool. Each callback compares its execution time with the next expected +/// cadence boundary, producing non-overlapping stall intervals when callback dispatch is delayed. +/// +internal sealed partial class ThreadPoolStallDetector : IDisposable +{ + internal static readonly TimeSpan DetectionPeriod = TimeSpan.FromMilliseconds(100); + + private static readonly TimerCallback TimerCallback = static state => ((ThreadPoolStallDetector)state!).OnTimer(); + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _retentionPeriod; + private readonly long _detectionPeriodTimestampLength; + private readonly ITimer _timer; + private readonly List _stalls = []; + private readonly SortedDictionary _pendingQueries = []; +#if NET9_0_OR_GREATER + private readonly Lock _lock = new(); +#else + private readonly object _lock = new(); +#endif + private TaskCompletionSource? _nextSample; + private long _lastSampleTimestamp; + private long _nextExpectedTimestamp; + private int _stallHead; + private bool _disposed; + + public ThreadPoolStallDetector( + ILogger logger, + TimeProvider timeProvider, + TimeSpan detectionPeriod, + TimeSpan retentionPeriod) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(detectionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(retentionPeriod, TimeSpan.Zero); + + _logger = logger; + _timeProvider = timeProvider; + _retentionPeriod = retentionPeriod; + _detectionPeriodTimestampLength = GetTimestampLength(detectionPeriod); + _lastSampleTimestamp = timeProvider.GetTimestamp(); + _nextExpectedTimestamp = _lastSampleTimestamp + _detectionPeriodTimestampLength; + _timer = timeProvider.CreateTimer(TimerCallback, this, detectionPeriod, detectionPeriod); + } + + public async ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + AddPendingQuery(startTimestamp); + } + + try + { + while (true) + { + Task nextSample; + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_lastSampleTimestamp >= endTimestamp) + { + return GetStallDurationCore(startTimestamp, endTimestamp); + } + + nextSample = (_nextSample ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task; + } + + await nextSample.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + finally + { + lock (_lock) + { + RemovePendingQuery(startTimestamp); + RemoveExpired(_timeProvider.GetTimestamp()); + } + } + } + + public TimeSpan GetStallDuration(long startTimestamp, long endTimestamp) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + return GetStallDurationCore(startTimestamp, endTimestamp); + } + } + + public TimeSpan GetMaximumStallDuration(long startTimestamp, long endTimestamp) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + var maximumTimestampLength = 0L; + for (var i = _stallHead; i < _stalls.Count; i++) + { + var stall = _stalls[i]; + if (stall.EndTimestamp <= startTimestamp) + { + continue; + } + + if (stall.StartTimestamp >= endTimestamp) + { + break; + } + + maximumTimestampLength = Math.Max( + maximumTimestampLength, + stall.EndTimestamp - stall.StartTimestamp); + } + + return _timeProvider.GetElapsedTime(0, maximumTimestampLength); + } + } + + public void Dispose() + { + TaskCompletionSource? nextSample; + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + nextSample = _nextSample; + } + + _timer.Dispose(); + nextSample?.TrySetCanceled(); + } + + private void OnTimer() + { + try + { + Sample(); + } + catch (Exception exception) + { + LogStallDetectorError(_logger, exception); + } + } + + private void Sample() + { + TaskCompletionSource? completedSample = null; + lock (_lock) + { + if (_disposed) + { + return; + } + + var timestamp = _timeProvider.GetTimestamp(); + if (timestamp > _lastSampleTimestamp) + { + if (timestamp > _nextExpectedTimestamp) + { + AddStall(new(_nextExpectedTimestamp, timestamp)); + } + + if (timestamp >= _nextExpectedTimestamp) + { + var elapsedPeriods = (timestamp - _nextExpectedTimestamp) / _detectionPeriodTimestampLength + 1; + _nextExpectedTimestamp = checked( + _nextExpectedTimestamp + elapsedPeriods * _detectionPeriodTimestampLength); + } + + _lastSampleTimestamp = timestamp; + RemoveExpired(timestamp); + completedSample = _nextSample; + _nextSample = null; + } + } + + completedSample?.TrySetResult(); + } + + private void AddStall(StallInterval stall) + { + if (_stalls.Count > _stallHead + && _stalls[^1].EndTimestamp >= stall.StartTimestamp) + { + var prior = _stalls[^1]; + _stalls[^1] = prior with { EndTimestamp = Math.Max(prior.EndTimestamp, stall.EndTimestamp) }; + } + else + { + _stalls.Add(stall); + } + } + + private TimeSpan GetStallDurationCore(long startTimestamp, long endTimestamp) + { + var durationTimestampLength = 0L; + for (var i = _stallHead; i < _stalls.Count; i++) + { + var stall = _stalls[i]; + if (stall.EndTimestamp <= startTimestamp) + { + continue; + } + + if (stall.StartTimestamp >= endTimestamp) + { + break; + } + + var overlapStartTimestamp = Math.Max(startTimestamp, stall.StartTimestamp); + var overlapEndTimestamp = Math.Min(endTimestamp, stall.EndTimestamp); + durationTimestampLength = checked( + durationTimestampLength + overlapEndTimestamp - overlapStartTimestamp); + } + + return _timeProvider.GetElapsedTime(0, durationTimestampLength); + } + + private void RemoveExpired(long nowTimestamp) + { + var oldestTimestamp = nowTimestamp - GetTimestampLength(_retentionPeriod); + if (_pendingQueries.Count > 0) + { + using var enumerator = _pendingQueries.GetEnumerator(); + _ = enumerator.MoveNext(); + oldestTimestamp = Math.Min(oldestTimestamp, enumerator.Current.Key); + } + + while (_stallHead < _stalls.Count && _stalls[_stallHead].EndTimestamp <= oldestTimestamp) + { + _stallHead++; + } + + if (_stallHead >= 1024 && _stallHead >= _stalls.Count / 2) + { + _stalls.RemoveRange(0, _stallHead); + _stallHead = 0; + } + } + + private void AddPendingQuery(long startTimestamp) + { + if (_pendingQueries.TryGetValue(startTimestamp, out var count)) + { + _pendingQueries[startTimestamp] = count + 1; + } + else + { + _pendingQueries.Add(startTimestamp, 1); + } + } + + private void RemovePendingQuery(long startTimestamp) + { + var count = _pendingQueries[startTimestamp]; + if (count == 1) + { + _pendingQueries.Remove(startTimestamp); + } + else + { + _pendingQueries[startTimestamp] = count - 1; + } + } + + private void ValidateInterval(long startTimestamp, long endTimestamp) + { + if (endTimestamp < startTimestamp) + { + throw new ArgumentOutOfRangeException( + nameof(endTimestamp), + endTimestamp, + "The interval end must not precede its start."); + } + } + + private long GetTimestampLength(TimeSpan duration) + => checked((long)Math.Ceiling(duration.TotalSeconds * _timeProvider.TimestampFrequency)); + + private readonly record struct StallInterval(long StartTimestamp, long EndTimestamp); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Exception monitoring .NET Thread Pool stalls" + )] + private static partial void LogStallDetectorError(ILogger logger, Exception exception); +} diff --git a/src/Orleans.Runtime/Silo/Watchdog.cs b/src/Orleans.Runtime/Silo/Watchdog.cs index 194990564ed..2de14c3d78f 100644 --- a/src/Orleans.Runtime/Silo/Watchdog.cs +++ b/src/Orleans.Runtime/Silo/Watchdog.cs @@ -107,6 +107,15 @@ private void CheckRuntimeHealth() var pauseDurationSinceLastTick = GC.GetTotalPauseDuration() - _cumulativeGCPauseDuration; var timeSinceLastTick = _timeProvider.GetElapsedTime(_platformWatchdogTimestamp); + if (pauseDurationSinceLastTick > TimeSpan.Zero) + { + _healthEventRecorder.RecordHealthEvent( + LocalSiloHealthCheckKind.GarbageCollectionPause, + score: 0, + complaint: $"The .NET runtime paused for garbage collection for {pauseDurationSinceLastTick}.", + duration: pauseDurationSinceLastTick); + } + if (timeSinceLastTick > PlatformWatchdogHeartbeatPeriod.Multiply(2)) { var gc = new[] { GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2) }; diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs index a72a0c42016..79191e0ed92 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthEventHistoryTests.cs @@ -173,108 +173,6 @@ public void Aggregate_ClampsWithoutOverflow() Assert.Equal(LocalSiloHealthMonitor.MaxScore, status.Score); } - [Fact] - public void AggregatePauses_ReportsPerKindDurationsAndDeduplicatedTotal() - { - var history = CreateHistory(); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(3)), - LocalSiloHealthCheckKind.GarbageCollectionPause, - duration: TimeSpan.FromSeconds(2))); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(8)), - LocalSiloHealthCheckKind.GarbageCollectionPause, - duration: TimeSpan.FromSeconds(2))); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(7)), - LocalSiloHealthCheckKind.RuntimeStall, - duration: TimeSpan.FromSeconds(5))); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(9)), - LocalSiloHealthCheckKind.ThreadPoolQueueDelay, - duration: TimeSpan.FromSeconds(1))); - - var end = TimestampAt(TimeSpan.FromSeconds(10)); - var status = history.AggregatePauses( - _startTimestamp, - end, - end, - LocalSiloHealthCheckCategory.Local); - - Assert.Equal(TimeSpan.FromSeconds(7), status.TotalPauseDuration); - Assert.Equal(TimeSpan.FromSeconds(4), status.GetDuration(LocalSiloHealthCheckKind.GarbageCollectionPause)); - Assert.Equal(TimeSpan.FromSeconds(5), status.GetDuration(LocalSiloHealthCheckKind.RuntimeStall)); - Assert.Equal(TimeSpan.Zero, status.GetDuration(LocalSiloHealthCheckKind.ThreadPoolQueueDelay)); - Assert.Equal(TimeSpan.Zero, status.GetDuration(LocalSiloHealthCheckKind.ComponentHealthCheckStall)); - } - - [Fact] - public void AggregatePauses_UnionsOverlappingPausesOfSameKind() - { - var history = CreateHistory(); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(4)), - LocalSiloHealthCheckKind.RuntimeStall, - duration: TimeSpan.FromSeconds(3))); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(6)), - LocalSiloHealthCheckKind.RuntimeStall, - duration: TimeSpan.FromSeconds(3))); - - var end = TimestampAt(TimeSpan.FromSeconds(6)); - var status = history.AggregatePauses( - _startTimestamp, - end, - end, - LocalSiloHealthCheckCategory.Local); - - Assert.Equal(TimeSpan.FromSeconds(5), status.TotalPauseDuration); - Assert.Equal(TimeSpan.FromSeconds(5), status.GetDuration(LocalSiloHealthCheckKind.RuntimeStall)); - } - - [Fact] - public void AggregatePauses_ClipsPausesToRequestedInterval() - { - var history = CreateHistory(); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(6)), - LocalSiloHealthCheckKind.ComponentHealthCheckStall, - duration: TimeSpan.FromSeconds(5))); - - var start = TimestampAt(TimeSpan.FromSeconds(3)); - var end = TimestampAt(TimeSpan.FromSeconds(5)); - var status = history.AggregatePauses( - start, - end, - TimestampAt(TimeSpan.FromSeconds(6)), - LocalSiloHealthCheckCategory.Local); - - Assert.Equal(TimeSpan.FromSeconds(2), status.TotalPauseDuration); - Assert.Equal(TimeSpan.FromSeconds(2), status.GetDuration(LocalSiloHealthCheckKind.ComponentHealthCheckStall)); - } - - [Fact] - public void AggregatePauses_RetainsEventsForActiveLongRunningInterval() - { - var history = CreateHistory(); - history.BeginPauseCollection(_startTimestamp); - history.Add(Event( - TimestampAt(TimeSpan.FromSeconds(30)), - LocalSiloHealthCheckKind.GarbageCollectionPause, - duration: TimeSpan.FromSeconds(10))); - - var end = TimestampAt(TimeSpan.FromMinutes(2)); - var status = history.AggregatePauses( - _startTimestamp, - end, - end, - LocalSiloHealthCheckCategory.Local); - history.EndPauseCollection(_startTimestamp, end); - - Assert.Equal(TimeSpan.FromSeconds(10), status.TotalPauseDuration); - Assert.Equal(TimeSpan.FromSeconds(10), status.GetDuration(LocalSiloHealthCheckKind.GarbageCollectionPause)); - } - private LocalSiloHealthEventHistory CreateHistory() => new(_timeProvider, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); @@ -282,8 +180,7 @@ private LocalSiloHealthEvent Event( long timestamp, LocalSiloHealthCheckKind kind, int score = 1, - string? source = null, - TimeSpan? duration = null) + string? source = null) => new( timestamp, kind, @@ -291,7 +188,7 @@ private LocalSiloHealthEvent Event( source, score, Complaint: null, - duration); + Duration: null); private long TimestampAt(TimeSpan elapsed) => _startTimestamp diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs index 2d0ca0c5609..6bef37b7f9d 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs @@ -41,6 +41,18 @@ public LocalSiloHealthMonitorTests() }))); } + [Fact] + public void GetTimestamp_UsesMembershipTimeProvider() + { + var monitor = CreateMonitor(); + + Assert.Equal(_startTimestamp, monitor.GetTimestamp()); + + _timeProvider.Advance(TimeSpan.FromMilliseconds(250)); + + Assert.Equal(TimestampAt(TimeSpan.FromMilliseconds(250)), monitor.GetTimestamp()); + } + [Fact] public void GetLocalHealthStatus_RejectsNegativePeriod() { @@ -350,9 +362,14 @@ public void GetLocalHealthStatus_IncludesDurationEventWhichOverlapsInterval() var healthEvent = Assert.Single( status.Events, item => item.Kind == LocalSiloHealthCheckKind.RuntimeStall); - Assert.Equal(2, status.Score); + Assert.Equal(4, status.Score); Assert.Equal(TimestampAt(TimeSpan.FromSeconds(3)), healthEvent.Timestamp); Assert.Equal(TimeSpan.FromSeconds(2), healthEvent.Duration); + var stallEvent = Assert.Single( + status.Events, + item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay); + Assert.Equal(2, stallEvent.Score); + Assert.Equal(TimeSpan.FromMilliseconds(2900), stallEvent.Duration); } [Fact] diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index b9be0333c38..88eb72f500c 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -144,9 +144,7 @@ public async Task SiloHealthMonitor_SuccessfulProbe() Assert.Equal(0, probeResult.FailedProbeCount); Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); - _localSiloHealthMonitor.Received(1).CapturePauseTimestamp(); - _localSiloHealthMonitor.Received(1).EndPauseCollection(default); - _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetPauseStatus(default); + await _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetStallDurationAsync(default, default, default); await Shutdown(); } @@ -222,14 +220,15 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); - _localSiloHealthMonitor.Received(1).GetPauseStatus(default); - _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().EndPauseCollection(default); - + await _localSiloHealthMonitor.Received(1).GetStallDurationAsync( + Arg.Any(), + Arg.Any(), + Arg.Any()); await Shutdown(); } [Fact] - public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() + public async Task SiloHealthMonitor_TimeoutDuringLocalStall_WaitsForDetectorCycleAndIsInconclusive() { var timeProvider = new Microsoft.Extensions.Time.Testing.FakeTimeProvider( new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero)); @@ -250,7 +249,11 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() return pendingProbe.Task; }); var probeResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var detectorQuery = new TaskCompletionSource<(long Start, long End)>(TaskCreationOptions.RunContinuationsAsynchronously); + var detectorResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var timer = new DilationProbeTimer(); + const long ExpectedStartTimestamp = 123; + const long ExpectedEndTimestamp = 456; var monitor = new SiloHealthMonitor( _targetSilo, (_, result) => @@ -266,10 +269,14 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() _membershipService, _localSiloDetails, timeProvider); - _localSiloHealthMonitor.CapturePauseTimestamp().Returns(_ => timeProvider.GetTimestamp()); - _localSiloHealthMonitor.GetPauseStatus(default).ReturnsForAnyArgs(new LocalSiloPauseStatus( - options.ProbeTimeout.Multiply(0.25), - [new(LocalSiloHealthCheckKind.GarbageCollectionPause, options.ProbeTimeout.Multiply(0.25))])); + _localSiloHealthMonitor.GetTimestamp().Returns(ExpectedStartTimestamp, ExpectedEndTimestamp); + _localSiloHealthMonitor + .GetStallDurationAsync(default, default, default) + .ReturnsForAnyArgs(call => + { + detectorQuery.TrySetResult((call.ArgAt(0), call.ArgAt(1))); + return new ValueTask(detectorResult.Task); + }); try { @@ -281,6 +288,12 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() timeProvider.Advance(options.ProbeTimeout); Assert.True(cancellationToken.IsCancellationRequested); + var interval = await detectorQuery.Task; + Assert.Equal(ExpectedStartTimestamp, interval.Start); + Assert.Equal(ExpectedEndTimestamp, interval.End); + Assert.False(probeResult.Task.IsCompleted); + + detectorResult.SetResult(options.ProbeTimeout.Multiply(0.25)); var result = await probeResult.Task; Assert.Equal(ProbeResultStatus.Unknown, result.Status); Assert.Equal(0, result.FailedProbeCount); @@ -299,9 +312,6 @@ public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive() public async Task SiloHealthMonitor_FailedProbe_Exception() { _clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2); - _localSiloHealthMonitor.GetPauseStatus(default).ReturnsForAnyArgs(new LocalSiloPauseStatus( - _clusterMembershipOptions.ProbeTimeout, - [new(LocalSiloHealthCheckKind.RuntimeStall, _clusterMembershipOptions.ProbeTimeout)])); _prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope")); _prober.ProbeIndirectly(default!, default!, default, default).ThrowsAsyncForAnyArgs(new InvalidOperationException("No")); @@ -322,8 +332,7 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); - _localSiloHealthMonitor.Received(2).EndPauseCollection(default); - _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetPauseStatus(default); + await _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetStallDurationAsync(default, default, default); await Shutdown(); } diff --git a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs new file mode 100644 index 00000000000..95989c782df --- /dev/null +++ b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs @@ -0,0 +1,217 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.MembershipService; +using TestExtensions; +using Xunit; + +namespace NonSilo.Tests.Membership; + +[TestCategory("BVT"), TestCategory("Membership")] +public sealed class ThreadPoolStallDetectorTests +{ + private static readonly DateTimeOffset Start = new(2026, 1, 1, 12, 0, 0, TimeSpan.Zero); + private static readonly TimeSpan DetectionPeriod = TimeSpan.FromMilliseconds(100); + + [Fact] + public async Task TimerCallback_DetectsDelayBeginningImmediatelyAfterPriorCycle() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(400), duration); + } + + [Fact] + public async Task GetStallDurationAsync_WaitsForSampleAfterIntervalEnd() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + Assert.False(durationTask.IsCompleted); + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.Zero, await durationTask); + } + + [Fact] + public async Task GetStallDurationAsync_ExcludesDelayAfterIntervalEnd() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(100)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.Zero, await durationTask); + } + + [Fact] + public async Task GetStallDurationAsync_AggregatesMultipleDelayedCycles() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(300)); + timeProvider.FireTimer(); + timeProvider.Advance(TimeSpan.FromMilliseconds(250)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(350), duration); + } + + [Fact] + public async Task TimerCallback_AggregatesRepeatedLatenessAgainstFixedCadence() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(150)); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(150), duration); + } + + [Fact] + public void GetMaximumStallDuration_ReturnsFullOverlappingStall() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var initialTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(600)); + timeProvider.FireTimer(); + + var duration = detector.GetMaximumStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(500)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(600))); + + Assert.Equal(TimeSpan.FromMilliseconds(500), duration); + } + + [Fact] + public async Task PendingQuery_RetainsStallsPastConfiguredHistory() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(300)); + timeProvider.FireTimer(); + for (var i = 0; i < 4; i++) + { + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + Assert.False(durationTask.IsCompleted); + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.FromMilliseconds(200), await durationTask); + Assert.Equal(TimeSpan.Zero, detector.GetStallDuration(startTimestamp, timeProvider.GetTimestamp())); + } + + private static ThreadPoolStallDetector CreateDetector( + TimeProvider timeProvider, + TimeSpan? retentionPeriod = null) + => new( + NullLogger.Instance, + timeProvider, + DetectionPeriod, + retentionPeriod ?? TimeSpan.FromMinutes(1)); + + private static long TimestampAt(TimeProvider timeProvider, long initialTimestamp, TimeSpan elapsed) + => initialTimestamp + (long)(elapsed.TotalSeconds * timeProvider.TimestampFrequency); + + private sealed class ManualTimerTimeProvider(DateTimeOffset start) : FakeTimeProvider(start) + { + private TimerCallback? _callback; + private object? _state; + + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + Assert.Null(_callback); + Assert.Equal(DetectionPeriod, dueTime); + Assert.Equal(DetectionPeriod, period); + _callback = callback; + _state = state; + return new ManualTimer(); + } + + public void FireTimer() + { + Assert.NotNull(_callback); + _callback(_state); + } + + private sealed class ManualTimer : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => default; + } + } +} diff --git a/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs b/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs index f41b2e3ffe8..f1bfb8f4105 100644 --- a/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs +++ b/test/Orleans.Runtime.Tests/Diagnostics/WatchdogTests.cs @@ -15,6 +15,24 @@ public class WatchdogTests { private static readonly DateTimeOffset Start = new(2026, 1, 1, 12, 0, 0, TimeSpan.Zero); + [Fact, TestCategory("BVT")] + public void CheckRuntimeHealth_RecordsGarbageCollectionPauseEvent() + { + using var fixture = new WatchdogFixture(); + fixture.SetWatchdogTimestamp("_platformWatchdogTimestamp"); + fixture.SetWatchdogTimestamp("_componentWatchdogTimestamp"); + fixture.SetField("_cumulativeGCPauseDuration", GC.GetTotalPauseDuration() - TimeSpan.FromTicks(1)); + + fixture.CheckRuntimeHealth(); + + var healthEvent = Assert.Single(fixture.Events); + Assert.Equal(LocalSiloHealthCheckKind.GarbageCollectionPause, healthEvent.Kind); + Assert.Equal(0, healthEvent.Score); + Assert.Contains("garbage collection", healthEvent.Complaint, StringComparison.OrdinalIgnoreCase); + Assert.True(healthEvent.Duration > TimeSpan.Zero); + Assert.Null(healthEvent.Source); + } + [Fact, TestCategory("BVT")] public void CheckRuntimeHealth_RecordsRuntimeStallEvent() { From 86a8acda01565ea7b9f9398c5ebae07393ef8ebf Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 16:11:57 -0700 Subject: [PATCH 10/12] fix(membership): retain minimum stall history --- .../ThreadPoolStallDetector.cs | 5 +- .../ThreadPoolStallDetectorTests.cs | 60 +++++++++++++++++-- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs index 322e09d857e..c1abae731ce 100644 --- a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs +++ b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs @@ -22,6 +22,7 @@ internal sealed partial class ThreadPoolStallDetector : IDisposable private readonly TimeProvider _timeProvider; private readonly TimeSpan _retentionPeriod; private readonly long _detectionPeriodTimestampLength; + private readonly int _minimumRetainedStallCount; private readonly ITimer _timer; private readonly List _stalls = []; private readonly SortedDictionary _pendingQueries = []; @@ -51,6 +52,7 @@ public ThreadPoolStallDetector( _timeProvider = timeProvider; _retentionPeriod = retentionPeriod; _detectionPeriodTimestampLength = GetTimestampLength(detectionPeriod); + _minimumRetainedStallCount = checked((int)Math.Ceiling(retentionPeriod / detectionPeriod)); _lastSampleTimestamp = timeProvider.GetTimestamp(); _nextExpectedTimestamp = _lastSampleTimestamp + _detectionPeriodTimestampLength; _timer = timeProvider.CreateTimer(TimerCallback, this, detectionPeriod, detectionPeriod); @@ -250,7 +252,8 @@ private void RemoveExpired(long nowTimestamp) oldestTimestamp = Math.Min(oldestTimestamp, enumerator.Current.Key); } - while (_stallHead < _stalls.Count && _stalls[_stallHead].EndTimestamp <= oldestTimestamp) + while (_stalls.Count - _stallHead > _minimumRetainedStallCount + && _stalls[_stallHead].EndTimestamp <= oldestTimestamp) { _stallHead++; } diff --git a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs index 95989c782df..0f867108e7c 100644 --- a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs +++ b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs @@ -33,6 +33,25 @@ public async Task TimerCallback_DetectsDelayBeginningImmediatelyAfterPriorCycle( Assert.Equal(TimeSpan.FromMilliseconds(400), duration); } + [Fact] + public async Task LongStallExceedingHistoryDuration_IsDetected() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var startTimestamp = timeProvider.GetTimestamp(); + + timeProvider.Advance(TimeSpan.FromSeconds(2)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(1900), duration); + } + [Fact] public async Task GetStallDurationAsync_WaitsForSampleAfterIntervalEnd() { @@ -142,11 +161,9 @@ public async Task PendingQuery_RetainsStallsPastConfiguredHistory() var timeProvider = new ManualTimerTimeProvider(Start); using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); var startTimestamp = timeProvider.GetTimestamp(); - timeProvider.Advance(TimeSpan.FromMilliseconds(300)); - timeProvider.FireTimer(); - for (var i = 0; i < 4; i++) + for (var i = 0; i < 5; i++) { - timeProvider.Advance(DetectionPeriod); + timeProvider.Advance(i == 0 ? TimeSpan.FromMilliseconds(150) : DetectionPeriod); timeProvider.FireTimer(); } @@ -162,8 +179,39 @@ public async Task PendingQuery_RetainsStallsPastConfiguredHistory() timeProvider.Advance(TimeSpan.FromMilliseconds(50)); timeProvider.FireTimer(); - Assert.Equal(TimeSpan.FromMilliseconds(200), await durationTask); - Assert.Equal(TimeSpan.Zero, detector.GetStallDuration(startTimestamp, timeProvider.GetTimestamp())); + Assert.Equal(TimeSpan.FromMilliseconds(250), await durationTask); + Assert.Equal( + TimeSpan.Zero, + detector.GetStallDuration( + TimestampAt(timeProvider, startTimestamp, TimeSpan.FromMilliseconds(100)), + TimestampAt(timeProvider, startTimestamp, TimeSpan.FromMilliseconds(150)))); + } + + [Fact] + public void HistoryRetention_PreservesMinimumCycleCountAcrossLongStall() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var initialTimestamp = timeProvider.GetTimestamp(); + for (var i = 0; i < 6; i++) + { + timeProvider.Advance(i == 0 ? TimeSpan.FromMilliseconds(150) : DetectionPeriod); + timeProvider.FireTimer(); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(1350)); + timeProvider.FireTimer(); + + Assert.Equal( + TimeSpan.Zero, + detector.GetStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(200)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(250)))); + Assert.Equal( + TimeSpan.FromMilliseconds(50), + detector.GetStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(300)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(350)))); } private static ThreadPoolStallDetector CreateDetector( From 0e818ec6e267d61001265aed98effc95ea862b02 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 16:47:56 -0700 Subject: [PATCH 11/12] fix(membership): address stall detector review --- .../LocalSiloHealthMonitor.cs | 20 +++++++++++-------- .../MembershipService/SiloHealthMonitor.cs | 3 +-- .../Membership/LocalSiloHealthMonitorTests.cs | 12 +++++++++++ 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index d4a9c02fc9c..47e71e001db 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -394,14 +394,14 @@ private void CheckThreadPoolStalls( List events, List complaints) { - var delay = _stallDetector.GetMaximumStallDuration( + var stallDuration = _stallDetector.GetMaximumStallDuration( SubtractTimestamp(timestamp, MinimumCheckPeriod), timestamp); - var score = (int)delay.TotalSeconds; + var score = (int)stallDuration.TotalSeconds; string? complaint = null; if (score >= 1) { - complaint = $".NET Thread Pool is exhibiting delays of {delay.TotalSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses."; + complaint = $".NET Thread Pool execution stalled for {stallDuration.TotalSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses."; complaints.Add(complaint); } @@ -412,7 +412,7 @@ private void CheckThreadPoolStalls( Source: null, score, complaint, - delay, + stallDuration, score >= 10 ? LogLevel.Error : LogLevel.Warning)); } @@ -653,7 +653,7 @@ private void LogHealthDetails(ImmutableArray events) if (healthEvent.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay) { - LogThreadPoolDelay(healthEvent.LogLevel, healthEvent.Duration?.TotalSeconds ?? 0); + LogThreadPoolStall(healthEvent.LogLevel, healthEvent.Duration?.TotalSeconds ?? 0); } else { @@ -685,12 +685,16 @@ public async Task OnStop(CancellationToken ct) } } - public void Dispose() => _stallDetector.Dispose(); + public void Dispose() + { + _degradationCheckTimer.Dispose(); + _stallDetector.Dispose(); + } [LoggerMessage( - Message = ".NET Thread Pool is exhibiting delays of {ThreadPoolQueueDelaySeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses." + Message = ".NET Thread Pool execution stalled for {ThreadPoolStallSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses." )] - private partial void LogThreadPoolDelay(LogLevel logLevel, double threadPoolQueueDelaySeconds); + private partial void LogThreadPoolStall(LogLevel logLevel, double threadPoolStallSeconds); [LoggerMessage( Message = "{Kind} health check for {Source} reported: {Complaint}" diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 22c75ec54f7..d154549c9d4 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -206,8 +206,7 @@ private async Task Run() isDirectProbe = !options.EnableIndirectProbes || _failedProbes < options.NumMissedProbesLimit - 1 || otherNodes.Length == 0; localDegradationScore = GetLocalDegradationScore(previousProbePeriod); var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); - var currentProbeStartTimestamp = _timeProvider.GetTimestamp(); - probeStartTimestamp = currentProbeStartTimestamp; + probeStartTimestamp = _timeProvider.GetTimestamp(); var stallStartTimestamp = isDirectProbe ? _localSiloHealthMonitor.GetTimestamp() : 0; diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs index 6bef37b7f9d..80fb0689a03 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs @@ -608,6 +608,18 @@ public void RecordHealthEvent_RejectsNegativeScore() Assert.DoesNotContain(status.Complaints, complaint => complaint == "rejected"); } + [Fact] + public void Dispose_DisposesDegradationTimer() + { + var timer = new DelegateAsyncTimer(_ => Task.FromResult(false)); + var monitor = CreateMonitor( + new DelegateAsyncTimerFactory((_, _) => timer)); + + monitor.Dispose(); + + Assert.Equal(1, timer.DisposedCounter); + } + private LocalSiloHealthMonitor CreateMonitor(params IHealthCheckParticipant[] participants) { var timerFactory = new DelegateAsyncTimerFactory( From afb540a100ba71882b40a7fdc092c62c8810179a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sat, 22 Aug 2026 17:14:41 -0700 Subject: [PATCH 12/12] fix(membership): report thread pool stalls accurately --- .../LocalSiloHealthEventHistory.cs | 2 +- .../LocalSiloHealthMonitor.cs | 8 ++++---- .../ThreadPoolStallDetector.cs | 11 +++++----- .../Membership/LocalSiloHealthMonitorTests.cs | 6 +++--- .../ThreadPoolStallDetectorTests.cs | 20 +++++++++++++++++++ 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs index b2a76373bd1..6d25ebc780d 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs @@ -242,7 +242,7 @@ private bool Overlaps(LocalSiloHealthEvent healthEvent, long startTimestamp, lon private static bool IsStateful(LocalSiloHealthCheckKind kind) => kind is LocalSiloHealthCheckKind.MembershipStatus or LocalSiloHealthCheckKind.HealthCheckParticipant - or LocalSiloHealthCheckKind.ThreadPoolQueueDelay + or LocalSiloHealthCheckKind.ThreadPoolStall or LocalSiloHealthCheckKind.ProbeRequests or LocalSiloHealthCheckKind.ProbeResponses; diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index 47e71e001db..c7e261cd9ff 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -27,7 +27,7 @@ internal enum LocalSiloHealthCheckKind MembershipStatus, SiloSuspected, HealthCheckParticipant, - ThreadPoolQueueDelay, + ThreadPoolStall, ProbeRequests, ProbeResponses, GarbageCollectionPause, @@ -407,8 +407,8 @@ private void CheckThreadPoolStalls( events.Add(new( timestamp, - LocalSiloHealthCheckKind.ThreadPoolQueueDelay, - GetCategory(LocalSiloHealthCheckKind.ThreadPoolQueueDelay), + LocalSiloHealthCheckKind.ThreadPoolStall, + GetCategory(LocalSiloHealthCheckKind.ThreadPoolStall), Source: null, score, complaint, @@ -651,7 +651,7 @@ private void LogHealthDetails(ImmutableArray events) continue; } - if (healthEvent.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay) + if (healthEvent.Kind == LocalSiloHealthCheckKind.ThreadPoolStall) { LogThreadPoolStall(healthEvent.LogLevel, healthEvent.Duration?.TotalSeconds ?? 0); } diff --git a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs index c1abae731ce..10a93290b4a 100644 --- a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs +++ b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs @@ -186,12 +186,11 @@ private void Sample() AddStall(new(_nextExpectedTimestamp, timestamp)); } - if (timestamp >= _nextExpectedTimestamp) - { - var elapsedPeriods = (timestamp - _nextExpectedTimestamp) / _detectionPeriodTimestampLength + 1; - _nextExpectedTimestamp = checked( - _nextExpectedTimestamp + elapsedPeriods * _detectionPeriodTimestampLength); - } + var elapsedPeriods = Math.Max( + 1, + (timestamp - _nextExpectedTimestamp) / _detectionPeriodTimestampLength + 1); + _nextExpectedTimestamp = checked( + _nextExpectedTimestamp + elapsedPeriods * _detectionPeriodTimestampLength); _lastSampleTimestamp = timestamp; RemoveExpired(timestamp); diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs index 80fb0689a03..f3edb858af3 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs @@ -367,7 +367,7 @@ public void GetLocalHealthStatus_IncludesDurationEventWhichOverlapsInterval() Assert.Equal(TimeSpan.FromSeconds(2), healthEvent.Duration); var stallEvent = Assert.Single( status.Events, - item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay); + item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolStall); Assert.Equal(2, stallEvent.Score); Assert.Equal(TimeSpan.FromMilliseconds(2900), stallEvent.Duration); } @@ -571,14 +571,14 @@ public async Task RecordHealthEvent_DoesNotWaitForHealthCheckParticipants() } [Fact] - public void GetLocalHealthStatus_EmitsTypedThreadPoolQueueDelayEvent() + public void GetLocalHealthStatus_EmitsTypedThreadPoolStallEvent() { var monitor = CreateMonitor(); var status = monitor.GetLocalHealthStatus(TimeSpan.Zero, LocalSiloHealthCheckCategory.Local); var threadPoolEvent = Assert.Single( status.Events, - item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay); + item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolStall); Assert.Equal(_startTimestamp, threadPoolEvent.Timestamp); Assert.Equal(LocalSiloHealthCheckCategory.Local, threadPoolEvent.Category); diff --git a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs index 0f867108e7c..c04e62f4244 100644 --- a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs +++ b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs @@ -139,6 +139,26 @@ public async Task TimerCallback_AggregatesRepeatedLatenessAgainstFixedCadence() Assert.Equal(TimeSpan.FromMilliseconds(150), duration); } + [Fact] + public async Task TimerCallback_EarlyExecutionAdvancesExpectedCadence() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.Zero, duration); + } + [Fact] public void GetMaximumStallDuration_ReturnsFullOverlappingStall() {