diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs index b2a76373bd1..6d25ebc780d 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthEventHistory.cs @@ -242,7 +242,7 @@ private bool Overlaps(LocalSiloHealthEvent healthEvent, long startTimestamp, lon private static bool IsStateful(LocalSiloHealthCheckKind kind) => kind is LocalSiloHealthCheckKind.MembershipStatus or LocalSiloHealthCheckKind.HealthCheckParticipant - or LocalSiloHealthCheckKind.ThreadPoolQueueDelay + or LocalSiloHealthCheckKind.ThreadPoolStall or LocalSiloHealthCheckKind.ProbeRequests or LocalSiloHealthCheckKind.ProbeResponses; diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index f92c78624d7..c7e261cd9ff 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -27,7 +27,7 @@ internal enum LocalSiloHealthCheckKind MembershipStatus, SiloSuspected, HealthCheckParticipant, - ThreadPoolQueueDelay, + ThreadPoolStall, ProbeRequests, ProbeResponses, GarbageCollectionPause, @@ -53,6 +53,24 @@ public ImmutableArray Complaints internal interface ILocalSiloHealthMonitor { + /// + /// Returns a timestamp from the stall detector's time source. + /// + /// A timestamp suitable for use with . + long GetTimestamp(); + + /// + /// Waits for the stall detector to sample past the end of the interval, then returns the detected stall duration. + /// + /// The start of the interval. + /// The end of the interval. + /// A token which cancels the wait. + /// The detected stall duration. + ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken); + /// /// Returns the aggregate local health status over the provided interval. /// @@ -101,7 +119,7 @@ void RecordHealthEvent( /// Check that no other silo suspects this silo. /// Check for recently received successful ping responses (via ). /// Check for recently received ping requests (via ). - /// Check that the .NET Thread Pool is able to process work items within one second. + /// Check that the .NET Thread Pool executes periodic timer callbacks on schedule. /// Check that local async timers have been firing on-time (within 3 seconds of their due time). /// /// @@ -109,7 +127,8 @@ internal partial class LocalSiloHealthMonitor : ILifecycleParticipant, ILifecycleObserver, ILocalSiloHealthMonitor, - ILocalSiloHealthEventRecorder + ILocalSiloHealthEventRecorder, + IDisposable { internal const int MaxScore = 8; private static readonly TimeSpan HistoryDuration = TimeSpan.FromMinutes(1); @@ -122,7 +141,7 @@ internal partial class LocalSiloHealthMonitor : private readonly ILogger _log; private readonly ClusterMembershipOptions _clusterMembershipOptions; private readonly IAsyncTimer _degradationCheckTimer; - private readonly ThreadPoolMonitor _threadPoolMonitor; + private readonly ThreadPoolStallDetector _stallDetector; private readonly TimeProvider _timeProvider; #if NET9_0_OR_GREATER private readonly Lock _historyLock = new(); @@ -166,12 +185,32 @@ public LocalSiloHealthMonitor( MinimumCheckPeriod, nameof(LocalSiloHealthMonitor), timeProvider); - _threadPoolMonitor = new ThreadPoolMonitor(loggerFactory.CreateLogger(), timeProvider); + var stallRetentionPeriod = _clusterMembershipOptions.MaxProbeTimeout + ThreadPoolStallDetector.DetectionPeriod; + if (stallRetentionPeriod < HistoryDuration) + { + stallRetentionPeriod = HistoryDuration; + } + + _stallDetector = new( + loggerFactory.CreateLogger(), + timeProvider, + ThreadPoolStallDetector.DetectionPeriod, + stallRetentionPeriod); } /// public ImmutableArray Complaints { get; private set; } = []; + /// + public long GetTimestamp() => _timeProvider.GetTimestamp(); + + /// + public ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken) + => _stallDetector.GetStallDurationAsync(startTimestamp, endTimestamp, cancellationToken); + /// public LocalSiloHealthStatus GetLocalHealthStatus( long startTimestamp, @@ -265,7 +304,7 @@ private LocalSiloHealthStatus EnsureHealthCheck( var events = new List(_healthCheckParticipants.Count + 1); var complaints = new List(); CheckLocalHealthCheckParticipants(now.UtcDateTime, timestamp, events, complaints); - CheckThreadPoolQueueDelay(timestamp, events, complaints); + CheckThreadPoolStalls(timestamp, events, complaints); AddEvents(timestamp, events); var score = GetScore(events); @@ -350,28 +389,30 @@ private void AddEvents(long timestamp, List events) } } - private void CheckThreadPoolQueueDelay( + private void CheckThreadPoolStalls( long timestamp, List events, List complaints) { - var delay = _threadPoolMonitor.MeasureQueueDelay(); - var score = (int)delay.TotalSeconds; + var stallDuration = _stallDetector.GetMaximumStallDuration( + SubtractTimestamp(timestamp, MinimumCheckPeriod), + timestamp); + var score = (int)stallDuration.TotalSeconds; string? complaint = null; if (score >= 1) { - complaint = $".NET Thread Pool is exhibiting delays of {delay.TotalSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses."; + complaint = $".NET Thread Pool execution stalled for {stallDuration.TotalSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses."; complaints.Add(complaint); } events.Add(new( timestamp, - LocalSiloHealthCheckKind.ThreadPoolQueueDelay, - GetCategory(LocalSiloHealthCheckKind.ThreadPoolQueueDelay), + LocalSiloHealthCheckKind.ThreadPoolStall, + GetCategory(LocalSiloHealthCheckKind.ThreadPoolStall), Source: null, score, complaint, - delay, + stallDuration, score >= 10 ? LogLevel.Error : LogLevel.Warning)); } @@ -610,9 +651,9 @@ private void LogHealthDetails(ImmutableArray events) continue; } - if (healthEvent.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay) + if (healthEvent.Kind == LocalSiloHealthCheckKind.ThreadPoolStall) { - LogThreadPoolDelay(healthEvent.LogLevel, healthEvent.Duration?.TotalSeconds ?? 0); + LogThreadPoolStall(healthEvent.LogLevel, healthEvent.Duration?.TotalSeconds ?? 0); } else { @@ -644,79 +685,16 @@ public async Task OnStop(CancellationToken ct) } } - /// - /// Measures queue delay on the .NET . - /// - private class ThreadPoolMonitor + public void Dispose() { - private static readonly WaitCallback Callback = state => ((ThreadPoolMonitor)state!).Execute(); -#if NET9_0_OR_GREATER - private readonly Lock _lockObj = new(); -#else - private readonly object _lockObj = new(); -#endif - private readonly ILogger _log; - private readonly TimeProvider _timeProvider; - private bool _scheduled; - private TimeSpan _lastQueueDelay; - private long _queueDelayTimestamp; - - public ThreadPoolMonitor(ILogger log, TimeProvider timeProvider) - { - _log = log; - _timeProvider = timeProvider; - } - - public TimeSpan MeasureQueueDelay() - { - bool shouldSchedule; - TimeSpan delay; - lock (_lockObj) - { - var currentQueueDelay = _scheduled ? _timeProvider.GetElapsedTime(_queueDelayTimestamp) : TimeSpan.Zero; - delay = currentQueueDelay > _lastQueueDelay ? currentQueueDelay : _lastQueueDelay; - - if (!_scheduled) - { - _scheduled = true; - shouldSchedule = true; - _queueDelayTimestamp = _timeProvider.GetTimestamp(); - } - else - { - shouldSchedule = false; - } - } - - if (shouldSchedule) - { - _ = ThreadPool.UnsafeQueueUserWorkItem(Callback, this); - } - - return delay; - } - - private void Execute() - { - try - { - lock (_lockObj) - { - _scheduled = false; - _lastQueueDelay = _timeProvider.GetElapsedTime(_queueDelayTimestamp); - } - } - catch (Exception exception) - { - LocalSiloHealthMonitor.LogThreadPoolDelayMonitorError(_log, exception); - } - } + _degradationCheckTimer.Dispose(); + _stallDetector.Dispose(); } [LoggerMessage( - Message = ".NET Thread Pool is exhibiting delays of {ThreadPoolQueueDelaySeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses." + Message = ".NET Thread Pool execution stalled for {ThreadPoolStallSeconds}s. This can indicate .NET Thread Pool starvation, very long .NET GC pauses, or other runtime or machine pauses." )] - private partial void LogThreadPoolDelay(LogLevel logLevel, double threadPoolQueueDelaySeconds); + private partial void LogThreadPoolStall(LogLevel logLevel, double threadPoolStallSeconds); [LoggerMessage( Message = "{Kind} health check for {Source} reported: {Complaint}" @@ -763,12 +741,6 @@ private partial void LogRecordedHealthIssue( )] private partial void LogSelfMonitoringDegraded(int score, int maxScore, string complaints); - [LoggerMessage( - Level = LogLevel.Error, - Message = "Exception monitoring .NET thread pool delay" - )] - private static partial void LogThreadPoolDelayMonitorError(ILogger logger, Exception exception); - [LoggerMessage( Level = LogLevel.Error, Message = "Error while monitoring local silo health" diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index ae69ed79e3c..d154549c9d4 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -207,12 +207,15 @@ private async Task Run() localDegradationScore = GetLocalDegradationScore(previousProbePeriod); var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); probeStartTimestamp = _timeProvider.GetTimestamp(); + var stallStartTimestamp = isDirectProbe + ? _localSiloHealthMonitor.GetTimestamp() + : 0; using var cancellation = new CancellationTokenSource(timeout, _timeProvider); if (isDirectProbe) { // Probe the silo directly. - probeResult = await this.ProbeDirectly(cancellation.Token).ConfigureAwait(false); + probeResult = await this.ProbeDirectly(cancellation.Token, timeout, stallStartTimestamp).ConfigureAwait(false); } else { @@ -321,8 +324,10 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i /// Probes the remote silo. /// /// A token to cancel and fail the probe attempt. + /// The timeout used for this probe, for local stall evaluation. + /// The start of the local stall collection interval. /// The number of failed probes since the last successful probe. - private async Task ProbeDirectly(CancellationToken cancellation) + private async Task ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, long stallStartTimestamp) { var id = ++_nextProbeId; LogTraceGoingToSendPing(_log, id, TargetSiloAddress); @@ -330,6 +335,7 @@ private async Task ProbeDirectly(CancellationToken cancellation) var roundTripTimer = TimeProviderValueStopwatch.StartNew(_timeProvider); ProbeResult probeResult; Exception? failureException; + var probeTimedOut = false; try { await _prober.Probe(TargetSiloAddress, id, cancellation).WaitAsync(cancellation); @@ -337,6 +343,7 @@ private async Task ProbeDirectly(CancellationToken cancellation) } catch (OperationCanceledException exception) { + probeTimedOut = cancellation.IsCancellationRequested; failureException = new OperationCanceledException( $"The ping attempt was cancelled after {roundTripTimer.GetElapsedTime(out _)}. Ping #{id}", exception); @@ -346,7 +353,15 @@ private async Task ProbeDirectly(CancellationToken cancellation) failureException = exception; } var roundTripTime = roundTripTimer.GetElapsedTime(out _); - + var stallDurationDuring = TimeSpan.Zero; + if (probeTimedOut) + { + var probeEndTimestamp = _localSiloHealthMonitor.GetTimestamp(); + stallDurationDuring = await _localSiloHealthMonitor.GetStallDurationAsync( + stallStartTimestamp, + probeEndTimestamp, + _stoppingCancellation.Token).ConfigureAwait(false); + } if (failureException is null) { _messagingInstruments?.OnPingReplyReceived(TargetSiloAddress); @@ -361,12 +376,22 @@ private async Task ProbeDirectly(CancellationToken cancellation) } else { - _messagingInstruments?.OnPingReplyMissed(TargetSiloAddress); - - var failedProbes = ++_failedProbes; - LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes); + // Check if a local stall consumed a significant portion of the probe timeout. + // If so, the local silo may have been unable to process the response in time, + // so we treat this as an inconclusive result rather than a failure. + if (probeTimedOut && stallDurationDuring >= probeTimeout.Multiply(0.25)) + { + LogWarningProbeFailureDuringLocalStall(_log, id, TargetSiloAddress, roundTripTime, stallDurationDuring, _failedProbes); + probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown); + } + else + { + _messagingInstruments?.OnPingReplyMissed(TargetSiloAddress); + var failedProbes = ++_failedProbes; + LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes); - probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed); + probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed); + } } return probeResult; @@ -536,5 +561,11 @@ public enum ProbeResultStatus Message = "Exception monitoring silo {SiloAddress}" )] private static partial void LogErrorExceptionMonitoringSilo(ILogger logger, Exception exception, SiloAddress siloAddress); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but local execution stalled for {StallDuration} during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}." + )] + private static partial void LogWarningProbeFailureDuringLocalStall(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan stallDuration, int failedProbeCount); } } diff --git a/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs new file mode 100644 index 00000000000..10a93290b4a --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/ThreadPoolStallDetector.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Orleans.Runtime.MembershipService; + +/// +/// Detects delays in .NET Thread Pool execution using a periodic timer callback. +/// +/// +/// Timer callbacks execute on the Thread Pool. Each callback compares its execution time with the next expected +/// cadence boundary, producing non-overlapping stall intervals when callback dispatch is delayed. +/// +internal sealed partial class ThreadPoolStallDetector : IDisposable +{ + internal static readonly TimeSpan DetectionPeriod = TimeSpan.FromMilliseconds(100); + + private static readonly TimerCallback TimerCallback = static state => ((ThreadPoolStallDetector)state!).OnTimer(); + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _retentionPeriod; + private readonly long _detectionPeriodTimestampLength; + private readonly int _minimumRetainedStallCount; + private readonly ITimer _timer; + private readonly List _stalls = []; + private readonly SortedDictionary _pendingQueries = []; +#if NET9_0_OR_GREATER + private readonly Lock _lock = new(); +#else + private readonly object _lock = new(); +#endif + private TaskCompletionSource? _nextSample; + private long _lastSampleTimestamp; + private long _nextExpectedTimestamp; + private int _stallHead; + private bool _disposed; + + public ThreadPoolStallDetector( + ILogger logger, + TimeProvider timeProvider, + TimeSpan detectionPeriod, + TimeSpan retentionPeriod) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(detectionPeriod, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(retentionPeriod, TimeSpan.Zero); + + _logger = logger; + _timeProvider = timeProvider; + _retentionPeriod = retentionPeriod; + _detectionPeriodTimestampLength = GetTimestampLength(detectionPeriod); + _minimumRetainedStallCount = checked((int)Math.Ceiling(retentionPeriod / detectionPeriod)); + _lastSampleTimestamp = timeProvider.GetTimestamp(); + _nextExpectedTimestamp = _lastSampleTimestamp + _detectionPeriodTimestampLength; + _timer = timeProvider.CreateTimer(TimerCallback, this, detectionPeriod, detectionPeriod); + } + + public async ValueTask GetStallDurationAsync( + long startTimestamp, + long endTimestamp, + CancellationToken cancellationToken) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + AddPendingQuery(startTimestamp); + } + + try + { + while (true) + { + Task nextSample; + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_lastSampleTimestamp >= endTimestamp) + { + return GetStallDurationCore(startTimestamp, endTimestamp); + } + + nextSample = (_nextSample ??= new(TaskCreationOptions.RunContinuationsAsynchronously)).Task; + } + + await nextSample.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + finally + { + lock (_lock) + { + RemovePendingQuery(startTimestamp); + RemoveExpired(_timeProvider.GetTimestamp()); + } + } + } + + public TimeSpan GetStallDuration(long startTimestamp, long endTimestamp) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + return GetStallDurationCore(startTimestamp, endTimestamp); + } + } + + public TimeSpan GetMaximumStallDuration(long startTimestamp, long endTimestamp) + { + lock (_lock) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ValidateInterval(startTimestamp, endTimestamp); + var maximumTimestampLength = 0L; + for (var i = _stallHead; i < _stalls.Count; i++) + { + var stall = _stalls[i]; + if (stall.EndTimestamp <= startTimestamp) + { + continue; + } + + if (stall.StartTimestamp >= endTimestamp) + { + break; + } + + maximumTimestampLength = Math.Max( + maximumTimestampLength, + stall.EndTimestamp - stall.StartTimestamp); + } + + return _timeProvider.GetElapsedTime(0, maximumTimestampLength); + } + } + + public void Dispose() + { + TaskCompletionSource? nextSample; + lock (_lock) + { + if (_disposed) + { + return; + } + + _disposed = true; + nextSample = _nextSample; + } + + _timer.Dispose(); + nextSample?.TrySetCanceled(); + } + + private void OnTimer() + { + try + { + Sample(); + } + catch (Exception exception) + { + LogStallDetectorError(_logger, exception); + } + } + + private void Sample() + { + TaskCompletionSource? completedSample = null; + lock (_lock) + { + if (_disposed) + { + return; + } + + var timestamp = _timeProvider.GetTimestamp(); + if (timestamp > _lastSampleTimestamp) + { + if (timestamp > _nextExpectedTimestamp) + { + AddStall(new(_nextExpectedTimestamp, timestamp)); + } + + var elapsedPeriods = Math.Max( + 1, + (timestamp - _nextExpectedTimestamp) / _detectionPeriodTimestampLength + 1); + _nextExpectedTimestamp = checked( + _nextExpectedTimestamp + elapsedPeriods * _detectionPeriodTimestampLength); + + _lastSampleTimestamp = timestamp; + RemoveExpired(timestamp); + completedSample = _nextSample; + _nextSample = null; + } + } + + completedSample?.TrySetResult(); + } + + private void AddStall(StallInterval stall) + { + if (_stalls.Count > _stallHead + && _stalls[^1].EndTimestamp >= stall.StartTimestamp) + { + var prior = _stalls[^1]; + _stalls[^1] = prior with { EndTimestamp = Math.Max(prior.EndTimestamp, stall.EndTimestamp) }; + } + else + { + _stalls.Add(stall); + } + } + + private TimeSpan GetStallDurationCore(long startTimestamp, long endTimestamp) + { + var durationTimestampLength = 0L; + for (var i = _stallHead; i < _stalls.Count; i++) + { + var stall = _stalls[i]; + if (stall.EndTimestamp <= startTimestamp) + { + continue; + } + + if (stall.StartTimestamp >= endTimestamp) + { + break; + } + + var overlapStartTimestamp = Math.Max(startTimestamp, stall.StartTimestamp); + var overlapEndTimestamp = Math.Min(endTimestamp, stall.EndTimestamp); + durationTimestampLength = checked( + durationTimestampLength + overlapEndTimestamp - overlapStartTimestamp); + } + + return _timeProvider.GetElapsedTime(0, durationTimestampLength); + } + + private void RemoveExpired(long nowTimestamp) + { + var oldestTimestamp = nowTimestamp - GetTimestampLength(_retentionPeriod); + if (_pendingQueries.Count > 0) + { + using var enumerator = _pendingQueries.GetEnumerator(); + _ = enumerator.MoveNext(); + oldestTimestamp = Math.Min(oldestTimestamp, enumerator.Current.Key); + } + + while (_stalls.Count - _stallHead > _minimumRetainedStallCount + && _stalls[_stallHead].EndTimestamp <= oldestTimestamp) + { + _stallHead++; + } + + if (_stallHead >= 1024 && _stallHead >= _stalls.Count / 2) + { + _stalls.RemoveRange(0, _stallHead); + _stallHead = 0; + } + } + + private void AddPendingQuery(long startTimestamp) + { + if (_pendingQueries.TryGetValue(startTimestamp, out var count)) + { + _pendingQueries[startTimestamp] = count + 1; + } + else + { + _pendingQueries.Add(startTimestamp, 1); + } + } + + private void RemovePendingQuery(long startTimestamp) + { + var count = _pendingQueries[startTimestamp]; + if (count == 1) + { + _pendingQueries.Remove(startTimestamp); + } + else + { + _pendingQueries[startTimestamp] = count - 1; + } + } + + private void ValidateInterval(long startTimestamp, long endTimestamp) + { + if (endTimestamp < startTimestamp) + { + throw new ArgumentOutOfRangeException( + nameof(endTimestamp), + endTimestamp, + "The interval end must not precede its start."); + } + } + + private long GetTimestampLength(TimeSpan duration) + => checked((long)Math.Ceiling(duration.TotalSeconds * _timeProvider.TimestampFrequency)); + + private readonly record struct StallInterval(long StartTimestamp, long EndTimestamp); + + [LoggerMessage( + Level = LogLevel.Error, + Message = "Exception monitoring .NET Thread Pool stalls" + )] + private static partial void LogStallDetectorError(ILogger logger, Exception exception); +} diff --git a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs index 2d0ca0c5609..f3edb858af3 100644 --- a/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/LocalSiloHealthMonitorTests.cs @@ -41,6 +41,18 @@ public LocalSiloHealthMonitorTests() }))); } + [Fact] + public void GetTimestamp_UsesMembershipTimeProvider() + { + var monitor = CreateMonitor(); + + Assert.Equal(_startTimestamp, monitor.GetTimestamp()); + + _timeProvider.Advance(TimeSpan.FromMilliseconds(250)); + + Assert.Equal(TimestampAt(TimeSpan.FromMilliseconds(250)), monitor.GetTimestamp()); + } + [Fact] public void GetLocalHealthStatus_RejectsNegativePeriod() { @@ -350,9 +362,14 @@ public void GetLocalHealthStatus_IncludesDurationEventWhichOverlapsInterval() var healthEvent = Assert.Single( status.Events, item => item.Kind == LocalSiloHealthCheckKind.RuntimeStall); - Assert.Equal(2, status.Score); + Assert.Equal(4, status.Score); Assert.Equal(TimestampAt(TimeSpan.FromSeconds(3)), healthEvent.Timestamp); Assert.Equal(TimeSpan.FromSeconds(2), healthEvent.Duration); + var stallEvent = Assert.Single( + status.Events, + item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolStall); + Assert.Equal(2, stallEvent.Score); + Assert.Equal(TimeSpan.FromMilliseconds(2900), stallEvent.Duration); } [Fact] @@ -554,14 +571,14 @@ public async Task RecordHealthEvent_DoesNotWaitForHealthCheckParticipants() } [Fact] - public void GetLocalHealthStatus_EmitsTypedThreadPoolQueueDelayEvent() + public void GetLocalHealthStatus_EmitsTypedThreadPoolStallEvent() { var monitor = CreateMonitor(); var status = monitor.GetLocalHealthStatus(TimeSpan.Zero, LocalSiloHealthCheckCategory.Local); var threadPoolEvent = Assert.Single( status.Events, - item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolQueueDelay); + item => item.Kind == LocalSiloHealthCheckKind.ThreadPoolStall); Assert.Equal(_startTimestamp, threadPoolEvent.Timestamp); Assert.Equal(LocalSiloHealthCheckCategory.Local, threadPoolEvent.Category); @@ -591,6 +608,18 @@ public void RecordHealthEvent_RejectsNegativeScore() Assert.DoesNotContain(status.Complaints, complaint => complaint == "rejected"); } + [Fact] + public void Dispose_DisposesDegradationTimer() + { + var timer = new DelegateAsyncTimer(_ => Task.FromResult(false)); + var monitor = CreateMonitor( + new DelegateAsyncTimerFactory((_, _) => timer)); + + monitor.Dispose(); + + Assert.Equal(1, timer.DisposedCounter); + } + private LocalSiloHealthMonitor CreateMonitor(params IHealthCheckParticipant[] participants) { var timerFactory = new DelegateAsyncTimerFactory( diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index ec08323dedd..88eb72f500c 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -144,6 +144,7 @@ public async Task SiloHealthMonitor_SuccessfulProbe() Assert.Equal(0, probeResult.FailedProbeCount); Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); + await _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetStallDurationAsync(default, default, default); await Shutdown(); } @@ -219,10 +220,94 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); - + await _localSiloHealthMonitor.Received(1).GetStallDurationAsync( + Arg.Any(), + Arg.Any(), + Arg.Any()); await Shutdown(); } + [Fact] + public async Task SiloHealthMonitor_TimeoutDuringLocalStall_WaitsForDetectorCycleAndIsInconclusive() + { + var timeProvider = new Microsoft.Extensions.Time.Testing.FakeTimeProvider( + new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero)); + var options = new ClusterMembershipOptions + { + ProbeTimeout = TimeSpan.FromSeconds(5), + MaxProbeTimeout = TimeSpan.FromSeconds(45), + EnableIndirectProbes = false, + }; + var optionsMonitor = Substitute.For>(); + optionsMonitor.CurrentValue.Returns(options); + var prober = Substitute.For(); + var probeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pendingProbe = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + prober.Probe(default!, default, default).ReturnsForAnyArgs(call => + { + probeEntered.TrySetResult(call.ArgAt(2)); + return pendingProbe.Task; + }); + var probeResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var detectorQuery = new TaskCompletionSource<(long Start, long End)>(TaskCreationOptions.RunContinuationsAsynchronously); + var detectorResult = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var timer = new DilationProbeTimer(); + const long ExpectedStartTimestamp = 123; + const long ExpectedEndTimestamp = 456; + var monitor = new SiloHealthMonitor( + _targetSilo, + (_, result) => + { + probeResult.TrySetResult(result); + return Task.CompletedTask; + }, + optionsMonitor, + _loggerFactory, + prober, + new DelegateAsyncTimerFactory((_, _) => timer), + _localSiloHealthMonitor, + _membershipService, + _localSiloDetails, + timeProvider); + _localSiloHealthMonitor.GetTimestamp().Returns(ExpectedStartTimestamp, ExpectedEndTimestamp); + _localSiloHealthMonitor + .GetStallDurationAsync(default, default, default) + .ReturnsForAnyArgs(call => + { + detectorQuery.TrySetResult((call.ArgAt(0), call.ArgAt(1))); + return new ValueTask(detectorResult.Task); + }); + + try + { + monitor.Start(); + var firstTick = await timer.Ticks.ReadAsync(); + firstTick.Completion.SetResult(true); + var cancellationToken = await probeEntered.Task; + + timeProvider.Advance(options.ProbeTimeout); + + Assert.True(cancellationToken.IsCancellationRequested); + var interval = await detectorQuery.Task; + Assert.Equal(ExpectedStartTimestamp, interval.Start); + Assert.Equal(ExpectedEndTimestamp, interval.End); + Assert.False(probeResult.Task.IsCompleted); + + detectorResult.SetResult(options.ProbeTimeout.Multiply(0.25)); + var result = await probeResult.Task; + Assert.Equal(ProbeResultStatus.Unknown, result.Status); + Assert.Equal(0, result.FailedProbeCount); + Assert.Equal(0, ((ITestAccessor)monitor).MissedProbes); + + var nextTick = await timer.Ticks.ReadAsync(); + nextTick.Completion.SetResult(false); + } + finally + { + await monitor.StopAsync(CancellationToken.None); + } + } + [Fact] public async Task SiloHealthMonitor_FailedProbe_Exception() { @@ -247,6 +332,7 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); + await _localSiloHealthMonitor.DidNotReceiveWithAnyArgs().GetStallDurationAsync(default, default, default); await Shutdown(); } diff --git a/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs new file mode 100644 index 00000000000..c04e62f4244 --- /dev/null +++ b/test/Orleans.Core.Tests/Membership/ThreadPoolStallDetectorTests.cs @@ -0,0 +1,285 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Orleans.Runtime.MembershipService; +using TestExtensions; +using Xunit; + +namespace NonSilo.Tests.Membership; + +[TestCategory("BVT"), TestCategory("Membership")] +public sealed class ThreadPoolStallDetectorTests +{ + private static readonly DateTimeOffset Start = new(2026, 1, 1, 12, 0, 0, TimeSpan.Zero); + private static readonly TimeSpan DetectionPeriod = TimeSpan.FromMilliseconds(100); + + [Fact] + public async Task TimerCallback_DetectsDelayBeginningImmediatelyAfterPriorCycle() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(500)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(400), duration); + } + + [Fact] + public async Task LongStallExceedingHistoryDuration_IsDetected() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var startTimestamp = timeProvider.GetTimestamp(); + + timeProvider.Advance(TimeSpan.FromSeconds(2)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(1900), duration); + } + + [Fact] + public async Task GetStallDurationAsync_WaitsForSampleAfterIntervalEnd() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + Assert.False(durationTask.IsCompleted); + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.Zero, await durationTask); + } + + [Fact] + public async Task GetStallDurationAsync_ExcludesDelayAfterIntervalEnd() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + timeProvider.Advance(TimeSpan.FromMilliseconds(100)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.Zero, await durationTask); + } + + [Fact] + public async Task GetStallDurationAsync_AggregatesMultipleDelayedCycles() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(300)); + timeProvider.FireTimer(); + timeProvider.Advance(TimeSpan.FromMilliseconds(250)); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(350), duration); + } + + [Fact] + public async Task TimerCallback_AggregatesRepeatedLatenessAgainstFixedCadence() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(150)); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.FromMilliseconds(150), duration); + } + + [Fact] + public async Task TimerCallback_EarlyExecutionAdvancesExpectedCadence() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var startTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(99)); + timeProvider.FireTimer(); + timeProvider.Advance(DetectionPeriod); + timeProvider.FireTimer(); + var endTimestamp = timeProvider.GetTimestamp(); + + var duration = await detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None); + + Assert.Equal(TimeSpan.Zero, duration); + } + + [Fact] + public void GetMaximumStallDuration_ReturnsFullOverlappingStall() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider); + var initialTimestamp = timeProvider.GetTimestamp(); + timeProvider.Advance(TimeSpan.FromMilliseconds(600)); + timeProvider.FireTimer(); + + var duration = detector.GetMaximumStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(500)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(600))); + + Assert.Equal(TimeSpan.FromMilliseconds(500), duration); + } + + [Fact] + public async Task PendingQuery_RetainsStallsPastConfiguredHistory() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var startTimestamp = timeProvider.GetTimestamp(); + for (var i = 0; i < 5; i++) + { + timeProvider.Advance(i == 0 ? TimeSpan.FromMilliseconds(150) : DetectionPeriod); + timeProvider.FireTimer(); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + var endTimestamp = timeProvider.GetTimestamp(); + var durationTask = detector.GetStallDurationAsync( + startTimestamp, + endTimestamp, + CancellationToken.None).AsTask(); + + Assert.False(durationTask.IsCompleted); + + timeProvider.Advance(TimeSpan.FromMilliseconds(50)); + timeProvider.FireTimer(); + + Assert.Equal(TimeSpan.FromMilliseconds(250), await durationTask); + Assert.Equal( + TimeSpan.Zero, + detector.GetStallDuration( + TimestampAt(timeProvider, startTimestamp, TimeSpan.FromMilliseconds(100)), + TimestampAt(timeProvider, startTimestamp, TimeSpan.FromMilliseconds(150)))); + } + + [Fact] + public void HistoryRetention_PreservesMinimumCycleCountAcrossLongStall() + { + var timeProvider = new ManualTimerTimeProvider(Start); + using var detector = CreateDetector(timeProvider, TimeSpan.FromMilliseconds(500)); + var initialTimestamp = timeProvider.GetTimestamp(); + for (var i = 0; i < 6; i++) + { + timeProvider.Advance(i == 0 ? TimeSpan.FromMilliseconds(150) : DetectionPeriod); + timeProvider.FireTimer(); + } + + timeProvider.Advance(TimeSpan.FromMilliseconds(1350)); + timeProvider.FireTimer(); + + Assert.Equal( + TimeSpan.Zero, + detector.GetStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(200)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(250)))); + Assert.Equal( + TimeSpan.FromMilliseconds(50), + detector.GetStallDuration( + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(300)), + TimestampAt(timeProvider, initialTimestamp, TimeSpan.FromMilliseconds(350)))); + } + + private static ThreadPoolStallDetector CreateDetector( + TimeProvider timeProvider, + TimeSpan? retentionPeriod = null) + => new( + NullLogger.Instance, + timeProvider, + DetectionPeriod, + retentionPeriod ?? TimeSpan.FromMinutes(1)); + + private static long TimestampAt(TimeProvider timeProvider, long initialTimestamp, TimeSpan elapsed) + => initialTimestamp + (long)(elapsed.TotalSeconds * timeProvider.TimestampFrequency); + + private sealed class ManualTimerTimeProvider(DateTimeOffset start) : FakeTimeProvider(start) + { + private TimerCallback? _callback; + private object? _state; + + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + Assert.Null(_callback); + Assert.Equal(DetectionPeriod, dueTime); + Assert.Equal(DetectionPeriod, period); + _callback = callback; + _state = state; + return new ManualTimer(); + } + + public void FireTimer() + { + Assert.NotNull(_callback); + _callback(_state); + } + + private sealed class ManualTimer : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => default; + } + } +}