Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ public ImmutableArray<string> Complaints

internal interface ILocalSiloHealthMonitor
{
/// <summary>
/// Gets the total duration of local runtime pauses since process start.
/// </summary>
TimeSpan TotalPauseDuration { get; }

/// <summary>
/// Returns the aggregate local health status over the provided interval.
/// </summary>
Expand Down Expand Up @@ -172,6 +177,9 @@ public LocalSiloHealthMonitor(
/// <inheritdoc />
public ImmutableArray<string> Complaints { get; private set; } = [];

/// <inheritdoc />
public TimeSpan TotalPauseDuration => GC.GetTotalPauseDuration();

/// <inheritdoc />
public LocalSiloHealthStatus GetLocalHealthStatus(
long startTimestamp,
Expand Down
38 changes: 31 additions & 7 deletions src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,14 @@ private async Task Run()
var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached);
probeStartTimestamp = _timeProvider.GetTimestamp();
using var cancellation = new CancellationTokenSource(timeout, _timeProvider);
var pauseDurationBefore = isDirectProbe
? _localSiloHealthMonitor.TotalPauseDuration
: TimeSpan.Zero;

if (isDirectProbe)
{
// Probe the silo directly.
probeResult = await this.ProbeDirectly(cancellation.Token).ConfigureAwait(false);
probeResult = await this.ProbeDirectly(cancellation.Token, timeout, pauseDurationBefore).ConfigureAwait(false);
}
else
{
Expand Down Expand Up @@ -321,22 +324,26 @@ internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, i
/// Probes the remote silo.
/// </summary>
/// <param name="cancellation">A token to cancel and fail the probe attempt.</param>
/// <param name="probeTimeout">The timeout used for this probe, for local pause evaluation.</param>
/// <param name="pauseDurationBefore">The cumulative local pause duration when the probe timeout was armed.</param>
/// <returns>The number of failed probes since the last successful probe.</returns>
private async Task<ProbeResult> ProbeDirectly(CancellationToken cancellation)
private async Task<ProbeResult> ProbeDirectly(CancellationToken cancellation, TimeSpan probeTimeout, TimeSpan pauseDurationBefore)
{
var id = ++_nextProbeId;
LogTraceGoingToSendPing(_log, id, TargetSiloAddress);

var roundTripTimer = TimeProviderValueStopwatch.StartNew(_timeProvider);
ProbeResult probeResult;
Exception? failureException;
var probeTimedOut = false;
try
{
await _prober.Probe(TargetSiloAddress, id, cancellation).WaitAsync(cancellation);
failureException = null;
}
catch (OperationCanceledException exception)
{
probeTimedOut = cancellation.IsCancellationRequested;
failureException = new OperationCanceledException(
$"The ping attempt was cancelled after {roundTripTimer.GetElapsedTime(out _)}. Ping #{id}",
exception);
Expand All @@ -361,12 +368,23 @@ private async Task<ProbeResult> ProbeDirectly(CancellationToken cancellation)
}
else
{
_messagingInstruments?.OnPingReplyMissed(TargetSiloAddress);

var failedProbes = ++_failedProbes;
LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes);
// Check if a local pause consumed a significant portion of the probe timeout.
// If so, the local silo may have been unable to process the response in time,
// so we treat this as an inconclusive result rather than a failure.
var pauseDurationDuring = _localSiloHealthMonitor.TotalPauseDuration - pauseDurationBefore;
if (probeTimedOut && pauseDurationDuring >= probeTimeout.Multiply(0.25))
{
LogWarningProbeFailureDuringLocalPause(_log, id, TargetSiloAddress, roundTripTime, pauseDurationDuring, _failedProbes);
probeResult = ProbeResult.CreateDirect(_failedProbes, ProbeResultStatus.Unknown);
}
else
{
_messagingInstruments?.OnPingReplyMissed(TargetSiloAddress);
var failedProbes = ++_failedProbes;
LogWarningDidNotGetResponseForProbe(_log, failureException, id, TargetSiloAddress, roundTripTime, failedProbes);

probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed);
probeResult = ProbeResult.CreateDirect(failedProbes, ProbeResultStatus.Failed);
}
}

return probeResult;
Expand Down Expand Up @@ -536,5 +554,11 @@ public enum ProbeResultStatus
Message = "Exception monitoring silo {SiloAddress}"
)]
private static partial void LogErrorExceptionMonitoringSilo(ILogger logger, Exception exception, SiloAddress siloAddress);

[LoggerMessage(
Level = LogLevel.Warning,
Message = "Probe #{Id} to silo {SiloAddress} failed after {Elapsed}, but a local pause of {PauseDuration} was detected during the probe. Treating as inconclusive. Consecutive failed probe count remains at {FailedProbeCount}."
)]
private static partial void LogWarningProbeFailureDuringLocalPause(ILogger logger, int id, SiloAddress siloAddress, TimeSpan elapsed, TimeSpan pauseDuration, int failedProbeCount);
}
}
69 changes: 69 additions & 0 deletions test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,79 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout()
await Shutdown();
}

[Fact]
public async Task SiloHealthMonitor_TimeoutDuringLocalPause_IsInconclusive()
{
var timeProvider = new Microsoft.Extensions.Time.Testing.FakeTimeProvider(
new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero));
var options = new ClusterMembershipOptions
{
ProbeTimeout = TimeSpan.FromSeconds(5),
MaxProbeTimeout = TimeSpan.FromSeconds(45),
EnableIndirectProbes = false,
};
var optionsMonitor = Substitute.For<IOptionsMonitor<ClusterMembershipOptions>>();
optionsMonitor.CurrentValue.Returns(options);
var prober = Substitute.For<IRemoteSiloProber>();
var probeEntered = new TaskCompletionSource<CancellationToken>(TaskCreationOptions.RunContinuationsAsynchronously);
var pendingProbe = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
prober.Probe(default!, default, default).ReturnsForAnyArgs(call =>
{
probeEntered.TrySetResult(call.ArgAt<CancellationToken>(2));
return pendingProbe.Task;
});
var probeResult = new TaskCompletionSource<ProbeResult>(TaskCreationOptions.RunContinuationsAsynchronously);
var timer = new DilationProbeTimer();
var monitor = new SiloHealthMonitor(
_targetSilo,
(_, result) =>
{
probeResult.TrySetResult(result);
return Task.CompletedTask;
},
optionsMonitor,
_loggerFactory,
prober,
new DelegateAsyncTimerFactory((_, _) => timer),
_localSiloHealthMonitor,
_membershipService,
_localSiloDetails,
timeProvider);
var totalPauseDuration = TimeSpan.Zero;
_localSiloHealthMonitor.TotalPauseDuration.Returns(_ => totalPauseDuration);

try
{
monitor.Start();
var firstTick = await timer.Ticks.ReadAsync();
firstTick.Completion.SetResult(true);
var cancellationToken = await probeEntered.Task;

totalPauseDuration = options.ProbeTimeout.Multiply(0.25);
timeProvider.Advance(options.ProbeTimeout);

Assert.True(cancellationToken.IsCancellationRequested);
var result = await probeResult.Task;
Assert.Equal(ProbeResultStatus.Unknown, result.Status);
Assert.Equal(0, result.FailedProbeCount);
Assert.Equal(0, ((ITestAccessor)monitor).MissedProbes);

var nextTick = await timer.Ticks.ReadAsync();
nextTick.Completion.SetResult(false);
}
finally
{
await monitor.StopAsync(CancellationToken.None);
}
}

[Fact]
public async Task SiloHealthMonitor_FailedProbe_Exception()
{
_clusterMembershipOptions.ProbeTimeout = TimeSpan.FromSeconds(2);
var pauseSample = 0;
_localSiloHealthMonitor.TotalPauseDuration.Returns(_ =>
Interlocked.Increment(ref pauseSample) % 2 == 1 ? TimeSpan.Zero : _clusterMembershipOptions.ProbeTimeout);

_prober.Probe(default!, default).ThrowsAsyncForAnyArgs(new Exception("nope"));
_prober.ProbeIndirectly(default!, default!, default, default).ThrowsAsyncForAnyArgs(new InvalidOperationException("No"));
Expand Down
Loading