diff --git a/docs/site/src/content/docs/implementation/cluster-management.md b/docs/site/src/content/docs/implementation/cluster-management.md index 033c80e80f6..174120ccfbb 100644 --- a/docs/site/src/content/docs/implementation/cluster-management.md +++ b/docs/site/src/content/docs/implementation/cluster-management.md @@ -1,7 +1,7 @@ --- title: Cluster membership protocol description: Understand Orleans membership storage, failure detection, ordered views, and death-vote invariants. -ms.date: 08/02/2026 +ms.date: 08/11/2026 ms.topic: concept-article --- @@ -41,6 +41,10 @@ The periodic `IAmAlive` value is not the peer heartbeat. It is a timestamp writt Active silos monitor peers selected from the membership view. `ClusterHealthMonitor` sends probes over silo-to-silo messaging, tracks consecutive failures, and can use indirect probes to distinguish a failed target from an unhealthy observer. A failed monitor writes a timestamped vote into the target's membership row. +Each observer maintains a [Phi Accrual failure detector](https://paperhub.s3.amazonaws.com/f516fdfa940caa08c679d3946b273128.pdf) for each peer. The detector models successful direct-probe round-trip times and estimates the timeout at which the probability of a later response is sufficiently low. The timeout starts at and adapts after enough observations. Failures are excluded because they only show that the response exceeded the current timeout, while indirect results are excluded because they measure a different observer's network path. + +The learned timeout also determines probe cadence. Each probe is scheduled relative to the previous probe's start, so a quick response waits for the remainder of the current timeout while a probe which consumes its timeout is followed immediately by the next attempt. Local-health and indirect-hop extensions are applied to the learned timeout before it is clamped between and . Debugger-specific extensions are applied after the clamp so a paused process is not accused because of the configured production bound. + ```mermaid sequenceDiagram participant A as Monitoring silo A @@ -70,7 +74,9 @@ The defaults are defined by | 10 | Number of peers monitored by each silo | -| | 5 seconds | Baseline direct probe timeout | +| | 5 seconds | Initial timeout and probe period before the peer has supplied enough evidence | +| | Half the initial timeout (2.5 seconds by default) | Lower bound for an effective probe timeout | +| | Four times the initial timeout (20 seconds by default) | Upper bound for an effective probe timeout | | | 3 | Failed probes before a death vote | | | 2 | Fresh votes required to mark a member dead | | | 2 minutes | Lifetime of a death vote | diff --git a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs index 4773a33fcd7..246e7dbdbd6 100644 --- a/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs +++ b/src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs @@ -7,6 +7,9 @@ namespace Orleans.Configuration /// public class ClusterMembershipOptions { + private TimeSpan? _minProbeTimeout; + private TimeSpan? _maxProbeTimeout; + /// /// Gets or sets the number of missed "I am alive" updates in the table from a silo that causes warning to be logged. /// @@ -23,11 +26,38 @@ public class ClusterMembershipOptions public bool LivenessEnabled { get; set; } = true; /// - /// Gets or sets both the period between sending a liveness probe to any given host as well as the timeout for each probe. + /// Gets or sets the initial timeout for liveness probes. /// - /// Probes time out and a new probe is sent every 5 seconds by default. + /// + /// The effective probe timeout and the period between probe starts are adjusted independently for each monitored silo + /// based on observed direct-probe response times and are bounded by and + /// . + /// + /// The initial probe timeout is 5 seconds by default. public TimeSpan ProbeTimeout { get; set; } = TimeSpan.FromSeconds(5); + /// + /// Gets or sets the minimum effective timeout for a liveness probe. + /// + /// Half of by default. + public TimeSpan MinProbeTimeout + { + get => _minProbeTimeout ?? TimeSpan.FromTicks(ProbeTimeout.Ticks / 2); + set => _minProbeTimeout = value; + } + + /// + /// Gets or sets the maximum effective timeout for a liveness probe. + /// + /// Four times by default. + public TimeSpan MaxProbeTimeout + { + get => _maxProbeTimeout ?? (ProbeTimeout.Ticks <= TimeSpan.MaxValue.Ticks / 4 + ? TimeSpan.FromTicks(ProbeTimeout.Ticks * 4) + : TimeSpan.MaxValue); + set => _maxProbeTimeout = value; + } + /// /// Gets or sets the period between fetching updates from the membership table. /// @@ -131,7 +161,7 @@ public class ClusterMembershipOptions public TimeSpan LocalHealthDegradationMonitoringPeriod { get; set; } = TimeSpan.FromSeconds(10); /// - /// Gets or sets a value indicating whether to extend the effective value based upon current local health degradation. + /// Gets or sets a value indicating whether to extend the effective probe timeout based upon current local health degradation. /// public bool ExtendProbeTimeoutDuringDegradation { get; set; } = true; @@ -145,7 +175,7 @@ public class ClusterMembershipOptions /// /// /// When enabled, if an active connection to a silo has recently received messages within the monitoring window - /// ( × ), votes to suspect that silo will be suppressed + /// ( × ), votes to suspect that silo will be suppressed /// since the connection activity demonstrates the silo is alive. This helps prevent false death declarations /// when probes fail due to local issues such as GC pauses or thread pool saturation. /// diff --git a/src/Orleans.Runtime/Configuration/ClusterMembershipOptionsExtensions.cs b/src/Orleans.Runtime/Configuration/ClusterMembershipOptionsExtensions.cs new file mode 100644 index 00000000000..2b6d168660b --- /dev/null +++ b/src/Orleans.Runtime/Configuration/ClusterMembershipOptionsExtensions.cs @@ -0,0 +1,10 @@ +using System; +using Orleans.Internal; + +namespace Orleans.Configuration; + +internal static class ClusterMembershipOptionsExtensions +{ + internal static TimeSpan GetFailureDetectionTimeout(this ClusterMembershipOptions options) => + options.MaxProbeTimeout.Multiply(options.NumMissedProbesLimit); +} diff --git a/src/Orleans.Runtime/Configuration/Validators/SiloClusteringValidator.cs b/src/Orleans.Runtime/Configuration/Validators/SiloClusteringValidator.cs index 9bca42e1917..03504af3c08 100644 --- a/src/Orleans.Runtime/Configuration/Validators/SiloClusteringValidator.cs +++ b/src/Orleans.Runtime/Configuration/Validators/SiloClusteringValidator.cs @@ -13,6 +13,7 @@ namespace Orleans.Runtime.Configuration /// internal class SiloClusteringValidator : IConfigurationValidator { + private const uint MaxSupportedTimeoutMilliseconds = 0xfffffffe; private readonly IServiceProvider serviceProvider; public SiloClusteringValidator(IServiceProvider serviceProvider) @@ -51,6 +52,56 @@ public void ValidateConfiguration() throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxDefunctSiloEntries)} ({clusterMembershipOptions.MaxDefunctSiloEntries}) must be greater than or equal to 0, or null."); } + if (clusterMembershipOptions.ProbeTimeout <= TimeSpan.Zero) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be greater than 0."); + } + + if (clusterMembershipOptions.ProbeTimeout.TotalMilliseconds > MaxSupportedTimeoutMilliseconds) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be less than or equal to {TimeSpan.FromMilliseconds(MaxSupportedTimeoutMilliseconds)}."); + } + + if (clusterMembershipOptions.MinProbeTimeout <= TimeSpan.Zero) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout}) must be greater than 0."); + } + + if (clusterMembershipOptions.MaxProbeTimeout < clusterMembershipOptions.MinProbeTimeout) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout}) must be greater than or equal to {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout})."); + } + + if (clusterMembershipOptions.MaxProbeTimeout.TotalMilliseconds > MaxSupportedTimeoutMilliseconds) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout}) must be less than or equal to {TimeSpan.FromMilliseconds(MaxSupportedTimeoutMilliseconds)}."); + } + + if (clusterMembershipOptions.ProbeTimeout < clusterMembershipOptions.MinProbeTimeout + || clusterMembershipOptions.ProbeTimeout > clusterMembershipOptions.MaxProbeTimeout) + { + throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be between {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout}) and {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout})."); + } + + var maxProbeCycleTime = clusterMembershipOptions.MaxProbeTimeout; + var failureDetectionTimeoutTicks = 0L; + if (clusterMembershipOptions.NumMissedProbesLimit > 0 + && maxProbeCycleTime.Ticks > TimeSpan.MaxValue.Ticks / clusterMembershipOptions.NumMissedProbesLimit) + { + throw new OrleansConfigurationException($"The maximum probe cycle time ({maxProbeCycleTime}) multiplied by {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.NumMissedProbesLimit)} ({clusterMembershipOptions.NumMissedProbesLimit}) must not exceed {TimeSpan.MaxValue}."); + } + else if (clusterMembershipOptions.NumMissedProbesLimit > 0) + { + failureDetectionTimeoutTicks = maxProbeCycleTime.Ticks * clusterMembershipOptions.NumMissedProbesLimit; + } + + var tableRefreshTimeoutTicks = clusterMembershipOptions.TableRefreshTimeout.Ticks; + if (tableRefreshTimeoutTicks > 0 + && tableRefreshTimeoutTicks > (TimeSpan.MaxValue.Ticks - failureDetectionTimeoutTicks) / 2) + { + throw new OrleansConfigurationException($"The failure detection timeout plus twice {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.TableRefreshTimeout)} ({clusterMembershipOptions.TableRefreshTimeout}) must not exceed {TimeSpan.MaxValue}."); + } + if (clusterMembershipOptions.LivenessEnabled) { if (clusterMembershipOptions.NumVotesForDeathDeclaration > clusterMembershipOptions.NumProbedSilos) diff --git a/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs b/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs index fdbb1527c35..5e14307388d 100644 --- a/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs +++ b/src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs @@ -140,7 +140,7 @@ internal static TimeSpan CalculateDeadSiloLeaseDuration( TimeSpan rangeLeaseDuration, ClusterMembershipOptions membershipOptions) { - var failureDetectionDuration = membershipOptions.ProbeTimeout * membershipOptions.NumMissedProbesLimit; + var failureDetectionDuration = membershipOptions.GetFailureDetectionTimeout(); return rangeLeaseDuration > failureDetectionDuration ? rangeLeaseDuration - failureDetectionDuration : TimeSpan.Zero; diff --git a/src/Orleans.Runtime/MembershipService/ClusterHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/ClusterHealthMonitor.cs index 628d3f8786c..39274a845dd 100644 --- a/src/Orleans.Runtime/MembershipService/ClusterHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/ClusterHealthMonitor.cs @@ -102,7 +102,7 @@ private async Task ProcessMembershipUpdates() if (!newMonitoredSilos.ContainsKey(pair.Key)) { using var cancellation = new CancellationTokenSource( - this.clusterMembershipOptions.CurrentValue.ProbeTimeout, + this.clusterMembershipOptions.CurrentValue.MaxProbeTimeout, this.timeProvider); await pair.Value.StopAsync(cancellation.Token); } @@ -359,7 +359,7 @@ private async Task OnProbeResultInternal(SiloHealthMonitor monitor, ProbeResult /// /// Checks whether a connection to the specified silo has received a message within the monitoring window - /// ( × ). + /// (the maximum probe cycle time multiplied by ). /// If so, the silo is demonstrably alive and the vote should be suppressed. /// private bool IsConnectionActiveWithinMonitoringWindow(SiloAddress targetSilo) @@ -370,7 +370,7 @@ private bool IsConnectionActiveWithinMonitoringWindow(SiloAddress targetSilo) return false; } - var monitoringWindow = options.ProbeTimeout.Multiply(options.NumMissedProbesLimit); + var monitoringWindow = options.GetFailureDetectionTimeout(); if (this.connectionManager.GetElapsedSinceLastMessageReceived(targetSilo) is { } elapsed && elapsed <= monitoringWindow) { diff --git a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs index a73ed12321c..f92c78624d7 100644 --- a/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs @@ -326,7 +326,7 @@ private LocalSiloHealthStatus EnsureNetworkHealthCheck( } // Only consider certain checks if the silo has been a member of a multi-silo cluster for a certain period. - var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit); + var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout(); if (_clusteredSinceTimestamp is { } clusteredSince && _timeProvider.GetElapsedTime(clusteredSince, timestamp) > recencyWindow) { diff --git a/src/Orleans.Runtime/MembershipService/PhiAccrualFailureDetector.cs b/src/Orleans.Runtime/MembershipService/PhiAccrualFailureDetector.cs new file mode 100644 index 00000000000..29035d183dc --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/PhiAccrualFailureDetector.cs @@ -0,0 +1,104 @@ +using System; + +namespace Orleans.Runtime.MembershipService; + +internal sealed class PhiAccrualFailureDetector +{ + private const double Threshold = 8; + private const int MaxSampleSize = 100; + private const int MinimumSampleCount = 4; + private static readonly double ThresholdStandardDeviations = CalculateThresholdStandardDeviations(); + + private readonly double[] _samples = new double[MaxSampleSize]; + private readonly TimeSpan _initialTimeout; + private readonly double _minimumStandardDeviationMilliseconds; + private int _nextSampleIndex; + private int _sampleCount; + private double _sampleSum; + private double _squaredSampleSum; + + public PhiAccrualFailureDetector(TimeSpan initialTimeout) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(initialTimeout, TimeSpan.Zero); + + _initialTimeout = initialTimeout; + _minimumStandardDeviationMilliseconds = initialTimeout.TotalMilliseconds / (2 * ThresholdStandardDeviations); + } + + public int SampleCount => _sampleCount; + + public void RecordResponseTime(TimeSpan responseTime) + { + ArgumentOutOfRangeException.ThrowIfLessThan(responseTime, TimeSpan.Zero); + + var sample = responseTime.TotalMilliseconds; + if (_sampleCount == MaxSampleSize) + { + var replacedSample = _samples[_nextSampleIndex]; + _sampleSum -= replacedSample; + _squaredSampleSum -= replacedSample * replacedSample; + } + else + { + _sampleCount++; + } + + _samples[_nextSampleIndex] = sample; + _nextSampleIndex = (_nextSampleIndex + 1) % MaxSampleSize; + _sampleSum += sample; + _squaredSampleSum += sample * sample; + } + + public TimeSpan GetTimeout() => + _sampleCount < MinimumSampleCount + ? _initialTimeout + : TimeSpan.FromMilliseconds(Math.Min(GetEstimatedTimeoutMilliseconds(), TimeSpan.MaxValue.TotalMilliseconds)); + + public TimeSpan GetTimeout(TimeSpan minimumTimeout, TimeSpan maximumTimeout, int extensionFactor) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(minimumTimeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThan(maximumTimeout, minimumTimeout); + ArgumentOutOfRangeException.ThrowIfLessThan(extensionFactor, 1); + + var timeoutTicks = GetTimeout().Ticks * (double)extensionFactor; + return TimeSpan.FromTicks((long)Math.Clamp(timeoutTicks, minimumTimeout.Ticks, maximumTimeout.Ticks)); + } + + internal static double CalculatePhi(double elapsedMilliseconds, double meanMilliseconds, double standardDeviationMilliseconds) + { + // Logistic approximation of the normal CDF used by Akka's Phi Accrual implementation. + var y = (elapsedMilliseconds - meanMilliseconds) / standardDeviationMilliseconds; + var e = Math.Exp(-y * (1.5976 + (0.070566 * y * y))); + return elapsedMilliseconds > meanMilliseconds + ? -Math.Log10(e / (1 + e)) + : -Math.Log10(1 - (1 / (1 + e))); + } + + private double GetEstimatedTimeoutMilliseconds() + { + var mean = _sampleSum / _sampleCount; + var variance = Math.Max(0, (_squaredSampleSum / _sampleCount) - (mean * mean)); + var standardDeviation = Math.Max(Math.Sqrt(variance), _minimumStandardDeviationMilliseconds); + return mean + (ThresholdStandardDeviations * standardDeviation); + } + + private static double CalculateThresholdStandardDeviations() + { + var lower = 0d; + var upper = 10d; + for (var i = 0; i < 64; i++) + { + var midpoint = (lower + upper) / 2; + if (CalculatePhi(midpoint, 0, 1) < Threshold) + { + lower = midpoint; + } + else + { + upper = midpoint; + } + } + + return upper; + } +} diff --git a/src/Orleans.Runtime/MembershipService/ProbingSiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/ProbingSiloHealthMonitor.cs index 3a30921da8f..4202625f4a4 100644 --- a/src/Orleans.Runtime/MembershipService/ProbingSiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/ProbingSiloHealthMonitor.cs @@ -51,7 +51,7 @@ public int CheckReceivedProbeRequests(DateTime now, int activeNodeCount, out str } var sinceLastProbeRequest = _probeRequestMonitor.ElapsedSinceLastProbeRequest; - var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit); + var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout(); if (!sinceLastProbeRequest.HasValue) { @@ -96,7 +96,7 @@ public int CheckReceivedProbeResponses(DateTime now, int monitoredNodeCount, out return 0; } - var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit); + var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout(); if (!elapsedSinceLastResponse.HasValue) { diff --git a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs index 2536f16b707..ae69ed79e3c 100644 --- a/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs +++ b/src/Orleans.Runtime/MembershipService/SiloHealthMonitor.cs @@ -20,6 +20,7 @@ namespace Orleans.Runtime.MembershipService /// internal partial class SiloHealthMonitor : ITestAccessor, IHealthCheckable, IDisposable, IAsyncDisposable { + private static readonly TimeSpan MaxSupportedTimerTimeout = TimeSpan.FromMilliseconds(0xfffffffe); private readonly ILogger _log; private readonly IOptionsMonitor _clusterMembershipOptions; private readonly IRemoteSiloProber _prober; @@ -37,6 +38,7 @@ internal partial class SiloHealthMonitor : ITestAccessor, IHealthCheckable, IDis private readonly IAsyncTimer _pingTimer; private long? _lastSuccessfulResponseTimestamp; private readonly Func _onProbeResult; + private PhiAccrualFailureDetector? _failureDetector; private Task? _runTask; /// @@ -93,6 +95,8 @@ public SiloHealthMonitor( internal interface ITestAccessor { int MissedProbes { get; } + int ProbeTimeoutSampleCount { get; } + TimeSpan CurrentProbeTimeout { get; } } /// @@ -106,6 +110,8 @@ internal interface ITestAccessor public bool IsCanceled => _stoppingCancellation.IsCancellationRequested; int ITestAccessor.MissedProbes => _failedProbes; + int ITestAccessor.ProbeTimeoutSampleCount => _failureDetector?.SampleCount ?? 0; + TimeSpan ITestAccessor.CurrentProbeTimeout => _failureDetector?.GetTimeout() ?? _clusterMembershipOptions.CurrentValue.ProbeTimeout; /// /// Start the monitor. @@ -170,11 +176,15 @@ private async Task Run() MembershipTableSnapshot? activeMembersSnapshot = default; SiloAddress[]? otherNodes = default; var options = _clusterMembershipOptions.CurrentValue; - TimeSpan? overrideDelay = RandomTimeSpan.Next(options.ProbeTimeout); - while (await _pingTimer.NextTick(overrideDelay)) + var failureDetector = _failureDetector = new PhiAccrualFailureDetector(options.ProbeTimeout); + var previousProbePeriod = options.ProbeTimeout; + TimeSpan? nextProbeDelay = RandomTimeSpan.Next(previousProbePeriod); + while (await _pingTimer.NextTick(nextProbeDelay)) { ProbeResult probeResult; - overrideDelay = default; + var isDirectProbe = true; + var localDegradationScore = 0; + long? probeStartTimestamp = null; var now = _timeProvider.GetUtcNow().UtcDateTime; try @@ -193,8 +203,10 @@ private async Task Run() .ToArray(); } - var isDirectProbe = !options.EnableIndirectProbes || _failedProbes < options.NumMissedProbesLimit - 1 || otherNodes.Length == 0; - var timeout = GetTimeout(isDirectProbe); + isDirectProbe = !options.EnableIndirectProbes || _failedProbes < options.NumMissedProbesLimit - 1 || otherNodes.Length == 0; + localDegradationScore = GetLocalDegradationScore(previousProbePeriod); + var timeout = CalculateProbeTimeout(failureDetector, options, localDegradationScore, isDirectProbe, Debugger.IsAttached); + probeStartTimestamp = _timeProvider.GetTimestamp(); using var cancellation = new CancellationTokenSource(timeout, _timeProvider); if (isDirectProbe) @@ -210,7 +222,8 @@ private async Task Run() // Select a timeout which will allow the intermediary node to attempt to probe the target node and still respond to this node // if the remote node does not respond in time. // Attempt to account for local health degradation by extending the timeout period. - probeResult = await this.ProbeIndirectly(intermediary, timeout, cancellation.Token).ConfigureAwait(false); + var directProbeTimeout = CalculateIndirectProbeTargetTimeout(timeout, localDegradationScore); + probeResult = await this.ProbeIndirectly(intermediary, directProbeTimeout, cancellation.Token).ConfigureAwait(false); // If the intermediary is not entirely healthy, remove it from consideration and continue to probe. // Note that all recused silos will be included in the consideration set the next time cluster membership changes. @@ -218,7 +231,6 @@ private async Task Run() { LogInformationRecusingUnhealthyIntermediary(_log, intermediary); otherNodes = [.. otherNodes.Where(node => !node.Equals(intermediary))]; - overrideDelay = TimeSpan.FromMilliseconds(250); } } @@ -231,37 +243,78 @@ private async Task Run() { LogErrorExceptionMonitoringSilo(_log, exception, TargetSiloAddress); } + finally + { + previousProbePeriod = CalculateProbeTimeout( + failureDetector, + options, + localDegradationScore, + isDirectProbe, + Debugger.IsAttached); + nextProbeDelay = probeStartTimestamp is { } timestamp + ? CalculateNextProbeDelay(previousProbePeriod, _timeProvider.GetElapsedTime(timestamp)) + : previousProbePeriod; + } } - TimeSpan GetTimeout(bool isDirectProbe) + int GetLocalDegradationScore(TimeSpan period) { - var additionalTimeout = 0; - - if (options.ExtendProbeTimeoutDuringDegradation) + if (!options.ExtendProbeTimeoutDuringDegradation) { - // This query must happen before the probe because its result determines the probe timeout. - // Outcome-reporting health checks, such as an intermediary's health score, run after probing. - var localHealth = _localSiloHealthMonitor.GetLocalHealthStatus( - options.ProbeTimeout, - LocalSiloHealthCheckCategory.Local); - additionalTimeout += localHealth.Score; + return 0; } - if (!isDirectProbe) - { - // Indirect probes need extra time to account for the additional hop. - additionalTimeout += 1; - } + // This query must happen before the probe because its result determines the probe timeout. + // Outcome-reporting health checks, such as an intermediary's health score, run after probing. + return _localSiloHealthMonitor.GetLocalHealthStatus( + period, + LocalSiloHealthCheckCategory.Local).Score; + } + } - // When the debugger is attached, extend probe times so that silos are not terminated - // due to debugging pauses. - if (Debugger.IsAttached) - { - additionalTimeout += 25; - } + internal static TimeSpan CalculateProbeTimeout( + PhiAccrualFailureDetector failureDetector, + ClusterMembershipOptions options, + int localDegradationScore, + bool isDirectProbe, + bool isDebuggerAttached) + { + var extensionFactor = 1 + localDegradationScore; + if (!isDirectProbe) + { + extensionFactor++; + } - return options.ProbeTimeout.Multiply(1 + additionalTimeout); + var timeout = failureDetector.GetTimeout(options.MinProbeTimeout, options.MaxProbeTimeout, extensionFactor); + if (isDebuggerAttached) + { + var debuggerExtensionTicks = failureDetector.GetTimeout().Ticks * 25d; + var extendedTimeoutTicks = Math.Min(MaxSupportedTimerTimeout.Ticks, timeout.Ticks + debuggerExtensionTicks); + timeout = TimeSpan.FromTicks((long)extendedTimeoutTicks); + } + + return timeout; + } + + internal static TimeSpan CalculateNextProbeDelay(TimeSpan probePeriod, TimeSpan elapsed) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(probePeriod, TimeSpan.Zero); + if (elapsed < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(elapsed), elapsed, "Elapsed time must not be negative."); } + + return elapsed < probePeriod ? probePeriod - elapsed : TimeSpan.Zero; + } + + internal static TimeSpan CalculateIndirectProbeTargetTimeout(TimeSpan timeout, int localDegradationScore) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfNegative(localDegradationScore); + + var extensionFactor = 1 + localDegradationScore; + var responseAllowanceTicks = Math.Max(1, timeout.Ticks / (extensionFactor + 1)); + return TimeSpan.FromTicks(timeout.Ticks - responseAllowanceTicks); } /// @@ -303,6 +356,7 @@ private async Task ProbeDirectly(CancellationToken cancellation) _failedProbes = 0; _lastSuccessfulResponseTimestamp = _timeProvider.GetTimestamp(); LastRoundTripTime = roundTripTime; + _failureDetector!.RecordResponseTime(roundTripTime); probeResult = ProbeResult.CreateDirect(0, ProbeResultStatus.Succeeded); } else diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index 74b856391ec..4dd96e5d5f3 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -28,7 +28,7 @@ void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) Task OnStart(CancellationToken _) { // This gives time for the cluster to be voted Dead and for membership updates to propagate that out - negativeCachePeriod = clusterMembershipOptions.Value.ProbeTimeout * clusterMembershipOptions.Value.NumMissedProbesLimit + negativeCachePeriod = clusterMembershipOptions.Value.GetFailureDetectionTimeout() + (2 * clusterMembershipOptions.Value.TableRefreshTimeout); task = Task.Run(() => this.ProcessMembershipUpdates(_cts.Token)); return Task.CompletedTask; @@ -144,4 +144,3 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) Message = "Stopping membership update processor")] private static partial void LogDebugStoppingMembershipProcessor(ILogger logger); } - diff --git a/src/Orleans.TestingHost/InProcTestCluster.cs b/src/Orleans.TestingHost/InProcTestCluster.cs index 40e4b8bfa2a..c4889ab9375 100644 --- a/src/Orleans.TestingHost/InProcTestCluster.cs +++ b/src/Orleans.TestingHost/InProcTestCluster.cs @@ -413,7 +413,7 @@ public static TimeSpan GetLivenessStabilizationTime(ClusterMembershipOptions clu if (didKill) { // in case of hard kill (kill and not Stop), we should give silos time to detect failures first. - stabilizationTime = TestingUtils.Multiply(clusterMembershipOptions.ProbeTimeout, clusterMembershipOptions.NumMissedProbesLimit); + stabilizationTime = TestingUtils.Multiply(clusterMembershipOptions.MaxProbeTimeout, clusterMembershipOptions.NumMissedProbesLimit); } if (clusterMembershipOptions.UseLivenessGossip) { diff --git a/src/Orleans.TestingHost/TestCluster.cs b/src/Orleans.TestingHost/TestCluster.cs index c640b57a019..67f4c22e9cc 100644 --- a/src/Orleans.TestingHost/TestCluster.cs +++ b/src/Orleans.TestingHost/TestCluster.cs @@ -485,7 +485,7 @@ public static TimeSpan GetLivenessStabilizationTime(ClusterMembershipOptions clu if (didKill) { // in case of hard kill (kill and not Stop), we should give silos time to detect failures first. - stabilizationTime = TestingUtils.Multiply(clusterMembershipOptions.ProbeTimeout, clusterMembershipOptions.NumMissedProbesLimit); + stabilizationTime = TestingUtils.Multiply(clusterMembershipOptions.MaxProbeTimeout, clusterMembershipOptions.NumMissedProbesLimit); } if (clusterMembershipOptions.UseLivenessGossip) { diff --git a/src/api/Orleans.Core/Orleans.Core.cs b/src/api/Orleans.Core/Orleans.Core.cs index 16150922ebc..f95bcbd4953 100644 --- a/src/api/Orleans.Core/Orleans.Core.cs +++ b/src/api/Orleans.Core/Orleans.Core.cs @@ -461,6 +461,10 @@ public partial class ClusterMembershipOptions public System.TimeSpan MaxJoinAttemptTime { get { throw null; } set { } } + public System.TimeSpan MaxProbeTimeout { get { throw null; } set { } } + + public System.TimeSpan MinProbeTimeout { get { throw null; } set { } } + public int NumMissedProbesLimit { get { throw null; } set { } } public int NumMissedTableIAmAliveLimit { get { throw null; } set { } } diff --git a/test/Orleans.Core.Tests/Membership/ClusterMembershipOptionsTests.cs b/test/Orleans.Core.Tests/Membership/ClusterMembershipOptionsTests.cs new file mode 100644 index 00000000000..d5deab6cf39 --- /dev/null +++ b/test/Orleans.Core.Tests/Membership/ClusterMembershipOptionsTests.cs @@ -0,0 +1,47 @@ +using Orleans.Configuration; +using Xunit; + +namespace NonSilo.Tests.Membership; + +[TestCategory("BVT"), TestCategory("Membership")] +public class ClusterMembershipOptionsTests +{ + [Fact] + public void ProbeTimeoutBoundsTrackInitialTimeout() + { + var options = new ClusterMembershipOptions(); + + Assert.Equal(TimeSpan.FromSeconds(5), options.ProbeTimeout); + Assert.Equal(TimeSpan.FromSeconds(2.5), options.MinProbeTimeout); + Assert.Equal(TimeSpan.FromSeconds(20), options.MaxProbeTimeout); + + options.ProbeTimeout = TimeSpan.FromSeconds(8); + + Assert.Equal(TimeSpan.FromSeconds(4), options.MinProbeTimeout); + Assert.Equal(TimeSpan.FromSeconds(32), options.MaxProbeTimeout); + } + + [Fact] + public void ExplicitProbeTimeoutBoundsDoNotTrackInitialTimeout() + { + var options = new ClusterMembershipOptions + { + MinProbeTimeout = TimeSpan.FromSeconds(2), + MaxProbeTimeout = TimeSpan.FromSeconds(12), + }; + + options.ProbeTimeout = TimeSpan.FromSeconds(8); + + Assert.Equal(TimeSpan.FromSeconds(2), options.MinProbeTimeout); + Assert.Equal(TimeSpan.FromSeconds(12), options.MaxProbeTimeout); + } + + [Fact] + public void ProbeTimeoutIsNotObsolete() + { + var property = typeof(ClusterMembershipOptions).GetProperty("ProbeTimeout"); + + Assert.NotNull(property); + Assert.Empty(property.GetCustomAttributes(typeof(ObsoleteAttribute), inherit: true)); + } +} diff --git a/test/Orleans.Core.Tests/Membership/PhiAccrualFailureDetectorTests.cs b/test/Orleans.Core.Tests/Membership/PhiAccrualFailureDetectorTests.cs new file mode 100644 index 00000000000..3f6e697c021 --- /dev/null +++ b/test/Orleans.Core.Tests/Membership/PhiAccrualFailureDetectorTests.cs @@ -0,0 +1,109 @@ +using Orleans.Configuration; +using Orleans.Runtime.MembershipService; +using Xunit; + +namespace NonSilo.Tests.Membership; + +[TestCategory("BVT"), TestCategory("Membership")] +public class PhiAccrualFailureDetectorTests +{ + [Fact] + public void UsesInitialTimeoutUntilEnoughEvidenceIsAvailable() + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + + for (var i = 0; i < 3; i++) + { + detector.RecordResponseTime(TimeSpan.Zero); + } + + Assert.Equal(TimeSpan.FromSeconds(5), detector.GetTimeout()); + } + + [Fact] + public void StableFastResponsesLowerTimeout() + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + + for (var i = 0; i < 4; i++) + { + detector.RecordResponseTime(TimeSpan.Zero); + } + + Assert.Equal(TimeSpan.FromSeconds(2.5), detector.GetTimeout()); + } + + [Fact] + public void StableSlowResponsesRaiseTimeout() + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + + for (var i = 0; i < 4; i++) + { + detector.RecordResponseTime(TimeSpan.FromSeconds(4)); + } + + Assert.Equal(TimeSpan.FromSeconds(6.5), detector.GetTimeout()); + } + + [Fact] + public void EffectiveTimeoutIsClampedAfterExtensions() + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + + for (var i = 0; i < 4; i++) + { + detector.RecordResponseTime(TimeSpan.FromSeconds(4)); + } + + var timeout = detector.GetTimeout(TimeSpan.FromSeconds(2.5), TimeSpan.FromSeconds(10), extensionFactor: 2); + + Assert.Equal(TimeSpan.FromSeconds(10), timeout); + } + + [Fact] + public void HistoryIsBounded() + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + for (var i = 0; i < 100; i++) + { + detector.RecordResponseTime(TimeSpan.Zero); + } + + for (var i = 0; i < 100; i++) + { + detector.RecordResponseTime(TimeSpan.FromSeconds(4)); + } + + Assert.Equal(100, detector.SampleCount); + Assert.Equal(TimeSpan.FromSeconds(6.5), detector.GetTimeout()); + } + + [Theory] + [InlineData(0, true, false, 5)] + [InlineData(1, true, false, 10)] + [InlineData(0, false, false, 10)] + [InlineData(0, true, true, 130)] + public void MonitorExtensionsAndClampAreAppliedInOrder( + int localDegradationScore, + bool isDirectProbe, + bool isDebuggerAttached, + double expectedSeconds) + { + var detector = new PhiAccrualFailureDetector(TimeSpan.FromSeconds(5)); + var options = new ClusterMembershipOptions + { + MinProbeTimeout = TimeSpan.FromSeconds(2.5), + MaxProbeTimeout = TimeSpan.FromSeconds(10), + }; + + var timeout = SiloHealthMonitor.CalculateProbeTimeout( + detector, + options, + localDegradationScore, + isDirectProbe, + isDebuggerAttached); + + Assert.Equal(TimeSpan.FromSeconds(expectedSeconds), timeout); + } +} diff --git a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs index 880833d9c53..86d9d0ef57c 100644 --- a/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs +++ b/test/Orleans.Core.Tests/Membership/SiloHealthMonitorTests.cs @@ -149,6 +149,58 @@ public async Task SiloHealthMonitor_SuccessfulProbe() await Shutdown(); } + [Fact] + public async Task SiloHealthMonitor_SuccessfulDirectProbesAdaptTimeout() + { + _prober.Probe(default!, default).ReturnsForAnyArgs(Task.CompletedTask); + _monitor.Start(); + + for (var i = 0; i < 4; i++) + { + var timerCall = await _timerCalls.Reader.ReadAsync(); + timerCall.Completion.TrySetResult(true); + var probeResult = await _probeResults.Reader.ReadAsync(); + Assert.Equal(ProbeResultStatus.Succeeded, probeResult.Status); + Assert.True(probeResult.IsDirectProbe); + } + + var testAccessor = (ITestAccessor)_monitor; + Assert.Equal(4, testAccessor.ProbeTimeoutSampleCount); + Assert.InRange( + testAccessor.CurrentProbeTimeout, + _clusterMembershipOptions.MinProbeTimeout, + _clusterMembershipOptions.ProbeTimeout); + + await Shutdown(); + } + + [Fact] + public async Task SiloHealthMonitor_SuccessfulIndirectProbeDoesNotAdaptTimeout() + { + _clusterMembershipOptions.NumMissedProbesLimit = 1; + var intermediary = Silo("127.0.0.1:1234@1234"); + await _membershipTable.InsertRow( + Entry(intermediary, SiloStatus.Active, DateTime.UtcNow), + _membershipTable.Version.Next()); + await _membershipService.Refresh(); + _prober.ProbeIndirectly(default!, default!, default, default).ReturnsForAnyArgs(new IndirectProbeResponse + { + Succeeded = true, + ProbeResponseTime = TimeSpan.FromMilliseconds(1), + }); + + _monitor.Start(); + var timerCall = await _timerCalls.Reader.ReadAsync(); + timerCall.Completion.TrySetResult(true); + var probeResult = await _probeResults.Reader.ReadAsync(); + + Assert.Equal(ProbeResultStatus.Succeeded, probeResult.Status); + Assert.False(probeResult.IsDirectProbe); + Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); + + await Shutdown(); + } + [Fact] public async Task SiloHealthMonitor_FailedProbe_Timeout() { @@ -167,6 +219,7 @@ public async Task SiloHealthMonitor_FailedProbe_Timeout() Assert.Equal(1, probeResult.FailedProbeCount); Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); + Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); await Shutdown(); } @@ -194,6 +247,7 @@ public async Task SiloHealthMonitor_FailedProbe_Exception() Assert.Equal(1, probeResult.FailedProbeCount); Assert.True(probeResult.IsDirectProbe); Assert.Equal(0, probeResult.IntermediaryHealthDegradationScore); + Assert.Equal(0, ((ITestAccessor)_monitor).ProbeTimeoutSampleCount); await Shutdown(); } @@ -260,6 +314,7 @@ public async Task SiloHealthMonitor_Indirect_FailedProbe() var args = probeCall.GetArguments(); var intermediary = Assert.IsType(args[0]); Assert.Equal(otherSilo, intermediary); + Assert.Equal(TimeSpan.FromSeconds(2), Assert.IsType(args[2])); // Ensure that negative results from unhealthy intermediaries are not considered. _prober.ProbeIndirectly(default!, default!, default, default).ReturnsForAnyArgs(new IndirectProbeResponse @@ -362,6 +417,7 @@ public async Task SiloHealthMonitor_DirectProbeTimeoutUsesConfiguredLocalHealthD var options = new ClusterMembershipOptions { ProbeTimeout = TimeSpan.FromSeconds(5), + MaxProbeTimeout = TimeSpan.FromSeconds(45), ExtendProbeTimeoutDuringDegradation = extendProbeTimeout, EnableIndirectProbes = false, }; @@ -402,7 +458,7 @@ public async Task SiloHealthMonitor_DirectProbeTimeoutUsesConfiguredLocalHealthD { monitor.Start(); var firstTick = await timer.Ticks.ReadAsync(); - firstTick.SetResult(true); + firstTick.Completion.SetResult(true); var cancellationToken = await probeEntered.Task; if (extendProbeTimeout) @@ -431,6 +487,10 @@ public async Task SiloHealthMonitor_DirectProbeTimeoutUsesConfiguredLocalHealthD Assert.Equal(1, result.FailedProbeCount); Assert.True(result.IsDirectProbe); Assert.Equal(0, result.IntermediaryHealthDegradationScore); + + var nextTick = await timer.Ticks.ReadAsync(); + Assert.Equal(TimeSpan.Zero, nextTick.DelayOverride); + nextTick.Completion.SetResult(false); } finally { @@ -438,13 +498,29 @@ public async Task SiloHealthMonitor_DirectProbeTimeoutUsesConfiguredLocalHealthD } } + [Theory] + [InlineData(5, 1, 4)] + [InlineData(5, 5, 0)] + [InlineData(5, 6, 0)] + public void CalculateNextProbeDelay_MeasuresFromPreviousProbeStart( + int probePeriodSeconds, + int elapsedSeconds, + int expectedDelaySeconds) + { + var delay = CalculateNextProbeDelay( + TimeSpan.FromSeconds(probePeriodSeconds), + TimeSpan.FromSeconds(elapsedSeconds)); + + Assert.Equal(TimeSpan.FromSeconds(expectedDelaySeconds), delay); + } + private sealed class DilationProbeTimer : IAsyncTimer { private readonly object _lock = new(); - private readonly Channel> _ticks = Channel.CreateUnbounded>(); + private readonly Channel _ticks = Channel.CreateUnbounded(); private bool _disposed; - public ChannelReader> Ticks => _ticks.Reader; + public ChannelReader Ticks => _ticks.Reader; public Task NextTick(TimeSpan? overrideDelay = null) { @@ -456,7 +532,7 @@ public Task NextTick(TimeSpan? overrideDelay = null) } var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - if (!_ticks.Writer.TryWrite(completion)) + if (!_ticks.Writer.TryWrite(new(overrideDelay, completion))) { throw new InvalidOperationException("Unable to publish the next timer tick."); } @@ -482,14 +558,16 @@ public void Dispose() _disposed = true; _ticks.Writer.TryComplete(); - while (_ticks.Reader.TryRead(out var completion)) + while (_ticks.Reader.TryRead(out var tick)) { - completion.TrySetResult(false); + tick.Completion.TrySetResult(false); } } } } + private readonly record struct ProbeTimerTick(TimeSpan? DelayOverride, TaskCompletionSource Completion); + private static SiloAddress Silo(string value) => SiloAddress.FromParsableString(value); private static MembershipEntry Entry(SiloAddress address, SiloStatus status, DateTimeOffset iAmAliveTime = default) => new() diff --git a/test/Orleans.Core.Tests/SiloBuilderTests.cs b/test/Orleans.Core.Tests/SiloBuilderTests.cs index 071db563369..c1d359d2cfd 100644 --- a/test/Orleans.Core.Tests/SiloBuilderTests.cs +++ b/test/Orleans.Core.Tests/SiloBuilderTests.cs @@ -153,6 +153,80 @@ await Assert.ThrowsAsync(async () => .Configure(options => options.MaxDefunctSiloEntries = -1); }).RunConsoleAsync(); }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => options.ProbeTimeout = TimeSpan.Zero); + }).RunConsoleAsync(); + }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => options.MinProbeTimeout = TimeSpan.FromSeconds(6)); + }).RunConsoleAsync(); + }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => + { + options.MinProbeTimeout = TimeSpan.FromSeconds(1); + options.MaxProbeTimeout = TimeSpan.FromSeconds(4); + }); + }).RunConsoleAsync(); + }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => options.MaxProbeTimeout = TimeSpan.FromDays(50)); + }).RunConsoleAsync(); + }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => + { + options.ProbeTimeout = TimeSpan.FromDays(1); + options.MaxProbeTimeout = TimeSpan.FromDays(49); + options.NumMissedProbesLimit = int.MaxValue; + }); + }).RunConsoleAsync(); + }); + + await Assert.ThrowsAsync(async () => + { + await new HostBuilder().UseOrleans((ctx, siloBuilder) => + { + siloBuilder + .UseLocalhostClustering() + .Configure(options => + { + options.ProbeTimeout = TimeSpan.FromTicks(4_611_686_018); + options.MaxProbeTimeout = options.ProbeTimeout; + options.NumMissedProbesLimit = 2_000_000_000; + }); + }).RunConsoleAsync(); + }); } /// diff --git a/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryLeaseTests.cs b/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryLeaseTests.cs index 3124ed19f80..bcb3ab246fd 100644 --- a/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryLeaseTests.cs +++ b/test/Orleans.GrainDirectory.Tests/GrainDirectory/GrainDirectoryLeaseTests.cs @@ -181,7 +181,7 @@ public void DefaultRangeLeaseDuration_IsThirtySeconds() => Assert.Equal(TimeSpan.FromSeconds(30), new GrainDirectoryOptions().RangeLeaseDuration); [Fact] - public void DefaultRangeLeaseDuration_LeavesFifteenSecondsAfterFailureDetection() + public void DefaultRangeLeaseDuration_IsConsumedByWorstCaseFailureDetection() { var membershipOptions = new ClusterMembershipOptions(); @@ -189,7 +189,7 @@ public void DefaultRangeLeaseDuration_LeavesFifteenSecondsAfterFailureDetection( new GrainDirectoryOptions().RangeLeaseDuration, membershipOptions); - Assert.Equal(TimeSpan.FromSeconds(15), duration); + Assert.Equal(TimeSpan.Zero, duration); } [Fact] @@ -401,6 +401,7 @@ private static (InProcessTestCluster Cluster, FakeTimeProvider TimeProvider) Cre siloBuilder.Services.Configure(options => { options.ProbeTimeout = TimeSpan.FromSeconds(1); + options.MaxProbeTimeout = TimeSpan.FromSeconds(1); options.NumMissedProbesLimit = 1; }); siloBuilder.Services.PostConfigure(o => o.RangeLeaseDuration = rangeLeaseDuration ?? RangeLeaseDuration); diff --git a/test/Orleans.Runtime.Internal.Tests/MembershipTests/ClientIdPartitionDataRebuildTests.cs b/test/Orleans.Runtime.Internal.Tests/MembershipTests/ClientIdPartitionDataRebuildTests.cs index f8fcf4d2d51..d3be84bfdd7 100644 --- a/test/Orleans.Runtime.Internal.Tests/MembershipTests/ClientIdPartitionDataRebuildTests.cs +++ b/test/Orleans.Runtime.Internal.Tests/MembershipTests/ClientIdPartitionDataRebuildTests.cs @@ -145,6 +145,7 @@ public void Configure(ISiloBuilder hostBuilder) { options.NumMissedProbesLimit = 1; options.ProbeTimeout = TimeSpan.FromMilliseconds(500); + options.MaxProbeTimeout = TimeSpan.FromMilliseconds(500); options.NumVotesForDeathDeclaration = 1; });