feat(membership): adapt probe timeouts

docs/site/src/content/docs/implementation/cluster-management.md

@@ -1,7 +1,7 @@
 ---
 title: Cluster membership protocol
 description: Understand Orleans membership storage, failure detection, ordered views, and death-vote invariants.
-ms.date: 08/02/2026
+ms.date: 08/11/2026
 ms.topic: concept-article
 ---
 
@@ -41,6 +41,10 @@ The periodic `IAmAlive` value is not the peer heartbeat. It is a timestamp writt
 
 Active silos monitor peers selected from the membership view. `ClusterHealthMonitor` sends probes over silo-to-silo messaging, tracks consecutive failures, and can use indirect probes to distinguish a failed target from an unhealthy observer. A failed monitor writes a timestamped vote into the target's membership row.
 
+Each observer maintains a [Phi Accrual failure detector](https://paperhub.s3.amazonaws.com/f516fdfa940caa08c679d3946b273128.pdf) for each peer. The detector models successful direct-probe round-trip times and estimates the timeout at which the probability of a later response is sufficiently low. The timeout starts at <xref:Orleans.Configuration.ClusterMembershipOptions.ProbeTimeout?displayProperty=nameWithType> and adapts after enough observations. Failures are excluded because they only show that the response exceeded the current timeout, while indirect results are excluded because they measure a different observer's network path.
+
+The learned timeout also determines probe cadence. Each probe is scheduled relative to the previous probe's start, so a quick response waits for the remainder of the current timeout while a probe which consumes its timeout is followed immediately by the next attempt. Local-health and indirect-hop extensions are applied to the learned timeout before it is clamped between <xref:Orleans.Configuration.ClusterMembershipOptions.MinProbeTimeout?displayProperty=nameWithType> and <xref:Orleans.Configuration.ClusterMembershipOptions.MaxProbeTimeout?displayProperty=nameWithType>. Debugger-specific extensions are applied after the clamp so a paused process is not accused because of the configured production bound.
+
 ```mermaid
 sequenceDiagram
     participant A as Monitoring silo A
@@ -70,7 +74,9 @@ The defaults are defined by <xref:Orleans.Configuration.ClusterMembershipOptions
 | Option | Default | Protocol role |
 | --- | ---: | --- |
 | <xref:Orleans.Configuration.ClusterMembershipOptions.NumProbedSilos?displayProperty=nameWithType> | 10 | Number of peers monitored by each silo |
-| <xref:Orleans.Configuration.ClusterMembershipOptions.ProbeTimeout?displayProperty=nameWithType> | 5 seconds | Baseline direct probe timeout |
+| <xref:Orleans.Configuration.ClusterMembershipOptions.ProbeTimeout?displayProperty=nameWithType> | 5 seconds | Initial timeout and probe period before the peer has supplied enough evidence |
+| <xref:Orleans.Configuration.ClusterMembershipOptions.MinProbeTimeout?displayProperty=nameWithType> | Half the initial timeout (2.5 seconds by default) | Lower bound for an effective probe timeout |
+| <xref:Orleans.Configuration.ClusterMembershipOptions.MaxProbeTimeout?displayProperty=nameWithType> | Four times the initial timeout (20 seconds by default) | Upper bound for an effective probe timeout |
 | <xref:Orleans.Configuration.ClusterMembershipOptions.NumMissedProbesLimit?displayProperty=nameWithType> | 3 | Failed probes before a death vote |
 | <xref:Orleans.Configuration.ClusterMembershipOptions.NumVotesForDeathDeclaration?displayProperty=nameWithType> | 2 | Fresh votes required to mark a member dead |
 | <xref:Orleans.Configuration.ClusterMembershipOptions.DeathVoteExpirationTimeout?displayProperty=nameWithType> | 2 minutes | Lifetime of a death vote |

src/Orleans.Core/Configuration/Options/ClusterMembershipOptions.cs

@@ -7,6 +7,9 @@ namespace Orleans.Configuration
     /// </summary>
     public class ClusterMembershipOptions
     {
+        private TimeSpan? _minProbeTimeout;
+        private TimeSpan? _maxProbeTimeout;
+
         /// <summary>
         /// Gets or sets the number of missed "I am alive" updates in the table from a silo that causes warning to be logged.
         /// </summary>
@@ -23,11 +26,38 @@ public class ClusterMembershipOptions
         public bool LivenessEnabled { get; set; } = true;
 
         /// <summary>
-        /// Gets or sets both the period between sending a liveness probe to any given host as well as the timeout for each probe.
+        /// Gets or sets the initial timeout for liveness probes.
         /// </summary>
-        /// <value>Probes time out and a new probe is sent every 5 seconds by default.</value>
+        /// <remarks>
+        /// The effective probe timeout and the period between probe starts are adjusted independently for each monitored silo
+        /// based on observed direct-probe response times and are bounded by <see cref="MinProbeTimeout"/> and
+        /// <see cref="MaxProbeTimeout"/>.
+        /// </remarks>
+        /// <value>The initial probe timeout is 5 seconds by default.</value>
         public TimeSpan ProbeTimeout { get; set; } = TimeSpan.FromSeconds(5);
 
+        /// <summary>
+        /// Gets or sets the minimum effective timeout for a liveness probe.
+        /// </summary>
+        /// <value>Half of <see cref="ProbeTimeout"/> by default.</value>
+        public TimeSpan MinProbeTimeout
+        {
+            get => _minProbeTimeout ?? TimeSpan.FromTicks(ProbeTimeout.Ticks / 2);
+            set => _minProbeTimeout = value;
+        }
+
+        /// <summary>
+        /// Gets or sets the maximum effective timeout for a liveness probe.
+        /// </summary>
+        /// <value>Four times <see cref="ProbeTimeout"/> by default.</value>
+        public TimeSpan MaxProbeTimeout
+        {
+            get => _maxProbeTimeout ?? (ProbeTimeout.Ticks <= TimeSpan.MaxValue.Ticks / 4
+                ? TimeSpan.FromTicks(ProbeTimeout.Ticks * 4)
+                : TimeSpan.MaxValue);
+            set => _maxProbeTimeout = value;
+        }
+
         /// <summary>
         /// Gets or sets the period between fetching updates from the membership table.
         /// </summary>
@@ -131,7 +161,7 @@ public class ClusterMembershipOptions
         public TimeSpan LocalHealthDegradationMonitoringPeriod { get; set; } = TimeSpan.FromSeconds(10);
 
         /// <summary>
-        /// Gets or sets a value indicating whether to extend the effective <see cref="ProbeTimeout"/> value based upon current local health degradation.
+        /// Gets or sets a value indicating whether to extend the effective probe timeout based upon current local health degradation.
         /// </summary>
         public bool ExtendProbeTimeoutDuringDegradation { get; set; } = true;
 
@@ -145,7 +175,7 @@ public class ClusterMembershipOptions
         /// </summary>
         /// <remarks>
         /// When enabled, if an active connection to a silo has recently received messages within the monitoring window
-        /// (<see cref="ProbeTimeout"/> × <see cref="NumMissedProbesLimit"/>), votes to suspect that silo will be suppressed
+        /// (<see cref="MaxProbeTimeout"/> × <see cref="NumMissedProbesLimit"/>), votes to suspect that silo will be suppressed
         /// since the connection activity demonstrates the silo is alive. This helps prevent false death declarations
         /// when probes fail due to local issues such as GC pauses or thread pool saturation.
         /// </remarks>

src/Orleans.Runtime/Configuration/ClusterMembershipOptionsExtensions.cs

@@ -0,0 +1,10 @@
+using System;
+using Orleans.Internal;
+
+namespace Orleans.Configuration;
+
+internal static class ClusterMembershipOptionsExtensions
+{
+    internal static TimeSpan GetFailureDetectionTimeout(this ClusterMembershipOptions options) =>
+        options.MaxProbeTimeout.Multiply(options.NumMissedProbesLimit);
+}

src/Orleans.Runtime/Configuration/Validators/SiloClusteringValidator.cs

@@ -13,6 +13,7 @@ namespace Orleans.Runtime.Configuration
     /// </summary>
     internal class SiloClusteringValidator : IConfigurationValidator
     {
+        private const uint MaxSupportedTimeoutMilliseconds = 0xfffffffe;
         private readonly IServiceProvider serviceProvider;
 
         public SiloClusteringValidator(IServiceProvider serviceProvider)
@@ -51,6 +52,56 @@ public void ValidateConfiguration()
                 throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxDefunctSiloEntries)} ({clusterMembershipOptions.MaxDefunctSiloEntries}) must be greater than or equal to 0, or null.");
             }
 
+            if (clusterMembershipOptions.ProbeTimeout <= TimeSpan.Zero)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be greater than 0.");
+            }
+
+            if (clusterMembershipOptions.ProbeTimeout.TotalMilliseconds > MaxSupportedTimeoutMilliseconds)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be less than or equal to {TimeSpan.FromMilliseconds(MaxSupportedTimeoutMilliseconds)}.");
+            }
+
+            if (clusterMembershipOptions.MinProbeTimeout <= TimeSpan.Zero)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout}) must be greater than 0.");
+            }
+
+            if (clusterMembershipOptions.MaxProbeTimeout < clusterMembershipOptions.MinProbeTimeout)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout}) must be greater than or equal to {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout}).");
+            }
+
+            if (clusterMembershipOptions.MaxProbeTimeout.TotalMilliseconds > MaxSupportedTimeoutMilliseconds)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout}) must be less than or equal to {TimeSpan.FromMilliseconds(MaxSupportedTimeoutMilliseconds)}.");
+            }
+
+            if (clusterMembershipOptions.ProbeTimeout < clusterMembershipOptions.MinProbeTimeout
+                || clusterMembershipOptions.ProbeTimeout > clusterMembershipOptions.MaxProbeTimeout)
+            {
+                throw new OrleansConfigurationException($"{nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.ProbeTimeout)} ({clusterMembershipOptions.ProbeTimeout}) must be between {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MinProbeTimeout)} ({clusterMembershipOptions.MinProbeTimeout}) and {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.MaxProbeTimeout)} ({clusterMembershipOptions.MaxProbeTimeout}).");
+            }
+
+            var maxProbeCycleTime = clusterMembershipOptions.MaxProbeTimeout;
+            var failureDetectionTimeoutTicks = 0L;
+            if (clusterMembershipOptions.NumMissedProbesLimit > 0
+                && maxProbeCycleTime.Ticks > TimeSpan.MaxValue.Ticks / clusterMembershipOptions.NumMissedProbesLimit)
+            {
+                throw new OrleansConfigurationException($"The maximum probe cycle time ({maxProbeCycleTime}) multiplied by {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.NumMissedProbesLimit)} ({clusterMembershipOptions.NumMissedProbesLimit}) must not exceed {TimeSpan.MaxValue}.");
+            }
+            else if (clusterMembershipOptions.NumMissedProbesLimit > 0)
+            {
+                failureDetectionTimeoutTicks = maxProbeCycleTime.Ticks * clusterMembershipOptions.NumMissedProbesLimit;
+            }
+
+            var tableRefreshTimeoutTicks = clusterMembershipOptions.TableRefreshTimeout.Ticks;
+            if (tableRefreshTimeoutTicks > 0
+                && tableRefreshTimeoutTicks > (TimeSpan.MaxValue.Ticks - failureDetectionTimeoutTicks) / 2)
+            {
+                throw new OrleansConfigurationException($"The failure detection timeout plus twice {nameof(ClusterMembershipOptions)}.{nameof(ClusterMembershipOptions.TableRefreshTimeout)} ({clusterMembershipOptions.TableRefreshTimeout}) must not exceed {TimeSpan.MaxValue}.");
+            }
+
             if (clusterMembershipOptions.LivenessEnabled)
             {
                 if (clusterMembershipOptions.NumVotesForDeathDeclaration > clusterMembershipOptions.NumProbedSilos)

src/Orleans.Runtime/GrainDirectory/DistributedGrainDirectory.cs

@@ -140,7 +140,7 @@ internal static TimeSpan CalculateDeadSiloLeaseDuration(
         TimeSpan rangeLeaseDuration,
         ClusterMembershipOptions membershipOptions)
     {
-        var failureDetectionDuration = membershipOptions.ProbeTimeout * membershipOptions.NumMissedProbesLimit;
+        var failureDetectionDuration = membershipOptions.GetFailureDetectionTimeout();
         return rangeLeaseDuration > failureDetectionDuration
             ? rangeLeaseDuration - failureDetectionDuration
             : TimeSpan.Zero;

src/Orleans.Runtime/MembershipService/ClusterHealthMonitor.cs

@@ -102,7 +102,7 @@ private async Task ProcessMembershipUpdates()
                         if (!newMonitoredSilos.ContainsKey(pair.Key))
                         {
                             using var cancellation = new CancellationTokenSource(
-                                this.clusterMembershipOptions.CurrentValue.ProbeTimeout,
+                                this.clusterMembershipOptions.CurrentValue.MaxProbeTimeout,
                                 this.timeProvider);
                             await pair.Value.StopAsync(cancellation.Token);
                         }
@@ -359,7 +359,7 @@ private async Task OnProbeResultInternal(SiloHealthMonitor monitor, ProbeResult
 
         /// <summary>
         /// Checks whether a connection to the specified silo has received a message within the monitoring window
-        /// (<see cref="ClusterMembershipOptions.ProbeTimeout"/> × <see cref="ClusterMembershipOptions.NumMissedProbesLimit"/>).
+        /// (the maximum probe cycle time multiplied by <see cref="ClusterMembershipOptions.NumMissedProbesLimit"/>).
         /// If so, the silo is demonstrably alive and the vote should be suppressed.
         /// </summary>
         private bool IsConnectionActiveWithinMonitoringWindow(SiloAddress targetSilo)
@@ -370,7 +370,7 @@ private bool IsConnectionActiveWithinMonitoringWindow(SiloAddress targetSilo)
                 return false;
             }
 
-            var monitoringWindow = options.ProbeTimeout.Multiply(options.NumMissedProbesLimit);
+            var monitoringWindow = options.GetFailureDetectionTimeout();
 
             if (this.connectionManager.GetElapsedSinceLastMessageReceived(targetSilo) is { } elapsed && elapsed <= monitoringWindow)
             {

src/Orleans.Runtime/MembershipService/LocalSiloHealthMonitor.cs

@@ -326,7 +326,7 @@ private LocalSiloHealthStatus EnsureNetworkHealthCheck(
                 }
 
                 // Only consider certain checks if the silo has been a member of a multi-silo cluster for a certain period.
-                var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit);
+                var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout();
                 if (_clusteredSinceTimestamp is { } clusteredSince
                     && _timeProvider.GetElapsedTime(clusteredSince, timestamp) > recencyWindow)
                 {

src/Orleans.Runtime/MembershipService/PhiAccrualFailureDetector.cs

@@ -0,0 +1,104 @@
+using System;
+
+namespace Orleans.Runtime.MembershipService;
+
+internal sealed class PhiAccrualFailureDetector
+{
+    private const double Threshold = 8;
+    private const int MaxSampleSize = 100;
+    private const int MinimumSampleCount = 4;
+    private static readonly double ThresholdStandardDeviations = CalculateThresholdStandardDeviations();
+
+    private readonly double[] _samples = new double[MaxSampleSize];
+    private readonly TimeSpan _initialTimeout;
+    private readonly double _minimumStandardDeviationMilliseconds;
+    private int _nextSampleIndex;
+    private int _sampleCount;
+    private double _sampleSum;
+    private double _squaredSampleSum;
+
+    public PhiAccrualFailureDetector(TimeSpan initialTimeout)
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(initialTimeout, TimeSpan.Zero);
+
+        _initialTimeout = initialTimeout;
+        _minimumStandardDeviationMilliseconds = initialTimeout.TotalMilliseconds / (2 * ThresholdStandardDeviations);
+    }
+
+    public int SampleCount => _sampleCount;
+
+    public void RecordResponseTime(TimeSpan responseTime)
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThan(responseTime, TimeSpan.Zero);
+
+        var sample = responseTime.TotalMilliseconds;
+        if (_sampleCount == MaxSampleSize)
+        {
+            var replacedSample = _samples[_nextSampleIndex];
+            _sampleSum -= replacedSample;
+            _squaredSampleSum -= replacedSample * replacedSample;
+        }
+        else
+        {
+            _sampleCount++;
+        }
+
+        _samples[_nextSampleIndex] = sample;
+        _nextSampleIndex = (_nextSampleIndex + 1) % MaxSampleSize;
+        _sampleSum += sample;
+        _squaredSampleSum += sample * sample;
+    }
+
+    public TimeSpan GetTimeout() =>
+        _sampleCount < MinimumSampleCount
+            ? _initialTimeout
+            : TimeSpan.FromMilliseconds(Math.Min(GetEstimatedTimeoutMilliseconds(), TimeSpan.MaxValue.TotalMilliseconds));
+
+    public TimeSpan GetTimeout(TimeSpan minimumTimeout, TimeSpan maximumTimeout, int extensionFactor)
+    {
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(minimumTimeout, TimeSpan.Zero);
+        ArgumentOutOfRangeException.ThrowIfLessThan(maximumTimeout, minimumTimeout);
+        ArgumentOutOfRangeException.ThrowIfLessThan(extensionFactor, 1);
+
+        var timeoutTicks = GetTimeout().Ticks * (double)extensionFactor;
+        return TimeSpan.FromTicks((long)Math.Clamp(timeoutTicks, minimumTimeout.Ticks, maximumTimeout.Ticks));
+    }
+
+    internal static double CalculatePhi(double elapsedMilliseconds, double meanMilliseconds, double standardDeviationMilliseconds)
+    {
+        // Logistic approximation of the normal CDF used by Akka's Phi Accrual implementation.
+        var y = (elapsedMilliseconds - meanMilliseconds) / standardDeviationMilliseconds;
+        var e = Math.Exp(-y * (1.5976 + (0.070566 * y * y)));
+        return elapsedMilliseconds > meanMilliseconds
+            ? -Math.Log10(e / (1 + e))
+            : -Math.Log10(1 - (1 / (1 + e)));
+    }
+
+    private double GetEstimatedTimeoutMilliseconds()
+    {
+        var mean = _sampleSum / _sampleCount;
+        var variance = Math.Max(0, (_squaredSampleSum / _sampleCount) - (mean * mean));
+        var standardDeviation = Math.Max(Math.Sqrt(variance), _minimumStandardDeviationMilliseconds);
+        return mean + (ThresholdStandardDeviations * standardDeviation);
+    }
+
+    private static double CalculateThresholdStandardDeviations()
+    {
+        var lower = 0d;
+        var upper = 10d;
+        for (var i = 0; i < 64; i++)
+        {
+            var midpoint = (lower + upper) / 2;
+            if (CalculatePhi(midpoint, 0, 1) < Threshold)
+            {
+                lower = midpoint;
+            }
+            else
+            {
+                upper = midpoint;
+            }
+        }
+
+        return upper;
+    }
+}

src/Orleans.Runtime/MembershipService/ProbingSiloHealthMonitor.cs

@@ -51,7 +51,7 @@ public int CheckReceivedProbeRequests(DateTime now, int activeNodeCount, out str
         }
 
         var sinceLastProbeRequest = _probeRequestMonitor.ElapsedSinceLastProbeRequest;
-        var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit);
+        var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout();
 
         if (!sinceLastProbeRequest.HasValue)
         {
@@ -96,7 +96,7 @@ public int CheckReceivedProbeResponses(DateTime now, int monitoredNodeCount, out
             return 0;
         }
 
-        var recencyWindow = _clusterMembershipOptions.ProbeTimeout.Multiply(_clusterMembershipOptions.NumMissedProbesLimit);
+        var recencyWindow = _clusterMembershipOptions.GetFailureDetectionTimeout();
 
         if (!elapsedSinceLastResponse.HasValue)
         {