From ad717f094d6ddbe9974e4a0310604c854c6a4740 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 9 Mar 2026 13:46:37 -0500 Subject: [PATCH 1/3] Add failing tests proving heartbeat surrender bug (#3404) --- .../LeaseActorSpec.cs | 25 ++++++++++++++++++- .../LeaseActorSpec.cs | 25 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs b/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs index 0a5376e77..6be1f5cb2 100644 --- a/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs +++ b/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs @@ -363,6 +363,29 @@ public void HeartBeatFailShouldCallLeaseLostCallback() }); } + [Fact(DisplayName = "Bug #3404: should retry heartbeat on transient failure without surrendering lease")] + public void ShouldRetryHeartbeatOnTransientFailureWithoutSurrenderingLease() + { + RunTest(() => + { + Exception? callbackError = null; + AcquireLease(e => callbackError = e); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + var transientFailure = new LeaseException("Transient failure"); + UpdateProbe.Reply(new Status.Failure(transientFailure)); + + // Correct behavior: retain lease and retry while TTL still allows it. + Granted.Value.Should().BeTrue(); + callbackError.Should().BeNull(); + + var retry = UpdateProbe.ExpectMsg<(string, ETag)>(LeaseSettings.TimeoutSettings.HeartbeatInterval * 3); + retry.Should().Be((OwnerName, CurrentVersion)); + }); + } + [Fact(DisplayName = "lock should be acquire-able after heart beat conflict")] public void LockShouldAcquireAfterHeartBeatConflict() { @@ -756,4 +779,4 @@ protected void HeartBeatFailure() }); } } -} \ No newline at end of file +} diff --git a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs index 3426f5629..1865ced07 100644 --- a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs +++ b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs @@ -364,6 +364,29 @@ public void HeartBeatFailShouldCallLeaseLostCallback() }); } + [Fact(DisplayName = "Bug #3404: should retry heartbeat on transient failure without surrendering lease")] + public void ShouldRetryHeartbeatOnTransientFailureWithoutSurrenderingLease() + { + RunTest(() => + { + Exception? callbackError = null; + AcquireLease(e => callbackError = e); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + var transientFailure = new LeaseException("Transient failure"); + UpdateProbe.Reply(new Status.Failure(transientFailure)); + + // Correct behavior: retain lease and retry while TTL still allows it. + Granted.Value.Should().BeTrue(); + callbackError.Should().BeNull(); + + var retry = UpdateProbe.ExpectMsg<(string, string)>(LeaseSettings.TimeoutSettings.HeartbeatInterval * 3); + retry.Should().Be((OwnerName, CurrentVersion)); + }); + } + [Fact(DisplayName = "lock should be acquire-able after heart beat conflict")] public void LockShouldAcquireAfterHeartBeatConflict() { @@ -757,4 +780,4 @@ protected void HeartBeatFailure() }); } } -} \ No newline at end of file +} From acc23a1e5ae3241f7739c8f9e4aee003acc85a6f Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Thu, 5 Mar 2026 21:55:27 +0700 Subject: [PATCH 2/3] Retry heartbeat on transient failure instead of immediately surrendering lease (cherry picked from commit 0cf2b4e3ea9be5a56686d2e302b2875bc8af4a2c) --- .../LeaseActorSpec.cs | 130 ++++++++++++++---- .../Akka.Coordination.Azure/LeaseActor.cs | 25 +++- 2 files changed, 122 insertions(+), 33 deletions(-) diff --git a/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs b/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs index 6be1f5cb2..067c4a09a 100644 --- a/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs +++ b/src/coordination/azure/Akka.Coordination.Azure.Tests/LeaseActorSpec.cs @@ -324,18 +324,13 @@ public void HeartBeatFailShouldSetGrantedToFalse() { RunTest(() => { - var failure = new LeaseException("Failed to communicate with API server"); AcquireLease(); ExpectHeartBeat(); Granted.Value.Should().BeTrue(); - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(failure)); - AwaitAssert(() => - { - Granted.Value.Should().BeFalse(); - }); + // With retry logic, multiple failures are needed to exhaust the TTL window + HeartBeatFailure(); + Granted.Value.Should().BeFalse(); }); } @@ -353,9 +348,8 @@ public void HeartBeatFailShouldCallLeaseLostCallback() ExpectHeartBeat(); Granted.Value.Should().BeTrue(); - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(failure)); + // With retry logic, drive failures until TTL expires and callback fires + DriveHeartBeatFailures(failure); AwaitAssert(() => { callbackCalled.Should().Be(failure); @@ -363,26 +357,95 @@ public void HeartBeatFailShouldCallLeaseLostCallback() }); } - [Fact(DisplayName = "Bug #3404: should retry heartbeat on transient failure without surrendering lease")] - public void ShouldRetryHeartbeatOnTransientFailureWithoutSurrenderingLease() + [Fact(DisplayName = "transient heartbeat failure should retry and stay granted")] + public void TransientHeartBeatFailureShouldRetryAndStayGranted() { RunTest(() => { - Exception? callbackError = null; - AcquireLease(e => callbackError = e); + AcquireLease(); ExpectHeartBeat(); Granted.Value.Should().BeTrue(); + // First heartbeat fails transiently UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - var transientFailure = new LeaseException("Transient failure"); - UpdateProbe.Reply(new Status.Failure(transientFailure)); + UpdateProbe.Reply(new Status.Failure(new LeaseException("Transient failure"))); - // Correct behavior: retain lease and retry while TTL still allows it. + // Should still be granted — actor retries within TTL window Granted.Value.Should().BeTrue(); - callbackError.Should().BeNull(); - var retry = UpdateProbe.ExpectMsg<(string, ETag)>(LeaseSettings.TimeoutSettings.HeartbeatInterval * 3); - retry.Should().Be((OwnerName, CurrentVersion)); + // Retry heartbeat succeeds + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + // Should still be granted and heartbeating normally + Granted.Value.Should().BeTrue(); + + // Normal heartbeat continues + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + }); + } + + [Fact(DisplayName = "multiple transient heartbeat failures should recover on success")] + public void MultipleTransientHeartBeatFailuresShouldRecover() + { + RunTest(() => + { + AcquireLease(); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + // Multiple consecutive failures, all within TTL window + for (var i = 0; i < 3; i++) + { + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + UpdateProbe.Reply(new Status.Failure(new LeaseException($"Transient failure {i}"))); + Granted.Value.Should().BeTrue(); + } + + // Recovery: next heartbeat succeeds + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + // Lease is still held + Granted.Value.Should().BeTrue(); + }); + } + + [Fact(DisplayName = "heartbeat failure should not call lease lost callback during retry")] + public void HeartBeatFailureShouldNotCallLeaseLostCallbackDuringRetry() + { + RunTest(() => + { + var callbackCalled = false; + AcquireLease(e => + { + callbackCalled = true; + }); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + // Single transient failure within TTL + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + UpdateProbe.Reply(new Status.Failure(new LeaseException("Transient failure"))); + + // Callback should NOT be called — TTL still valid + callbackCalled.Should().BeFalse(); + Granted.Value.Should().BeTrue(); + + // Recovery + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + callbackCalled.Should().BeFalse(); }); } @@ -768,15 +831,28 @@ protected void HeartBeatConflict() }); } + /// + /// Drives heartbeat failures until the TTL window expires and the actor surrenders the lease. + /// With the retry logic, the actor retries heartbeats within the TTL window before surrendering. + /// protected void HeartBeatFailure() { - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(new LeaseException("Failed to communicate with API server"))); - AwaitAssert(() => + DriveHeartBeatFailures(new LeaseException("Failed to communicate with API server")); + } + + /// + /// Keeps replying with the given failure to all heartbeat retry attempts + /// until the actor exhausts the TTL window and surrenders the lease. + /// + protected void DriveHeartBeatFailures(Exception failure) + { + while (Granted.Value) { - Granted.Value.Should().BeFalse(); - }); + UpdateProbe.ExpectMsg<(string, ETag)>(TimeSpan.FromSeconds(2)); + UpdateProbe.Reply(new Status.Failure(failure)); + Task.Delay(10).Wait(); + } + AwaitAssert(() => Granted.Value.Should().BeFalse()); } } } diff --git a/src/coordination/azure/Akka.Coordination.Azure/LeaseActor.cs b/src/coordination/azure/Akka.Coordination.Azure/LeaseActor.cs index 1be51041c..18ce17abd 100644 --- a/src/coordination/azure/Akka.Coordination.Azure/LeaseActor.cs +++ b/src/coordination/azure/Akka.Coordination.Azure/LeaseActor.cs @@ -109,19 +109,22 @@ public OperationInProgress( public sealed class GrantedVersion: IData { - public GrantedVersion(ETag version, Action leaseLostCallback) + public GrantedVersion(ETag version, Action leaseLostCallback, DateTime? lastSuccessfulHeartbeat = null) { Version = version; LeaseLostCallback = leaseLostCallback; + LastSuccessfulHeartbeat = lastSuccessfulHeartbeat ?? DateTime.UtcNow; } public ETag Version { get; } public Action LeaseLostCallback { get; } + public DateTime LastSuccessfulHeartbeat { get; } - public GrantedVersion Copy(ETag? version = null, Action? leaseLostCallback = null) + public GrantedVersion Copy(ETag? version = null, Action? leaseLostCallback = null, DateTime? lastSuccessfulHeartbeat = null) => new GrantedVersion( version: version ?? Version, - leaseLostCallback: leaseLostCallback ?? LeaseLostCallback); + leaseLostCallback: leaseLostCallback ?? LeaseLostCallback, + lastSuccessfulHeartbeat: lastSuccessfulHeartbeat ?? LastSuccessfulHeartbeat); } public interface ICommand { } @@ -397,7 +400,7 @@ public LeaseActor(IAzureApi client, LeaseSettings settings, string leaseName, At if(_log.IsDebugEnabled) _log.Debug("Heartbeat: lease time updated: Version {0}", resource.Value.Version); Timers!.StartSingleTimer("heartbeat", Heartbeat.Instance, settings.TimeoutSettings.HeartbeatInterval); - return Stay().Using(gv.Copy(version: resource.Value.Version)); + return Stay().Using(gv.Copy(version: resource.Value.Version, lastSuccessfulHeartbeat: DateTime.UtcNow)); case WriteResponse {Response: Left resource}: _log.Warning("Conflict during heartbeat to lease {0}. Lease assumed to be released.", resource.Value); @@ -406,8 +409,18 @@ public LeaseActor(IAzureApi client, LeaseSettings settings, string leaseName, At return GoTo(Idle.Instance).Using(ReadRequired.Instance); case Status.Failure failure: - // FIXME, retry if timeout far enough off: https://github.com/lightbend/akka-commercial-addons/issues/501 - _log.Warning(failure.Cause, "Failure during heartbeat to lease. Lease assumed to be released."); + var timeSinceLastHeartbeat = DateTime.UtcNow - gv.LastSuccessfulHeartbeat; + if (timeSinceLastHeartbeat < _timeoutOffset - settings.TimeoutSettings.HeartbeatInterval) + { + _log.Warning(failure.Cause, + "Transient failure during heartbeat to lease {0}. TTL still valid ({1:F0}s remaining). Retrying.", + leaseName, + (_timeoutOffset - timeSinceLastHeartbeat).TotalSeconds); + Timers!.StartSingleTimer("heartbeat", Heartbeat.Instance, settings.TimeoutSettings.HeartbeatInterval); + return Stay(); + } + _log.Warning(failure.Cause, + "Failure during heartbeat to lease {0}. TTL window expired, releasing lease.", leaseName); localGranted.GetAndSet(false); ExecuteLeaseLockCallback(leaseLost, failure.Cause); return GoTo(Idle.Instance).Using(ReadRequired.Instance); From ea61692db9c6a4eaa22334a0ae748bed194f7e2e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 9 Mar 2026 14:33:11 -0500 Subject: [PATCH 3/3] Extend heartbeat retry fix to Kubernetes LeaseActor --- .../LeaseActorSpec.cs | 122 ++++++++++++++---- .../LeaseActor.cs | 28 +++- 2 files changed, 116 insertions(+), 34 deletions(-) diff --git a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs index 1865ced07..e3c993e7c 100644 --- a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs +++ b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/LeaseActorSpec.cs @@ -325,18 +325,13 @@ public void HeartBeatFailShouldSetGrantedToFalse() { RunTest(() => { - var failure = new LeaseException("Failed to communicate with API server"); AcquireLease(); ExpectHeartBeat(); Granted.Value.Should().BeTrue(); - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(failure)); - AwaitAssert(() => - { - Granted.Value.Should().BeFalse(); - }); + // With retry logic, multiple failures are needed to exhaust the TTL window + HeartBeatFailure(); + Granted.Value.Should().BeFalse(); }); } @@ -354,9 +349,8 @@ public void HeartBeatFailShouldCallLeaseLostCallback() ExpectHeartBeat(); Granted.Value.Should().BeTrue(); - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(failure)); + // With retry logic, drive failures until TTL expires and callback fires + DriveHeartBeatFailures(failure); AwaitAssert(() => { callbackCalled.Should().Be(failure); @@ -364,26 +358,95 @@ public void HeartBeatFailShouldCallLeaseLostCallback() }); } - [Fact(DisplayName = "Bug #3404: should retry heartbeat on transient failure without surrendering lease")] - public void ShouldRetryHeartbeatOnTransientFailureWithoutSurrenderingLease() + [Fact(DisplayName = "transient heartbeat failure should retry and stay granted")] + public void TransientHeartBeatFailureShouldRetryAndStayGranted() { RunTest(() => { - Exception? callbackError = null; - AcquireLease(e => callbackError = e); + AcquireLease(); ExpectHeartBeat(); Granted.Value.Should().BeTrue(); + // First heartbeat fails transiently UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - var transientFailure = new LeaseException("Transient failure"); - UpdateProbe.Reply(new Status.Failure(transientFailure)); + UpdateProbe.Reply(new Status.Failure(new LeaseException("Transient failure"))); - // Correct behavior: retain lease and retry while TTL still allows it. + // Should still be granted - actor retries within TTL window Granted.Value.Should().BeTrue(); - callbackError.Should().BeNull(); - var retry = UpdateProbe.ExpectMsg<(string, string)>(LeaseSettings.TimeoutSettings.HeartbeatInterval * 3); - retry.Should().Be((OwnerName, CurrentVersion)); + // Retry heartbeat succeeds + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + // Should still be granted and heartbeating normally + Granted.Value.Should().BeTrue(); + + // Normal heartbeat continues + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + }); + } + + [Fact(DisplayName = "multiple transient heartbeat failures should recover on success")] + public void MultipleTransientHeartBeatFailuresShouldRecover() + { + RunTest(() => + { + AcquireLease(); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + // Multiple consecutive failures, all within TTL window + for (var i = 0; i < 3; i++) + { + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + UpdateProbe.Reply(new Status.Failure(new LeaseException($"Transient failure {i}"))); + Granted.Value.Should().BeTrue(); + } + + // Recovery: next heartbeat succeeds + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + // Lease is still held + Granted.Value.Should().BeTrue(); + }); + } + + [Fact(DisplayName = "heartbeat failure should not call lease lost callback during retry")] + public void HeartBeatFailureShouldNotCallLeaseLostCallbackDuringRetry() + { + RunTest(() => + { + var callbackCalled = false; + AcquireLease(e => + { + callbackCalled = true; + }); + ExpectHeartBeat(); + Granted.Value.Should().BeTrue(); + + // Single transient failure within TTL + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + UpdateProbe.Reply(new Status.Failure(new LeaseException("Transient failure"))); + + // Callback should NOT be called - TTL still valid + callbackCalled.Should().BeFalse(); + Granted.Value.Should().BeTrue(); + + // Recovery + UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); + IncrementVersion(); + UpdateProbe.Reply( + new Right( + new LeaseResource(OwnerName, CurrentVersion, CurrentTime))); + + callbackCalled.Should().BeFalse(); }); } @@ -771,13 +834,18 @@ protected void HeartBeatConflict() protected void HeartBeatFailure() { - UpdateProbe.ExpectMsg((OwnerName, CurrentVersion)); - IncrementVersion(); - UpdateProbe.Reply(new Status.Failure(new LeaseException("Failed to communicate with API server"))); - AwaitAssert(() => + DriveHeartBeatFailures(new LeaseException("Failed to communicate with API server")); + } + + protected void DriveHeartBeatFailures(Exception failure) + { + while (Granted.Value) { - Granted.Value.Should().BeFalse(); - }); + UpdateProbe.ExpectMsg<(string, string)>(TimeSpan.FromSeconds(2)); + UpdateProbe.Reply(new Status.Failure(failure)); + Task.Delay(10).Wait(); + } + AwaitAssert(() => Granted.Value.Should().BeFalse()); } } } diff --git a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi/LeaseActor.cs b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi/LeaseActor.cs index d62bd83d3..68b3054c3 100644 --- a/src/coordination/kubernetes/Akka.Coordination.KubernetesApi/LeaseActor.cs +++ b/src/coordination/kubernetes/Akka.Coordination.KubernetesApi/LeaseActor.cs @@ -108,19 +108,22 @@ public OperationInProgress( public sealed class GrantedVersion: IData { - public GrantedVersion(string version, Action leaseLostCallback) + public GrantedVersion(string version, Action leaseLostCallback, DateTime? lastSuccessfulHeartbeat = null) { Version = version; LeaseLostCallback = leaseLostCallback; + LastSuccessfulHeartbeat = lastSuccessfulHeartbeat ?? DateTime.UtcNow; } public string Version { get; } public Action LeaseLostCallback { get; } + public DateTime LastSuccessfulHeartbeat { get; } - public GrantedVersion Copy(string? version = null, Action? leaseLostCallback = null) + public GrantedVersion Copy(string? version = null, Action? leaseLostCallback = null, DateTime? lastSuccessfulHeartbeat = null) => new GrantedVersion( version: version ?? Version, - leaseLostCallback: leaseLostCallback ?? LeaseLostCallback); + leaseLostCallback: leaseLostCallback ?? LeaseLostCallback, + lastSuccessfulHeartbeat: lastSuccessfulHeartbeat ?? LastSuccessfulHeartbeat); } public interface ICommand { } @@ -393,7 +396,7 @@ public LeaseActor(IKubernetesApi client, LeaseSettings settings, string leaseNam throw new LeaseException($"response from API server has different owner for success: {resource}"); _log.Debug("Heartbeat: lease time updated: Version {0}", resource.Value.Version); Timers!.StartSingleTimer("heartbeat", Heartbeat.Instance, settings.TimeoutSettings.HeartbeatInterval); - return Stay().Using(gv.Copy(version: resource.Value.Version)); + return Stay().Using(gv.Copy(version: resource.Value.Version, lastSuccessfulHeartbeat: DateTime.UtcNow)); case WriteResponse {Response: Left resource}: _log.Warning("Conflict during heartbeat to lease {0}. Lease assumed to be released.", resource.Value); @@ -402,8 +405,19 @@ public LeaseActor(IKubernetesApi client, LeaseSettings settings, string leaseNam return GoTo(Idle.Instance).Using(ReadRequired.Instance); case Status.Failure failure: - // FIXME, retry if timeout far enough off: https://github.com/lightbend/akka-commercial-addons/issues/501 - _log.Warning(failure.Cause, "Failure during heartbeat to lease. Lease assumed to be released."); + var timeSinceLastHeartbeat = DateTime.UtcNow - gv.LastSuccessfulHeartbeat; + var retryWindow = _heartbeatTimeout - _heartbeatOffset - settings.TimeoutSettings.HeartbeatInterval; + if (timeSinceLastHeartbeat < retryWindow) + { + _log.Warning(failure.Cause, + "Transient failure during heartbeat to lease {0}. TTL still valid ({1:F0}s remaining). Retrying.", + leaseName, + (retryWindow - timeSinceLastHeartbeat).TotalSeconds); + Timers!.StartSingleTimer("heartbeat", Heartbeat.Instance, settings.TimeoutSettings.HeartbeatInterval); + return Stay(); + } + _log.Warning(failure.Cause, + "Failure during heartbeat to lease {0}. TTL window expired, releasing lease.", leaseName); localGranted.GetAndSet(false); ExecuteLeaseLockCallback(leaseLost, failure.Cause); return GoTo(Idle.Instance).Using(ReadRequired.Instance); @@ -555,4 +569,4 @@ private bool HasLeaseTimedOut(DateTime leaseTime) public ITimerScheduler Timers { get; set; } } -} \ No newline at end of file +}