From 77990a0eb9bf17dda4347a7cdfa0e27245e6d7c6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 03:19:35 +0000 Subject: [PATCH 01/10] Akka.Cluster: add removal tombstones to Gossip and consult them when merging A member status on its own cannot tell "this member was removed" apart from "this node has not heard about it yet". Merge had only the status to go on, so a member the leader had removed was put back whenever a lagging peer still held it as Leaving. The resurrected member is dead, so it can never mark itself seen, and convergence stops for good. Gossip now carries a tombstone per removed node - keyed by UniqueAddress, stamped with the epoch milliseconds of the removal. That is positive evidence a removal happened, and it travels with the gossip so any peer can apply it. - Gossip.RemoveAll strips a node from members, seen, reachability and the vector clock and records its tombstone in one step, so no gossip that breaks the invariants is ever built. - Gossip.Merge unions tombstones first, then prunes the merged vector clock by them. Union alone is not enough: a clock entry for a removed node is resurrected by the merge exactly the way the member is. - Gossip.MergeTombstones unions tombstones without touching anything else, for the gossip-reception branches that pick a whole gossip as the winner. - Gossip.PruneTombstones drops expired entries and returns the same instance when it drops nothing, so a caller can skip the update by reference check. - Member.PickHighestPriority gains a tombstone-aware overload. The one-sided drop condition is widened with OR, not replaced: every member dropped before is still dropped, and no input keeps a member the old code dropped. - AssertInvariants now requires members and tombstones to be disjoint. Merge picks a two-sided member by status alone, which is only correct while that holds. Timestamps order nothing; they only decide when a tombstone expires. Union keeps the later timestamp on collision, so two nodes merging the same pair of gossips in opposite order reach the same state. --- src/core/Akka.Cluster/Gossip.cs | 176 +++++++++++++++++++++++++++++--- src/core/Akka.Cluster/Member.cs | 17 ++- 2 files changed, 180 insertions(+), 13 deletions(-) diff --git a/src/core/Akka.Cluster/Gossip.cs b/src/core/Akka.Cluster/Gossip.cs index 7979e2418dc..9ac9b9e32f8 100644 --- a/src/core/Akka.Cluster/Gossip.cs +++ b/src/core/Akka.Cluster/Gossip.cs @@ -52,6 +52,13 @@ internal sealed class Gossip /// public static readonly ImmutableSortedSet EmptyMembers = ImmutableSortedSet.Create(); + /// + /// An empty set of tombstones. Declared before because the constructor + /// chain reads it. + /// + public static readonly ImmutableDictionary EmptyTombstones = + ImmutableDictionary.Empty; + /// /// An empty object. /// @@ -71,6 +78,7 @@ public static Gossip Create(ImmutableSortedSet members) readonly ImmutableSortedSet _members; readonly GossipOverview _overview; readonly VectorClock _version; + readonly ImmutableDictionary _tombstones; /// /// The current members of the cluster @@ -85,6 +93,23 @@ public static Gossip Create(ImmutableSortedSet members) /// public VectorClock Version { get { return _version; } } + /// + /// The nodes this cluster has removed, keyed by and stamped with the + /// epoch milliseconds of the removal. + /// + /// A member status on its own cannot tell "this member was removed" apart from "this node has + /// not heard about it yet", so merging two gossips has no way to decide what to do with a member + /// that only one side holds. A tombstone answers that directly: it is positive evidence that the + /// removal happened, and it travels in the gossip so every peer can apply it. + /// + /// The key carries the node UID, so a restarted node at the same host and port is never blocked + /// by the tombstone of its predecessor. + /// + /// The timestamp orders nothing. It exists only so the leader can drop tombstones older than + /// akka.cluster.prune-gossip-tombstones-after. + /// + public ImmutableDictionary Tombstones { get { return _tombstones; } } + /// /// TBD /// @@ -106,10 +131,23 @@ public Gossip(ImmutableSortedSet members, GossipOverview overview) : thi /// TBD /// TBD public Gossip(ImmutableSortedSet members, GossipOverview overview, VectorClock version) + : this(members, overview, version, EmptyTombstones) { } + + /// + /// TBD + /// + /// TBD + /// TBD + /// TBD + /// The nodes this cluster has removed. See . + /// TBD + public Gossip(ImmutableSortedSet members, GossipOverview overview, VectorClock version, + ImmutableDictionary tombstones) { _members = members; _overview = overview; _version = version; + _tombstones = tombstones; _membersMap = new Lazy>( () => members.ToImmutableDictionary(m => m.UniqueAddress, m => m)); @@ -129,11 +167,13 @@ public Gossip(ImmutableSortedSet members, GossipOverview overview, Vecto /// TBD /// TBD /// TBD + /// The nodes this cluster has removed. See . /// TBD public Gossip Copy(ImmutableSortedSet members = null, GossipOverview overview = null, - VectorClock version = null) + VectorClock version = null, ImmutableDictionary tombstones = null) { - return new Gossip(members ?? _members, overview ?? _overview, version ?? _version); + return new Gossip(members ?? _members, overview ?? _overview, version ?? _version, + tombstones ?? _tombstones); } private void AssertInvariants() @@ -159,6 +199,14 @@ private void AssertInvariants() IfTrueThrow(!seenButNotMember.IsEmpty, expected: "Nodes not part of cluster have marked the Gossip as seen", actual: string.Join(", ", seenButNotMember.Select(a => a.ToString()))); + + // Members and tombstones must stay disjoint. Merge depends on it: a member that appears on + // both sides is picked by status alone, without consulting tombstones, which is only correct + // while no gossip carries a member and a tombstone for the same node. + var tombstonedButStillMember = _members.Where(m => _tombstones.ContainsKey(m.UniqueAddress)).ToList(); + IfTrueThrow(tombstonedButStillMember.Count > 0, + expected: "Removed nodes must not be members", + actual: string.Join(", ", tombstonedButStillMember.Select(m => m.ToString()))); return; void IfTrueThrow(bool func, string expected, string actual) @@ -261,20 +309,123 @@ public Gossip MergeSeen(Gossip that) public Gossip Merge(Gossip that) { //TODO: Member ordering import? - // 1. merge vector clocks - var mergedVClock = _version.Merge(that._version); - - // 2. merge members by selecting the single Member with highest MemberStatus out of the Member groups - var mergedMembers = EmptyMembers.Union(Member.PickHighestPriority(this._members, that._members)); - - // 3. merge reachability table by picking records with highest version + // 1. merge tombstones - steps 2 and 3 both read the result + var mergedTombstones = UnionTombstones(_tombstones, that._tombstones); + var tombstonedNodes = mergedTombstones.Count == 0 + ? ImmutableHashSet.Empty + : mergedTombstones.Keys.ToImmutableHashSet(); + + // 2. merge vector clocks, then drop the entry of every removed node. + // A clock entry is resurrected by the union the same way a member is, so filtering members + // alone would leave the removed node in the clock forever. + var mergedVClock = tombstonedNodes.Aggregate( + _version.Merge(that._version), + (clock, node) => clock.Prune(VectorClock.Node.Create(ClusterCoreDaemon.VclockName(node)))); + + // 3. merge members by selecting the single Member with highest MemberStatus out of the Member groups + var mergedMembers = EmptyMembers.Union(Member.PickHighestPriority(this._members, that._members, tombstonedNodes)); + + // 4. merge reachability table by picking records with highest version var mergedReachability = _overview.Reachability.Merge(mergedMembers.Select(m => m.UniqueAddress).ToImmutableSortedSet(), that._overview.Reachability); - // 4. Nobody can have seen this new gossip yet + // 5. Nobody can have seen this new gossip yet var mergedSeen = ImmutableHashSet.Create(); - return new Gossip(mergedMembers, new GossipOverview(mergedSeen, mergedReachability), mergedVClock); + return new Gossip(mergedMembers, new GossipOverview(mergedSeen, mergedReachability), mergedVClock, + mergedTombstones); + } + + /// + /// Adds the tombstones of to this gossip, keeping the later timestamp + /// when both sides hold the same node. + /// + /// already unions tombstones, but it only runs on the concurrent branch of + /// gossip reception. The other three branches pick a whole gossip as the winner, which would + /// drop the loser's tombstones. Unioning them here instead keeps a tombstone alive no matter + /// which branch a given exchange takes. + /// + /// The gossip to take tombstones from. + /// This gossip when it already covers every tombstone of . + public Gossip MergeTombstones(Gossip that) + { + if (that._tombstones.Count == 0) return this; + + // A node this gossip still holds as a member has not been removed here, so its tombstone is + // not adopted - members and tombstones must stay disjoint. Every tombstone write bumps the + // vector clock, so the winning gossip of these branches is never behind on removals and this + // case cannot arise; the filter is a guard, not a decision. + var merged = UnionTombstones(_tombstones, that._tombstones.Where(kv => !HasMember(kv.Key))); + if (ReferenceEquals(merged, _tombstones)) return this; + return Copy(tombstones: merged); + } + + /// + /// Drops every tombstone stamped at or before . + /// + /// Epoch milliseconds. Tombstones at or before this are dropped. + /// This gossip when nothing was dropped. The caller compares by reference to decide + /// whether the gossip needs to be published, so this identity matters. + public Gossip PruneTombstones(long removeEarlierThan) + { + if (_tombstones.Count == 0) return this; + + var pruned = _tombstones.Where(kv => kv.Value <= removeEarlierThan).Select(kv => kv.Key).ToList(); + if (pruned.Count == 0) return this; + + return Copy(tombstones: _tombstones.RemoveRange(pruned)); + } + + /// + /// Removes nodes from the cluster and records a tombstone for each of them. + /// + /// Members, the seen table, the reachability table and the vector clock are all stripped in one + /// step, so no intermediate gossip that breaks the invariants is ever built. + /// + /// The nodes being removed. + /// Epoch milliseconds of the removal. + /// This gossip when is empty. + public Gossip RemoveAll(IImmutableSet nodes, long removalTimestamp) + { + if (nodes.Count == 0) return this; + + var newMembers = _members.Where(m => !nodes.Contains(m.UniqueAddress)).ToImmutableSortedSet(); + var newOverview = _overview.Copy( + seen: _overview.Seen.Except(nodes), + reachability: _overview.Reachability.Remove(nodes)); + + // Clear the VectorClock when a member is removed. The change made by the leader is stamped + // and will propagate as is if there are no other changes on other nodes. + var newVersion = nodes.Aggregate(_version, + (clock, node) => clock.Prune(VectorClock.Node.Create(ClusterCoreDaemon.VclockName(node)))); + + var newTombstones = _tombstones; + foreach (var node in nodes) + newTombstones = newTombstones.SetItem(node, removalTimestamp); + + return new Gossip(newMembers, newOverview, newVersion, newTombstones); + } + + /// + /// Unions two tombstone sets, keeping the later timestamp when both hold the same node. + /// + /// Two nodes merge the same pair of gossips in opposite order, so the result has to be the same + /// either way. Taking the later timestamp is commutative and associative, and it errs towards + /// keeping a tombstone longer, which is the safe direction. + /// + private static ImmutableDictionary UnionTombstones( + ImmutableDictionary current, + IEnumerable> other) + { + var merged = current; + foreach (var kv in other) + { + if (!merged.TryGetValue(kv.Key, out var existing)) + merged = merged.Add(kv.Key, kv.Value); + else if (kv.Value > existing) + merged = merged.SetItem(kv.Key, kv.Value); + } + return merged; } /// @@ -381,7 +532,8 @@ public Gossip MarkAsDown(Member member) public override string ToString() { var members = string.Join(", ", _members.Select(m => m.ToString())); - return $"Gossip(members = [{members}], overview = {_overview}, version = {_version}"; + var tombstones = string.Join(", ", _tombstones.Select(t => $"{t.Key} -> {t.Value}")); + return $"Gossip(members = [{members}], overview = {_overview}, version = {_version}, tombstones = [{tombstones}])"; } } diff --git a/src/core/Akka.Cluster/Member.cs b/src/core/Akka.Cluster/Member.cs index 90d3cfc6835..4ec257fade6 100644 --- a/src/core/Akka.Cluster/Member.cs +++ b/src/core/Akka.Cluster/Member.cs @@ -357,6 +357,17 @@ public static Member PickNextTransition(Member a, Member b) /// The second collection of members. /// An immutable hash set containing the members with the highest priority. public static ImmutableHashSet PickHighestPriority(IEnumerable a, IEnumerable b) + => PickHighestPriority(a, b, ImmutableHashSet.Empty); + + /// + /// Combines and sorts two lists of into a single list ordered by highest prioirity + /// + /// The first collection of members. + /// The second collection of members. + /// Nodes the cluster has removed. + /// An immutable hash set containing the members with the highest priority. + public static ImmutableHashSet PickHighestPriority(IEnumerable a, IEnumerable b, + IImmutableSet tombstones) { // group all members by Address => Seq[Member] var groupedByAddress = (a.Concat(b)).GroupBy(x => x.UniqueAddress); @@ -368,8 +379,12 @@ public static ImmutableHashSet PickHighestPriority(IEnumerable a if (g.Count() == 2) acc.Add(HighestPriorityOf(g.First(), g.Skip(1).First())); else { + // Present on one side only. Either the other side removed it, or the other side has not + // heard of it yet. A tombstone says which. Down and Exiting are terminal, so they are + // still read as evidence of a removal even without one. var m = g.First(); - if (!MembershipState.RemoveUnreachableWithMemberStatus.Contains(m.Status)) acc.Add(m); + if (!tombstones.Contains(m.UniqueAddress) + && !MembershipState.RemoveUnreachableWithMemberStatus.Contains(m.Status)) acc.Add(m); } } return acc.ToImmutableHashSet(); From 5123e4a3f896b8fc1164a1eb6ae2469e5e5ca108 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 03:19:50 +0000 Subject: [PATCH 02/10] Akka.Cluster: put gossip tombstones on the wire Adds `repeated Tombstone tombstones = 7` to message Gossip, plus the Tombstone message itself. Field 7 was the one vacant slot in Gossip - allAppVersions already sits at 8 - so nothing needs renumbering. Codegen runs at build time; there is no checked-in generated file to update. The serializer builds its address table from members only, and a tombstoned node is by definition not a member. Tombstone addresses are appended to that table after the member loop, so a tombstone can index into it like everything else. Getting this wrong is silent: it produces tombstones pointing at other nodes' addresses rather than an error. Gossip written before this field decodes to an empty tombstone set, which reduces the merge to its previous status-only rule. --- .../Serialization/ClusterMessageSerializer.cs | 36 +++++++++++++++++-- src/protobuf/ClusterMessages.proto | 10 ++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/core/Akka.Cluster/Serialization/ClusterMessageSerializer.cs b/src/core/Akka.Cluster/Serialization/ClusterMessageSerializer.cs index f778cc08579..b8fec3211c4 100644 --- a/src/core/Akka.Cluster/Serialization/ClusterMessageSerializer.cs +++ b/src/core/Akka.Cluster/Serialization/ClusterMessageSerializer.cs @@ -368,7 +368,26 @@ private static Proto.Msg.Gossip GossipToProto(Gossip gossip) membersProtos.Add(MemberToProto(m)); } - + + // A tombstoned node is never a member, so its address is missing from the table the + // member loop just built. Append it, so the tombstone can index into the same table. + var tombstoneProtos = new List(gossip.Tombstones.Count); + foreach (var t in gossip.Tombstones) + { + if (!addressMapping.ContainsKey(t.Key)) + { + addressMapping.Add(t.Key, addrIndex); + addrIndex += 1; + addressesToProto.Add(UniqueAddressToProto(t.Key)); + } + + tombstoneProtos.Add(new Proto.Msg.Tombstone + { + AddressIndex = MapUniqueAddress(t.Key), + Timestamp = t.Value + }); + } + //var addressMapping = allAddresses.ZipWithIndex(); //var roleMapping = allRoles.ZipWithIndex(); var allHashes = gossip.Version.Versions.Keys.Select(x => x.ToString()).ToArray(); @@ -389,6 +408,7 @@ private static Proto.Msg.Gossip GossipToProto(Gossip gossip) message.Members.AddRange(membersProtos); message.Overview = overview; message.Version = VectorClockToProto(gossip.Version, hashMapping); + message.Tombstones.AddRange(tombstoneProtos); message.AllAppVersions.AddRange(allAppVersions); return message; @@ -420,7 +440,19 @@ private static Gossip GossipFrom(Proto.Msg.Gossip gossip) var seen = gossip.Overview.Seen.Select(x => addressMapping[x]).ToImmutableHashSet(); var overview = new GossipOverview(seen, reachability); - return new Gossip(members, overview, VectorClockFrom(gossip.Version, hashMapping)); + // gossip written before tombstones existed simply carries none + var tombstones = ImmutableDictionary.Empty; + if (gossip.Tombstones.Count > 0) + { + var builder = tombstones.ToBuilder(); + foreach (var t in gossip.Tombstones) + { + builder[addressMapping[t.AddressIndex]] = t.Timestamp; + } + tombstones = builder.ToImmutable(); + } + + return new Gossip(members, overview, VectorClockFrom(gossip.Version, hashMapping), tombstones); Member MemberFromProto(Proto.Msg.Member member) => Member.Create( diff --git a/src/protobuf/ClusterMessages.proto b/src/protobuf/ClusterMessages.proto index 81dd452e659..b745ad5355b 100644 --- a/src/protobuf/ClusterMessages.proto +++ b/src/protobuf/ClusterMessages.proto @@ -108,9 +108,19 @@ message Gossip { repeated Member members = 4; GossipOverview overview = 5; VectorClock version = 6; + repeated Tombstone tombstones = 7; repeated string allAppVersions = 8; } +/* Records that a member was removed from the cluster. + * addressIndex points into allAddresses, which carries every tombstoned + * address after the member addresses. timestamp is epoch milliseconds, + * used only to expire the tombstone. */ +message Tombstone { + int32 addressIndex = 1; + int64 timestamp = 2; +} + // Gossip Overview message GossipOverview { /* This is the address indexes for the nodes that have seen this gossip */ From 10ebdaa323e09b14e1af6240b4437b88cd6b40f3 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 03:19:50 +0000 Subject: [PATCH 03/10] Akka.Cluster: write tombstones on removal and expire them on the leader The leader records a tombstone for every node it removes, on both removal paths - unreachable Down/Exiting members and confirmed-Exiting members. Both already ran through the same block in LeaderActionsOnConvergence; that block now calls Gossip.RemoveAll instead of deriving members, seen, reachability and the vector clock separately. ReceiveGossip unions tombstones on all four comparison branches. Only the concurrent branch merges; the other three pick one whole gossip as the winner and would otherwise drop the loser's tombstones, letting removals decay out of the cluster over time. That failure mode presents as the original bug returning intermittently, which no unit test would catch. Expired tombstones are dropped on the leader at the end of the same method. Pruning has to run on converged ticks where nothing else changed, so it sits outside the change guard: the method now computes the updated gossip - or the local one when nothing changed - prunes it, and publishes only when the result differs from the local gossip by reference. PruneTombstones returns the same instance when it drops nothing, which is what makes that check work. Without it every converged leader tick would bump the vector clock and reset the seen table, and the cluster would never sit still. Retention is akka.cluster.prune-gossip-tombstones-after, default 24h. The bound that matters is how long a gossip carrying the stale member can survive before it merges back in, and under a partition that is unbounded - so the value is set against partition length, not bandwidth. A partition outlasting the window re-opens the hole silently. UpdateLatestGossip now records the invariant it carries: every change this node makes to the gossip goes through it and bumps the vector clock, which is what lets peers tell whose removal set is newer. --- src/core/Akka.Cluster/ClusterDaemon.cs | 96 +++++++++++-------- src/core/Akka.Cluster/ClusterSettings.cs | 13 +++ .../Akka.Cluster/Configuration/Cluster.conf | 7 ++ 3 files changed, 77 insertions(+), 39 deletions(-) diff --git a/src/core/Akka.Cluster/ClusterDaemon.cs b/src/core/Akka.Cluster/ClusterDaemon.cs index 48461fe191c..908b2e12d8f 100644 --- a/src/core/Akka.Cluster/ClusterDaemon.cs +++ b/src/core/Akka.Cluster/ClusterDaemon.cs @@ -1899,21 +1899,24 @@ public ReceiveGossipType ReceiveGossip(GossipEnvelope envelope) switch (comparison) { + // Three of the four branches below pick one whole gossip as the winner, which on its own + // would throw away the loser's tombstones. Tombstones are unioned in every branch instead, + // so a removal cannot decay out of the cluster along a path of ordinary gossip exchanges. case VectorClock.Ordering.Same: //same version talkback = !_exitingTasksInProgress && !remoteGossip.SeenByNode(_selfUniqueAddress); - winningGossip = remoteGossip.MergeSeen(localGossip); + winningGossip = remoteGossip.MergeSeen(localGossip).MergeTombstones(localGossip); gossipType = ReceiveGossipType.Same; break; case VectorClock.Ordering.Before: //local is newer - winningGossip = localGossip; + winningGossip = localGossip.MergeTombstones(remoteGossip); talkback = true; gossipType = ReceiveGossipType.Older; break; case VectorClock.Ordering.After: //remote is newer - winningGossip = remoteGossip; + winningGossip = remoteGossip.MergeTombstones(localGossip); talkback = !_exitingTasksInProgress && !remoteGossip.SeenByNode(_selfUniqueAddress); gossipType = ReceiveGossipType.Newer; break; @@ -2294,7 +2297,6 @@ public void LeaderActionsOnConvergence() var localGossip = LatestGossip; var localMembers = localGossip.Members; var localOverview = localGossip.Overview; - var localSeen = localOverview.Seen; var enoughMembers = IsMinNrOfMembersFulfilled(); @@ -2338,49 +2340,55 @@ public void LeaderActionsOnConvergence() return null; }).Where(m => m != null).ToImmutableSortedSet(); - if (!removedUnreachable.IsEmpty || !removedExitingConfirmed.IsEmpty || !changedMembers.IsEmpty) + var hasChanges = !removedUnreachable.IsEmpty || !removedExitingConfirmed.IsEmpty || !changedMembers.IsEmpty; + var updatedGossip = localGossip; + + if (hasChanges) { // handle changes // replace changed members - var newMembers = Member.PickNextTransition(changedMembers, localMembers) - .Except(removedUnreachable) - .Where(x => !removedExitingConfirmed.Contains(x.UniqueAddress)) - .ToImmutableSortedSet(); + var updatedMembers = Member.PickNextTransition(changedMembers, localMembers).ToImmutableSortedSet(); - // removing REMOVED nodes from the `seen` table var removed = removedUnreachable.Select(u => u.UniqueAddress) .ToImmutableHashSet() .Union(removedExitingConfirmed); - var newSeen = localSeen.Except(removed); - // removing REMOVED nodes from the `reachability` table - var newReachability = localOverview.Reachability.Remove(removed); - var newOverview = localOverview.Copy(seen: newSeen, reachability: newReachability); - - // Clear the VectorClock when member is removed. The change made by the leader is stamped - // and will propagate as is if there are no other changes on other nodes. - // If other concurrent changes on other nodes (e.g. join) the pruning is also - // taken care of when receiving gossips. - var newVersion = removed.Aggregate(localGossip.Version, (v, node) => - { - return v.Prune(VectorClock.Node.Create(VclockName(node))); - }); - var newGossip = localGossip.Copy(members: newMembers, overview: newOverview, version: newVersion); - if (!_exitingTasksInProgress && newGossip.GetMember(_selfUniqueAddress).Status == MemberStatus.Exiting) - { - // Leader is moving itself from Leaving to Exiting. - // ExitingCompleted will be received via CoordinatedShutdown to continue - // the leaving process. Meanwhile the gossip state is not marked as seen. - - _exitingTasksInProgress = true; - if (_coordShutdown.ShutdownReason == null) - _cluster.LogInfo("Exiting (leader), starting coordinated shutdown."); - _selfExiting.TrySetResult(Done.Instance); - _coordShutdown.Run(CoordinatedShutdown.ClusterLeavingReason.Instance); - } + // RemoveAll strips each removed node from the members, the `seen` table, the `reachability` + // table and the vector clock, and records a tombstone for it, all in one step. The tombstone + // is what stops a peer that still holds the member from putting it back on the next merge. + updatedGossip = localGossip.Copy(members: updatedMembers) + .RemoveAll(removed, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + } - UpdateLatestGossip(newGossip); + // Expired tombstones are dropped here, on the leader, on a tick that has already established + // convergence. A quiet tick drops nothing, PruneTombstones hands back the same instance, and the + // reference check below skips the update - otherwise every converged tick would bump the vector + // clock and reset the `seen` table, and the cluster would never settle. + var removeTombstonesEarlierThan = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + - (long)_cluster.Settings.PruneGossipTombstonesAfter.TotalMilliseconds; + var newGossip = updatedGossip.PruneTombstones(removeTombstonesEarlierThan); + + if (ReferenceEquals(newGossip, localGossip)) + return; + + if (hasChanges && !_exitingTasksInProgress && newGossip.GetMember(_selfUniqueAddress).Status == MemberStatus.Exiting) + { + // Leader is moving itself from Leaving to Exiting. + // ExitingCompleted will be received via CoordinatedShutdown to continue + // the leaving process. Meanwhile the gossip state is not marked as seen. + + _exitingTasksInProgress = true; + if (_coordShutdown.ShutdownReason == null) + _cluster.LogInfo("Exiting (leader), starting coordinated shutdown."); + _selfExiting.TrySetResult(Done.Instance); + _coordShutdown.Run(CoordinatedShutdown.ClusterLeavingReason.Instance); + } + + UpdateLatestGossip(newGossip); + + if (hasChanges) + { _exitingConfirmed = _exitingConfirmed.Except(removedExitingConfirmed); // log status changes @@ -2398,10 +2406,12 @@ public void LeaderActionsOnConvergence() { _cluster.LogInfo("Leader is removing confirmed Exiting node [{0}]", m.Address); } + } - PublishMembershipState(); + PublishMembershipState(); + + if (hasChanges) GossipExitingMembersToOldest(changedMembers.Where(i => i.Status == MemberStatus.Exiting).ToArray()); - } return; @@ -2601,6 +2611,14 @@ public static IReadOnlyCollection GossipTargetsForExitingMembers(Gossip /// /// Updates the local gossip with the latest received from over the network. /// + /// + /// Every change this node makes to the gossip goes through here, and every call increments this + /// node's vector clock entry. Tombstone writes and tombstone pruning must keep using this path. + /// A tombstone written without a clock bump would be invisible to the causal comparison in + /// , so peers could not tell which side holds the newer removal set. + /// unions tombstones on every branch, which keeps a stray + /// unstamped write from losing tombstones outright, but the ordering guarantee still comes from here. + /// /// The new gossip to merge with our own. public void UpdateLatestGossip(Gossip gossip) { diff --git a/src/core/Akka.Cluster/ClusterSettings.cs b/src/core/Akka.Cluster/ClusterSettings.cs index c01ef3977b0..dd9d65d3c2d 100644 --- a/src/core/Akka.Cluster/ClusterSettings.cs +++ b/src/core/Akka.Cluster/ClusterSettings.cs @@ -53,6 +53,10 @@ public ClusterSettings(Config config, string systemName) GossipInterval = clusterConfig.GetTimeSpan("gossip-interval", null); GossipTimeToLive = clusterConfig.GetTimeSpan("gossip-time-to-live", null); LeaderActionsInterval = clusterConfig.GetTimeSpan("leader-actions-interval", null); + PruneGossipTombstonesAfter = clusterConfig.GetTimeSpan("prune-gossip-tombstones-after", null); + if (PruneGossipTombstonesAfter <= TimeSpan.Zero) + throw new ConfigurationException( + $"[akka.cluster.prune-gossip-tombstones-after] must be a timespan greater than 0s. Received [{PruneGossipTombstonesAfter}]"); UnreachableNodesReaperInterval = clusterConfig.GetTimeSpan("unreachable-nodes-reaper-interval", null); PublishStatsInterval = clusterConfig.GetTimeSpanWithOffSwitch("publish-stats-interval"); @@ -199,6 +203,15 @@ TimeSpan GetWeaklyUpDuration() /// public TimeSpan LeaderActionsInterval { get; } + /// + /// How long the gossip remembers that it removed a member. + /// + /// The record stops a peer that has not caught up from putting the removed member back into the + /// gossip. A partition that outlasts this window re-opens that hole, so the value should sit well + /// above the longest partition the cluster is expected to heal from. Defaults to 24 hours. + /// + public TimeSpan PruneGossipTombstonesAfter { get; } + /// /// TBD /// diff --git a/src/core/Akka.Cluster/Configuration/Cluster.conf b/src/core/Akka.Cluster/Configuration/Cluster.conf index b2133ef6120..cccbbed992b 100644 --- a/src/core/Akka.Cluster/Configuration/Cluster.conf +++ b/src/core/Akka.Cluster/Configuration/Cluster.conf @@ -143,6 +143,13 @@ akka { # discard incoming gossip messages if not handled within this duration gossip-time-to-live = 2s + # how long the gossip remembers that it removed a member. + # The record stops a peer that has not caught up from putting the removed member + # back into the gossip. If the cluster is partitioned for longer than this, a + # removed node can be re-added by stale gossip, so keep this well above the + # longest partition the cluster is expected to heal from. + prune-gossip-tombstones-after = 24h + # how often should the leader perform maintenance tasks? leader-actions-interval = 1s From db6512cc3e913aaa2da6d6d29150d9268f2dc66b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 03:38:24 +0000 Subject: [PATCH 04/10] Akka.Cluster: tests for gossip removal tombstones GossipSpec - The removal-resurrection case built straight from Gossip values: the leader has removed a member and holds its tombstone, a lagging peer still holds it as Leaving. The merge must not put it back, in either direction. - The same setup without the tombstone, asserting the member IS kept. That case both proves the test above bites and pins the correct behaviour: with no evidence of a removal, a one-sided Leaving member may be a node the other side has not heard about yet, and dropping it would strand a live process. - Union is commutative, and keeps the later timestamp on collision. - The merged vector clock is pruned for every tombstoned node. Each side carries a clock entry for both removed nodes and a tombstone the other lacks; with only the member filter in place, this is the single test that fails. - One-sided Down and Exiting members are still dropped with no tombstones present, so the OR did not turn into a replacement. - A new incarnation at the same host and port survives its predecessor's tombstone. - RemoveAll strips members, seen, reachability as observer and as subject, and the vector clock entry, while leaving other nodes' clock entries alone. - Pruning boundary, and the same-instance identity that the leader's no-op check depends on. - The three non-merge gossip comparison branches each keep both sides' tombstones, and the union refuses to adopt a tombstone for a node it still holds as a member. ClusterMessageSerializerSpec - Round trip with tombstones, over GossipEnvelope and over Welcome. - A tombstoned address that shares a host and port with a member survives with its own UID, which is what catches a serializer resolving tombstones through the member address table. That bug is silent - it yields the wrong address, not an error. - A gossip proto with field 7 cleared decodes to an empty tombstone set with members and version intact. ClusterConfigSpec asserts the 24h default. The API approval files gain the two new public members: the PickHighestPriority overload and ClusterSettings .PruneGossipTombstonesAfter. Also fixes Gossip.Prune, which rebuilt the gossip through the constructor and so dropped tombstones on the pre-merge clock pruning path. --- ...APISpec.ApproveCluster.DotNet.verified.txt | 2 + ...oreAPISpec.ApproveCluster.Net.verified.txt | 2 + .../Akka.Cluster.Tests/ClusterConfigSpec.cs | 1 + src/core/Akka.Cluster.Tests/GossipSpec.cs | 249 ++++++++++++++++++ .../ClusterMessageSerializerSpec.cs | 91 +++++++ src/core/Akka.Cluster/Gossip.cs | 2 +- 6 files changed, 346 insertions(+), 1 deletion(-) diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.DotNet.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.DotNet.verified.txt index af790c4801f..c6e251644ad 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.DotNet.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.DotNet.verified.txt @@ -213,6 +213,7 @@ namespace Akka.Cluster public System.Collections.Immutable.ImmutableDictionary MinNrOfMembersOfRole { get; } public int MonitoredByNrOfMembers { get; } public System.TimeSpan PeriodicTasksInitialDelay { get; } + public System.TimeSpan PruneGossipTombstonesAfter { get; } public System.TimeSpan? PublishStatsInterval { get; } public int ReduceGossipDifferentViewProbability { get; } public System.TimeSpan? RetryUnsuccessfulJoinAfter { get; } @@ -257,6 +258,7 @@ namespace Akka.Cluster public override string ToString() { } public static Akka.Cluster.Member HighestPriorityOf(Akka.Cluster.Member m1, Akka.Cluster.Member m2) { } public static System.Collections.Immutable.ImmutableHashSet PickHighestPriority(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b) { } + public static System.Collections.Immutable.ImmutableHashSet PickHighestPriority(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b, System.Collections.Immutable.IImmutableSet tombstones) { } public static Akka.Cluster.Member PickNextTransition(Akka.Cluster.Member a, Akka.Cluster.Member b) { } public static System.Collections.Immutable.ImmutableSortedSet PickNextTransition(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b) { } } diff --git a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.Net.verified.txt b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.Net.verified.txt index 42b4e03e0a0..fd0a4eef4a5 100644 --- a/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.Net.verified.txt +++ b/src/core/Akka.API.Tests/verify/CoreAPISpec.ApproveCluster.Net.verified.txt @@ -213,6 +213,7 @@ namespace Akka.Cluster public System.Collections.Immutable.ImmutableDictionary MinNrOfMembersOfRole { get; } public int MonitoredByNrOfMembers { get; } public System.TimeSpan PeriodicTasksInitialDelay { get; } + public System.TimeSpan PruneGossipTombstonesAfter { get; } public System.TimeSpan? PublishStatsInterval { get; } public int ReduceGossipDifferentViewProbability { get; } public System.TimeSpan? RetryUnsuccessfulJoinAfter { get; } @@ -257,6 +258,7 @@ namespace Akka.Cluster public override string ToString() { } public static Akka.Cluster.Member HighestPriorityOf(Akka.Cluster.Member m1, Akka.Cluster.Member m2) { } public static System.Collections.Immutable.ImmutableHashSet PickHighestPriority(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b) { } + public static System.Collections.Immutable.ImmutableHashSet PickHighestPriority(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b, System.Collections.Immutable.IImmutableSet tombstones) { } public static Akka.Cluster.Member PickNextTransition(Akka.Cluster.Member a, Akka.Cluster.Member b) { } public static System.Collections.Immutable.ImmutableSortedSet PickNextTransition(System.Collections.Generic.IEnumerable a, System.Collections.Generic.IEnumerable b) { } } diff --git a/src/core/Akka.Cluster.Tests/ClusterConfigSpec.cs b/src/core/Akka.Cluster.Tests/ClusterConfigSpec.cs index 967b6ab180f..8b1890d903e 100644 --- a/src/core/Akka.Cluster.Tests/ClusterConfigSpec.cs +++ b/src/core/Akka.Cluster.Tests/ClusterConfigSpec.cs @@ -37,6 +37,7 @@ public void Clustering_must_be_able_to_parse_generic_cluster_config_elements() settings.PeriodicTasksInitialDelay.Should().Be(1.Seconds()); settings.GossipInterval.Should().Be(1.Seconds()); settings.GossipTimeToLive.Should().Be(2.Seconds()); + settings.PruneGossipTombstonesAfter.Should().Be(TimeSpan.FromHours(24)); settings.HeartbeatInterval.Should().Be(1.Seconds()); settings.MonitoredByNrOfMembers.Should().Be(9); settings.HeartbeatExpectedResponseAfter.Should().Be(1.Seconds()); diff --git a/src/core/Akka.Cluster.Tests/GossipSpec.cs b/src/core/Akka.Cluster.Tests/GossipSpec.cs index 905f029e030..c4cb354fb2b 100644 --- a/src/core/Akka.Cluster.Tests/GossipSpec.cs +++ b/src/core/Akka.Cluster.Tests/GossipSpec.cs @@ -174,6 +174,255 @@ public void A_gossip_must_merge_members_by_removing_removed_members() merged2.Members.Should().BeEquivalentTo(merged1.Members); } + // --------------------------------------------------------------------------------------------- + // Removal tombstones + // --------------------------------------------------------------------------------------------- + + private static ImmutableDictionary Tombstone(Member m, long timestamp = 1000L) => + ImmutableDictionary.Empty.Add(m.UniqueAddress, timestamp); + + private static VectorClock.Node VclockNodeOf(Member m) => VectorClock.Node.Create(VclockName(m.UniqueAddress)); + + [Fact(DisplayName = "Merge should not resurrect a removed member that a lagging peer still holds as Leaving")] + public void A_gossip_must_not_resurrect_a_tombstoned_member() + { + // the leader removed c1 while it was Exiting, and recorded a tombstone for it + var leaderGossip = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), + VectorClock.Create(), Tombstone(c1)); + + // a peer that has not caught up still holds c1 as Leaving - the status alone gives the merge + // nothing to go on, because Leaving is not terminal + var laggingPeerGossip = Gossip.Create(ImmutableSortedSet.Create(a1, b1, c1)); + + leaderGossip.Merge(laggingPeerGossip).Members.Should().BeEquivalentTo(ImmutableHashSet.Create(a1, b1)); + laggingPeerGossip.Merge(leaderGossip).Members.Should().BeEquivalentTo(ImmutableHashSet.Create(a1, b1)); + } + + [Fact(DisplayName = "Merge should resurrect a one-sided Leaving member when no tombstone says otherwise")] + public void A_gossip_must_keep_a_one_sided_leaving_member_without_a_tombstone() + { + // exactly the setup above minus the tombstone. c1 is kept, which is both the bug being fixed + // and the correct answer here: without evidence of a removal, a one-sided Leaving member may + // simply be a node the other side has not heard about yet, and dropping it would strand a live + // process that can never rejoin. + var g1 = Gossip.Create(ImmutableSortedSet.Create(a1, b1)); + var g2 = Gossip.Create(ImmutableSortedSet.Create(a1, b1, c1)); + + g1.Merge(g2).Members.Should().Contain(c1); + g2.Merge(g1).Members.Should().Contain(c1); + } + + [Fact(DisplayName = "Merge should union tombstones the same way in both directions")] + public void A_gossip_must_union_tombstones_commutatively() + { + var g1 = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 100L)); + var g2 = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(d1, 200L)); + + var merged1 = g1.Merge(g2); + var merged2 = g2.Merge(g1); + + merged1.Tombstones.Should().Equal(merged2.Tombstones); + merged1.Tombstones.Keys.Should().BeEquivalentTo(new[] { c1.UniqueAddress, d1.UniqueAddress }); + merged1.Members.Should().BeEquivalentTo(merged2.Members); + } + + [Fact(DisplayName = "Merge should keep the later timestamp when both sides tombstone the same node")] + public void A_gossip_must_keep_the_later_timestamp_on_a_tombstone_collision() + { + var g1 = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 100L)); + var g2 = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 500L)); + + g1.Merge(g2).Tombstones[c1.UniqueAddress].Should().Be(500L); + g2.Merge(g1).Tombstones[c1.UniqueAddress].Should().Be(500L); + } + + [Fact(DisplayName = "Merge should prune the merged vector clock for every tombstoned node")] + public void A_gossip_must_prune_the_merged_vector_clock_by_tombstones() + { + // each side carries a clock entry for BOTH removed nodes, and a tombstone the other side lacks. + // Unioning the clocks would put both entries back, exactly the way unioning members would put + // the members back. + var clock = VectorClock.Create() + .Increment(VclockNodeOf(c1)) + .Increment(VclockNodeOf(d1)) + .Increment(VclockNodeOf(a1)); + + var g1 = new Gossip(ImmutableSortedSet.Create(a1, d1), new GossipOverview(), clock, Tombstone(c1)); + var g2 = new Gossip(ImmutableSortedSet.Create(a1, c1), new GossipOverview(), clock, Tombstone(d1)); + + foreach (var merged in new[] { g1.Merge(g2), g2.Merge(g1) }) + { + merged.Members.Should().BeEquivalentTo(ImmutableHashSet.Create(a1)); + merged.Version.Versions.Keys.Should().NotContain(VclockNodeOf(c1)); + merged.Version.Versions.Keys.Should().NotContain(VclockNodeOf(d1)); + merged.Version.Versions.Keys.Should().Contain(VclockNodeOf(a1)); + } + } + + [Fact(DisplayName = "Merge should still drop one-sided Down and Exiting members with no tombstones present")] + public void A_gossip_must_still_drop_one_sided_terminal_members_without_tombstones() + { + // the drop condition was widened with OR, not replaced + var g1 = Gossip.Create(ImmutableSortedSet.Create(a1, b1)); + var g2 = Gossip.Create(ImmutableSortedSet.Create(a1, b1, c3, e3)); + + foreach (var merged in new[] { g1.Merge(g2), g2.Merge(g1) }) + { + merged.Members.Should().BeEquivalentTo(ImmutableHashSet.Create(a1, b1)); + merged.Tombstones.Should().BeEmpty(); + } + } + + [Fact(DisplayName = "Merge should let a new incarnation join even while its predecessor is tombstoned")] + public void A_gossip_must_not_block_a_new_incarnation_with_an_old_tombstone() + { + // same host and port, different UID. The tombstone key carries the UID, so it says nothing + // about the new incarnation. + var oldIncarnation = TestMember.Create(c1.Address, MemberStatus.Leaving, uid: 1); + var newIncarnation = TestMember.Create(c1.Address, MemberStatus.Joining, uid: 2); + + var g1 = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(oldIncarnation)); + var g2 = Gossip.Create(ImmutableSortedSet.Create(a1, newIncarnation)); + + foreach (var merged in new[] { g1.Merge(g2), g2.Merge(g1) }) + { + merged.Members.Should().Contain(newIncarnation); + merged.Tombstones.Keys.Should().NotContain(newIncarnation.UniqueAddress); + merged.Tombstones.Keys.Should().Contain(oldIncarnation.UniqueAddress); + } + } + + [Fact(DisplayName = "RemoveAll should strip a node from members, seen, reachability and the vector clock at once")] + public void A_gossip_must_strip_everything_when_removing_a_node() + { + var reachability = Reachability.Empty + .Unreachable(c1.UniqueAddress, b1.UniqueAddress) // c1 as observer + .Unreachable(b1.UniqueAddress, c1.UniqueAddress); // c1 as subject + var clock = VectorClock.Create().Increment(VclockNodeOf(a1)).Increment(VclockNodeOf(c1)); + + var g = new Gossip(ImmutableSortedSet.Create(a1, b1, c1), new GossipOverview(reachability), clock) + .Seen(a1.UniqueAddress) + .Seen(c1.UniqueAddress); + + var removed = g.RemoveAll(ImmutableHashSet.Create(c1.UniqueAddress), 4242L); + + removed.Members.Should().BeEquivalentTo(ImmutableSortedSet.Create(a1, b1)); + removed.Overview.Seen.Should().BeEquivalentTo(ImmutableHashSet.Create(a1.UniqueAddress)); + removed.Overview.Reachability.AllObservers.Should().NotContain(c1.UniqueAddress); + removed.Overview.Reachability.AllUnreachable.Should().NotContain(c1.UniqueAddress); + removed.Version.Versions.Keys.Should().NotContain(VclockNodeOf(c1)); + removed.Version.Versions.Keys.Should().Contain(VclockNodeOf(a1)); + removed.Tombstones[c1.UniqueAddress].Should().Be(4242L); + } + + [Fact(DisplayName = "RemoveAll should hand back the same gossip when there is nothing to remove")] + public void A_gossip_must_return_the_same_instance_when_removing_nothing() + { + var g = Gossip.Create(ImmutableSortedSet.Create(a1, b1)); + g.RemoveAll(ImmutableHashSet.Empty, 1L).Should().BeSameAs(g); + } + + [Fact(DisplayName = "PruneTombstones should drop entries at or before the threshold and keep later ones")] + public void A_gossip_must_prune_expired_tombstones() + { + var tombstones = ImmutableDictionary.Empty + .Add(c1.UniqueAddress, 100L) + .Add(d1.UniqueAddress, 101L); + var g = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), tombstones); + + g.PruneTombstones(99L).Should().BeSameAs(g); + + var atThreshold = g.PruneTombstones(100L); + atThreshold.Tombstones.Keys.Should().BeEquivalentTo(new[] { d1.UniqueAddress }); + + g.PruneTombstones(101L).Tombstones.Should().BeEmpty(); + } + + [Fact(DisplayName = "PruneTombstones should hand back the same gossip when nothing expired")] + public void A_gossip_must_return_the_same_instance_when_pruning_nothing() + { + // the leader publishes only when this reference changes, so a quiet tick must not allocate + var g = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 100L)); + + g.PruneTombstones(99L).Should().BeSameAs(g); + + var noTombstones = Gossip.Create(ImmutableSortedSet.Create(a1)); + noTombstones.PruneTombstones(long.MaxValue).Should().BeSameAs(noTombstones); + } + + // The four comparison branches of gossip reception. Only the concurrent one merges; the other + // three pick a whole gossip as the winner, so each has to union tombstones explicitly or removals + // decay out of the cluster. + + [Fact(DisplayName = "The Same branch should keep tombstones from both sides")] + public void A_gossip_must_carry_tombstones_through_the_same_branch() + { + var version = VectorClock.Create().Increment(VclockNodeOf(a1)); + var local = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), version, Tombstone(c1)); + var remote = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), version, Tombstone(d1)); + + var winner = remote.MergeSeen(local).MergeTombstones(local); + + winner.Tombstones.Keys.Should().BeEquivalentTo(new[] { c1.UniqueAddress, d1.UniqueAddress }); + } + + [Fact(DisplayName = "The Older branch should keep the remote side's tombstones")] + public void A_gossip_must_carry_tombstones_through_the_older_branch() + { + var local = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1)); + var remote = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + Tombstone(d1)); + + // local wins, but remote's tombstone survives + local.MergeTombstones(remote).Tombstones.Keys + .Should().BeEquivalentTo(new[] { c1.UniqueAddress, d1.UniqueAddress }); + } + + [Fact(DisplayName = "The Newer branch should keep the local side's tombstones")] + public void A_gossip_must_carry_tombstones_through_the_newer_branch() + { + var local = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1)); + var remote = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + Tombstone(d1)); + + // remote wins, but local's tombstone survives + remote.MergeTombstones(local).Tombstones.Keys + .Should().BeEquivalentTo(new[] { c1.UniqueAddress, d1.UniqueAddress }); + } + + [Fact(DisplayName = "MergeTombstones should not adopt a tombstone for a node it still holds as a member")] + public void A_gossip_must_keep_members_and_tombstones_disjoint_when_unioning() + { + var winner = Gossip.Create(ImmutableSortedSet.Create(a1, c1)); + var loser = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1)); + + var merged = winner.MergeTombstones(loser); + + merged.Should().BeSameAs(winner); + merged.Members.Should().Contain(c1); + } + + [Fact(DisplayName = "MergeTombstones should hand back the same gossip when it adds nothing")] + public void A_gossip_must_return_the_same_instance_when_unioning_nothing() + { + var g = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 500L)); + var older = new Gossip(ImmutableSortedSet.Create(a1), new GossipOverview(), VectorClock.Create(), + Tombstone(c1, 100L)); + + g.MergeTombstones(older).Should().BeSameAs(g); + g.MergeTombstones(Gossip.Create(ImmutableSortedSet.Create(a1))).Should().BeSameAs(g); + } + [Fact] public void A_gossip_must_have_leader_as_first_member_based_on_ordering_except_exiting_status() { diff --git a/src/core/Akka.Cluster.Tests/Serialization/ClusterMessageSerializerSpec.cs b/src/core/Akka.Cluster.Tests/Serialization/ClusterMessageSerializerSpec.cs index 4c7f5bd061e..eecbfc2803d 100644 --- a/src/core/Akka.Cluster.Tests/Serialization/ClusterMessageSerializerSpec.cs +++ b/src/core/Akka.Cluster.Tests/Serialization/ClusterMessageSerializerSpec.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System.Collections.Immutable; +using System.Linq; using Akka.Actor; using Akka.Cluster.Routing; using Akka.Cluster.Serialization; @@ -17,6 +18,7 @@ using Akka.Util; using Akka.Util.Internal; using Google.Protobuf; +using Proto = Akka.Cluster.Serialization.Proto; namespace Akka.Cluster.Tests.Serialization { @@ -96,6 +98,95 @@ public void Can_serialize_GossipEnvelope() message1.Gossip.Version.Versions.Should().Equal(deserialized.Gossip.Version.Versions); } + [Fact(DisplayName = "A gossip should round-trip its removal tombstones")] + public void Can_serialize_Gossip_tombstones() + { + var address = new Address("akka.tcp", "system", "some.host.org", 4711); + var uniqueAddress2 = new UniqueAddress(address, 18); + + // a1 stays a member; the two tombstoned nodes are not members, so their addresses only reach + // the wire if the serializer appends them to the address table + var removed1 = new UniqueAddress(new Address("akka.tcp", "sys", "gone1", 2552), 4711); + var removed2 = new UniqueAddress(new Address("akka.tcp", "sys", "gone2", 2552), 4712); + + var tombstones = ImmutableDictionary.Empty + .Add(removed1, 1234567890L) + .Add(removed2, 987654321L); + + var gossip = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + tombstones); + + var deserialized = AssertAndReturn(new GossipEnvelope(a1.UniqueAddress, uniqueAddress2, gossip)).Gossip; + + deserialized.Tombstones.Should().Equal(tombstones); + deserialized.Members.Should().BeEquivalentTo(gossip.Members); + } + + [Fact(DisplayName = "A tombstoned address should survive a round trip with its UID intact")] + public void Can_serialize_Gossip_tombstone_address_with_uid() + { + var address = new Address("akka.tcp", "system", "some.host.org", 4711); + var uniqueAddress2 = new UniqueAddress(address, 18); + + // same host and port as member b1, different UID. If the serializer resolved tombstones through + // the member address table it would silently hand back b1's UID here. + var removed = new UniqueAddress(b1.Address, b1.UniqueAddress.Uid + 1); + + var gossip = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + ImmutableDictionary.Empty.Add(removed, 42L)); + + var deserialized = AssertAndReturn(new GossipEnvelope(a1.UniqueAddress, uniqueAddress2, gossip)).Gossip; + + var key = deserialized.Tombstones.Keys.Should().ContainSingle().Subject; + key.Should().Be(removed); + key.Uid.Should().Be(removed.Uid); + deserialized.Members.Select(m => m.UniqueAddress).Should().BeEquivalentTo( + new[] { a1.UniqueAddress, b1.UniqueAddress }); + } + + [Fact(DisplayName = "Gossip written without the tombstone field should deserialize to an empty set")] + public void Can_deserialize_Gossip_without_tombstones() + { + var address = new Address("akka.tcp", "system", "some.host.org", 4711); + var uniqueAddress2 = new UniqueAddress(address, 18); + var removed = new UniqueAddress(new Address("akka.tcp", "sys", "gone", 2552), 4711); + + var gossip = new Gossip(ImmutableSortedSet.Create(a1, b1, c1), new GossipOverview(), + VectorClock.Create().Increment(new VectorClock.Node("node1")), + ImmutableDictionary.Empty.Add(removed, 99L)); + + var envelope = new GossipEnvelope(a1.UniqueAddress, uniqueAddress2, gossip); + var serializer = Sys.Serialization.FindSerializerFor(envelope); + var manifest = serializer.Manifest(envelope); + + var envelopeProto = Proto.Msg.GossipEnvelope.Parser.ParseFrom(serializer.ToBinary(envelope)); + var gossipProto = Proto.Msg.Gossip.Parser.ParseFrom(envelopeProto.SerializedGossip); + gossipProto.Tombstones.Should().NotBeEmpty(); + + // drop field 7 to get what a writer that predates tombstones produces + gossipProto.Tombstones.Clear(); + envelopeProto.SerializedGossip = ByteString.CopyFrom(gossipProto.ToByteArray()); + + var deserialized = (GossipEnvelope)serializer.FromBinary(envelopeProto.ToByteArray(), manifest); + + deserialized.Gossip.Tombstones.Should().BeEmpty(); + deserialized.Gossip.Members.Should().BeEquivalentTo(gossip.Members); + deserialized.Gossip.Version.Versions.Should().Equal(gossip.Version.Versions); + } + + [Fact(DisplayName = "A Welcome should round-trip its gossip's removal tombstones")] + public void Can_serialize_Welcome_with_tombstones() + { + var removed = new UniqueAddress(new Address("akka.tcp", "sys", "gone", 2552), 4711); + var tombstones = ImmutableDictionary.Empty.Add(removed, 55L); + var gossip = new Gossip(ImmutableSortedSet.Create(a1, b1), new GossipOverview(), VectorClock.Create(), + tombstones); + + var deserialized = AssertAndReturn(new InternalClusterAction.Welcome(a1.UniqueAddress, gossip)); + + deserialized.Gossip.Tombstones.Should().Equal(tombstones); + } + [Fact] public void Can_serialize_GossipStatus() { diff --git a/src/core/Akka.Cluster/Gossip.cs b/src/core/Akka.Cluster/Gossip.cs index 9ac9b9e32f8..978ae75aa28 100644 --- a/src/core/Akka.Cluster/Gossip.cs +++ b/src/core/Akka.Cluster/Gossip.cs @@ -513,7 +513,7 @@ public Gossip Prune(VectorClock.Node removedNode) if (newVersion.Equals(Version)) return this; else - return new Gossip(Members, Overview, newVersion); + return Copy(version: newVersion); } public Gossip MarkAsDown(Member member) From effd7c624b307da148f4282529100aff2599b4e6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 03:38:24 +0000 Subject: [PATCH 05/10] Record gossip removal tombstones in the v1.6 breaking changes ledger Wire addition to cluster gossip, the new retention setting, and the behavior change: a removed member can no longer be resurrected by stale gossip. --- BREAKING_CHANGES_V1.6.md | 1 + 1 file changed, 1 insertion(+) diff --git a/BREAKING_CHANGES_V1.6.md b/BREAKING_CHANGES_V1.6.md index 8aed1792fe6..5e9b1de496d 100644 --- a/BREAKING_CHANGES_V1.6.md +++ b/BREAKING_CHANGES_V1.6.md @@ -40,6 +40,7 @@ of `Behavior`, `Wire`, `API` (combine with `+`). | Status | PR / Branch | Component | Type | Change | Migration | |--------|-------------|-----------|------|--------|-----------| +| Planned | `fix/gossip-removal-tombstones` | `Akka.Cluster` | Wire + Behavior | Cluster gossip now carries a removal tombstone for every member the leader removes (new `repeated Tombstone tombstones = 7` on the `Gossip` proto message, previously the one vacant field number). A member the leader removed can no longer be put back into the gossip by a peer that has not caught up, which previously blocked convergence permanently and needed a full cluster restart to clear. Tombstones expire after the new `akka.cluster.prune-gossip-tombstones-after` setting (default `24h`), pruned by the leader on a converged tick. The field is additive and proto3 ignores unknown fields in both directions, so rolling upgrades are safe; an older node parses the gossip fine but drops the tombstones when it re-emits it, so the protection only holds between upgraded nodes and the fix is not fully in effect until every node is upgraded. | No action required. Raise `akka.cluster.prune-gossip-tombstones-after` if the cluster must survive partitions longer than 24 hours, or lower it if a cluster with heavy join/leave churn needs the gossip message kept small -- each tombstone adds a full address plus a timestamp to every gossip message until it expires. | | Planned | `feature/default-bounded-shard-rebalancing` | `Akka.Cluster.Sharding` | Behavior | The default `rebalance-absolute-limit` is now `20`, selecting the bounded shard allocation strategy instead of the legacy threshold-based strategy. | To retain the legacy strategy temporarily, explicitly set `akka.cluster.sharding.least-shard-allocation-strategy.rebalance-absolute-limit = 0`. Review `rebalance-threshold` and `max-simultaneous-rebalance`, which do not apply while the bounded strategy is active. | | Planned | `fix/artery-inbound-quarantine-check` | `Akka.Remote` (Artery) | Behavior | Quarantine is now enforced on the INBOUND path too. Previously Artery only gated outbound sends -- an envelope arriving FROM a uid this system has quarantined was still delivered, and the quarantined peer was only notified once, proactively, at the moment `Quarantine()` was called. A new `InboundQuarantineCheckStage`, woven into the inbound pipeline right after handshake, now drops every inbound envelope (ordinary or control, including system messages) whose origin uid is quarantined, and reactively re-sends a `Quarantined` control notice to the origin for each drop (except for a heartbeat or the peer's own `Quarantined` notice, to avoid a reply storm). The existing one-shot proactive notice in `Quarantine()` is unchanged. Additionally, an ordinary/large outbound stream that terminates while its association is quarantined no longer wedges permanently: the materialize-once gate is released (timer-driven auto-reconnect stays suppressed), so a quarantine-piercing `ActorSelection` send -- or any send after a new incarnation's handshake lifts the quarantine -- re-materializes the stream on demand and can reach a restarted peer at the same address. | No action required -- this is a bug fix restoring the documented "no further communication" guarantee of quarantine and the documented new-incarnation piercing behavior; code that (incorrectly) depended on a quarantined peer's replies still arriving is unsupported. | | Planned | `fix/artery-daemonmsgcreate-control-stream` | `Akka.Remote` (Artery) | Behavior | Remote deployment's `DaemonMsgCreate` now travels over Artery's CONTROL stream (as a plain envelope, no delivery/ack sequencing) instead of the ordinary stream, ordering it ahead of the `Watch` that remote deployment sends immediately afterwards. Previously the two rode independent, unordered TCP connections and `Watch` systematically arrived first, so the receiver replied `DeathWatchNotification(existenceConfirmed: false)` for a not-yet-created actor and the deployer reaped the freshly-deployed routee before its `Supervise` registration landed, emptying cluster router pools. Additionally (Pekko parity), inbound ordinary messages addressed to a remote-deployed recipient that has not been created yet are no longer dead-lettered immediately: the resolve is retried on a bounded schedule (20 attempts x 50ms, buffered per recipient in FIFO order) so first messages that arrive ahead of the in-flight `DaemonMsgCreate` are delivered once the actor exists; paths that never resolve are banned and dead-letter as before. | No action required -- this is a bug fix restoring correct create-before-watch ordering; code that (incorrectly) depended on the old race is unsupported. | From 630228a4ea824971d54861c47ceeee1c6f2cbe30 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 17:39:07 +0000 Subject: [PATCH 06/10] Akka.Cluster: property-based specs for gossip removal tombstones The example specs pin the cases a human thought of. These sample the same code over a few hundred to a few thousand random gossips per property and check laws: merge is commutative, idempotent and associative; a tombstone always beats a stale member; a removal never comes back over a random sequence of exchanges. CsCheck 4.8.0, referenced from Akka.Cluster.Tests only. Fixed iteration counts, so the class runs in about five seconds. A failure prints the seed to replay. Generators draw from a bounded universe of six nodes, two of which share a host and port and differ only by UID - the case that catches a serializer resolving a tombstone through the member address table. Member statuses honour the allowed transition table, tombstone timestamps come from a four value pool so collisions are frequent, and every timestamp is passed in rather than read off the clock. P1-P3 merge is commutative, idempotent and associative over members, tombstones with their timestamps, reachability and the vector clock. P4-P6 a tombstoned node loses; a one-sided live member with no tombstone is kept; a one-sided Down or Exiting member is still dropped. P7 disjointness, split by reception branch: the concurrent path lets the tombstone win, the winner-picked paths let the winner's member win. P8 no tombstoned node keeps a vector clock entry through a merge. P9 merged tombstones are the union, collisions take the later timestamp. P10 PruneTombstones drops what expired and hands back the same instance otherwise, which the leader's no-op check depends on. P11 RemoveAll writes a tombstone for every removal and strips it elsewhere. P12 proto round trip preserves members, tombstones, reachability and clock. P13-P14 random histories over three to five nodes exchanging gossip through the same branch selection ReceiveGossip uses, checked against a plain set of removed UIDs. Every node converges on that set, and no node ever loses a tombstone outside a prune. Three properties carry a documented caveat where the law they check is narrower than it first looks. Merge with itself also drops the clock entries of tombstoned nodes, so P2 compares against that form. The one-sided drop for Down and Exiting is not associative on its own, predating tombstones, so P3 draws non-terminal statuses and lets tombstones carry the removals. Reachability merge breaks an equal-version tie by argument order, so the two sides observe from disjoint observer sets. Each property counts the iterations that hit the case it is about and fails if that count is too low, so none of them can pass by never generating anything interesting. --- Directory.Build.props | 1 + .../Akka.Cluster.Tests.csproj | 1 + .../GossipTombstoneGenerators.cs | 352 ++++++++ .../GossipTombstonePropertySpecs.cs | 787 ++++++++++++++++++ .../GossipTombstoneSerializerPropertySpecs.cs | 103 +++ 5 files changed, 1244 insertions(+) create mode 100644 src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs create mode 100644 src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs create mode 100644 src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs diff --git a/Directory.Build.props b/Directory.Build.props index 4e80e1ea97d..46541023512 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -41,6 +41,7 @@ true 3.3.3 3.3.3 + 4.8.0 2.0.3 6.0.1 [6.0.*,) diff --git a/src/core/Akka.Cluster.Tests/Akka.Cluster.Tests.csproj b/src/core/Akka.Cluster.Tests/Akka.Cluster.Tests.csproj index bf4b318b182..f4374cf7c60 100644 --- a/src/core/Akka.Cluster.Tests/Akka.Cluster.Tests.csproj +++ b/src/core/Akka.Cluster.Tests/Akka.Cluster.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs b/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs new file mode 100644 index 00000000000..314fd1717f9 --- /dev/null +++ b/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs @@ -0,0 +1,352 @@ +//----------------------------------------------------------------------- +// +// Copyright (C) 2009-2022 Lightbend Inc. +// Copyright (C) 2013-2025 .NET Foundation +// +//----------------------------------------------------------------------- + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Akka.Actor; +using Akka.Util; +using CsCheck; +using static Akka.Cluster.ClusterCoreDaemon; + +namespace Akka.Cluster.Tests +{ + /// + /// Generators and equivalence helpers shared by the gossip removal-tombstone property specs. + /// + /// Everything under test is pure: is immutable, so a property only has to + /// build inputs and compare outputs. All randomness comes from CsCheck generators and every + /// timestamp is passed in, so a failing run replays from its printed seed. + /// + internal static class GossipTombstoneGenerators + { + /// + /// The bounded universe the properties draw from. Six nodes is enough to get interesting splits + /// while keeping the search space small enough that a few hundred iterations cover it. + /// + /// Index 2 and index 3 share a host and port and differ only by UID. That pair is the serializer + /// trap: a serializer that resolves a tombstone through the member address table hands back the + /// wrong UID for it, and the failure is silent. + /// + public static readonly ImmutableArray Universe = ImmutableArray.Create( + new UniqueAddress(new Address("akka.tcp", "sys", "a", 2552), 1), + new UniqueAddress(new Address("akka.tcp", "sys", "b", 2552), 2), + new UniqueAddress(new Address("akka.tcp", "sys", "c", 2552), 3), + new UniqueAddress(new Address("akka.tcp", "sys", "c", 2552), 4), + new UniqueAddress(new Address("akka.tcp", "sys", "d", 2552), 5), + new UniqueAddress(new Address("akka.tcp", "sys", "e", 2552), 6)); + + public static int NodeCount => Universe.Length; + + private static readonly ImmutableArray> Roles = ImmutableArray.Create( + ImmutableHashSet.Empty, + ImmutableHashSet.Create("r1"), + ImmutableHashSet.Create("r1", "r2"), + ImmutableHashSet.Create("r2"), + ImmutableHashSet.Empty, + ImmutableHashSet.Create("r3")); + + private static readonly ImmutableArray AppVersions = ImmutableArray.Create( + AppVersion.Zero, + AppVersion.Create("1.0.0"), + AppVersion.Create("1.1.0"), + AppVersion.Create("1.1.0"), + AppVersion.Create("2.0.0"), + AppVersion.Zero); + + /// + /// Vector clock nodes are MD5 hashes of the address, so they are computed once and reused. + /// + private static readonly ImmutableArray VclockNodes = + Universe.Select(a => VectorClock.Node.Create(VclockName(a))).ToImmutableArray(); + + /// Every status a live member may hold. + private static readonly ImmutableArray AllStatuses = ImmutableArray.Create( + MemberStatus.Joining, MemberStatus.WeaklyUp, MemberStatus.Up, + MemberStatus.Leaving, MemberStatus.Exiting, MemberStatus.Down); + + /// + /// The statuses the associativity property draws from. + /// + /// + /// drops a one-sided Down or Exiting member, and that rule is not associative on its + /// own: a node that is Up on one side and Down on another is Down after the first merge and then + /// dropped by the second, but survives when the same three sides are grouped the other way. That + /// predates tombstones. Leaving those two statuses out isolates the part associativity can speak + /// about, and removals still get exercised - through tombstones, which is the point. + /// + private static readonly ImmutableArray NonTerminalStatuses = ImmutableArray.Create( + MemberStatus.Joining, MemberStatus.WeaklyUp, MemberStatus.Up, MemberStatus.Leaving); + + public static UniqueAddress Node(int i) => Universe[i]; + + public static VectorClock.Node VclockNodeOf(UniqueAddress address) => + VclockNodes[Universe.IndexOf(address)]; + + public static Member MemberOf(int i, MemberStatus status) => + Member.Create(Universe[i], upNumber: i + 1, status, Roles[i], AppVersions[i]); + + /// How the sides of a merge may be shaped. + public enum Shape + { + /// Every side obeys the Gossip invariant: its members and its tombstones are disjoint. + Legal, + + /// + /// A side may carry the same node as a member and as a tombstone. The Gossip constructor + /// rejects that when AKKA_CLUSTER_ASSERT=on, but nothing else does, so the merge has to + /// hold up against it. + /// + Adversarial, + + /// + /// , and no member ever holds a terminal status. See + /// for why associativity needs this. + /// + NonTerminal + } + + /// + /// The raw draws behind one merge case. Kept flat so the whole case comes out of a single + /// generator and shrinks as one unit. + /// + internal sealed class Draw + { + public Draw(int[] baseStatus, bool[] isMember, int[] statusStep, bool[] isTombstone, + long[] timestamps, int[] clockTicks, bool[] seen, int[] reachOps) + { + BaseStatus = baseStatus; + IsMember = isMember; + StatusStep = statusStep; + IsTombstone = isTombstone; + Timestamps = timestamps; + ClockTicks = clockTicks; + Seen = seen; + ReachOps = reachOps; + } + + public int[] BaseStatus { get; } + public bool[] IsMember { get; } + public int[] StatusStep { get; } + public bool[] IsTombstone { get; } + public long[] Timestamps { get; } + public int[] ClockTicks { get; } + public bool[] Seen { get; } + public int[] ReachOps { get; } + } + + private const int ReachOpsPerSide = 4; + + /// + /// Generates gossips over the shared universe. + /// + /// Each node gets a base status; every side that holds it as a member holds it either at that + /// status or at one legal transition away from it, so no pair of sides disagrees in a way the + /// cluster could never produce. Membership is an independent draw per side, which is what + /// produces the overlapping splits the merge has to reconcile. + /// + public static Gen Sides(int sides, Shape shape = Shape.Legal) + { + var n = NodeCount; + var perSide = sides * n; + + var draw = Gen.Select( + Gen.Int[0, 5].Array[n], + Gen.Bool.Array[perSide], + Gen.Int[0, 4].Array[perSide], + Gen.Bool.Array[perSide], + // a four value pool guarantees collisions, which is what makes the max-timestamp rule bite + Gen.Long[1L, 4L].Array[perSide], + Gen.Int[0, 2].Array[perSide], + Gen.Bool.Array[perSide], + Gen.Int[0, 999].Array[sides * ReachOpsPerSide], + (a, b, c, d, e, f, g, h) => new Draw(a, b, c, d, e, f, g, h)); + + return draw.Select(d => Build(d, sides, shape)); + } + + private static Gossip[] Build(Draw d, int sides, Shape shape) + { + var n = NodeCount; + var statuses = shape == Shape.NonTerminal ? NonTerminalStatuses : AllStatuses; + + // per node: the base status, and per side: whether the side holds it and at which status + var memberOn = new bool[sides][]; + var statusOn = new MemberStatus[sides][]; + for (var s = 0; s < sides; s++) + { + memberOn[s] = new bool[n]; + statusOn[s] = new MemberStatus[n]; + } + + for (var i = 0; i < n; i++) + { + var baseStatus = statuses[d.BaseStatus[i] % statuses.Length]; + for (var s = 0; s < sides; s++) + { + var k = s * n + i; + memberOn[s][i] = d.IsMember[k]; + statusOn[s][i] = StepFrom(baseStatus, d.StatusStep[k], shape); + } + } + + // tombstones: a side never tombstones a node it holds as a member, unless the shape says otherwise + var tombstoneOn = new bool[sides][]; + for (var s = 0; s < sides; s++) + { + tombstoneOn[s] = new bool[n]; + for (var i = 0; i < n; i++) + { + var k = s * n + i; + tombstoneOn[s][i] = d.IsTombstone[k] + && (shape == Shape.Adversarial || !memberOn[s][i]); + } + } + + // Reachability observations are drawn from the nodes every side holds and nobody tombstones. + // + // Two reasons. Reachability.Merge breaks an equal-version tie by taking whichever side was + // passed second, so the same observer at the same version on both sides is not commutative - + // that predates tombstones, so each side gets its own observers here. And a record whose + // subject is not a member is filtered out by the merge, which would make associativity depend + // on the grouping. Neither has anything to do with removals. + var core = new List(); + for (var i = 0; i < n; i++) + { + var everywhere = true; + for (var s = 0; s < sides && everywhere; s++) + everywhere = memberOn[s][i] && !tombstoneOn[s][i]; + if (everywhere) core.Add(i); + } + + var result = new Gossip[sides]; + for (var s = 0; s < sides; s++) + { + var members = ImmutableSortedSet.Empty; + for (var i = 0; i < n; i++) + if (memberOn[s][i]) + members = members.Add(MemberOf(i, statusOn[s][i])); + + var tombstones = ImmutableDictionary.Empty; + for (var i = 0; i < n; i++) + if (tombstoneOn[s][i]) + tombstones = tombstones.Add(Universe[i], d.Timestamps[s * n + i]); + + var version = VectorClock.Create(); + for (var i = 0; i < n; i++) + for (var t = 0; t < d.ClockTicks[s * n + i]; t++) + version = version.Increment(VclockNodes[i]); + + // observers are partitioned across the sides so no two sides observe from the same node + var observers = core.Where(i => i % sides == s).ToList(); + var reachability = Reachability.Empty; + if (observers.Count > 0 && core.Count > 1) + { + for (var op = 0; op < ReachOpsPerSide; op++) + { + var v = d.ReachOps[s * ReachOpsPerSide + op]; + var observer = Universe[observers[v / 100 % observers.Count]]; + var subject = Universe[core[v / 10 % core.Count]]; + if (observer.Equals(subject)) continue; + reachability = (v % 3) switch + { + 0 => reachability.Unreachable(observer, subject), + 1 => reachability.Reachable(observer, subject), + _ => reachability.Terminated(observer, subject) + }; + } + } + + var seen = ImmutableHashSet.Empty; + for (var i = 0; i < n; i++) + if (memberOn[s][i] && d.Seen[s * n + i]) + seen = seen.Add(Universe[i]); + + result[s] = new Gossip(members, new GossipOverview(seen, reachability), version, tombstones); + } + + return result; + } + + /// + /// Picks a status for one side of one node: either the node's base status, or one legal + /// transition away from it. Sides never disagree by an illegal jump. + /// + private static MemberStatus StepFrom(MemberStatus baseStatus, int step, Shape shape) + { + if (step == 0) return baseStatus; + + var next = Member.AllowedTransitions[baseStatus] + .Where(s => s != MemberStatus.Removed) // a live gossip never holds a Removed member + .Where(s => shape != Shape.NonTerminal || NonTerminalStatuses.Contains(s)) + .OrderBy(s => (int)s) + .ToArray(); + + return next.Length == 0 ? baseStatus : next[(step - 1) % next.Length]; + } + + // ----------------------------------------------------------------------------------------- + // Equivalence + // ----------------------------------------------------------------------------------------- + + /// + /// The parts of a gossip two merge orders must agree on. Reachability records and merged member + /// sets come out in whatever order the merge built them, so both are compared as sets. + /// + public static string Describe(Gossip g) + { + var members = g.Members + .Select(m => $"{m.UniqueAddress}/{m.Status}/{m.UpNumber}/[{string.Join(",", m.Roles.OrderBy(r => r, StringComparer.Ordinal))}]/{m.AppVersion.Version}") + .OrderBy(x => x, StringComparer.Ordinal); + + var tombstones = g.Tombstones + .Select(t => $"{t.Key}={t.Value}") + .OrderBy(x => x, StringComparer.Ordinal); + + var records = g.Overview.Reachability.Records + .Select(r => $"{r.Observer}->{r.Subject}/{r.Status}/{r.Version}") + .OrderBy(x => x, StringComparer.Ordinal); + + var reachVersions = g.Overview.Reachability.Versions + .Select(v => $"{v.Key}={v.Value}") + .OrderBy(x => x, StringComparer.Ordinal); + + var version = g.Version.Versions + .Select(v => $"{v.Key}={v.Value}") + .OrderBy(x => x, StringComparer.Ordinal); + + return $"members=[{string.Join("; ", members)}]\n" + + $"tombstones=[{string.Join("; ", tombstones)}]\n" + + $"reachability=[{string.Join("; ", records)}] versions=[{string.Join("; ", reachVersions)}]\n" + + $"clock=[{string.Join("; ", version)}]"; + } + + /// Members, tombstones and clock only - what the sequence properties converge on. + public static string DescribeCore(Gossip g) + { + var members = g.Members.Select(m => $"{m.UniqueAddress}/{m.Status}") + .OrderBy(x => x, StringComparer.Ordinal); + var tombstones = g.Tombstones.Select(t => $"{t.Key}={t.Value}") + .OrderBy(x => x, StringComparer.Ordinal); + var version = g.Version.Versions.Select(v => $"{v.Key}={v.Value}") + .OrderBy(x => x, StringComparer.Ordinal); + + return $"members=[{string.Join("; ", members)}] tombstones=[{string.Join("; ", tombstones)}] " + + $"clock=[{string.Join("; ", version)}]"; + } + + public static string Print(IEnumerable gossips) => + string.Join("\n----\n", gossips.Select((g, i) => $"side {i}:\n{Describe(g)}")); + + /// The clock entry names of every node this gossip has tombstoned. + public static ImmutableHashSet TombstonedClockNodes(Gossip g) => + g.Tombstones.Keys.Select(VclockNodeOf).ToImmutableHashSet(); + } +} diff --git a/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs b/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs new file mode 100644 index 00000000000..04a82711d97 --- /dev/null +++ b/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs @@ -0,0 +1,787 @@ +//----------------------------------------------------------------------- +// +// Copyright (C) 2009-2022 Lightbend Inc. +// Copyright (C) 2013-2025 .NET Foundation +// +//----------------------------------------------------------------------- + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using CsCheck; +using FluentAssertions; +using Xunit; +using static Akka.Cluster.Tests.GossipTombstoneGenerators; + +namespace Akka.Cluster.Tests +{ + /// + /// Property-based specs for gossip removal tombstones. + /// + /// The example specs in GossipSpec pin the cases a human thought of. These sample the same + /// code over a few hundred random gossips per property and check laws instead of outcomes: + /// merge is commutative and associative, a tombstone always beats a stale member, and a removal + /// never comes back over a sequence of exchanges. + /// + /// Replaying a failure: CsCheck prints the failing seed, e.g. + /// Set seed: "0N0000000000" or -e CsCheck_Seed=0N0000000000 to reproduce + /// Paste it into the failing Sample call as seed: "0N0000000000", or set the + /// CsCheck_Seed environment variable and rerun. Iteration counts are fixed so the suite has a + /// predictable runtime; raise iter locally when hunting something rare. + /// + public class GossipTombstonePropertySpecs + { + // Fixed iteration counts, sized so the whole class runs in a few seconds. + private const int MergeIterations = 2000; + private const int SequenceIterations = 800; + + private static readonly ImmutableHashSet Terminal = + ImmutableHashSet.Create(MemberStatus.Down, MemberStatus.Exiting); + + // ----------------------------------------------------------------------------------------- + // P1 - commutativity + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P1: Merge is commutative over members, tombstones, reachability and the clock")] + public void P1_Merge_is_commutative() + { + Sides(2).Sample(g => + { + var ab = g[0].Merge(g[1]); + var ba = g[1].Merge(g[0]); + + // Describe covers members with their status, tombstone keys AND timestamps, reachability + // records and versions, and the merged vector clock. + Describe(ab).Should().Be(Describe(ba)); + }, iter: MergeIterations, print: Print); + } + + // ----------------------------------------------------------------------------------------- + // P2 - idempotence + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P2: Merge with itself changes nothing but the seen table and stale clock entries")] + public void P2_Merge_is_idempotent() + { + Sides(2).Sample(g => + { + var a = g[0]; + var merged = a.Merge(a); + + // Two documented differences, both by design: + // + // 1. Merge clears the seen table - "nobody can have seen this new gossip yet" - so the + // comparison is against a with its seen table cleared. + // 2. Merge drops the clock entry of every tombstoned node. A gossip can legitimately hold + // a tombstone and a stale clock entry for the same node: MergeTombstones adopts a + // tombstone without touching the clock. Merge is where that gets cleaned up, so the + // comparison prunes those entries from a as well. + var expected = a.ClearSeen(); + foreach (var node in TombstonedClockNodes(a)) + expected = expected.Prune(node); + + Describe(merged).Should().Be(Describe(expected)); + + // and the cleaned-up form really is a fixed point + Describe(merged.Merge(merged)).Should().Be(Describe(merged)); + }, iter: MergeIterations, print: Print); + } + + // ----------------------------------------------------------------------------------------- + // P3 - associativity + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P3: Merge is associative")] + public void P3_Merge_is_associative() + { + // Shape.NonTerminal keeps Down and Exiting out of the member statuses. The one-sided drop for + // those two is not associative on its own and predates tombstones - see the comment on + // GossipTombstoneGenerators.NonTerminalStatuses. Removals still get covered here, by the + // tombstones, which is what this suite is about. + Sides(3, Shape.NonTerminal).Sample(g => + { + var left = g[0].Merge(g[1]).Merge(g[2]); + var right = g[0].Merge(g[1].Merge(g[2])); + + Describe(left).Should().Be(Describe(right)); + }, iter: MergeIterations, print: Print); + } + + // ----------------------------------------------------------------------------------------- + // P4 - a tombstone beats a member on the other side + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P4: a node tombstoned on either side is gone from the merged members")] + public void P4_Tombstone_dominates_a_stale_member() + { + var covered = 0; + + Sides(2).Sample(g => + { + var merged = g[0].Merge(g[1]); + var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + for (var side = 0; side < 2; side++) + { + var other = g[1 - side]; + foreach (var tombstoned in g[side].Tombstones.Keys) + { + // whatever status the other side holds it at - Up, WeaklyUp, Joining, anything + mergedAddresses.Should().NotContain(tombstoned, + "a tombstone is positive evidence of a removal"); + + // and the removed node is gone from the reachability table too, which the + // merge gets by filtering reachability through the merged member set + merged.Overview.Reachability.AllObservers.Should().NotContain(tombstoned); + merged.Overview.Reachability.Versions.Keys.Should().NotContain(tombstoned); + + if (other.HasMember(tombstoned)) Interlocked.Increment(ref covered); + } + } + }, iter: MergeIterations, print: Print); + + // the property is worthless if the generator never produced the case it is about + covered.Should().BeGreaterThan(MergeIterations / 10, + "the generator must actually produce nodes tombstoned on one side and held as a member on the other"); + } + + // ----------------------------------------------------------------------------------------- + // P5 - a one-sided member with no tombstone is kept + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P5: a one-sided member with no tombstone and a live status is kept")] + public void P5_One_sided_live_member_is_retained() + { + var covered = 0; + + Sides(2).Sample(g => + { + var merged = g[0].Merge(g[1]); + var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + for (var side = 0; side < 2; side++) + { + var other = g[1 - side]; + foreach (var m in g[side].Members) + { + if (other.HasMember(m.UniqueAddress)) continue; + if (Terminal.Contains(m.Status)) continue; + if (g[0].Tombstones.ContainsKey(m.UniqueAddress)) continue; + if (g[1].Tombstones.ContainsKey(m.UniqueAddress)) continue; + + // With no evidence of a removal, a one-sided member may simply be a node the other + // side has not heard about yet. Dropping it would strand a live process. + mergedAddresses.Should().Contain(m.UniqueAddress); + Interlocked.Increment(ref covered); + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(MergeIterations / 10, + "the generator must actually produce one-sided live members with no tombstone"); + } + + // ----------------------------------------------------------------------------------------- + // P6 - a one-sided Down or Exiting member is still dropped + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P6: a one-sided Down or Exiting member is dropped with no tombstone present")] + public void P6_One_sided_terminal_member_is_dropped() + { + var covered = 0; + + Sides(2).Sample(g => + { + var merged = g[0].Merge(g[1]); + var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + for (var side = 0; side < 2; side++) + { + var other = g[1 - side]; + foreach (var m in g[side].Members) + { + if (other.HasMember(m.UniqueAddress)) continue; + if (!Terminal.Contains(m.Status)) continue; + + // the tombstone check widened the drop condition with an OR - it did not replace it + mergedAddresses.Should().NotContain(m.UniqueAddress); + Interlocked.Increment(ref covered); + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(MergeIterations / 10, + "the generator must actually produce one-sided Down and Exiting members"); + } + + // ----------------------------------------------------------------------------------------- + // P7 - members and tombstones never intersect + // + // Dominance is not the same on both gossip-reception paths, so this is four properties: + // + // P7a Merge (the Concurrent branch): the tombstone wins. + // P7b Merge, with a side that breaks the invariant by holding a node as a member AND as a + // tombstone: the tombstone still wins. + // P7c MergeTombstones (the Same/Before/After branches): the winner's member wins, because the + // implementation refuses to adopt a tombstone for a node it still holds. Both sides of the + // pair stay disjoint either way. + // P7d the one input Merge cannot make disjoint - and why the Gossip constructor forbids it. + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P7a: merged members and merged tombstones never intersect")] + public void P7a_Merged_members_and_tombstones_are_disjoint() + { + Sides(2).Sample(g => + { + foreach (var merged in new[] { g[0].Merge(g[1]), g[1].Merge(g[0]) }) + { + var addresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + addresses.Intersect(merged.Tombstones.Keys).Should().BeEmpty(); + } + }, iter: MergeIterations, print: Print); + } + + [Fact(DisplayName = "P7b: a side that holds a node as a member and as a tombstone still loses the member")] + public void P7b_Merge_survives_a_side_that_breaks_the_invariant() + { + var covered = 0; + + // Shape.Adversarial lets a side carry the same node as a member and as a tombstone. The Gossip + // constructor rejects that under AKKA_CLUSTER_ASSERT=on; nothing rejects it otherwise, so the + // merge is checked against it here. + Sides(2, Shape.Adversarial).Sample(g => + { + for (var side = 0; side < 2; side++) + { + var other = g[1 - side]; + foreach (var key in g[side].Tombstones.Keys) + { + // this side holds the node as a member and as a tombstone, and the other side + // does not hold it at all. The case both sides hold it is P7d. + if (!g[side].HasMember(key) || other.HasMember(key)) continue; + + foreach (var merged in new[] { g[0].Merge(g[1]), g[1].Merge(g[0]) }) + { + merged.Members.Select(m => m.UniqueAddress).Should().NotContain(key); + merged.Tombstones.Keys.Should().Contain(key); + } + + Interlocked.Increment(ref covered); + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(MergeIterations / 10, + "Shape.Adversarial must actually produce a side holding a node as member and tombstone"); + } + + [Fact(DisplayName = "P7c: MergeTombstones refuses a tombstone for a node the winner still holds")] + public void P7c_MergeTombstones_keeps_the_winner_disjoint() + { + var covered = 0; + + Sides(2).Sample(g => + { + foreach (var (winner, loser) in new[] { (g[0], g[1]), (g[1], g[0]) }) + { + var merged = winner.MergeTombstones(loser); + + // the winner's members are untouched, so disjointness is what the guard buys + merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet() + .Intersect(merged.Tombstones.Keys).Should().BeEmpty(); + merged.Members.SetEquals(winner.Members).Should().BeTrue(); + + foreach (var kv in loser.Tombstones) + { + if (winner.HasMember(kv.Key)) + { + // dropped on purpose. Every tombstone write bumps the clock, so a gossip that + // wins the causal comparison is never behind on removals and this branch is a + // guard rather than a decision. + merged.Tombstones.Keys.Should().NotContain(kv.Key); + Interlocked.Increment(ref covered); + } + else + { + merged.Tombstones.Keys.Should().Contain(kv.Key); + } + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(MergeIterations / 10, + "the generator must actually produce a loser tombstone for a node the winner holds"); + } + + [Fact(DisplayName = "P7d: Merge cannot drop a member both sides hold, even when one side tombstones it")] + public void P7d_Merge_cannot_decide_a_member_both_sides_hold() + { + var covered = 0; + + // Pins the precondition the Gossip constructor enforces. PickHighestPriority consults + // tombstones only on its one-sided branch; a node both sides hold as a member is picked by + // status alone. So a gossip that carries a member and a tombstone for the same node, merged + // with a gossip that holds that member, comes out non-disjoint. That is exactly the state + // AssertInvariants rejects, and why it rejects it. + Sides(2, Shape.Adversarial).Sample(g => + { + for (var side = 0; side < 2; side++) + { + var other = g[1 - side]; + foreach (var key in g[side].Tombstones.Keys) + { + if (!g[side].HasMember(key) || !other.HasMember(key)) continue; + + var merged = g[0].Merge(g[1]); + merged.Members.Select(m => m.UniqueAddress).Should().Contain(key); + merged.Tombstones.Keys.Should().Contain(key); + Interlocked.Increment(ref covered); + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(0, + "the generator must actually produce a node held as a member on both sides and tombstoned on one"); + } + + // ----------------------------------------------------------------------------------------- + // P8 - clock hygiene + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P8: no tombstoned node keeps a vector clock entry through a merge")] + public void P8_Merge_prunes_the_clock_of_tombstoned_nodes() + { + var covered = 0; + + Sides(2).Sample(g => + { + foreach (var merged in new[] { g[0].Merge(g[1]), g[1].Merge(g[0]) }) + { + foreach (var key in merged.Tombstones.Keys) + { + // Merging clocks resurrects an entry the same way merging members resurrects a + // member, so filtering members alone would leave the removed node in the clock + // forever. + merged.Version.Versions.Keys.Should().NotContain(VclockNodeOf(key)); + } + } + + // count the iterations where a side actually carried a clock entry for a tombstoned node + for (var side = 0; side < 2; side++) + { + if (g[side].Tombstones.Keys.Any(k => + g[0].Version.Versions.ContainsKey(VclockNodeOf(k)) || + g[1].Version.Versions.ContainsKey(VclockNodeOf(k)))) + { + Interlocked.Increment(ref covered); + break; + } + } + }, iter: MergeIterations, print: Print); + + covered.Should().BeGreaterThan(MergeIterations / 10, + "the generator must actually produce clock entries for tombstoned nodes"); + } + + // ----------------------------------------------------------------------------------------- + // P9 - the union law + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P9: merged tombstones are the union, and a collision resolves to the later timestamp")] + public void P9_Tombstones_union_with_the_max_timestamp() + { + var collisions = 0; + + Sides(2).Sample(g => + { + var expectedKeys = g[0].Tombstones.Keys.Union(g[1].Tombstones.Keys).ToImmutableHashSet(); + + foreach (var merged in new[] { g[0].Merge(g[1]), g[1].Merge(g[0]) }) + { + merged.Tombstones.Keys.ToImmutableHashSet().SetEquals(expectedKeys).Should().BeTrue(); + + foreach (var key in expectedKeys) + { + var expected = Math.Max( + g[0].Tombstones.TryGetValue(key, out var t0) ? t0 : long.MinValue, + g[1].Tombstones.TryGetValue(key, out var t1) ? t1 : long.MinValue); + merged.Tombstones[key].Should().Be(expected); + } + } + + // the winner-picked branches take the same union, minus the nodes the winner still holds + foreach (var (winner, loser) in new[] { (g[0], g[1]), (g[1], g[0]) }) + { + var merged = winner.MergeTombstones(loser); + var expectedUnion = winner.Tombstones.Keys + .Union(loser.Tombstones.Keys.Where(k => !winner.HasMember(k))) + .ToImmutableHashSet(); + + merged.Tombstones.Keys.ToImmutableHashSet().SetEquals(expectedUnion).Should().BeTrue(); + + foreach (var key in expectedUnion) + { + var expected = Math.Max( + winner.Tombstones.TryGetValue(key, out var w) ? w : long.MinValue, + loser.Tombstones.TryGetValue(key, out var l) && !winner.HasMember(key) + ? l + : long.MinValue); + merged.Tombstones[key].Should().Be(expected); + } + } + + foreach (var key in g[0].Tombstones.Keys) + if (g[1].Tombstones.ContainsKey(key)) + { + Interlocked.Increment(ref collisions); + break; + } + }, iter: MergeIterations, print: Print); + + collisions.Should().BeGreaterThan(MergeIterations / 10, + "the timestamp pool must be small enough that both sides tombstone the same node"); + } + + // ----------------------------------------------------------------------------------------- + // P10 - prune laws + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P10: PruneTombstones drops entries at or before the cutoff and is a no-op otherwise")] + public void P10_Prune_drops_expired_entries_only() + { + // Timestamps come out of [1, 4], so cutoffs from 0 to 5 cover both boundaries and everything + // between. removeEarlierThan is inclusive: the implementation drops entries at or before it. + Gen.Select(Sides(1), Gen.Long[0L, 5L], (g, cutoff) => (Gossip: g[0], Cutoff: cutoff)) + .Sample(t => + { + var pruned = t.Gossip.PruneTombstones(t.Cutoff); + + var expired = t.Gossip.Tombstones.Where(kv => kv.Value <= t.Cutoff).ToList(); + var kept = t.Gossip.Tombstones.Where(kv => kv.Value > t.Cutoff).ToList(); + + if (expired.Count == 0) + { + // The leader's convergence tick publishes only when this reference changes. Value + // equality is not enough: a fresh instance would bump the vector clock and reset + // the seen table on every quiet tick, and the cluster would never settle. + pruned.Should().BeSameAs(t.Gossip); + } + else + { + pruned.Tombstones.Keys.ToImmutableHashSet() + .SetEquals(kept.Select(kv => kv.Key)).Should().BeTrue(); + foreach (var kv in kept) + pruned.Tombstones[kv.Key].Should().Be(kv.Value); + } + + // pruning never touches anything else + pruned.Members.SetEquals(t.Gossip.Members).Should().BeTrue(); + pruned.Version.Versions.Should().Equal(t.Gossip.Version.Versions); + pruned.Overview.Seen.SetEquals(t.Gossip.Overview.Seen).Should().BeTrue(); + }, iter: MergeIterations, print: t => Describe(t.Gossip) + $"\ncutoff={t.Cutoff}"); + } + + // ----------------------------------------------------------------------------------------- + // P11 - the write path + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P11: RemoveAll writes a tombstone for every removed node and strips it everywhere else")] + public void P11_RemoveAll_writes_a_tombstone_for_every_removal() + { + // the removal set is drawn as a bitmask over the universe; the timestamp is supplied, never read + // off the clock + Gen.Select(Sides(1), Gen.Int[0, (1 << NodeCount) - 1], Gen.Long[100L, 200L], + (g, mask, ts) => (Gossip: g[0], Mask: mask, Timestamp: ts)) + .Sample(t => + { + var nodes = Enumerable.Range(0, NodeCount) + .Where(i => (t.Mask & (1 << i)) != 0) + .Select(Node) + .ToImmutableHashSet(); + + var removed = t.Gossip.RemoveAll(nodes, t.Timestamp); + + if (nodes.Count == 0) + { + removed.Should().BeSameAs(t.Gossip); + return; + } + + foreach (var node in nodes) + { + removed.Tombstones.Should().ContainKey(node); + removed.Tombstones[node].Should().Be(t.Timestamp); + removed.HasMember(node).Should().BeFalse(); + removed.Overview.Seen.Should().NotContain(node); + removed.Overview.Reachability.AllObservers.Should().NotContain(node); + removed.Overview.Reachability.AllUnreachable.Should().NotContain(node); + removed.Overview.Reachability.Versions.Keys.Should().NotContain(node); + removed.Version.Versions.Keys.Should().NotContain(VclockNodeOf(node)); + } + + // everything else is left alone + foreach (var m in t.Gossip.Members.Where(m => !nodes.Contains(m.UniqueAddress))) + removed.Members.Should().Contain(m); + + foreach (var kv in t.Gossip.Tombstones.Where(kv => !nodes.Contains(kv.Key))) + removed.Tombstones[kv.Key].Should().Be(kv.Value); + + foreach (var kv in t.Gossip.Version.Versions.Where(kv => + !nodes.Select(VclockNodeOf).Contains(kv.Key))) + removed.Version.Versions[kv.Key].Should().Be(kv.Value); + }, iter: MergeIterations, print: t => Describe(t.Gossip) + $"\nmask={t.Mask} ts={t.Timestamp}"); + } + + // ----------------------------------------------------------------------------------------- + // P13 / P14 - sequences of removals and exchanges + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P13: a removal never comes back over a random sequence of gossip exchanges")] + public void P13_Removals_do_not_resurrect_over_a_history() + { + var removals = 0; + + Histories().Sample(h => + { + var removed = RunHistory(h, checkMonotonicity: false, checkConvergence: true); + if (removed > 0) Interlocked.Increment(ref removals); + }, iter: SequenceIterations, print: h => h.ToString()); + + removals.Should().BeGreaterThan(SequenceIterations / 2, + "most histories must actually remove something"); + } + + [Fact(DisplayName = "P14: a node's tombstone set never shrinks except across a prune")] + public void P14_Tombstone_sets_only_grow() + { + var removals = 0; + + Histories().Sample(h => + { + var removed = RunHistory(h, checkMonotonicity: true, checkConvergence: false); + if (removed > 0) Interlocked.Increment(ref removals); + }, iter: SequenceIterations, print: h => h.ToString()); + + removals.Should().BeGreaterThan(SequenceIterations / 2, + "most histories must actually remove something"); + } + + /// + /// A random op sequence over N virtual nodes, each holding a gossip over the same initial + /// membership. Ops are ints, decoded by . + /// + private static Gen Histories() => + Gen.Select(Gen.Int[3, 5], Gen.Int[0, 899].Array[10, 30], (n, ops) => new History(n, ops)); + + private sealed class History + { + public History(int nodes, int[] ops) + { + Nodes = nodes; + Ops = ops; + } + + public int Nodes { get; } + public int[] Ops { get; } + + public override string ToString() => $"nodes={Nodes} ops=[{string.Join(",", Ops)}]"; + } + + /// + /// Replays a history against N gossips and an oracle - a plain set of removed UIDs. Returns how + /// many nodes the oracle saw removed, so the caller can check the histories were not all no-ops. + /// + /// Exchanges use the same branch selection ClusterCoreDaemon.ReceiveGossip uses, so the + /// property covers the paths a real cluster takes, not just . + /// Timestamps come from a counter passed in here, never from the wall clock. + /// + /// The N virtual nodes stay in the cluster for the whole history: a removal never targets one of + /// them. A node that keeps gossiping after being removed is not a history the cluster produces - + /// the removed node shuts down, and ReceiveGossip drops gossip that does not contain the + /// receiver and gossip from a sender the receiver no longer holds, so a zombie is deaf and + /// ignored in both directions. Modelling one instead would only measure that omission. + /// + private static int RunHistory(History history, bool checkMonotonicity, bool checkConvergence) + { + var n = history.Nodes; + + // every virtual node starts from the same gossip: the whole universe, all Up, one clock tick + // each so removals have a clock entry to prune + var initialMembers = Enumerable.Range(0, NodeCount) + .Select(i => MemberOf(i, MemberStatus.Up)) + .ToImmutableSortedSet(); + + var initialVersion = Enumerable.Range(0, NodeCount) + .Aggregate(VectorClock.Create(), (v, i) => v.Increment(VclockNodeOf(Node(i)))); + + var initial = new Gossip(initialMembers, new GossipOverview(), initialVersion); + + var gossips = Enumerable.Repeat(initial, n).ToArray(); + var identities = Enumerable.Range(0, n).Select(Node).ToArray(); + + var oracle = new HashSet(); + var timestamp = 1L; + + // a cutoff below every timestamp this history will ever write, so the prune op expires nothing + const long QuietCutoff = 0L; + + foreach (var raw in history.Ops) + { + var before = gossips.Select(g => g.Tombstones.Keys.ToImmutableHashSet()).ToArray(); + var kind = raw % 3; + var isPrune = kind == 2; + + switch (kind) + { + case 0: + { + // the leader removes a live member that is not one of the gossiping nodes + var actor = raw / 3 % n; + var g = gossips[actor]; + var candidates = g.Members + .Where(m => !identities.Contains(m.UniqueAddress)) + .ToArray(); + if (candidates.Length == 0) break; + + var victim = candidates[raw / 30 % candidates.Length].UniqueAddress; + var removed = g.RemoveAll(ImmutableHashSet.Create(victim), timestamp++); + + // ClusterCoreDaemon.UpdateLatestGossip stamps every local change with the node's + // own clock entry. Without that bump a removal would be invisible to the causal + // comparison below. + gossips[actor] = StampLocalChange(removed, identities[actor]); + oracle.Add(victim); + break; + } + + case 1: + { + var receiver = raw / 3 % n; + var sender = raw / 30 % n; + if (receiver == sender) break; + gossips[receiver] = Receive(gossips[receiver], gossips[sender], identities[receiver]); + break; + } + + default: + { + var node = raw / 3 % n; + var pruned = gossips[node].PruneTombstones(QuietCutoff); + + // nothing expired, so the leader's no-op guard must hold + pruned.Should().BeSameAs(gossips[node]); + gossips[node] = pruned; + break; + } + } + + // (a) and (b): after every op, no node holds a UID as a member and as a tombstone + for (var i = 0; i < n; i++) + { + var g = gossips[i]; + var addresses = g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + addresses.Intersect(g.Tombstones.Keys).Should().BeEmpty( + "node {0} must keep members and tombstones disjoint", i); + + foreach (var key in g.Tombstones.Keys) + g.HasMember(key).Should().BeFalse( + "node {0} carries a tombstone for {1}", i, key); + + // and nothing is invented out of thin air + addresses.Except(initialMembers.Select(m => m.UniqueAddress)).Should().BeEmpty(); + } + + if (checkMonotonicity && !isPrune) + { + for (var i = 0; i < n; i++) + before[i].Except(gossips[i].Tombstones.Keys).Should().BeEmpty( + "node {0} must not lose a tombstone outside a prune", i); + } + } + + if (!checkConvergence) return oracle.Count; + + // exchange every ordered pair until nothing moves + var rounds = 0; + while (rounds++ < 25) + { + var snapshot = gossips.Select(DescribeCore).ToArray(); + + for (var i = 0; i < n; i++) + for (var j = 0; j < n; j++) + { + if (i == j) continue; + gossips[i] = Receive(gossips[i], gossips[j], identities[i]); + } + + if (gossips.Select(DescribeCore).SequenceEqual(snapshot) && + snapshot.Distinct().Count() == 1) + break; + } + + var converged = gossips.Select(DescribeCore).Distinct().ToArray(); + converged.Length.Should().Be(1, + "every node must end on the same members, tombstones and clock, got:\n{0}", + string.Join("\n", converged)); + + var expectedMembers = initialMembers + .Where(m => !oracle.Contains(m.UniqueAddress)) + .Select(m => m.UniqueAddress) + .ToImmutableHashSet(); + + foreach (var g in gossips) + { + g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet() + .SetEquals(expectedMembers).Should().BeTrue(); + g.Tombstones.Keys.ToImmutableHashSet().SetEquals(oracle).Should().BeTrue(); + } + + return oracle.Count; + } + + /// + /// Mirrors ClusterCoreDaemon.UpdateLatestGossip: stamp the change with the node's own clock + /// entry, then mark the gossip as seen by nobody but this node. + /// + private static Gossip StampLocalChange(Gossip gossip, UniqueAddress self) => + gossip.Increment(VclockNodeOf(self)).OnlySeen(self); + + /// + /// Mirrors the four comparison branches of ClusterCoreDaemon.ReceiveGossip, including the + /// conflicting-gossip clock pruning the concurrent branch does before merging. + /// + private static Gossip Receive(Gossip local, Gossip remote, UniqueAddress self) + { + Gossip winner; + switch (remote.Version.CompareTo(local.Version)) + { + case VectorClock.Ordering.Same: + winner = remote.MergeSeen(local).MergeTombstones(local); + break; + case VectorClock.Ordering.Before: + winner = local.MergeTombstones(remote); + break; + case VectorClock.Ordering.After: + winner = remote.MergeTombstones(local); + break; + default: + winner = PruneConflicting(remote, local).Merge(PruneConflicting(local, remote)); + break; + } + + return winner.Seen(self); + } + + private static Gossip PruneConflicting(Gossip gossip, Gossip other) => + gossip.Members.Aggregate(gossip, (g, m) => + MembershipState.RemoveUnreachableWithMemberStatus.Contains(m.Status) && + !other.Members.Contains(m) + ? g.Prune(VclockNodeOf(m.UniqueAddress)) + : g); + } +} diff --git a/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs b/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs new file mode 100644 index 00000000000..be2a2706cdb --- /dev/null +++ b/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs @@ -0,0 +1,103 @@ +//----------------------------------------------------------------------- +// +// Copyright (C) 2009-2022 Lightbend Inc. +// Copyright (C) 2013-2025 .NET Foundation +// +//----------------------------------------------------------------------- + +#nullable enable + +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using Akka.Serialization; +using Akka.TestKit; +using CsCheck; +using FluentAssertions; +using Xunit; +using static Akka.Cluster.Tests.GossipTombstoneGenerators; + +namespace Akka.Cluster.Tests.Serialization +{ + /// + /// P12: proto round trip for gossip carrying removal tombstones. + /// + /// A tombstoned node is not a member, so its address is missing from the address table the member + /// loop builds. Getting that wrong is silent - the tombstone comes back pointing at some other node + /// - which is why this is a property rather than a handful of examples. + /// + /// Replaying a failure: CsCheck prints the failing seed. Pass it back as seed: "..." on the + /// Sample call, or set the CsCheck_Seed environment variable and rerun. + /// + public class GossipTombstoneSerializerPropertySpecs : AkkaSpec + { + private const int Iterations = 1000; + + public GossipTombstoneSerializerPropertySpecs(ITestOutputHelper output) + : base("akka.actor.provider = cluster", output) + { + } + + [Fact(DisplayName = "P12: gossip with tombstones survives a proto round trip intact")] + public void P12_Gossip_round_trips_with_tombstones() + { + var probe = new GossipEnvelope(Node(0), Node(1), Gossip.Empty); + var serializer = Sys.Serialization.FindSerializerFor(probe); + serializer.Should().BeOfType(); + var manifest = ((SerializerWithStringManifest)serializer).Manifest(probe); + + // how often the shared host-and-port pair actually landed on opposite sides of the member + // and tombstone tables + var sharedHostPortCovered = 0; + + Sides(1).Sample(g => + { + var gossip = g[0]; + var envelope = new GossipEnvelope(Node(0), Node(1), gossip); + + var bytes = serializer.ToBinary(envelope); + var back = (GossipEnvelope)serializer.FromBinary(bytes, manifest); + + // Describe compares members with status, upNumber, roles and app version; tombstone + // addresses AND timestamps; reachability records and versions; and the vector clock. + Describe(back.Gossip).Should().Be(Describe(gossip)); + + // spelled out for the case this property exists for: the tombstone key keeps its own UID + foreach (var kv in gossip.Tombstones) + { + var key = back.Gossip.Tombstones.Keys.Single(k => k.Equals(kv.Key)); + key.Uid.Should().Be(kv.Key.Uid); + key.Address.Should().Be(kv.Key.Address); + back.Gossip.Tombstones[key].Should().Be(kv.Value); + } + + // Node(2) and Node(3) share a host and port and differ only by UID. A serializer that + // looked a tombstone up by address would hand back the member's UID here. + var memberAddresses = gossip.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + if ((memberAddresses.Contains(Node(2)) && gossip.Tombstones.ContainsKey(Node(3))) || + (memberAddresses.Contains(Node(3)) && gossip.Tombstones.ContainsKey(Node(2)))) + Interlocked.Increment(ref sharedHostPortCovered); + }, iter: Iterations, print: Print); + + sharedHostPortCovered.Should().BeGreaterThan(Iterations / 20, + "the generator must put the shared host-and-port pair on both sides of the member/tombstone split"); + } + + [Fact(DisplayName = "P12: a Welcome carries the same tombstones through a round trip")] + public void P12_Welcome_round_trips_with_tombstones() + { + var probe = new InternalClusterAction.Welcome(Node(0), Gossip.Empty); + var serializer = Sys.Serialization.FindSerializerFor(probe); + var manifest = ((SerializerWithStringManifest)serializer).Manifest(probe); + + Sides(1).Sample(g => + { + var welcome = new InternalClusterAction.Welcome(Node(0), g[0]); + var bytes = serializer.ToBinary(welcome); + var back = (InternalClusterAction.Welcome)serializer.FromBinary(bytes, manifest); + + Describe(back.Gossip).Should().Be(Describe(g[0])); + }, iter: Iterations, print: Print); + } + } +} From 207ae79398be248472c86f8c390b7f5f9265feca Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 20:46:54 +0000 Subject: [PATCH 07/10] Akka.Cluster: keep only the winner's tombstones on the causally-ordered branches ReceiveGossip unioned the receiver's tombstones into the winning gossip on all four comparison branches, while PruneTombstones is called from one place - the leader's LeaderActionsOnConvergence. A prune was therefore undone by the next exchange with any peer, because peers never prune and always re-unioned. So prune-gossip-tombstones-after reclaimed nothing, tombstones rode every gossip message forever, and once one was past the window the leader looped on every tick: prune, clock bump, seen reset, re-converge, peers hand it back, prune again. That is exactly the never-settles failure the reference-equality guard in PruneTombstones exists to prevent. Tombstones are now unioned only where neither gossip descends from the other - the equal-clock branch, and the concurrent branch inside Gossip.Merge. The two branches that pick a strictly newer gossip keep the winner's tombstones and nothing else. The winner is not behind on removals: a removal bumps the removing node's own clock entry, so a gossip that dominates the loser's clock descends from every removal the loser knows about. The tombstones it does not carry are the ones its own leader pruned, and honouring those prunes is the point. The branch selection moved out of ReceiveGossip into ClusterCoreDaemon.SelectWinningGossip, unchanged apart from the two lines above. The property specs now call it instead of re-implementing it - they had their own copy of the branches, so deleting the production change left them all green. Three new properties: P15 a tombstone pruned on a converged tick does not come back. The pruning node stamps the prune the way UpdateLatestGossip does, which puts it strictly ahead of every peer, so every exchange afterwards takes a winner-picked branch - the branch that used to hand the tombstones straight back. P16 a simulated week of virtual time: sparse jumps forward, a constant per-node clock offset up to five minutes, and a prune tick run by a different drawn node each time, measured against the shipped prune-gossip-tombstones-after default rather than a cutoff invented for the test. Expired tombstones are reclaimed and stay reclaimed, and no tombstone is dropped sooner than the window minus the widest clock disagreement. P17 a removal holds when the node that recorded it is itself removed later. That is where Merge prunes the clock entry that recorded the removal, so it is the one place a gossip could come out strictly newer than a tombstone carrier without having descended from the removal. P15 and P16 both fail if the union is restored on the Before and After branches. P17 documents a resurrection the model can reach and a cluster cannot, because removing a member is a leader action and the leader needs convergence first; the same history resurrects the member with the union restored, so it is not something the union was buying. Also in the suite: the seen table joins the equivalence helper; reachability is built from one shared op history per observer with each side replaying a prefix of it, so sides share observers at different versions and Reachability.Merge's version arbitration is actually exercised; histories gained a status op so peers hold a victim at mixed statuses while its tombstone propagates; every op asserts the gossip carries no clock entry for a non-member, mirroring the "too many vector clock entries" check; P4, P7c and P11 assert no reachability record survives whose subject was removed; the serializer property gained a sample where a tombstone key is also a member, which is the arm of the address-table writer nothing else reached; coverage guards count iterations rather than occurrences; and the properties whose inputs the Gossip constructor rejects under AKKA_CLUSTER_ASSERT=on skip there instead of failing. --- BREAKING_CHANGES_V1.6.md | 4 +- .../GossipTombstoneGenerators.cs | 72 +- .../GossipTombstonePropertySpecs.cs | 881 ++++++++++++++++-- .../GossipTombstoneSerializerPropertySpecs.cs | 45 + src/core/Akka.Cluster/ClusterDaemon.cs | 120 ++- src/core/Akka.Cluster/Gossip.cs | 20 +- 6 files changed, 998 insertions(+), 144 deletions(-) diff --git a/BREAKING_CHANGES_V1.6.md b/BREAKING_CHANGES_V1.6.md index 5e9b1de496d..9d5a9c2f247 100644 --- a/BREAKING_CHANGES_V1.6.md +++ b/BREAKING_CHANGES_V1.6.md @@ -1,4 +1,4 @@ -# Akka.NET v1.6 Breaking Changes +# Akka.NET v1.6 Breaking Changes This document tracks **breaking changes** introduced into the `dev` branch during the Akka.NET **v1.6** development cycle, ahead of a stable `v1.6.0` release. @@ -40,7 +40,7 @@ of `Behavior`, `Wire`, `API` (combine with `+`). | Status | PR / Branch | Component | Type | Change | Migration | |--------|-------------|-----------|------|--------|-----------| -| Planned | `fix/gossip-removal-tombstones` | `Akka.Cluster` | Wire + Behavior | Cluster gossip now carries a removal tombstone for every member the leader removes (new `repeated Tombstone tombstones = 7` on the `Gossip` proto message, previously the one vacant field number). A member the leader removed can no longer be put back into the gossip by a peer that has not caught up, which previously blocked convergence permanently and needed a full cluster restart to clear. Tombstones expire after the new `akka.cluster.prune-gossip-tombstones-after` setting (default `24h`), pruned by the leader on a converged tick. The field is additive and proto3 ignores unknown fields in both directions, so rolling upgrades are safe; an older node parses the gossip fine but drops the tombstones when it re-emits it, so the protection only holds between upgraded nodes and the fix is not fully in effect until every node is upgraded. | No action required. Raise `akka.cluster.prune-gossip-tombstones-after` if the cluster must survive partitions longer than 24 hours, or lower it if a cluster with heavy join/leave churn needs the gossip message kept small -- each tombstone adds a full address plus a timestamp to every gossip message until it expires. | +| Planned | `fix/gossip-removal-tombstones` | `Akka.Cluster` | Wire + Behavior | Cluster gossip now carries a removal tombstone for every member the leader removes (new `repeated Tombstone tombstones = 7` on the `Gossip` proto message, previously the one vacant field number). A member the leader removed can no longer be put back into the gossip by a peer that has not caught up, which previously blocked convergence permanently and needed a full cluster restart to clear. Tombstones expire after the new `akka.cluster.prune-gossip-tombstones-after` setting (default `24h`), pruned by the leader on a converged tick and reclaimed for good: gossip reception keeps the tombstones of whichever gossip wins the causal comparison, so a peer that has not pruned yet cannot hand a pruned tombstone back. Tombstones are only unioned across the two gossips when neither descends from the other -- equal clocks, or concurrent clocks, where each side may hold a removal the other has not heard about. The field is additive and proto3 ignores unknown fields in both directions, so rolling upgrades are safe; an older node parses the gossip fine but drops the tombstones when it re-emits it, and if that stripped gossip is causally newer than an upgraded node's, the upgraded node adopts it and loses those tombstones too -- so the protection only holds between upgraded nodes and the fix is not fully in effect until every node is upgraded. | No action required. Raise `akka.cluster.prune-gossip-tombstones-after` if the cluster must survive partitions longer than 24 hours, or lower it if a cluster with heavy join/leave churn needs the gossip message kept small -- each tombstone adds a full address plus a timestamp to every gossip message until it expires. | | Planned | `feature/default-bounded-shard-rebalancing` | `Akka.Cluster.Sharding` | Behavior | The default `rebalance-absolute-limit` is now `20`, selecting the bounded shard allocation strategy instead of the legacy threshold-based strategy. | To retain the legacy strategy temporarily, explicitly set `akka.cluster.sharding.least-shard-allocation-strategy.rebalance-absolute-limit = 0`. Review `rebalance-threshold` and `max-simultaneous-rebalance`, which do not apply while the bounded strategy is active. | | Planned | `fix/artery-inbound-quarantine-check` | `Akka.Remote` (Artery) | Behavior | Quarantine is now enforced on the INBOUND path too. Previously Artery only gated outbound sends -- an envelope arriving FROM a uid this system has quarantined was still delivered, and the quarantined peer was only notified once, proactively, at the moment `Quarantine()` was called. A new `InboundQuarantineCheckStage`, woven into the inbound pipeline right after handshake, now drops every inbound envelope (ordinary or control, including system messages) whose origin uid is quarantined, and reactively re-sends a `Quarantined` control notice to the origin for each drop (except for a heartbeat or the peer's own `Quarantined` notice, to avoid a reply storm). The existing one-shot proactive notice in `Quarantine()` is unchanged. Additionally, an ordinary/large outbound stream that terminates while its association is quarantined no longer wedges permanently: the materialize-once gate is released (timer-driven auto-reconnect stays suppressed), so a quarantine-piercing `ActorSelection` send -- or any send after a new incarnation's handshake lifts the quarantine -- re-materializes the stream on demand and can reach a restarted peer at the same address. | No action required -- this is a bug fix restoring the documented "no further communication" guarantee of quarantine and the documented new-incarnation piercing behavior; code that (incorrectly) depended on a quarantined peer's replies still arriving is unsupported. | | Planned | `fix/artery-daemonmsgcreate-control-stream` | `Akka.Remote` (Artery) | Behavior | Remote deployment's `DaemonMsgCreate` now travels over Artery's CONTROL stream (as a plain envelope, no delivery/ack sequencing) instead of the ordinary stream, ordering it ahead of the `Watch` that remote deployment sends immediately afterwards. Previously the two rode independent, unordered TCP connections and `Watch` systematically arrived first, so the receiver replied `DeathWatchNotification(existenceConfirmed: false)` for a not-yet-created actor and the deployer reaped the freshly-deployed routee before its `Supervise` registration landed, emptying cluster router pools. Additionally (Pekko parity), inbound ordinary messages addressed to a remote-deployed recipient that has not been created yet are no longer dead-lettered immediately: the resolve is retried on a bounded schedule (20 attempts x 50ms, buffered per recipient in FIFO order) so first messages that arrive ahead of the in-flight `DaemonMsgCreate` are delivered once the actor exists; paths that never resolve are banned and dead-letter as before. | No action required -- this is a bug fix restoring correct create-before-watch ordering; code that (incorrectly) depended on the old race is unsupported. | diff --git a/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs b/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs index 314fd1717f9..4880af4b289 100644 --- a/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs +++ b/src/core/Akka.Cluster.Tests/GossipTombstoneGenerators.cs @@ -142,7 +142,8 @@ public Draw(int[] baseStatus, bool[] isMember, int[] statusStep, bool[] isTombst public int[] ReachOps { get; } } - private const int ReachOpsPerSide = 4; + /// How many observations one observer's shared op history holds. + private const int ReachOpsPerObserver = 4; /// /// Generates gossips over the shared universe. @@ -166,7 +167,8 @@ public static Gen Sides(int sides, Shape shape = Shape.Legal) Gen.Long[1L, 4L].Array[perSide], Gen.Int[0, 2].Array[perSide], Gen.Bool.Array[perSide], - Gen.Int[0, 999].Array[sides * ReachOpsPerSide], + // one shared op history per observer, then one prefix length per side per observer + Gen.Int[0, 999].Array[n * ReachOpsPerObserver + perSide], (a, b, c, d, e, f, g, h) => new Draw(a, b, c, d, e, f, g, h)); return draw.Select(d => Build(d, sides, shape)); @@ -211,12 +213,28 @@ private static Gossip[] Build(Draw d, int sides, Shape shape) } // Reachability observations are drawn from the nodes every side holds and nobody tombstones. + // A record whose subject is not a member is filtered out by Reachability.Merge through its + // `allowed` set, and which nodes are members depends on how a three-way merge is grouped, so + // leaving those in would break associativity on something that has nothing to do with + // removals. // - // Two reasons. Reachability.Merge breaks an equal-version tie by taking whichever side was - // passed second, so the same observer at the same version on both sides is not commutative - - // that predates tombstones, so each side gets its own observers here. And a record whose - // subject is not a member is filtered out by the merge, which would make associativity depend - // on the grouping. Neither has anything to do with removals. + // Every observer writes its own rows and nothing else in the cluster touches them, so an + // observer's row set is always a prefix of that observer's own op history. Each side replays + // a prefix of the same history per observer, which is what a real cluster looks like: two + // nodes that have seen different amounts of the same observer's work. Sides therefore share + // observers at different versions, which is what puts Reachability.Merge's version + // arbitration under test, and equal version means equal rows by construction - the invariant + // its tie-break relies on. Equal prefixes are drawn often, so the identical-rows tie is + // covered too. + // + // Known wart, deliberately not modelled: Reachability.Remove and Gossip.Merge's `allowed` + // filter both drop an observer's rows without touching that observer's version, so a real + // cluster can hold the same observer at the same version with different rows. Sketch: two + // gossips both hold observer O at version 3 with rows {X unreachable, Y unreachable}; one of + // them removes X, so its O drops to {Y unreachable} while O's version stays 3. + // Reachability.Merge breaks that tie by taking whichever side was passed second, so the merge + // is not commutative there. That predates tombstones, like the Down/Exiting note on + // NonTerminalStatuses. var core = new List(); for (var i = 0; i < n; i++) { @@ -226,6 +244,9 @@ private static Gossip[] Build(Draw d, int sides, Shape shape) if (everywhere) core.Add(i); } + // the shared per-observer op histories come first in the draw, prefix lengths after them + var prefixOffset = n * ReachOpsPerObserver; + var result = new Gossip[sides]; for (var s = 0; s < sides; s++) { @@ -244,23 +265,29 @@ private static Gossip[] Build(Draw d, int sides, Shape shape) for (var t = 0; t < d.ClockTicks[s * n + i]; t++) version = version.Increment(VclockNodes[i]); - // observers are partitioned across the sides so no two sides observe from the same node - var observers = core.Where(i => i % sides == s).ToList(); var reachability = Reachability.Empty; - if (observers.Count > 0 && core.Count > 1) + if (core.Count > 1) { - for (var op = 0; op < ReachOpsPerSide; op++) + foreach (var i in core) { - var v = d.ReachOps[s * ReachOpsPerSide + op]; - var observer = Universe[observers[v / 100 % observers.Count]]; - var subject = Universe[core[v / 10 % core.Count]]; - if (observer.Equals(subject)) continue; - reachability = (v % 3) switch + var observer = Universe[i]; + + // nobody observes itself, so the subject pool leaves the observer out rather than + // drawing it and skipping the op - a skipped op is a wasted draw + var subjects = core.Where(c => c != i).ToList(); + var take = d.ReachOps[prefixOffset + s * n + i] % (ReachOpsPerObserver + 1); + + for (var op = 0; op < take; op++) { - 0 => reachability.Unreachable(observer, subject), - 1 => reachability.Reachable(observer, subject), - _ => reachability.Terminated(observer, subject) - }; + var v = d.ReachOps[i * ReachOpsPerObserver + op]; + var subject = Universe[subjects[v / 10 % subjects.Count]]; + reachability = (v % 3) switch + { + 0 => reachability.Unreachable(observer, subject), + 1 => reachability.Reachable(observer, subject), + _ => reachability.Terminated(observer, subject) + }; + } } } @@ -322,9 +349,14 @@ public static string Describe(Gossip g) .Select(v => $"{v.Key}={v.Value}") .OrderBy(x => x, StringComparer.Ordinal); + var seen = g.Overview.Seen + .Select(a => a.ToString()) + .OrderBy(x => x, StringComparer.Ordinal); + return $"members=[{string.Join("; ", members)}]\n" + $"tombstones=[{string.Join("; ", tombstones)}]\n" + $"reachability=[{string.Join("; ", records)}] versions=[{string.Join("; ", reachVersions)}]\n" + + $"seen=[{string.Join("; ", seen)}]\n" + $"clock=[{string.Join("; ", version)}]"; } diff --git a/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs b/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs index 04a82711d97..6a9b93ee0fc 100644 --- a/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs +++ b/src/core/Akka.Cluster.Tests/GossipTombstonePropertySpecs.cs @@ -12,6 +12,7 @@ using System.Collections.Immutable; using System.Linq; using System.Threading; +using Akka.Cluster.Configuration; using CsCheck; using FluentAssertions; using Xunit; @@ -27,6 +28,14 @@ namespace Akka.Cluster.Tests /// merge is commutative and associative, a tombstone always beats a stale member, and a removal /// never comes back over a sequence of exchanges. /// + /// Time is data here, not a clock. Every timestamp a property writes is passed in, and P16 carries a + /// virtual now that its histories jump forward by drawn hours. Nothing under test reads a clock - + /// and both take the instant as a + /// parameter - so sampling sparse instants across a simulated week is sound and needs no TimeProvider. + /// The two wall-clock reads that remain are in ClusterCoreDaemon.LeaderActionsOnConvergence, + /// which is what supplies those parameters; that call site is out of this suite's reach by design, and + /// what it computes is exactly the arithmetic P16 checks. + /// /// Replaying a failure: CsCheck prints the failing seed, e.g. /// Set seed: "0N0000000000" or -e CsCheck_Seed=0N0000000000 to reproduce /// Paste it into the failing Sample call as seed: "0N0000000000", or set the @@ -38,10 +47,33 @@ public class GossipTombstonePropertySpecs // Fixed iteration counts, sized so the whole class runs in a few seconds. private const int MergeIterations = 2000; private const int SequenceIterations = 800; + private const int LineageIterations = 2000; + + /// + /// The real retention window, read off the shipped default rather than picked here, so P16 checks + /// the arithmetic the cluster actually runs. + /// + private static readonly TimeSpan PruneWindow = + new ClusterSettings(ClusterConfigFactory.Default(), "gossip-tombstone-property-specs") + .PruneGossipTombstonesAfter; + + /// + /// How far apart two nodes' clocks may sit in P16. Node clocks are drawn in [0, MaxSkew], so the + /// widest disagreement between any minting node and any pruning node is MaxSkew. + /// + private static readonly TimeSpan MaxSkew = TimeSpan.FromMinutes(5); private static readonly ImmutableHashSet Terminal = ImmutableHashSet.Create(MemberStatus.Down, MemberStatus.Exiting); + /// + /// Shape.Adversarial builds a gossip that holds a node as a member and as a tombstone. The Gossip + /// constructor throws on exactly that under AKKA_CLUSTER_ASSERT=on, so the properties about it + /// cannot run there - the input they are about cannot be constructed. + /// + private const string AdversarialSkipReason = + "AKKA_CLUSTER_ASSERT=on rejects the member-and-tombstone gossip this property is about"; + // ----------------------------------------------------------------------------------------- // P1 - commutativity // ----------------------------------------------------------------------------------------- @@ -49,15 +81,27 @@ public class GossipTombstonePropertySpecs [Fact(DisplayName = "P1: Merge is commutative over members, tombstones, reachability and the clock")] public void P1_Merge_is_commutative() { + var sharedObserverAtDifferentVersions = 0; + Sides(2).Sample(g => { var ab = g[0].Merge(g[1]); var ba = g[1].Merge(g[0]); // Describe covers members with their status, tombstone keys AND timestamps, reachability - // records and versions, and the merged vector clock. + // records and versions, the seen table, and the merged vector clock. Describe(ab).Should().Be(Describe(ba)); + + // the reachability generator is only worth anything if the two sides really do hold the + // same observer at different versions - that is what Reachability.Merge arbitrates + var v0 = g[0].Overview.Reachability.Versions; + var v1 = g[1].Overview.Reachability.Versions; + if (v0.Keys.Any(o => v1.TryGetValue(o, out var other) && other != v0[o])) + Interlocked.Increment(ref sharedObserverAtDifferentVersions); }, iter: MergeIterations, print: Print); + + sharedObserverAtDifferentVersions.Should().BeGreaterThan(MergeIterations / 10, + "the two sides must share observers at different versions often enough to arbitrate"); } // ----------------------------------------------------------------------------------------- @@ -124,6 +168,7 @@ public void P4_Tombstone_dominates_a_stale_member() { var merged = g[0].Merge(g[1]); var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + var hit = false; for (var side = 0; side < 2; side++) { @@ -134,14 +179,24 @@ public void P4_Tombstone_dominates_a_stale_member() mergedAddresses.Should().NotContain(tombstoned, "a tombstone is positive evidence of a removal"); - // and the removed node is gone from the reachability table too, which the - // merge gets by filtering reachability through the merged member set + // and the removed node is gone from the reachability table too, which the merge + // gets by filtering reachability through the merged member set: as an observer, + // as a subject, and in the versions table merged.Overview.Reachability.AllObservers.Should().NotContain(tombstoned); + // The subject check is a pin rather than a bite here: the generator only ever + // observes nodes every side holds as a member, so no side starts with a record + // about a node it has tombstoned. P11 covers the case with teeth, where RemoveAll + // strips a node that really was being observed. + merged.Overview.Reachability.Records.Select(r => r.Subject) + .Should().NotContain(tombstoned, + "a record about a removed node is stale, and an unreachable non-member blocks convergence"); merged.Overview.Reachability.Versions.Keys.Should().NotContain(tombstoned); - if (other.HasMember(tombstoned)) Interlocked.Increment(ref covered); + if (other.HasMember(tombstoned)) hit = true; } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); // the property is worthless if the generator never produced the case it is about @@ -162,6 +217,7 @@ public void P5_One_sided_live_member_is_retained() { var merged = g[0].Merge(g[1]); var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + var hit = false; for (var side = 0; side < 2; side++) { @@ -176,9 +232,11 @@ public void P5_One_sided_live_member_is_retained() // With no evidence of a removal, a one-sided member may simply be a node the other // side has not heard about yet. Dropping it would strand a live process. mergedAddresses.Should().Contain(m.UniqueAddress); - Interlocked.Increment(ref covered); + hit = true; } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); covered.Should().BeGreaterThan(MergeIterations / 10, @@ -199,6 +257,8 @@ public void P6_One_sided_terminal_member_is_dropped() var merged = g[0].Merge(g[1]); var mergedAddresses = merged.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + var hit = false; + for (var side = 0; side < 2; side++) { var other = g[1 - side]; @@ -209,9 +269,11 @@ public void P6_One_sided_terminal_member_is_dropped() // the tombstone check widened the drop condition with an OR - it did not replace it mergedAddresses.Should().NotContain(m.UniqueAddress); - Interlocked.Increment(ref covered); + hit = true; } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); covered.Should().BeGreaterThan(MergeIterations / 10, @@ -248,6 +310,8 @@ public void P7a_Merged_members_and_tombstones_are_disjoint() [Fact(DisplayName = "P7b: a side that holds a node as a member and as a tombstone still loses the member")] public void P7b_Merge_survives_a_side_that_breaks_the_invariant() { + Assert.SkipWhen(Cluster.IsAssertInvariantsEnabled, AdversarialSkipReason); + var covered = 0; // Shape.Adversarial lets a side carry the same node as a member and as a tombstone. The Gossip @@ -255,6 +319,8 @@ public void P7b_Merge_survives_a_side_that_breaks_the_invariant() // merge is checked against it here. Sides(2, Shape.Adversarial).Sample(g => { + var hit = false; + for (var side = 0; side < 2; side++) { var other = g[1 - side]; @@ -270,9 +336,11 @@ public void P7b_Merge_survives_a_side_that_breaks_the_invariant() merged.Tombstones.Keys.Should().Contain(key); } - Interlocked.Increment(ref covered); + hit = true; } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); covered.Should().BeGreaterThan(MergeIterations / 10, @@ -286,6 +354,8 @@ public void P7c_MergeTombstones_keeps_the_winner_disjoint() Sides(2).Sample(g => { + var hit = false; + foreach (var (winner, loser) in new[] { (g[0], g[1]), (g[1], g[0]) }) { var merged = winner.MergeTombstones(loser); @@ -295,15 +365,32 @@ public void P7c_MergeTombstones_keeps_the_winner_disjoint() .Intersect(merged.Tombstones.Keys).Should().BeEmpty(); merged.Members.SetEquals(winner.Members).Should().BeTrue(); + // Adopting a tombstone leaves reachability alone, and it does not need to touch it. + // The winner only adopts tombstones for nodes it does not hold as a member, and no + // gossip carries a reachability record for a non-member subject: Merge filters + // subjects through its `allowed` set and RemoveAll strips them outright. If that ever + // stopped being true, MergeTombstones would have to strip subjects too - an + // unreachable non-member blocks convergence forever, because MembershipState. + // Convergence resolves it to a Removed member and Removed is not a status it skips. + merged.Overview.Reachability.Records.Select(r => r.Subject) + .Should().NotIntersectWith(merged.Tombstones.Keys); + foreach (var kv in loser.Tombstones) { if (winner.HasMember(kv.Key)) { - // dropped on purpose. Every tombstone write bumps the clock, so a gossip that - // wins the causal comparison is never behind on removals and this branch is a - // guard rather than a decision. + // Dropped on purpose, and it is a real drop - the winner loses a removal it + // was told about. It is safe because ReceiveGossip only calls MergeTombstones + // with two equal clocks: a removal bumps the removing node's clock entry, so + // both sides descend from every removal either of them knows about, and a + // member that is back after a removal is one whose tombstone was already + // pruned. Adopting it again would undo that prune. + // + // The branches that pick a strictly newer gossip do not come through here at + // all - they keep the winner's tombstones and nothing else, which is what + // makes a prune stick. P15 covers that. merged.Tombstones.Keys.Should().NotContain(kv.Key); - Interlocked.Increment(ref covered); + hit = true; } else { @@ -311,6 +398,8 @@ public void P7c_MergeTombstones_keeps_the_winner_disjoint() } } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); covered.Should().BeGreaterThan(MergeIterations / 10, @@ -322,6 +411,8 @@ public void P7d_Merge_cannot_decide_a_member_both_sides_hold() { var covered = 0; + Assert.SkipWhen(Cluster.IsAssertInvariantsEnabled, AdversarialSkipReason); + // Pins the precondition the Gossip constructor enforces. PickHighestPriority consults // tombstones only on its one-sided branch; a node both sides hold as a member is picked by // status alone. So a gossip that carries a member and a tombstone for the same node, merged @@ -329,6 +420,8 @@ public void P7d_Merge_cannot_decide_a_member_both_sides_hold() // AssertInvariants rejects, and why it rejects it. Sides(2, Shape.Adversarial).Sample(g => { + var hit = false; + for (var side = 0; side < 2; side++) { var other = g[1 - side]; @@ -339,12 +432,14 @@ public void P7d_Merge_cannot_decide_a_member_both_sides_hold() var merged = g[0].Merge(g[1]); merged.Members.Select(m => m.UniqueAddress).Should().Contain(key); merged.Tombstones.Keys.Should().Contain(key); - Interlocked.Increment(ref covered); + hit = true; } } + + if (hit) Interlocked.Increment(ref covered); }, iter: MergeIterations, print: Print); - covered.Should().BeGreaterThan(0, + covered.Should().BeGreaterThan(MergeIterations / 10, "the generator must actually produce a node held as a member on both sides and tombstoned on one"); } @@ -519,6 +614,13 @@ public void P11_RemoveAll_writes_a_tombstone_for_every_removal() removed.Overview.Seen.Should().NotContain(node); removed.Overview.Reachability.AllObservers.Should().NotContain(node); removed.Overview.Reachability.AllUnreachable.Should().NotContain(node); + + // subjects too, at every status - AllUnreachable misses Terminated records, and a + // record about a node that is no longer a member stalls convergence: the + // convergence check resolves the subject to a Removed member, and Removed is not + // one of the statuses it skips + removed.Overview.Reachability.Records.Select(r => r.Subject) + .Should().NotContain(node); removed.Overview.Reachability.Versions.Keys.Should().NotContain(node); removed.Version.Versions.Keys.Should().NotContain(VclockNodeOf(node)); } @@ -544,15 +646,21 @@ public void P11_RemoveAll_writes_a_tombstone_for_every_removal() public void P13_Removals_do_not_resurrect_over_a_history() { var removals = 0; + var mixedStatus = 0; Histories().Sample(h => { - var removed = RunHistory(h, checkMonotonicity: false, checkConvergence: true); - if (removed > 0) Interlocked.Increment(ref removals); + var result = RunHistory(h, checkMonotonicity: false, checkConvergence: true); + if (result.Removed > 0) Interlocked.Increment(ref removals); + if (result.MixedStatusRemoval) Interlocked.Increment(ref mixedStatus); }, iter: SequenceIterations, print: h => h.ToString()); removals.Should().BeGreaterThan(SequenceIterations / 2, "most histories must actually remove something"); + + mixedStatus.Should().BeGreaterThan(SequenceIterations / 10, + "the status op must actually put peers on a different status than the removing node, " + + "which is the shape a real removal has"); } [Fact(DisplayName = "P14: a node's tombstone set never shrinks except across a prune")] @@ -562,33 +670,569 @@ public void P14_Tombstone_sets_only_grow() Histories().Sample(h => { - var removed = RunHistory(h, checkMonotonicity: true, checkConvergence: false); - if (removed > 0) Interlocked.Increment(ref removals); + var result = RunHistory(h, checkMonotonicity: true, checkConvergence: false); + if (result.Removed > 0) Interlocked.Increment(ref removals); }, iter: SequenceIterations, print: h => h.ToString()); removals.Should().BeGreaterThan(SequenceIterations / 2, "most histories must actually remove something"); } + // ----------------------------------------------------------------------------------------- + // P15 - a prune sticks + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P15: a tombstone pruned on a converged tick does not come back")] + public void P15_A_prune_is_not_undone_by_the_next_exchange() + { + var pruned = 0; + + Histories().Sample(h => + { + var result = RunHistory(h, checkMonotonicity: false, checkConvergence: true, + checkPruneStickiness: true); + if (result.Pruned > 0) Interlocked.Increment(ref pruned); + }, iter: SequenceIterations, print: h => h.ToString()); + + pruned.Should().BeGreaterThan(SequenceIterations / 10, + "the drawn cutoff must actually expire tombstones in a good share of histories"); + } + + // ----------------------------------------------------------------------------------------- + // P16 - a week of virtual time, skewed clocks and a moving leader + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P16: tombstones expire and stay expired across a simulated week of skewed clocks")] + public void P16_Tombstones_expire_across_virtual_time() + { + var expiredAndPruned = 0; + + TimeHistories().Sample(h => + { + if (RunTimeHistory(h)) Interlocked.Increment(ref expiredAndPruned); + }, iter: SequenceIterations, print: h => h.ToString()); + + expiredAndPruned.Should().BeGreaterThan(SequenceIterations / 10, + "histories must actually age a tombstone past the window and prune it mid-history"); + } + + /// + /// Histories for P16: the same op alphabet as plus a jump forward in + /// virtual time, and a per-node clock offset so no two nodes agree on what time it is. + /// + private static Gen TimeHistories() => + Gen.Select( + Gen.Int[3, 5], + Gen.Int[0, 1199].Array[20, 44], + Gen.Long[0L, (long)MaxSkew.TotalMilliseconds].Array[NodeCount], + Gen.Int[0, 4], + (n, ops, skew, pruner) => new TimeHistory(n, ops, skew, pruner)); + + private sealed class TimeHistory + { + public TimeHistory(int nodes, int[] ops, long[] skewMillis, int finalPruneNode) + { + Nodes = nodes; + Ops = ops; + SkewMillis = skewMillis; + FinalPruneNode = finalPruneNode; + } + + public int Nodes { get; } + public int[] Ops { get; } + + /// How far ahead of true virtual time each node's own clock runs. + public long[] SkewMillis { get; } + + public int FinalPruneNode { get; } + + public override string ToString() => + $"nodes={Nodes} ops=[{string.Join(",", Ops)}] skew=[{string.Join(",", SkewMillis)}] " + + $"finalPruner={FinalPruneNode}"; + } + + /// + /// Replays a history over a simulated week. + /// + /// Three things this adds over : + /// + /// 1. Virtual time. A jump op moves now forward by drawn hours, so a tombstone minted early really + /// does age past - the shipped 24 hour default, not a cutoff invented + /// here. The jumps are sparse rather than a tick per second; every function under test takes its + /// instant as a parameter, so sampling instants is as good as stepping through them. + /// 2. Clock skew. Each node's own clock runs a drawn amount ahead of true virtual time. A removal + /// stamps its tombstone with the removing node's skewed view, and a prune computes its cutoff + /// from the pruning node's skewed view, so the two ends of the retention window are read off + /// different clocks - which is what happens in a real cluster. + /// 3. A moving leader. The prune op is run by a drawn node, not a fixed one. + /// + /// A prune only runs from a converged state, because that is the only state + /// LeaderActionsOnConvergence runs in. That is what keeps the window from expiring a tombstone + /// before it has reached everybody, which would put the removed member back - real behaviour, and + /// the reason the setting defaults to a day, but not what this property is about. + /// + /// Returns true when a mid-history prune actually dropped something, so the caller can check the + /// week-scale part was not a no-op. + /// + private static bool RunTimeHistory(TimeHistory history) + { + var n = history.Nodes; + var window = (long)PruneWindow.TotalMilliseconds; + var maxSkew = (long)MaxSkew.TotalMilliseconds; + + // start far enough in so a cutoff never goes negative + var now = window * 2; + + var initialMembers = Enumerable.Range(0, NodeCount) + .Select(i => MemberOf(i, MemberStatus.Up)) + .ToImmutableSortedSet(); + + var initialVersion = Enumerable.Range(0, NodeCount) + .Aggregate(VectorClock.Create(), (v, i) => v.Increment(VclockNodeOf(Node(i)))); + + var gossips = Enumerable.Repeat( + new Gossip(initialMembers, new GossipOverview(), initialVersion), n).ToArray(); + var identities = Enumerable.Range(0, n).Select(Node).ToArray(); + + var oracle = new HashSet(); + + // The true virtual instant behind each tombstone timestamp a history has written. Two mints can + // land on the same key and value from different instants, because the skews differ; the + // earliest is kept, which only ever weakens the skew-safety bound below - never fakes a pass. + var mintedAt = new Dictionary<(UniqueAddress Key, long Timestamp), long>(); + + var prunedSomethingMidHistory = false; + + bool Converged() => gossips.Select(DescribeCore).Distinct().Count() == 1; + + // one gossip round: every ordered pair exchanges once + void Round() + { + for (var i = 0; i < n; i++) + for (var j = 0; j < n; j++) + { + if (i == j) continue; + gossips[i] = Receive(gossips[i], gossips[j], identities[i]); + } + } + + void Converge() + { + var rounds = 0; + while (rounds++ < 25) + { + var snapshot = gossips.Select(DescribeCore).ToArray(); + Round(); + if (gossips.Select(DescribeCore).SequenceEqual(snapshot) && Converged()) + break; + } + } + + void RecordMint(UniqueAddress key, long stamp, long instant) + { + if (!mintedAt.TryGetValue((key, stamp), out var earlier) || instant < earlier) + mintedAt[(key, stamp)] = instant; + } + + // Every drop has to satisfy the arithmetic the retention argument rests on: a tombstone is + // never dropped sooner than window minus the widest clock disagreement after it was written. + void AssertSkewSafety(Gossip before, Gossip after, long instant) + { + foreach (var kv in before.Tombstones) + { + if (after.Tombstones.ContainsKey(kv.Key)) continue; + + mintedAt.TryGetValue((kv.Key, kv.Value), out var mintInstant).Should().BeTrue( + "every tombstone this history holds was written by it"); + + (instant - mintInstant).Should().BeGreaterOrEqualTo(window - maxSkew, + "a tombstone for {0} written at {1} must not be dropped at {2}", kv.Key, mintInstant, instant); + } + } + + void Prune(int node, long instant) + { + var target = gossips[node]; + var cutoff = instant + history.SkewMillis[node] - window; + var pruned = target.PruneTombstones(cutoff); + + if (ReferenceEquals(pruned, target)) + { + target.Tombstones.Values.Where(v => v <= cutoff).Should().BeEmpty( + "PruneTombstones handed back the same instance, so nothing may have expired"); + return; + } + + AssertSkewSafety(target, pruned, instant); + gossips[node] = StampLocalChange(pruned, identities[node]); + } + + foreach (var raw in history.Ops) + { + // 0 removes, 1 and 2 exchange a single pair, 3 runs a whole gossip round, 4 jumps time + // forward, 5 is the leader's converged tick. Whole rounds are in the alphabet because a + // prune only fires from a converged state, and single pairwise exchanges rarely get there + // inside a short history. + switch (raw % 6) + { + case 0: + { + var actor = raw / 6 % n; + var g = gossips[actor]; + var candidates = g.Members + .Where(m => !identities.Contains(m.UniqueAddress)) + .ToArray(); + if (candidates.Length == 0) break; + + var victim = candidates[raw / 60 % candidates.Length].UniqueAddress; + var stamp = now + history.SkewMillis[actor]; + + gossips[actor] = StampLocalChange( + g.RemoveAll(ImmutableHashSet.Create(victim), stamp), identities[actor]); + RecordMint(victim, stamp, now); + oracle.Add(victim); + break; + } + + case 3: + Round(); + break; + + case 4: + // a sparse jump forward, up to a day and a half at a time + now += (1 + raw / 6 % 36) * 60L * 60L * 1000L; + break; + + case 5: + { + // the leader's converged tick, run by whichever node the draw picked + if (!Converged()) break; + var node = raw / 6 % n; + var before = gossips[node].Tombstones.Count; + Prune(node, now); + if (gossips[node].Tombstones.Count < before) prunedSomethingMidHistory = true; + break; + } + + default: + { + var receiver = raw / 6 % n; + var sender = raw / 60 % n; + if (receiver == sender) break; + gossips[receiver] = Receive(gossips[receiver], gossips[sender], identities[receiver]); + break; + } + } + + for (var i = 0; i < n; i++) + { + var g = gossips[i]; + var addresses = g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + addresses.Intersect(g.Tombstones.Keys).Should().BeEmpty( + "node {0} must keep members and tombstones disjoint", i); + addresses.Except(initialMembers.Select(m => m.UniqueAddress)).Should().BeEmpty(); + g.Version.Versions.Keys.Should().BeSubsetOf(addresses.Select(VclockNodeOf), + "node {0} must not keep a clock entry for a node that is not a member", i); + } + } + + Converge(); + Converged().Should().BeTrue("every node must settle on the same members, tombstones and clock"); + + // Liveness. Push past the window plus the widest clock disagreement, so every tombstone this + // history wrote is expired for any node that might prune it, then let a drawn node run its + // leader tick and the cluster exchange again. Nothing may hand the expired tombstones back. + now += window + maxSkew + 60L * 60L * 1000L; + + var finalPruner = history.FinalPruneNode % n; + Prune(finalPruner, now); + Converge(); + + Converged().Should().BeTrue("the cluster must settle again after the prune"); + + var expectedMembers = initialMembers + .Where(m => !oracle.Contains(m.UniqueAddress)) + .Select(m => m.UniqueAddress) + .ToImmutableHashSet(); + + foreach (var g in gossips) + { + g.Tombstones.Should().BeEmpty( + "every tombstone is past the retention window, and a peer that has not pruned must not hand one back"); + + g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet() + .SetEquals(expectedMembers).Should().BeTrue( + "dropping an expired tombstone must not put a removed member back once the removal has reached everyone"); + } + + return prunedSomethingMidHistory; + } + + // ----------------------------------------------------------------------------------------- + // P17 - the node that minted a tombstone is removed too + // ----------------------------------------------------------------------------------------- + + [Fact(DisplayName = "P17: a removal holds after the node that recorded it is itself removed")] + public void P17_Tombstones_outlive_the_node_that_minted_them() + { + var lineageErased = 0; + + LineageHistories().Sample(h => + { + if (RunLineageHistory(h)) Interlocked.Increment(ref lineageErased); + }, iter: LineageIterations, print: h => h.ToString()); + + lineageErased.Should().BeGreaterThan(LineageIterations / 10, + "histories must actually remove a node that had itself removed someone"); + } + + /// + /// Histories for P17. Every node gossips and every node can be removed, including the one that + /// recorded somebody else's removal. + /// + private static Gen LineageHistories() => + Gen.Select(Gen.Int[4, NodeCount], Gen.Int[0, 1199].Array[30, 60], + (n, ops) => new History(n, ops, 0L, 0)); + + /// + /// The case leaves out: a removal may target a node that gossips, so the + /// node that recorded a tombstone can itself be removed afterwards. + /// + /// That matters because prunes the clock entry of every tombstoned + /// node. A tombstone is written with a bump to the recording node's own clock entry, and that bump + /// is the evidence a later gossip descends from the removal. Remove the recording node and the + /// evidence is pruned out of both sides' clocks - so this is where a gossip could, in principle, + /// come out strictly newer than a tombstone carrier without ever having seen the removal. + /// + /// A removed node stops gossiping the moment anyone removes it, the way a real one shuts down. A + /// node that has not heard about the removal may still remove the same node again from its own + /// view, which is what a real cluster does through its own leader, so a second tombstone with a + /// later timestamp is a normal thing for a history to produce. + /// + /// One restriction, and it earns its keep. A node only removes somebody when its own clock is at + /// or ahead of every other survivor's - the model's stand-in for the rule that + /// LeaderActionsOnConvergence only runs on a converged tick. Without it the model reaches a + /// resurrection that predates tombstones and that no cluster can actually reach: + /// + /// four nodes a, b, c, d. b removes a: b writes a's tombstone, prunes a's clock entry and + /// bumps its own. c picks that up. d never does. d then removes b, which prunes b's clock + /// entry - the one bump that recorded a's removal - while d still carries a's own clock entry, + /// because d never saw a go. d's clock now strictly dominates c's, so c adopts d's gossip + /// whole and a walks back in as a member. + /// + /// A real d cannot get there: removing b is a leader action and needs convergence, convergence + /// needs every unreachable member to be Down or Exiting, and d holds the dead a at Up. d has to + /// Down a first, and then it removes a itself and writes its own tombstone for it. The + /// restriction is on the model, not on production - the same history resurrects a with the + /// tombstone union restored on the winner-picked branches, so this is not something the union was + /// buying. + /// + /// Returns true when the history reached the shape it is named after: a node that recorded a + /// tombstone was itself removed later. + /// + private static bool RunLineageHistory(History history) + { + var n = history.Nodes; + + var initialMembers = Enumerable.Range(0, n) + .Select(i => MemberOf(i, MemberStatus.Up)) + .ToImmutableSortedSet(); + + var initialVersion = Enumerable.Range(0, n) + .Aggregate(VectorClock.Create(), (v, i) => v.Increment(VclockNodeOf(Node(i)))); + + var gossips = Enumerable.Repeat( + new Gossip(initialMembers, new GossipOverview(), initialVersion), n).ToArray(); + var identities = Enumerable.Range(0, n).Select(Node).ToArray(); + + var alive = Enumerable.Repeat(true, n).ToArray(); + var recorded = new HashSet(); // nodes that have written at least one tombstone + var lineageErased = false; + var timestamp = 1L; + + int[] Live() => Enumerable.Range(0, n).Where(i => alive[i]).ToArray(); + + foreach (var raw in history.Ops) + { + var live = Live(); + + // 0 removes, 1 through 4 exchange, 5 downs. Exchanges dominate on purpose: a removal + // needs its actor to have caught up with everyone, and only an exchange gets it there. + switch (raw % 6) + { + case 0: + { + // a node removes a member of its own view - possibly one that still gossips, and + // possibly one somebody else already removed + if (live.Length <= 2) break; + + var actor = live[raw / 6 % live.Length]; + + // only a node that is not behind anyone removes: see the restriction note above + var caughtUp = live.All(i => i == actor || + gossips[actor].Version.CompareTo(gossips[i].Version) + is VectorClock.Ordering.Same or VectorClock.Ordering.After); + if (!caughtUp) break; + + var g = gossips[actor]; + var candidates = g.Members + .Where(m => !m.UniqueAddress.Equals(identities[actor])) + .ToArray(); + if (candidates.Length == 0) break; + + var victim = candidates[raw / 60 % candidates.Length].UniqueAddress; + var victimIndex = Array.IndexOf(identities, victim); + + gossips[actor] = StampLocalChange( + g.RemoveAll(ImmutableHashSet.Create(victim), timestamp++), identities[actor]); + + recorded.Add(actor); + if (victimIndex >= 0 && alive[victimIndex]) + { + alive[victimIndex] = false; + if (recorded.Contains(victimIndex)) lineageErased = true; + } + + break; + } + + case 5: + { + var node = live[raw / 6 % live.Length]; + var g = gossips[node]; + var candidates = g.Members + .Where(m => m.Status == MemberStatus.Up && + !m.UniqueAddress.Equals(identities[node])) + .ToArray(); + if (candidates.Length == 0) break; + + var toDown = candidates[raw / 60 % candidates.Length]; + gossips[node] = StampLocalChange( + g.Copy(members: g.Members.Remove(toDown).Add(toDown.Copy(MemberStatus.Down))), + identities[node]); + break; + } + + default: + { + if (live.Length < 2) break; + var receiver = live[raw / 6 % live.Length]; + var sender = live[raw / 60 % live.Length]; + if (receiver == sender) break; + gossips[receiver] = Receive(gossips[receiver], gossips[sender], identities[receiver]); + break; + } + } + + foreach (var i in Live()) + { + var g = gossips[i]; + var addresses = g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet(); + + addresses.Intersect(g.Tombstones.Keys).Should().BeEmpty( + "node {0} must keep members and tombstones disjoint", i); + + g.Version.Versions.Keys.Should().BeSubsetOf(addresses.Select(VclockNodeOf), + "node {0} must not keep a clock entry for a node that is not a member", i); + } + } + + // exchange between the survivors until nothing moves + var survivors = Live(); + var rounds = 0; + while (rounds++ < 25) + { + var snapshot = survivors.Select(i => DescribeCore(gossips[i])).ToArray(); + + foreach (var i in survivors) + foreach (var j in survivors) + { + if (i == j) continue; + gossips[i] = Receive(gossips[i], gossips[j], identities[i]); + } + + if (survivors.Select(i => DescribeCore(gossips[i])).SequenceEqual(snapshot) && + snapshot.Distinct().Count() == 1) + break; + } + + var states = survivors.Select(i => DescribeCore(gossips[i])).Distinct().ToArray(); + states.Length.Should().Be(1, + "every survivor must end on the same members, tombstones and clock, got:\n{0}", + string.Join("\n", states)); + + var expectedMembers = survivors.Select(i => identities[i]).ToImmutableHashSet(); + var removed = Enumerable.Range(0, n).Where(i => !alive[i]).Select(i => identities[i]) + .ToImmutableHashSet(); + + foreach (var i in survivors) + { + var g = gossips[i]; + + g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet() + .SetEquals(expectedMembers).Should().BeTrue( + "a removed node must not come back, not even one that had recorded a removal itself"); + + g.Tombstones.Keys.ToImmutableHashSet().SetEquals(removed).Should().BeTrue( + "every removal must still be on the record, including one whose recording node was removed after it"); + } + + return lineageErased; + } + /// /// A random op sequence over N virtual nodes, each holding a gossip over the same initial /// membership. Ops are ints, decoded by . + /// + /// A history also carries the cutoff and the node for the converged-tick prune P15 runs. Removal + /// timestamps start at 1 and step by one, and a history writes at most a handful of them, so a + /// cutoff drawn from [0, 11] lands below, inside and above that range. /// private static Gen Histories() => - Gen.Select(Gen.Int[3, 5], Gen.Int[0, 899].Array[10, 30], (n, ops) => new History(n, ops)); + Gen.Select(Gen.Int[3, 5], Gen.Int[0, 1199].Array[10, 30], Gen.Long[0L, 11L], Gen.Int[0, 4], + (n, ops, cutoff, pruner) => new History(n, ops, cutoff, pruner)); private sealed class History { - public History(int nodes, int[] ops) + public History(int nodes, int[] ops, long pruneCutoff, int pruneNode) { Nodes = nodes; Ops = ops; + PruneCutoff = pruneCutoff; + PruneNode = pruneNode; } public int Nodes { get; } public int[] Ops { get; } + public long PruneCutoff { get; } + public int PruneNode { get; } - public override string ToString() => $"nodes={Nodes} ops=[{string.Join(",", Ops)}]"; + public override string ToString() => + $"nodes={Nodes} ops=[{string.Join(",", Ops)}] cutoff={PruneCutoff} pruner={PruneNode}"; + } + + /// What a replayed history did, so a property can check it was not a no-op. + private readonly struct HistoryResult + { + public HistoryResult(int removed, int pruned, bool mixedStatusRemoval) + { + Removed = removed; + Pruned = pruned; + MixedStatusRemoval = mixedStatusRemoval; + } + + /// How many nodes the oracle saw removed. + public int Removed { get; } + + /// How many tombstones the converged-tick prune dropped. + public int Pruned { get; } + + /// + /// Whether the history removed a node while some other node held it at a different status - + /// the shape a real removal has, and the reason the status op exists. + /// + public bool MixedStatusRemoval { get; } } /// @@ -599,13 +1243,25 @@ public History(int nodes, int[] ops) /// property covers the paths a real cluster takes, not just . /// Timestamps come from a counter passed in here, never from the wall clock. /// - /// The N virtual nodes stay in the cluster for the whole history: a removal never targets one of - /// them. A node that keeps gossiping after being removed is not a history the cluster produces - - /// the removed node shuts down, and ReceiveGossip drops gossip that does not contain the - /// receiver and gossip from a sender the receiver no longer holds, so a zombie is deaf and - /// ignored in both directions. Modelling one instead would only measure that omission. + /// Four op kinds: a removal, an exchange, a prune, and a status change that pushes one node's + /// view of one member to Down. The status change is what gives a removal its real shape - by the + /// time a leader removes a node, its peers hold that node at a mix of Up and Down, and the + /// tombstone has to propagate through that mix. + /// + /// Two deliberate restrictions on what a history can build: + /// + /// 1. The N virtual nodes stay in the cluster for the whole history: a removal never targets one + /// of them. A node that keeps gossiping after being removed is not a history the cluster + /// produces - the removed node shuts down, and ReceiveGossip drops gossip that does not + /// contain the receiver and gossip from a sender the receiver no longer holds, so a zombie is + /// deaf and ignored in both directions. Modelling one instead would only measure that + /// omission. P17 covers removing a gossiping node, including the node that minted a + /// tombstone. + /// 2. Every node starts Up. Statuses only ever move towards Down from there, through the status + /// op, so no history opens on a member set the cluster could not have reached. /// - private static int RunHistory(History history, bool checkMonotonicity, bool checkConvergence) + private static HistoryResult RunHistory(History history, bool checkMonotonicity, bool checkConvergence, + bool checkPruneStickiness = false) { var n = history.Nodes; @@ -625,6 +1281,7 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec var oracle = new HashSet(); var timestamp = 1L; + var mixedStatusRemoval = false; // a cutoff below every timestamp this history will ever write, so the prune op expires nothing const long QuietCutoff = 0L; @@ -632,7 +1289,7 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec foreach (var raw in history.Ops) { var before = gossips.Select(g => g.Tombstones.Keys.ToImmutableHashSet()).ToArray(); - var kind = raw % 3; + var kind = raw % 4; var isPrune = kind == 2; switch (kind) @@ -640,14 +1297,24 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec case 0: { // the leader removes a live member that is not one of the gossiping nodes - var actor = raw / 3 % n; + var actor = raw / 4 % n; var g = gossips[actor]; var candidates = g.Members .Where(m => !identities.Contains(m.UniqueAddress)) .ToArray(); if (candidates.Length == 0) break; - var victim = candidates[raw / 30 % candidates.Length].UniqueAddress; + var victimMember = candidates[raw / 40 % candidates.Length]; + var victim = victimMember.UniqueAddress; + + // did anyone else hold the victim at a different status at this moment? + for (var i = 0; i < n && !mixedStatusRemoval; i++) + { + if (i == actor) continue; + var peer = gossips[i].Members.FirstOrDefault(m => m.UniqueAddress.Equals(victim)); + if (peer != null && peer.Status != victimMember.Status) mixedStatusRemoval = true; + } + var removed = g.RemoveAll(ImmutableHashSet.Create(victim), timestamp++); // ClusterCoreDaemon.UpdateLatestGossip stamps every local change with the node's @@ -660,16 +1327,16 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec case 1: { - var receiver = raw / 3 % n; - var sender = raw / 30 % n; + var receiver = raw / 4 % n; + var sender = raw / 40 % n; if (receiver == sender) break; gossips[receiver] = Receive(gossips[receiver], gossips[sender], identities[receiver]); break; } - default: + case 2: { - var node = raw / 3 % n; + var node = raw / 4 % n; var pruned = gossips[node].PruneTombstones(QuietCutoff); // nothing expired, so the leader's no-op guard must hold @@ -677,6 +1344,24 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec gossips[node] = pruned; break; } + + default: + { + // One node moves one member to Down in its own view. Only that node sees it until + // the change gossips out, which is how a peer ends up holding a node the leader is + // about to remove at a different status than the leader does. + var node = raw / 4 % n; + var g = gossips[node]; + var candidates = g.Members + .Where(m => m.Status == MemberStatus.Up) + .ToArray(); + if (candidates.Length == 0) break; + + var toDown = candidates[raw / 40 % candidates.Length]; + var downed = g.Copy(members: g.Members.Remove(toDown).Add(toDown.Copy(MemberStatus.Down))); + gossips[node] = StampLocalChange(downed, identities[node]); + break; + } } // (a) and (b): after every op, no node holds a UID as a member and as a tombstone @@ -694,6 +1379,18 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec // and nothing is invented out of thin air addresses.Except(initialMembers.Select(m => m.UniqueAddress)).Should().BeEmpty(); + + // (c) the clock never accumulates entries for nodes that are not members. This is the + // model's copy of the "Too many vector clock entries in gossip state" check + // ClusterCoreDaemon.AssertLatestGossip runs, stated as the set inclusion it really + // means rather than a count: every entry belongs to a node that is still a member. + // A removal prunes the entry of the node it removes, Merge prunes the entry of every + // tombstoned node, and the concurrent branch prunes the entry of a one-sided Down or + // Exiting member it is about to drop. Miss any of those and the gossip grows a tail of + // dead nodes that rides every message forever. + var memberClockNodes = addresses.Select(VclockNodeOf).ToImmutableHashSet(); + g.Version.Versions.Keys.Should().BeSubsetOf(memberClockNodes, + "node {0} must not keep a clock entry for a node that is not a member", i); } if (checkMonotonicity && !isPrune) @@ -704,30 +1401,39 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec } } - if (!checkConvergence) return oracle.Count; + if (!checkConvergence) return new HistoryResult(oracle.Count, 0, mixedStatusRemoval); // exchange every ordered pair until nothing moves - var rounds = 0; - while (rounds++ < 25) + void Converge() { - var snapshot = gossips.Select(DescribeCore).ToArray(); - - for (var i = 0; i < n; i++) - for (var j = 0; j < n; j++) + var rounds = 0; + while (rounds++ < 25) { - if (i == j) continue; - gossips[i] = Receive(gossips[i], gossips[j], identities[i]); + var snapshot = gossips.Select(DescribeCore).ToArray(); + + for (var i = 0; i < n; i++) + for (var j = 0; j < n; j++) + { + if (i == j) continue; + gossips[i] = Receive(gossips[i], gossips[j], identities[i]); + } + + if (gossips.Select(DescribeCore).SequenceEqual(snapshot) && + snapshot.Distinct().Count() == 1) + break; } + } - if (gossips.Select(DescribeCore).SequenceEqual(snapshot) && - snapshot.Distinct().Count() == 1) - break; + void AssertConverged() + { + var states = gossips.Select(DescribeCore).Distinct().ToArray(); + states.Length.Should().Be(1, + "every node must end on the same members, tombstones and clock, got:\n{0}", + string.Join("\n", states)); } - var converged = gossips.Select(DescribeCore).Distinct().ToArray(); - converged.Length.Should().Be(1, - "every node must end on the same members, tombstones and clock, got:\n{0}", - string.Join("\n", converged)); + Converge(); + AssertConverged(); var expectedMembers = initialMembers .Where(m => !oracle.Contains(m.UniqueAddress)) @@ -741,7 +1447,49 @@ private static int RunHistory(History history, bool checkMonotonicity, bool chec g.Tombstones.Keys.ToImmutableHashSet().SetEquals(oracle).Should().BeTrue(); } - return oracle.Count; + if (!checkPruneStickiness) return new HistoryResult(oracle.Count, 0, mixedStatusRemoval); + + // P15. Every node now holds the same members, tombstones and clock - the state the leader is + // in when it prunes, because LeaderActionsOnConvergence only runs on a converged tick. Prune + // one node with the drawn cutoff, stamp it the way UpdateLatestGossip does, and let the + // cluster exchange again. The stamp puts the pruning node strictly ahead of every peer, so + // every exchange from here takes a winner-picked branch - which is the branch that used to + // hand the pruned tombstones straight back. + var pruner = history.PruneNode % n; + var target = gossips[pruner]; + var afterPrune = target.PruneTombstones(history.PruneCutoff); + + if (ReferenceEquals(afterPrune, target)) + { + target.Tombstones.Values.Where(v => v <= history.PruneCutoff).Should().BeEmpty( + "PruneTombstones handed back the same instance, so nothing may have expired"); + return new HistoryResult(oracle.Count, 0, mixedStatusRemoval); + } + + var dropped = target.Tombstones.Keys.Except(afterPrune.Tombstones.Keys).ToImmutableHashSet(); + gossips[pruner] = StampLocalChange(afterPrune, identities[pruner]); + + Converge(); + AssertConverged(); + + var survivors = oracle.Except(dropped).ToImmutableHashSet(); + + foreach (var g in gossips) + { + foreach (var key in dropped) + g.Tombstones.Keys.Should().NotContain(key, + "a tombstone pruned by the leader must not be handed back by a peer that never pruned"); + + g.Tombstones.Keys.ToImmutableHashSet().SetEquals(survivors).Should().BeTrue( + "the prune drops the expired tombstones and keeps the rest"); + + // the prune dropped tombstones for nodes no gossip still holds as a member, so a dropped + // tombstone cannot let a member walk back in + g.Members.Select(m => m.UniqueAddress).ToImmutableHashSet() + .SetEquals(expectedMembers).Should().BeTrue(); + } + + return new HistoryResult(oracle.Count, dropped.Count, mixedStatusRemoval); } /// @@ -752,36 +1500,17 @@ private static Gossip StampLocalChange(Gossip gossip, UniqueAddress self) => gossip.Increment(VclockNodeOf(self)).OnlySeen(self); /// - /// Mirrors the four comparison branches of ClusterCoreDaemon.ReceiveGossip, including the - /// conflicting-gossip clock pruning the concurrent branch does before merging. + /// One gossip exchange, run through the production branch selection rather than a copy of it. + /// is the code path a real node takes for + /// every gossip it receives; only the parts ReceiveGossip does around it - the talkback + /// decision, logging, and marking the result as seen - are left out. Calling it directly is what + /// makes these sequence properties fail when that branch selection changes. /// private static Gossip Receive(Gossip local, Gossip remote, UniqueAddress self) { - Gossip winner; - switch (remote.Version.CompareTo(local.Version)) - { - case VectorClock.Ordering.Same: - winner = remote.MergeSeen(local).MergeTombstones(local); - break; - case VectorClock.Ordering.Before: - winner = local.MergeTombstones(remote); - break; - case VectorClock.Ordering.After: - winner = remote.MergeTombstones(local); - break; - default: - winner = PruneConflicting(remote, local).Merge(PruneConflicting(local, remote)); - break; - } - + var comparison = remote.Version.CompareTo(local.Version); + var winner = ClusterCoreDaemon.SelectWinningGossip(local, remote, comparison); return winner.Seen(self); } - - private static Gossip PruneConflicting(Gossip gossip, Gossip other) => - gossip.Members.Aggregate(gossip, (g, m) => - MembershipState.RemoveUnreachableWithMemberStatus.Contains(m.Status) && - !other.Members.Contains(m) - ? g.Prune(VclockNodeOf(m.UniqueAddress)) - : g); } } diff --git a/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs b/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs index be2a2706cdb..586ee9742f6 100644 --- a/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs +++ b/src/core/Akka.Cluster.Tests/Serialization/GossipTombstoneSerializerPropertySpecs.cs @@ -83,6 +83,51 @@ public void P12_Gossip_round_trips_with_tombstones() "the generator must put the shared host-and-port pair on both sides of the member/tombstone split"); } + [Fact(DisplayName = "P12: a gossip that tombstones one of its own members still round trips")] + public void P12_Gossip_round_trips_when_a_tombstone_is_also_a_member() + { + // Shape.Adversarial builds gossips that hold a node as a member AND as a tombstone. The Gossip + // constructor rejects exactly that under AKKA_CLUSTER_ASSERT=on, so the input cannot be built + // there. + Assert.SkipWhen(Cluster.IsAssertInvariantsEnabled, + "AKKA_CLUSTER_ASSERT=on rejects the member-and-tombstone gossip this property is about"); + + var probe = new GossipEnvelope(Node(0), Node(1), Gossip.Empty); + var serializer = Sys.Serialization.FindSerializerFor(probe); + var manifest = ((SerializerWithStringManifest)serializer).Manifest(probe); + + // how often a tombstone key was already in the address table the member loop built + var tombstoneWasAlreadyMapped = 0; + + Sides(1, Shape.Adversarial).Sample(g => + { + var gossip = g[0]; + var envelope = new GossipEnvelope(Node(0), Node(1), gossip); + + var bytes = serializer.ToBinary(envelope); + var back = (GossipEnvelope)serializer.FromBinary(bytes, manifest); + + Describe(back.Gossip).Should().Be(Describe(gossip)); + + foreach (var kv in gossip.Tombstones) + { + var key = back.Gossip.Tombstones.Keys.Single(k => k.Equals(kv.Key)); + key.Uid.Should().Be(kv.Key.Uid); + back.Gossip.Tombstones[key].Should().Be(kv.Value); + } + + // The writer appends a tombstone's address to the shared address table only when the + // member loop has not already put it there. A tombstone for a node this gossip still holds + // as a member takes the other arm - it reuses the member's slot instead of appending a + // second one - and nothing else in the suite reaches that arm. + if (gossip.Tombstones.Keys.Any(gossip.HasMember)) + Interlocked.Increment(ref tombstoneWasAlreadyMapped); + }, iter: Iterations, print: Print); + + tombstoneWasAlreadyMapped.Should().BeGreaterThan(Iterations / 10, + "Shape.Adversarial must actually produce a tombstone for a node the gossip holds as a member"); + } + [Fact(DisplayName = "P12: a Welcome carries the same tombstones through a round trip")] public void P12_Welcome_round_trips_with_tombstones() { diff --git a/src/core/Akka.Cluster/ClusterDaemon.cs b/src/core/Akka.Cluster/ClusterDaemon.cs index 908b2e12d8f..6746c1d2af7 100644 --- a/src/core/Akka.Cluster/ClusterDaemon.cs +++ b/src/core/Akka.Cluster/ClusterDaemon.cs @@ -1893,66 +1893,35 @@ public ReceiveGossipType ReceiveGossip(GossipEnvelope envelope) var comparison = remoteGossip.Version.CompareTo(localGossip.Version); - Gossip winningGossip; bool talkback; ReceiveGossipType gossipType; switch (comparison) { - // Three of the four branches below pick one whole gossip as the winner, which on its own - // would throw away the loser's tombstones. Tombstones are unioned in every branch instead, - // so a removal cannot decay out of the cluster along a path of ordinary gossip exchanges. case VectorClock.Ordering.Same: - //same version talkback = !_exitingTasksInProgress && !remoteGossip.SeenByNode(_selfUniqueAddress); - winningGossip = remoteGossip.MergeSeen(localGossip).MergeTombstones(localGossip); gossipType = ReceiveGossipType.Same; break; case VectorClock.Ordering.Before: //local is newer - winningGossip = localGossip.MergeTombstones(remoteGossip); talkback = true; gossipType = ReceiveGossipType.Older; break; case VectorClock.Ordering.After: //remote is newer - winningGossip = remoteGossip.MergeTombstones(localGossip); talkback = !_exitingTasksInProgress && !remoteGossip.SeenByNode(_selfUniqueAddress); gossipType = ReceiveGossipType.Newer; break; default: - // conflicting versions, merge - // We can see that a removal was done when it is not in one of the gossips has status - // Down or Exiting in the other gossip. - // Perform the same pruning (clear of VectorClock) as the leader did when removing a member. - // Removal of member itself is handled in merge (pickHighestPriority) - var prunedLocalGossip = localGossip.Members.Aggregate(localGossip, (g, m) => - { - if (RemoveUnreachableWithMemberStatus.Contains(m.Status) && !remoteGossip.Members.Contains(m)) - { - _log.Debug("Cluster Node [{0}] - Pruned conflicting local gossip: {1}", _cluster.SelfAddress, m); - return g.Prune(VectorClock.Node.Create(VclockName(m.UniqueAddress))); - } - return g; - }); - - var prunedRemoteGossip = remoteGossip.Members.Aggregate(remoteGossip, (g, m) => - { - if (RemoveUnreachableWithMemberStatus.Contains(m.Status) && !localGossip.Members.Contains(m)) - { - _log.Debug("Cluster Node [{0}] - Pruned conflicting remote gossip: {1}", _cluster.SelfAddress, m); - return g.Prune(VectorClock.Node.Create(VclockName(m.UniqueAddress))); - } - return g; - }); - - //conflicting versions, merge - winningGossip = prunedRemoteGossip.Merge(prunedLocalGossip); talkback = true; gossipType = ReceiveGossipType.Merge; break; } + var winningGossip = SelectWinningGossip(localGossip, remoteGossip, comparison, + m => _log.Debug("Cluster Node [{0}] - Pruned conflicting local gossip: {1}", _cluster.SelfAddress, m), + m => _log.Debug("Cluster Node [{0}] - Pruned conflicting remote gossip: {1}", _cluster.SelfAddress, m)); + // Don't mark gossip state as seen while exiting is in progress, e.g. // shutting down singleton actors. This delays removal of the member until // the exiting tasks have been completed. @@ -2036,6 +2005,79 @@ public ReceiveGossipType ReceiveGossip(GossipEnvelope envelope) return gossipType; } + /// + /// Picks the gossip that wins an exchange, given how the incoming clock compares to the local one. + /// + /// Split out of so the branch selection can be driven directly by + /// tests instead of re-implemented by them: a copy of this logic in a test would keep passing + /// after this one changed. + /// + /// The receiver's current gossip. + /// The gossip that just arrived. + /// How 's clock compares to the local one. + /// Called for every local member whose clock entry the concurrent branch prunes. + /// Called for every remote member whose clock entry the concurrent branch prunes. + internal static Gossip SelectWinningGossip( + Gossip localGossip, + Gossip remoteGossip, + VectorClock.Ordering comparison, + Action onPrunedLocal = null, + Action onPrunedRemote = null) + { + switch (comparison) + { + // Tombstones are unioned only on the two branches where neither gossip descends from the + // other: Same, where MergeTombstones does it, and the concurrent branch, where Gossip.Merge + // does. The two branches that pick a strictly newer gossip keep the winner's tombstones and + // nothing else. + // + // The winner is not behind on removals: a removal bumps the removing node's own clock + // entry, so a gossip that dominates the loser's clock descends from every removal the loser + // knows about. The tombstones the winner does not carry are the ones its own leader + // deliberately pruned, and honouring those prunes is the point - unioning them back in + // would undo every prune, and the leader would prune the same tombstones again on every + // tick forever. + case VectorClock.Ordering.Same: + return remoteGossip.MergeSeen(localGossip).MergeTombstones(localGossip); + + case VectorClock.Ordering.Before: + // local is newer + return localGossip; + + case VectorClock.Ordering.After: + // remote is newer + return remoteGossip; + + default: + // conflicting versions, merge + // We can see that a removal was done when it is not in one of the gossips has status + // Down or Exiting in the other gossip. + // Perform the same pruning (clear of VectorClock) as the leader did when removing a member. + // Removal of member itself is handled in merge (pickHighestPriority) + var prunedLocalGossip = localGossip.Members.Aggregate(localGossip, (g, m) => + { + if (RemoveUnreachableWithMemberStatus.Contains(m.Status) && !remoteGossip.Members.Contains(m)) + { + onPrunedLocal?.Invoke(m); + return g.Prune(VectorClock.Node.Create(VclockName(m.UniqueAddress))); + } + return g; + }); + + var prunedRemoteGossip = remoteGossip.Members.Aggregate(remoteGossip, (g, m) => + { + if (RemoveUnreachableWithMemberStatus.Contains(m.Status) && !localGossip.Members.Contains(m)) + { + onPrunedRemote?.Invoke(m); + return g.Prune(VectorClock.Node.Create(VclockName(m.UniqueAddress))); + } + return g; + }); + + return prunedRemoteGossip.Merge(prunedLocalGossip); + } + } + /// /// Sends gossip and schedules two future intervals for more gossip /// @@ -2614,10 +2656,10 @@ public static IReadOnlyCollection GossipTargetsForExitingMembers(Gossip /// /// Every change this node makes to the gossip goes through here, and every call increments this /// node's vector clock entry. Tombstone writes and tombstone pruning must keep using this path. - /// A tombstone written without a clock bump would be invisible to the causal comparison in - /// , so peers could not tell which side holds the newer removal set. - /// unions tombstones on every branch, which keeps a stray - /// unstamped write from losing tombstones outright, but the ordering guarantee still comes from here. + /// A tombstone written or pruned without a clock bump would be invisible to the causal comparison + /// in , so peers could not tell which side holds the newer removal set. + /// Only the equal-clock and concurrent branches union tombstones now, so an unstamped write has + /// nothing to fall back on: a peer that is causally ahead wins outright and the write is gone. /// /// The new gossip to merge with our own. public void UpdateLatestGossip(Gossip gossip) diff --git a/src/core/Akka.Cluster/Gossip.cs b/src/core/Akka.Cluster/Gossip.cs index 978ae75aa28..f5e1e184d35 100644 --- a/src/core/Akka.Cluster/Gossip.cs +++ b/src/core/Akka.Cluster/Gossip.cs @@ -340,10 +340,11 @@ public Gossip Merge(Gossip that) /// Adds the tombstones of to this gossip, keeping the later timestamp /// when both sides hold the same node. /// - /// already unions tombstones, but it only runs on the concurrent branch of - /// gossip reception. The other three branches pick a whole gossip as the winner, which would - /// drop the loser's tombstones. Unioning them here instead keeps a tombstone alive no matter - /// which branch a given exchange takes. + /// takes the same union on the concurrent branch of gossip reception. This + /// method covers the branch where the two clocks are equal, where neither gossip descends from + /// the other and each may hold a removal the other has not heard about. The branches that pick a + /// strictly newer gossip keep the winner's tombstones alone - see + /// ClusterCoreDaemon.ReceiveGossip for why the winner cannot be behind on removals. /// /// The gossip to take tombstones from. /// This gossip when it already covers every tombstone of . @@ -352,9 +353,14 @@ public Gossip MergeTombstones(Gossip that) if (that._tombstones.Count == 0) return this; // A node this gossip still holds as a member has not been removed here, so its tombstone is - // not adopted - members and tombstones must stay disjoint. Every tombstone write bumps the - // vector clock, so the winning gossip of these branches is never behind on removals and this - // case cannot arise; the filter is a guard, not a decision. + // not adopted: members and tombstones must stay disjoint, and Merge picks a member both sides + // hold by status alone, without consulting tombstones. + // + // That is a real drop, not a formality - the caller loses a removal it was told about. It is + // safe because this only runs with two equal clocks: a removal bumps the removing node's clock + // entry, so both sides descend from every removal either of them knows about, and a member + // that is back after a removal is one whose tombstone has already been pruned. Adopting that + // tombstone again would undo the prune. var merged = UnionTombstones(_tombstones, that._tombstones.Where(kv => !HasMember(kv.Key))); if (ReferenceEquals(merged, _tombstones)) return this; return Copy(tombstones: merged); From 0b1520599b645f3f089295c98777c79644c0ee35 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 21:27:42 +0000 Subject: [PATCH 08/10] ci: retrigger after superseded-run cancellations From 51181173ce8ce3fa55905e5923ae0260a39365ec Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 21:48:28 +0000 Subject: [PATCH 09/10] Akka.Cluster: fold tombstones into the Gossip constructor and document the file Addresses two review comments on #8484. Drop the four-argument Gossip constructor added by the tombstone work. Gossip is internal, so there is no API risk in changing the existing three-argument constructor instead: tombstones becomes an optional parameter that defaults to no tombstones. Every existing call site compiles unchanged, including the ones that pass tombstones positionally. Replace all 62 TBD placeholders in Gossip.cs with docs that say what each member actually does. The delegating constructors carry a one-line summary noting their defaults and inherit the rest from the primary. Notable details now written down: GetMember hands back a Removed placeholder for a node that is not in the ring, AddMember is keyed by address so it will not replace a member that differs only in status, YoungestMember treats a member that is not up yet as up-number zero, and Prune clears a node's vector clock entry without touching members or tombstones. --- src/core/Akka.Cluster/Gossip.cs | 164 ++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 72 deletions(-) diff --git a/src/core/Akka.Cluster/Gossip.cs b/src/core/Akka.Cluster/Gossip.cs index f5e1e184d35..89c5f3bdd78 100644 --- a/src/core/Akka.Cluster/Gossip.cs +++ b/src/core/Akka.Cluster/Gossip.cs @@ -85,11 +85,13 @@ public static Gossip Create(ImmutableSortedSet members) /// public ImmutableSortedSet Members { get { return _members; } } /// - /// TBD + /// The seen table and the reachability table for the current members. /// public GossipOverview Overview { get { return _overview; } } /// - /// TBD + /// The vector clock that orders this gossip against gossip from other nodes. Every node that + /// changes the cluster state bumps its own entry, so two clocks can be compared to tell which + /// gossip is newer - or that the two are concurrent and have to be merged. /// public VectorClock Version { get { return _version; } } @@ -111,43 +113,36 @@ public static Gossip Create(ImmutableSortedSet members) public ImmutableDictionary Tombstones { get { return _tombstones; } } /// - /// TBD + /// Creates a gossip for with an empty overview and a fresh vector clock. /// - /// TBD + /// public Gossip(ImmutableSortedSet members) : this(members, new GossipOverview(), VectorClock.Create()) { } /// - /// TBD + /// Creates a gossip for and with a fresh vector clock. /// - /// TBD - /// TBD + /// public Gossip(ImmutableSortedSet members, GossipOverview overview) : this(members, overview, VectorClock.Create()) { } /// - /// TBD + /// Creates the cluster state carried by a single gossip message. /// - /// TBD - /// TBD - /// TBD - /// TBD - public Gossip(ImmutableSortedSet members, GossipOverview overview, VectorClock version) - : this(members, overview, version, EmptyTombstones) { } - - /// - /// TBD - /// - /// TBD - /// TBD - /// TBD - /// The nodes this cluster has removed. See . - /// TBD + /// The members of the cluster, sorted by address. + /// The seen table and the reachability table for those members. + /// The vector clock that orders this gossip against gossip from other nodes. + /// The nodes this cluster has removed. See . Defaults to none. + /// + /// Thrown when the arguments break an invariant - a member with status , + /// a seen or reachability entry for a node that is not a member, or a node that is both a member and a + /// tombstone. Only checked when is set. + /// public Gossip(ImmutableSortedSet members, GossipOverview overview, VectorClock version, - ImmutableDictionary tombstones) + ImmutableDictionary tombstones = null) { _members = members; _overview = overview; _version = version; - _tombstones = tombstones; + _tombstones = tombstones ?? EmptyTombstones; _membersMap = new Lazy>( () => members.ToImmutableDictionary(m => m.UniqueAddress, m => m)); @@ -162,13 +157,13 @@ public Gossip(ImmutableSortedSet members, GossipOverview overview, Vecto } /// - /// TBD + /// Creates a gossip that matches this one except for the arguments that are supplied. /// - /// TBD - /// TBD - /// TBD - /// The nodes this cluster has removed. See . - /// TBD + /// The new members, or null to keep the current ones. + /// The new overview, or null to keep the current one. + /// The new vector clock, or null to keep the current one. + /// The new tombstones, or null to keep the current ones. See . + /// A new gossip. This method always allocates, even when nothing changed. public Gossip Copy(ImmutableSortedSet members = null, GossipOverview overview = null, VectorClock version = null, ImmutableDictionary tombstones = null) { @@ -219,10 +214,11 @@ void IfTrueThrow(bool func, string expected, string actual) Lazy> _membersMap; /// - /// Increments the version for this 'Node'. + /// Bumps this node's entry in the vector clock, marking the gossip as changed here. Callers do + /// this whenever they alter the cluster state so peers can tell the new gossip descends from the old. /// - /// TBD - /// TBD + /// The vector clock entry to bump, normally the caller's own. + /// A copy of this gossip with the bumped clock. public Gossip Increment(VectorClock.Node node) { return Copy(version: _version.Increment(node)); @@ -231,8 +227,9 @@ public Gossip Increment(VectorClock.Node node) /// /// Adds a member to the member node ring. /// - /// TBD - /// TBD + /// The member to add. + /// This gossip when the ring already holds that node. Membership is keyed by address, so + /// this will not replace an existing member that differs only in status - use for that. public Gossip AddMember(Member member) { if (_members.Contains(member)) return this; @@ -242,8 +239,8 @@ public Gossip AddMember(Member member) /// /// Marks the gossip as seen by this node (address) by updating the address entry in the 'gossip.overview.seen' /// - /// TBD - /// TBD + /// The node that has seen this version of the gossip. + /// This gossip when the node had already seen it. public Gossip Seen(UniqueAddress node) { if (SeenByNode(node)) return this; @@ -253,8 +250,8 @@ public Gossip Seen(UniqueAddress node) /// /// Marks the gossip as seen by only this node (address) by replacing the 'gossip.overview.seen' /// - /// TBD - /// TBD + /// The only node that has seen this version of the gossip. + /// A copy of this gossip whose seen table holds alone. public Gossip OnlySeen(UniqueAddress node) { return Copy(overview: _overview.Copy(seen: ImmutableHashSet.Create(node))); @@ -435,12 +432,16 @@ private static ImmutableDictionary UnionTombstones( } /// - /// TBD + /// The reachability table with every record written by a member with status + /// stripped out. A downed node's own view no longer counts, + /// so its old unreachability records must not keep another node marked unreachable forever. + /// Computed once on first read. /// public Lazy ReachabilityExcludingDownedObservers { get; } /// - /// TBD + /// Every role held by any current member, deduplicated. Drives the per-role leader table, and the + /// serializer uses it to build the role table that member entries index into. /// public ImmutableHashSet AllRoles { @@ -448,7 +449,8 @@ public ImmutableHashSet AllRoles } /// - /// TBD + /// true when the cluster has exactly one member. The daemon skips gossiping and reaping + /// unreachable members in that case - there is no peer to talk to and nobody to watch. /// public bool IsSingletonCluster { @@ -459,8 +461,8 @@ public bool IsSingletonCluster /// Returns `true` if should be able to reach /// based on the unreachability data. /// - /// TBD - /// TBD + /// The observing node. + /// The node being observed. A node that is not a member is never reachable. public bool IsReachable(UniqueAddress fromAddress, UniqueAddress toAddress) { if (!HasMember(toAddress)) @@ -471,10 +473,11 @@ public bool IsReachable(UniqueAddress fromAddress, UniqueAddress toAddress) } /// - /// TBD + /// Looks up a member by address. /// - /// TBD - /// TBD + /// The address to look up. + /// The member, or a placeholder with status when the + /// node is not in the ring. Callers that need to tell the two apart should use . public Member GetMember(UniqueAddress node) { return _membersMap.Value.GetOrElse(node, @@ -482,10 +485,10 @@ public Member GetMember(UniqueAddress node) } /// - /// TBD + /// Checks whether a node is in the member ring. /// - /// TBD - /// TBD + /// The address to look for. + /// true when the node is a member, whatever its status. public bool HasMember(UniqueAddress node) { return _membersMap.Value.ContainsKey(node); @@ -493,7 +496,10 @@ public bool HasMember(UniqueAddress node) /// - /// TBD + /// The member that joined most recently, i.e. the one with the highest up-number. A member that + /// has not reached yet carries an up-number of + /// and is counted as 0 here, so it does not outrank a member that is already up. The leader reads + /// this to pick the next up-number to hand out. /// /// /// This exception is thrown when there are no members in the cluster. @@ -509,10 +515,14 @@ public Member YoungestMember } /// - /// TBD + /// Drops a node's entry from the vector clock, leaving members and tombstones alone. + /// + /// This is the clock half of what does. Gossip reception calls it when two + /// concurrent gossips disagree about a removal, so the merged clock matches the one the leader + /// produced when it removed the node. /// - /// TBD - /// TBD + /// The clock entry to drop, named after the removed node. + /// This gossip when the clock had no entry for that node. public Gossip Prune(VectorClock.Node removedNode) { var newVersion = Version.Prune(removedNode); @@ -522,6 +532,12 @@ public Gossip Prune(VectorClock.Node removedNode) return Copy(version: newVersion); } + /// + /// Sets a member's status to and drops it from the seen table, so a + /// downed node no longer counts towards convergence. + /// + /// The member to mark as down. + /// A copy of this gossip with the member replaced. public Gossip MarkAsDown(Member member) { // replace member (changed status) @@ -552,21 +568,22 @@ internal class GossipOverview readonly Reachability _reachability; /// - /// TBD + /// Creates an overview that nobody has seen and where every node is reachable. /// + /// public GossipOverview() : this(ImmutableHashSet.Create(), Reachability.Empty) { } /// - /// TBD + /// Creates an overview from a reachability table that nobody has seen yet. /// - /// TBD + /// public GossipOverview(Reachability reachability) : this(ImmutableHashSet.Create(), reachability) { } /// - /// TBD + /// Creates an overview from a seen table and a reachability table. /// - /// TBD - /// TBD + /// The nodes that have seen the current version of the gossip. + /// Which observers consider which members unreachable. public GossipOverview(ImmutableHashSet seen, Reachability reachability) { _seen = seen; @@ -574,22 +591,24 @@ public GossipOverview(ImmutableHashSet seen, Reachability reachab } /// - /// TBD + /// Creates an overview that matches this one except for the arguments that are supplied. /// - /// TBD - /// TBD - /// TBD + /// The new seen table, or null to keep the current one. + /// The new reachability table, or null to keep the current one. + /// A new overview. This method always allocates, even when nothing changed. public GossipOverview Copy(ImmutableHashSet seen = null, Reachability reachability = null) { return new GossipOverview(seen ?? _seen, reachability ?? _reachability); } /// - /// TBD + /// The nodes that have seen the current version of the gossip. Convergence is reached once every + /// member that counts has seen it. Merging two gossips yields a version nobody has seen, so the + /// table starts over as empty. /// public ImmutableHashSet Seen { get { return _seen; } } /// - /// TBD + /// Which observers consider which members unreachable, as reported by each node's failure detector. /// public Reachability Reachability { get { return _reachability; } } @@ -652,19 +671,20 @@ class GossipStatus : IClusterMessage readonly VectorClock _version; /// - /// TBD + /// The node that sent this status - either the one opening the chat or the one replying to it. /// public UniqueAddress From { get { return _from; } } /// - /// TBD + /// The vector clock of the sender's gossip. The receiver compares it against its own to decide + /// whether to reply with a , reply with its own status, or say nothing. /// public VectorClock Version { get { return _version; } } /// - /// TBD + /// Creates a status message advertising what the sender knows. /// - /// TBD - /// TBD + /// The sending node. + /// The vector clock of the sender's gossip. public GossipStatus(UniqueAddress from, VectorClock version) { _from = from; From 7bbd43d9043d8491c5dd146c2a5377ab5b3b2d9f Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 26 Aug 2026 22:54:11 +0000 Subject: [PATCH 10/10] ci: retrigger after singleton-restart timing flake on the Windows unit lane