Skip to content

Fix event-subscription agents left unassigned after node scale-down (#3341) #3346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeremydmiller merged 2 commits into from
Jul 8, 2026
+81 −22
Merged

Fix event-subscription agents left unassigned after node scale-down (#3341) #3346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dbca10c
Fix event-subscription agents left unassigned after node scale-down (…
jeremydmiller Jul 8, 2026
09ed539
Merge remote-tracking branch 'origin/main' into fix/3341-orphaned-sha…
jeremydmiller Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 42 additions & 3 deletions src/Testing/CoreTests/Runtime/Agents/distribute_by_group_affinity.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,10 @@ public void a_group_lands_only_on_a_node_capable_of_running_it()
[Fact]
public void when_no_node_can_host_the_whole_group_members_fall_back_individually()
{
// Mirrors the blue/green even path: no single node is capable of every member, so each member goes
// to its own least-loaded capable node, and a member NO node declares stays unassigned (parked).
// No single node is capable of every member, so each member goes to its own least-loaded capable
// node. A member NO node declares a capability for is NOT parked — for a sharded store an empty
// candidate set is a stale-snapshot artifact, so it falls back to the least-loaded node overall
// rather than silently stranding the shard (GH-3341).
var t1 = Agent("db1", "t1");
var t2 = Agent("db1", "t2");
var t3 = Agent("db1", "t3");
Expand All @@ -136,7 +138,8 @@ public void when_no_node_can_host_the_whole_group_members_fall_back_individually

grid.AgentFor(t1).AssignedNode.ShouldBe(node1);
grid.AgentFor(t2).AssignedNode.ShouldBe(node2);
grid.AgentFor(t3).AssignedNode.ShouldBeNull("an agent no node declares a capability for is parked, exactly like the even path");
grid.AgentFor(t3).AssignedNode.ShouldNotBeNull(
"an agent no node declares is still assigned to a surviving node, never silently stranded (GH-3341)");
}

[Fact]
Expand Down Expand Up @@ -171,6 +174,42 @@ public void a_node_already_running_a_groups_agents_stays_a_candidate_despite_a_s
hosts.Distinct().Count().ShouldBe(3, "three equal databases across three nodes must land one per node");
}

[Fact]
public void a_shard_orphaned_by_a_departed_node_is_reassigned_not_stranded()
{
// GH-3341: on scale-down, a shard database whose agents ran on a now-departed node — and whose
// URIs are absent from every SURVIVING node's capability snapshot (each node captures its
// event-subscription capabilities once at startup, so a database provisioned after the survivors
// started is missing from their snapshots even though every node can run it) — was left silently
// unassigned. The shard stopped projecting with no running agent, no error log, and no self-heal
// until a rolling restart. It must instead be reassigned to a surviving node.
var db1 = new[] { Agent("db1", "t1"), Agent("db1", "t2") };
var orphaned = new[] { Agent("db2", "t1"), Agent("db2", "t2") };

var grid = new AssignmentGrid();
// Two survivors with DIVERGENT snapshots (node 2 started later and also knows db3), so
// AllNodesHaveSameCapabilities is false and the capability-matching branch runs. Neither lists db2.
var node1 = grid.WithNode(1, Guid.NewGuid()).HasCapabilities(db1);
node1.Running(db1);
grid.WithNode(2, Guid.NewGuid()).HasCapabilities(db1.Append(Agent("db3", "t1")));

// db2's agents were running on a 3rd node that just departed; the leader re-enumerates them via
// AllKnownAgentsAsync and adds them unassigned. No surviving node declares them as capabilities.
grid.WithAgents(orphaned);

grid.DistributeByGroupAffinity("event-subscriptions", DatabaseKey);

foreach (var uri in orphaned)
{
grid.AgentFor(uri).AssignedNode.ShouldNotBeNull(
"an orphaned shard must be reassigned to a surviving node, not silently stranded (GH-3341)");
}

// And db2 stays whole on one node — the connection-pool affinity this method exists to provide.
orphaned.Select(u => grid.AgentFor(u).AssignedNode).Distinct().Count()
.ShouldBe(1, "the orphaned shard's agents must be co-located on a single node");
}

[Fact]
public void an_incumbent_node_keeps_its_group_up_to_the_ceiling()
{
Expand Down
58 changes: 39 additions & 19 deletions src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,32 +193,52 @@ public void DistributeByGroupAffinity(string scheme, Func<Uri, string> groupKey,

if (candidates.Count == 0)
{
// No single node can host the whole group — mirror the blue/green even path's per-agent
// behavior: an already-running member stays where it is (minimal disruption), an
// unassigned member goes to its least-loaded capable node, and a member no node declares
// a capability for is simply left alone (candidate == null there too).
foreach (var member in members)
// GH-3341: a whole group whose members are all unassigned AND declared by no node is a
// stale-snapshot artifact, not a genuine blue/green gap. A node captures its
// event-subscription capabilities once at startup (StartLocalAgentProcessingAsync), so a
// shard database provisioned after every surviving node started is absent from all their
// snapshots even though every node can run it — the agents are still enumerated as
// supported by AllKnownAgentsAsync. When such a group's incumbent was a departed node, the
// OriginalNode grandfathering above cannot rescue it, and the per-member fallback below
// would park every member: the shard silently stops projecting with no running agent, no
// log, and no self-heal until a restart refreshes the snapshots. Treat the whole group as
// assignable to any node so it always has a home, kept together to preserve the
// connection-pool affinity this method exists to provide.
if (members.All(m => m.AssignedNode == null && m.CandidateNodes.Count == 0))
{
if (member.AssignedNode != null)
candidates = nodes;
}
else
{
// Mixed capabilities (genuine blue/green): an already-running member stays where it is
// (minimal disruption), an unassigned member with a capable node goes to its
// least-loaded one, and an unassigned member no node declares falls back to the
// least-loaded node overall rather than being silently stranded (GH-3341).
foreach (var member in members)
{
load[member.AssignedNode] = load.GetValueOrDefault(member.AssignedNode) + 1;
continue;
}

var candidate = member.CandidateNodes
.OrderBy(n => load.GetValueOrDefault(n))
.ThenBy(n => n.IsLeader)
.ThenBy(n => n.AssignedId)
.FirstOrDefault();
if (member.AssignedNode != null)
{
load[member.AssignedNode] = load.GetValueOrDefault(member.AssignedNode) + 1;
continue;
}

var candidate = member.CandidateNodes
.OrderBy(n => load.GetValueOrDefault(n))
.ThenBy(n => n.IsLeader)
.ThenBy(n => n.AssignedId)
.FirstOrDefault()
?? nodes
.OrderBy(n => load.GetValueOrDefault(n))
.ThenBy(n => n.IsLeader)
.ThenBy(n => n.AssignedId)
.First();

if (candidate != null)
{
candidate.Assign(member);
load[candidate] += 1;
}
}

continue;
continue;
}
}

// Minimal disruption, mirroring DistributeEvenly: the node already running the WHOLE group
Expand Down
Loading