GH-5163 Unplaced agents carry an UnplacedReason; leader logs agents awaiting capacity

docs/guide/durability/leadership-and-troubleshooting.md

@@ -207,6 +207,9 @@ receive line were the same number, a node sitting right at the threshold would s
 immediately look like a valid target again, and get the agent right back on the next evaluation. The band gives the
 reading somewhere to settle.
 
+If 10 points is the wrong band for you, set `NodeAcceptThreshold` to the load below which a node receives agents again.
+Leave it null to keep the default of `NodeOverloadThreshold - 10`, and keep it below the overload threshold.
+
 `OverloadShedBatchSize` (1 by default) caps how many agents per scheme come off an overloaded node in any one
 evaluation. Shedding only ever happens when some *other* node can take the work, too. An overloaded node with nowhere
 to shed to keeps what it's running, because a node that's struggling is still better than no node at all.
@@ -240,6 +243,12 @@ That's the right call, but be aware that it's a *quiet* symptom compared to a cr
 is restarting, and the only outward sign is that some work isn't happening. If you're enabling this feature, it's worth
 setting up an alert on agents that stay unassigned across several evaluations.
 
+To make that easier to tell apart from other problems, the leader logs an Information message naming the agents waiting
+for capacity (once per change in that count, not on every evaluation), and each waiting agent in the `AssignmentGrid`
+handed to `IWolverineObserver.AssignmentsChanged()` carries `UnplacedReason.AwaitingCapacity`. An agent with no node
+that can run it shows `NoCapableNode`, a paused one `Paused`, and one held back by a newly elected leader
+`TakeoverHold`.
+
 ### Which Databases Support This
 
 All of them. PostgreSQL, Sql Server, MySQL, Oracle, Sqlite, RavenDb, and Azure Cosmos Db all persist and read back a

src/Testing/CoreTests/Runtime/Agents/SimulatedCluster.cs

@@ -133,6 +133,11 @@ public SimulatedCluster(int nodeCount, FakeAgentFamily family, int seed,
     }
 
     public WolverineOptions Options { get; private set; }
+
+    /// <summary>
+    ///     The grid the leader built on its most recent evaluation.
+    /// </summary>
+    public AssignmentGrid? LastAssignments => _controller.LastAssignments;
     public int Seed { get; }
 
     /// <summary>

src/Testing/CoreTests/Runtime/Agents/unplaced_agents_carry_a_reason_5163.cs

@@ -0,0 +1,186 @@
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using Shouldly;
+using Wolverine.Runtime.Agents;
+using Xunit;
+
+namespace CoreTests.Runtime.Agents;
+
+/// <summary>
+///     GH-5163. An agent the leader leaves without a node says why, so a monitor can tell an agent waiting on
+///     capacity from one that never started.
+/// </summary>
+public class unplaced_agents_carry_a_reason_5163
+{
+    private readonly Uri blue1 = new("blue://1");
+    private readonly Uri blue2 = new("blue://2");
+    private readonly Uri blue3 = new("blue://3");
+    private readonly Uri blue4 = new("blue://4");
+
+    [Fact]
+    public void agents_left_waiting_because_no_node_is_accepting_are_awaiting_capacity()
+    {
+        var grid = new AssignmentGrid();
+
+        var node1 = grid.WithNode(1, Guid.NewGuid()).Running(blue1);
+        var node2 = grid.WithNode(2, Guid.NewGuid()).Running(blue2);
+
+        foreach (var node in grid.Nodes)
+        {
+            node.IsOverloaded = true;
+            node.IsAcceptingAgents = false;
+        }
+
+        grid.WithAgents(blue3, blue4);
+        grid.DistributeEvenly("blue");
+
+        grid.AgentFor(blue3).UnplacedReason.ShouldBe(UnplacedReason.AwaitingCapacity);
+        grid.AgentFor(blue4).UnplacedReason.ShouldBe(UnplacedReason.AwaitingCapacity);
+
+        // The agents still running where they were have nothing to explain
+        grid.AgentFor(blue1).AssignedNode.ShouldBe(node1);
+        grid.AgentFor(blue1).UnplacedReason.ShouldBeNull();
+        grid.AgentFor(blue2).AssignedNode.ShouldBe(node2);
+        grid.AgentFor(blue2).UnplacedReason.ShouldBeNull();
+    }
+
+    [Fact]
+    public void the_reason_is_cleared_once_the_agent_is_placed()
+    {
+        var grid = new AssignmentGrid();
+        var node = grid.WithNode(1, Guid.NewGuid());
+        node.IsAcceptingAgents = false;
+
+        grid.WithAgents(blue1, blue2);
+        grid.DistributeEvenly("blue");
+        grid.AgentFor(blue1).UnplacedReason.ShouldBe(UnplacedReason.AwaitingCapacity);
+
+        // The node recovers and the next pass places everything
+        node.IsAcceptingAgents = true;
+        grid.DistributeEvenly("blue");
+
+        grid.AgentFor(blue1).AssignedNode.ShouldBe(node);
+        grid.AgentFor(blue1).UnplacedReason.ShouldBeNull();
+        grid.AgentFor(blue2).UnplacedReason.ShouldBeNull();
+    }
+
+    [Fact]
+    public void an_agent_no_node_declares_has_no_capable_node()
+    {
+        var grid = new AssignmentGrid();
+
+        // Mixed capabilities, so this takes the blue/green path, and nobody declares blue3
+        grid.WithNode(1, Guid.NewGuid()).HasCapabilities([blue1, blue2]);
+        grid.WithNode(2, Guid.NewGuid()).HasCapabilities([blue1]);
+        grid.WithAgents(blue1, blue2, blue3);
+
+        grid.DistributeEvenlyWithBlueGreenSemantics("blue");
+
+        grid.AgentFor(blue1).UnplacedReason.ShouldBeNull();
+        grid.AgentFor(blue2).UnplacedReason.ShouldBeNull();
+        grid.AgentFor(blue3).AssignedNode.ShouldBeNull();
+        grid.AgentFor(blue3).UnplacedReason.ShouldBe(UnplacedReason.NoCapableNode);
+    }
+
+    [Fact]
+    public void agents_with_no_capable_node_under_affinity_distribution_say_so()
+    {
+        var grid = new AssignmentGrid();
+        grid.WithNode(1, Guid.NewGuid()).Running(blue1);
+        grid.WithNode(2, Guid.NewGuid());
+        grid.WithAgents(blue2);
+
+        grid.DistributeEvenlyWithAffinity("blue", _ => null, _ => false);
+
+        grid.AgentFor(blue1).AssignedNode.ShouldBeNull();
+        grid.AgentFor(blue1).UnplacedReason.ShouldBe(UnplacedReason.NoCapableNode);
+        grid.AgentFor(blue2).UnplacedReason.ShouldBe(UnplacedReason.NoCapableNode);
+    }
+
+    [Fact]
+    public void a_paused_agent_is_paused()
+    {
+        var grid = new AssignmentGrid();
+        grid.WithNode(1, Guid.NewGuid()).Running(blue1, blue2);
+
+        var restrictions = new AgentRestrictions([]);
+        restrictions.PauseAgent(blue1);
+        grid.ApplyRestrictions(restrictions);
+
+        grid.DistributeEvenly("blue");
+
+        grid.AgentFor(blue1).AssignedNode.ShouldBeNull();
+        grid.AgentFor(blue1).UnplacedReason.ShouldBe(UnplacedReason.Paused);
+        grid.AgentFor(blue2).UnplacedReason.ShouldBeNull();
+    }
+
+    [Fact]
+    public void the_leader_logs_agents_awaiting_capacity_once_per_change()
+    {
+        var grid = new AssignmentGrid();
+        var node = grid.WithNode(1, Guid.NewGuid());
+        node.IsAcceptingAgents = false;
+        grid.WithAgents(blue1, blue2);
+        grid.DistributeEvenly("blue");
+
+        var logger = Substitute.For<ILogger>();
+        logger.IsEnabled(LogLevel.Information).Returns(true);
+
+        var reported = 0;
+        NodeAgentController.ReportAgentsAwaitingCapacity(grid, logger, ref reported);
+        reported.ShouldBe(2);
+
+        // The same parked steady state on the next evaluation says nothing new
+        NodeAgentController.ReportAgentsAwaitingCapacity(grid, logger, ref reported);
+        logger.ReceivedWithAnyArgs(1).Log(default, default, default!, default, default!);
+
+        // Capacity comes back, so the next one that parks agents is news again
+        node.IsAcceptingAgents = true;
+        grid.DistributeEvenly("blue");
+        NodeAgentController.ReportAgentsAwaitingCapacity(grid, logger, ref reported);
+        reported.ShouldBe(0);
+        logger.ReceivedWithAnyArgs(1).Log(default, default, default!, default, default!);
+    }
+
+    [Fact]
+    public async Task the_leader_marks_agents_parked_when_every_node_is_over_the_accept_threshold()
+    {
+        await using var cluster = new SimulatedCluster(nodeCount: 2, agentCount: 6, seed: 5163);
+        cluster.Options.Durability.CapacityAwareAssignment = true;
+        cluster.Options.Durability.NodeOverloadThreshold = 90;
+
+        // Below the overload line, so nothing sheds, but above a custom accept line, so nothing is received
+        cluster.Options.Durability.NodeAcceptThreshold = 50;
+        foreach (var nodeId in cluster.NodeIds) cluster.AdvertiseLoad(nodeId, 60);
+
+        await cluster.RunRoundAsync();
+
+        var unplaced = cluster.LastAssignments!.UnassignedAgents;
+        unplaced.ShouldNotBeEmpty();
+        unplaced.ShouldAllBe(x => x.UnplacedReason == UnplacedReason.AwaitingCapacity);
+        cluster.RunningAgents.ShouldBeEmpty();
+
+        // Back under the accept line: everything is placed and nothing is left with a reason
+        foreach (var nodeId in cluster.NodeIds) cluster.AdvertiseLoad(nodeId, 40);
+        await cluster.RunUntilConvergedAsync(20);
+
+        cluster.LastAssignments!.UnassignedAgents.ShouldBeEmpty();
+        cluster.LastAssignments.AllAgents.ShouldAllBe(x => x.UnplacedReason == null);
+    }
+
+    [Fact]
+    public async Task agents_held_back_by_a_new_leader_are_takeover_holds()
+    {
+        // A cold cluster's first node with a wave larger than one start batch holds first-time placement back
+        // for LeaderColdStartHoldEvaluations evaluations, so its peers can register
+        await using var cluster = new SimulatedCluster(nodeCount: 1, agentCount: 20, seed: 5163);
+        cluster.Options.Durability.AgentStartBatchSize = 5;
+        cluster.Options.Durability.LeaderColdStartHoldEvaluations = 1;
+
+        await cluster.RunRoundAsync();
+
+        var held = cluster.LastAssignments!.UnassignedAgents.Where(x => x.Uri.Scheme == "fake").ToArray();
+        held.ShouldNotBeEmpty();
+        held.ShouldAllBe(x => x.UnplacedReason == UnplacedReason.TakeoverHold);
+    }
+}

src/Wolverine/DurabilitySettings.cs

@@ -941,11 +941,20 @@ public bool EnableMessageDeduplication
 
     /// <summary>
     ///     Advertised load percentage at or above which a node is considered overloaded: it begins
-    ///     shedding agents, and stops receiving new ones starting 10 points below this value.
+    ///     shedding agents, and stops receiving new ones starting 10 points below this value (see
+    ///     <see cref="NodeAcceptThreshold" />).
     ///     Default 90.
     /// </summary>
     public double NodeOverloadThreshold { get; set; } = 90;
 
+    /// <summary>
+    ///     GH-5163. Advertised load percentage below which a node receives new agents under
+    ///     <see cref="CapacityAwareAssignment" />. When null (the default) this is 10 points below
+    ///     <see cref="NodeOverloadThreshold" />. Keep it below the overload threshold so placement and
+    ///     shedding do not fight around a single value.
+    /// </summary>
+    public double? NodeAcceptThreshold { get; set; }
+
     /// <summary>
     ///     Maximum number of agents per scheme the leader moves off an overloaded node in one
     ///     assignment evaluation. Default 1. Shedding only happens when some other node can take the

src/Wolverine/Runtime/Agents/AssignmentGrid.Agent.cs

@@ -30,6 +30,19 @@ internal Agent(Uri uri, Node? originalNode)
         /// </summary>
         public bool IsPaused { get; set; }
 
+        private UnplacedReason? _unplacedReason;
+
+        /// <summary>
+        ///     GH-5163. Why this agent was left without a node on this evaluation, when the grid knows. Always
+        ///     null while the agent has an <see cref="AssignedNode" />, so a reason recorded by one pass never
+        ///     outlives a later placement.
+        /// </summary>
+        public UnplacedReason? UnplacedReason
+        {
+            get => AssignedNode == null ? _unplacedReason : null;
+            internal set => _unplacedReason = value;
+        }
+
         /// <summary>
         ///     The node that was executing this agent at the time the assignment
         ///     grid was being determined

src/Wolverine/Runtime/Agents/AssignmentGrid.Distribution.cs

@@ -63,6 +63,13 @@ public void DistributeEvenly(string scheme, Func<Uri, bool> filter)
             // to zero (3 -> 2 -> 1 -> 0) and stay there for as long as the pressure held, taking the
             // durability agents and daemon shards with it. An overloaded node still running its work
             // is strictly better than that.
+            //
+            // GH-5163: say why, so a monitor can tell "waiting on capacity" from "never started".
+            foreach (var agent in agents)
+            {
+                if (agent.AssignedNode == null) agent.UnplacedReason = UnplacedReason.AwaitingCapacity;
+            }
+
             return;
         }
 
@@ -647,6 +654,7 @@ public void DistributeEvenlyWithAffinity(string scheme, Func<Uri, Node?> preferr
 
         if (capableNodes.Count == 0)
         {
+            foreach (var agent in agents) agent.UnplacedReason = UnplacedReason.NoCapableNode;
             return;
         }
 

src/Wolverine/Runtime/Agents/AssignmentGrid.Node.cs

@@ -258,6 +258,7 @@ public void Assign(Uri agentUri)
             }
 
             agent.AssignedNode = this;
+            agent.UnplacedReason = null;
             add(agent);
         }
 
@@ -274,6 +275,7 @@ public void Assign(Agent agent)
             }
 
             agent.AssignedNode = this;
+            agent.UnplacedReason = null;
             add(agent);
         }
 

src/Wolverine/Runtime/Agents/AssignmentGrid.cs

@@ -218,6 +218,13 @@ public IReadOnlyList<Agent> MatchAgentsToCapableNodesFor(string scheme, Func<Uri
             {
                 agent.CandidateNodes.RemoveAll(x => x.IsWarmup);
             }
+
+            // GH-5163. Only visible while the agent stays unassigned; a later placement (the GH-3341 rescue,
+            // say) hides it again.
+            if (agent.CandidateNodes.Count == 0)
+            {
+                agent.UnplacedReason = UnplacedReason.NoCapableNode;
+            }
         }
 
         detachFromIneligibleWarmupNodes(agents);
@@ -389,6 +396,7 @@ public void ApplyRestrictions(AgentRestrictions restrictions)
             }
 
             agent.IsPaused = true;
+            agent.UnplacedReason = UnplacedReason.Paused;
             if (agent.AssignedNode != null)
             {
                 agent.Detach();