Surface paused-shard failures to the assignment plane + critter stack bump (#3637, #3638, #3519)

Directory.Packages.props

@@ -36,20 +36,20 @@
     <PackageVersion Include="Grpc.StatusProto" Version="2.76.0" />
     <PackageVersion Include="Grpc.Tools" Version="2.76.0" />
     <PackageVersion Include="HtmlTags" Version="9.0.0" />
-    <PackageVersion Include="JasperFx" Version="2.34.0" />
-    <PackageVersion Include="JasperFx.Events" Version="2.34.0" />
-    <PackageVersion Include="JasperFx.Events.SourceGenerator" Version="2.34.0" />
+    <PackageVersion Include="JasperFx" Version="2.36.1" />
+    <PackageVersion Include="JasperFx.Events" Version="2.36.1" />
+    <PackageVersion Include="JasperFx.Events.SourceGenerator" Version="2.36.1" />
     <!-- RuntimeCompiler is on its own 5.x line (the Roslyn compiler package) — not the 2.1.x
          family; it stays at 5.0.0. -->
     <PackageVersion Include="JasperFx.RuntimeCompiler" Version="5.0.0" />
-    <PackageVersion Include="JasperFx.SourceGenerator" Version="2.34.0" />
+    <PackageVersion Include="JasperFx.SourceGenerator" Version="2.36.1" />
     <PackageVersion Include="Lamar.Microsoft.DependencyInjection" Version="16.0.0" />
-    <PackageVersion Include="Marten" Version="9.18.0" />
+    <PackageVersion Include="Marten" Version="9.20.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="6.1.3" />
-    <PackageVersion Include="Polecat" Version="[5.5.0,6.0.0)" />
+    <PackageVersion Include="Polecat" Version="[5.7.0,6.0.0)" />
     <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.46.1" />
-    <PackageVersion Include="Marten.AspNetCore" Version="9.18.0" />
-    <PackageVersion Include="Marten.Newtonsoft" Version="9.18.0" />
+    <PackageVersion Include="Marten.AspNetCore" Version="9.20.0" />
+    <PackageVersion Include="Marten.Newtonsoft" Version="9.20.0" />
     <PackageVersion Include="MemoryPack" Version="1.21.3" />
     <PackageVersion Include="MessagePack" Version="3.1.7" />
     <PackageVersion Include="Meziantou.Extensions.Logging.Xunit" Version="1.0.15" />

docs/guide/durability/marten/distribution.md

@@ -185,3 +185,53 @@ var host = await Host.CreateDefaultBuilder()
 <sup><a href='https://github.com/JasperFx/wolverine/blob/main/src/Persistence/MartenTests/Distribution/with_ancillary_stores.cs#L59-L103' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_using_distributed_projections_with_ancillary_stores' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
+
+## When a Projection Fails <Badge type="tip" text="6.x" />
+
+A projection or subscription shard that throws while applying an event is *paused* by the Marten/Polecat
+daemon rather than skipped, unless you have opted into skipping through `ErrorHandlingOptions`
+(`SkipApplyErrors` and friends). A paused shard makes no further progress, and Wolverine deliberately does
+not restart it — restarting would fail on the exact same event, so the shard would thrash instead of
+advance.
+
+Wolverine surfaces the paused shard so it does not simply go quiet:
+
+* The agent's health check reports the failure **category**, the sequence number and type of the event it
+  died on, and the root exception type — enough to act on without going to dig through logs.
+* `IWolverineObserver.AgentPaused(Uri agentUri, ShardFailure? failure)` fires once per transition into the
+  failed state (and again if the shard recovers and later fails anew). Implement it on a custom observer to
+  raise your own alert; [CritterWatch](https://critterwatch.io) uses this hook.
+* A `NodeRecordType.AgentPaused` record is written to the node-record log with the classified reason, so
+  the failure is readable after the fact and from another process.
+* `IEventSubscriptionAgent.Failure` exposes the same `ShardFailure` value directly. It is a plain,
+  serializable record — category, the failing event, the exception message and full detail — not an
+  `Exception`, so it survives being shipped to a monitoring UI.
+
+The category tells you what to do about it:
+
+| Category | What it means |
+|----------|---------------|
+| `ApplyEvent` | Your projection code threw on an event — the classic "poison pill". Needs a code fix, or `SkipApplyErrors`. |
+| `EventSerialization` | The store could not deserialize or upcast a stored event body. Needs a serializer or data fix. |
+| `UnknownEventType` | A stored event alias resolves to no known .NET type in *this* deployment — usually a missing registration or a rollback. |
+| `ProgressionOutOfOrder` | The shard's progression row moved underneath it, which almost always means two processes are running the same shard. |
+| `Other` | A database outage, a timeout, or a bug. No single event can be blamed. |
+
+Only `Other` is treated as potentially self-healing, so it is the only category Wolverine's stall detector
+will auto-restart. The rest are left alone until you resolve the underlying problem, at which point
+restarting or rewinding the agent picks it back up.
+
+## Agent Start Retries <Badge type="tip" text="6.x" />
+
+An agent's very first assignment can race the subsystems it depends on coming up — an event-subscription
+shard evaluated before its store's high-water detection is running, for instance, which on a multi-store
+host could leave a different shard idle on every boot. Wolverine retries a failed agent start locally a
+couple of times before leaving it to the next assignment reevaluation:
+
+```csharp
+opts.Durability.AgentStartRetryAttempts = 2;                       // default
+opts.Durability.AgentStartRetryDelay = TimeSpan.FromMilliseconds(250);  // default, multiplied by attempt number
+```
+
+Set `AgentStartRetryAttempts` to `0` to disable the local retry entirely. A failure that outlives the
+retries is logged and picked up again on the next `CheckAssignmentPeriod` tick, exactly as before.

docs/guide/durability/polecat/distribution.md

@@ -80,3 +80,11 @@ Other requirements:
   runs — just all of them on the single node. `Serverless` and `MediatorOnly` start no agents at all.
 * In `Balanced` mode you cannot disable external transports with `StubAllExternalTransports()`, because the nodes
   need the control queue to communicate
+
+## When a Projection Fails <Badge type="tip" text="6.x" />
+
+The failure handling for a paused projection or subscription shard — the classified `ShardFailure` on
+`IEventSubscriptionAgent`, the `IWolverineObserver.AgentPaused` hook, the `NodeRecordType.AgentPaused`
+record, and the rule that only a self-healing failure is auto-restarted — is shared by both event store
+integrations. See [When a Projection Fails](/guide/durability/marten/distribution#when-a-projection-fails)
+on the Marten page for the details; everything there applies identically to Polecat.

src/Testing/CoreTests/Runtime/Agents/agent_start_retry_on_startup_race.cs

@@ -0,0 +1,180 @@
+using JasperFx;
+using JasperFx.Core;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using Shouldly;
+using Wolverine.Runtime;
+using Wolverine.Runtime.Agents;
+using Xunit;
+
+namespace CoreTests.Runtime.Agents;
+
+/// <summary>
+/// Regression coverage for GH-3519. On a multi-store Marten host, one event-subscription agent — a
+/// different one on every boot — failed its very first assignment start because it was evaluated before
+/// its store's high-water detection was up, and then sat wedged for the life of the process. The daemon
+/// side is fixed in JasperFx.Events 2.36.x: the start failure now arrives as a <c>ShardStartException</c>
+/// that names its cause (jasperfx#534) and the half-started shard is released rather than orphaned
+/// (jasperfx#540), so the next attempt succeeds. What was left on this side was WHEN that next attempt
+/// happens — the node retried only on the next assignment reevaluation, so the loser of a sub-second
+/// startup race idled for a full CheckAssignmentPeriod (30s by default) while its high-water climbed.
+/// </summary>
+public class agent_start_retry_on_startup_race
+{
+    private readonly WolverineOptions _options;
+    private readonly IWolverineRuntime _runtime;
+    private readonly CancellationTokenSource _cancellation = new();
+
+    public agent_start_retry_on_startup_race()
+    {
+        _options = new WolverineOptions { ApplicationAssembly = GetType().Assembly };
+        _options.Durability.Mode = DurabilityMode.Solo;
+        _options.Durability.DurabilityAgentEnabled = false;
+        _options.Durability.CheckAssignmentPeriod = 1.Hours();
+
+        // Keep the test fast; the retry COUNT is what's under test, not the pacing.
+        _options.Durability.AgentStartRetryDelay = 1.Milliseconds();
+
+        _runtime = Substitute.For<IWolverineRuntime>();
+        _runtime.Options.Returns(_options);
+        _runtime.DurabilitySettings.Returns(_options.Durability);
+        _runtime.Observer.Returns(Substitute.For<IWolverineObserver>());
+    }
+
+    private NodeAgentController controllerFor(params FlakyAgent[] agents)
+    {
+        var family = new FlakyAgentFamily("event-subscriptions");
+        foreach (var agent in agents)
+        {
+            family.Add(agent);
+        }
+
+        return new NodeAgentController(_runtime, Substitute.For<INodeAgentPersistence>(), [family],
+            NullLogger<NodeAgentController>.Instance, _cancellation.Token);
+    }
+
+    [Fact]
+    public async Task recovers_from_a_start_that_loses_the_first_assignment_race()
+    {
+        var uri = new Uri("event-subscriptions://marten/iincidentsstore/localhost.postgres/incident/all");
+
+        // Exactly the reported shape: high-water detection isn't up yet on the first attempt and is by
+        // the second.
+        var agent = new FlakyAgent(uri, failuresBeforeSuccess: 1);
+        var controller = controllerFor(agent);
+
+        await controller.StartAgentAsync(uri);
+
+        agent.AttemptCount.ShouldBe(2);
+        agent.Status.ShouldBe(AgentStatus.Running);
+        controller.Agents.ContainsKey(uri).ShouldBeTrue();
+    }
+
+    [Fact]
+    public async Task gives_up_after_the_configured_attempts_and_preserves_the_daemon_s_reason()
+    {
+        var uri = new Uri("event-subscriptions://marten/incident/all");
+        var agent = new FlakyAgent(uri, failuresBeforeSuccess: int.MaxValue);
+        var controller = controllerFor(agent);
+
+        var ex = await Should.ThrowAsync<AgentStartingException>(() => controller.StartAgentAsync(uri));
+
+        // Default is 2 retries on top of the initial attempt. A failure that outlives them is left to
+        // the next assignment reevaluation rather than retried harder here.
+        agent.AttemptCount.ShouldBe(3);
+
+        // The daemon's reason has to survive the wrapping, or we are back to the causeless "Unable to
+        // start a subscription agent" that made this issue undiagnosable in the first place.
+        ex.InnerException.ShouldNotBeNull();
+        ex.InnerException.Message.ShouldContain("Incident:All");
+        ex.InnerException.Message.ShouldContain("High-water detection is not running yet");
+
+        controller.Agents.ContainsKey(uri).ShouldBeFalse();
+    }
+
+    [Fact]
+    public async Task retries_can_be_turned_off()
+    {
+        var uri = new Uri("event-subscriptions://marten/incident/all");
+        _options.Durability.AgentStartRetryAttempts = 0;
+
+        var agent = new FlakyAgent(uri, failuresBeforeSuccess: 1);
+        var controller = controllerFor(agent);
+
+        await Should.ThrowAsync<AgentStartingException>(() => controller.StartAgentAsync(uri));
+
+        agent.AttemptCount.ShouldBe(1);
+    }
+
+    [Fact]
+    public async Task a_healthy_agent_still_starts_on_the_first_attempt()
+    {
+        var uri = new Uri("event-subscriptions://marten/incident/all");
+        var agent = new FlakyAgent(uri, failuresBeforeSuccess: 0);
+        var controller = controllerFor(agent);
+
+        await controller.StartAgentAsync(uri);
+
+        agent.AttemptCount.ShouldBe(1);
+    }
+
+    private class FlakyAgentFamily : IAgentFamily
+    {
+        private readonly Dictionary<Uri, FlakyAgent> _agents = new();
+
+        public FlakyAgentFamily(string scheme) => Scheme = scheme;
+
+        public void Add(FlakyAgent agent) => _agents[agent.Uri] = agent;
+
+        public string Scheme { get; }
+
+        public ValueTask<IReadOnlyList<Uri>> AllKnownAgentsAsync()
+            => ValueTask.FromResult<IReadOnlyList<Uri>>(_agents.Keys.ToList());
+
+        public ValueTask<IAgent> BuildAgentAsync(Uri uri, IWolverineRuntime wolverineRuntime)
+            => ValueTask.FromResult<IAgent>(_agents[uri]);
+
+        public ValueTask<IReadOnlyList<Uri>> SupportedAgentsAsync()
+            => ValueTask.FromResult<IReadOnlyList<Uri>>(_agents.Keys.ToList());
+
+        public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) => ValueTask.CompletedTask;
+    }
+
+    private class FlakyAgent : IAgent
+    {
+        private readonly int _failuresBeforeSuccess;
+
+        public FlakyAgent(Uri uri, int failuresBeforeSuccess)
+        {
+            Uri = uri;
+            _failuresBeforeSuccess = failuresBeforeSuccess;
+        }
+
+        public int AttemptCount { get; private set; }
+
+        public Uri Uri { get; }
+        public AgentStatus Status { get; private set; } = AgentStatus.Stopped;
+
+        public Task StartAsync(CancellationToken cancellationToken)
+        {
+            AttemptCount++;
+            if (AttemptCount <= _failuresBeforeSuccess)
+            {
+                // Stands in for the ShardStartException JasperFxAsyncDaemon.StartAgentAsync(ShardName)
+                // now throws instead of a bare, causeless Exception (its constructors are internal to
+                // JasperFx.Events, so this reproduces the message shape rather than the type).
+                throw new Exception(
+                    "Unable to start a subscription agent for 'Incident:All'. High-water detection is not running yet, so the shard could not be positioned.");
+            }
+
+            Status = AgentStatus.Running;
+            return Task.CompletedTask;
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken)
+        {
+            Status = AgentStatus.Stopped;
+            return Task.CompletedTask;
+        }
+    }
+}