Expose the classified pause/stop reason of a shard (#565)

src/JasperFx.Events/Daemon/ApplyEventException.cs

@@ -1,6 +1,12 @@
 namespace JasperFx.Events.Daemon;
 
-public class ApplyEventException: Exception
+/// <summary>
+/// Thrown when user projection or subscription code fails while applying a single event — the classic
+/// "poison pill". Carries the offending event, which is what lets both the
+/// <see cref="ErrorHandlingOptions.SkipApplyErrors"/> dead-letter path and (jasperfx#565) the pause
+/// reporting name it.
+/// </summary>
+public class ApplyEventException: Exception, IEventFailureContext
 {
     public ApplyEventException(IEvent @event, Exception innerException): base(
         $"Failure to apply event #{@event.Sequence} Id({@event.Id})", innerException)
@@ -9,4 +15,22 @@ public ApplyEventException(IEvent @event, Exception innerException): base(
     }
 
     public IEvent Event { get; }
+
+    ShardFailureCategory IEventFailureContext.Category => ShardFailureCategory.ApplyEvent;
+
+    long IEventFailureContext.Sequence => Event.Sequence;
+
+    string? IEventFailureContext.EventTypeName => Event.EventTypeName;
+
+    Guid? IEventFailureContext.EventId => Event.Id;
+
+    // Guid.Empty is how a string-keyed stream reports "no Guid id", and an empty key is the mirror image
+    // on a Guid-keyed stream. Normalize both away so a consumer never renders a meaningless value.
+    Guid? IEventFailureContext.StreamId => Event.StreamId == Guid.Empty ? null : Event.StreamId;
+
+    string? IEventFailureContext.StreamKey => string.IsNullOrEmpty(Event.StreamKey) ? null : Event.StreamKey;
+
+    string? IEventFailureContext.TenantId => Event.TenantId;
+
+    long? IEventFailureContext.Version => Event.Version;
 }

src/JasperFx.Events/Daemon/DeadLetterEvent.cs

@@ -5,14 +5,21 @@ namespace JasperFx.Events.Daemon;
 
 public class DeadLetterEvent
 {
-#pragma warning disable CS8618 
+#pragma warning disable CS8618
     public DeadLetterEvent()
-#pragma warning restore CS8618 
+#pragma warning restore CS8618
     {
     }
 
     public DeadLetterEvent(IEvent e, ShardName shardName, ApplyEventException ex)
     {
+        // jasperfx#565: assign the identity here rather than leaving it to the store's document identity
+        // generation, so the id of a dead letter is known to the process that created it BEFORE the
+        // (background, retried) write lands. Stores only generate an id when the value is empty, so
+        // pre-assigning changes nothing about how the row is persisted. Version 7 keeps the ids
+        // time-ordered, which is what the store's index would have wanted anyway.
+        Id = Guid.CreateVersion7();
+
         ProjectionName = shardName.Name;
         ShardName = shardName.ShardKey;
         Timestamp = DateTimeOffset.UtcNow;
@@ -42,6 +49,34 @@ public DeadLetterEvent(IEvent e, ShardName shardName, ApplyEventException ex)
     /// </summary>
     public string? TenantId { get; set; }
 
+    /// <summary>
+    /// jasperfx#565: does this dead letter describe the same failing event as <paramref name="failure" />
+    /// on the shard named by <paramref name="shardName" />?
+    ///
+    /// <para>
+    /// This is the traceability link between the two halves of a per-event failure, which are recorded on
+    /// different paths and never at the same time. A shard that PAUSES (the error options do not skip)
+    /// reports a <see cref="ShardFailure" /> and writes nothing here — the event was not skipped, so
+    /// inflating the dead-letter counts stores use as their "projection is unhealthy" signal would be a
+    /// lie, and a restart loop would rewrite the row on every attempt. A shard that SKIPS
+    /// (<see cref="ErrorHandlingOptions.SkipApplyErrors" /> and friends) writes a dead letter and keeps
+    /// running. Same event, same projection, same shard, same sequence, same tenant — so an operator (or
+    /// CritterWatch) that has one can find the other, whether the deployment flipped the skip flag after
+    /// the pause or the other way round.
+    /// </para>
+    /// </summary>
+    public bool DescribesSameFailureAs(ShardName shardName, ShardFailure failure)
+    {
+        if (failure.Event == null) return false;
+
+        return ProjectionName == shardName.Name
+               && ShardName == shardName.ShardKey
+               && EventSequence == failure.Event.Sequence
+               // A failure detected before the event materialized may not know its tenant; don't let that
+               // veto a match the sequence already established.
+               && (failure.Event.TenantId == null || TenantId == null || TenantId == failure.Event.TenantId);
+    }
+
     public override string ToString()
     {
         return

src/JasperFx.Events/Daemon/EventFailureDetails.cs

@@ -0,0 +1,102 @@
+namespace JasperFx.Events.Daemon;
+
+/// <summary>
+/// jasperfx#565: the identity of the single event that broke a shard, lifted off an exception into a
+/// plain, serializable value. External supervisors (Wolverine's assignment plane, CritterWatch) ship
+/// this over a wire and render it in a UI, which an <see cref="Exception"/> can't reliably do — and an
+/// exception also drags along whatever object graph its data referenced.
+///
+/// <para>
+/// Every member except <see cref="Sequence"/> is nullable: a serialization failure is raised while
+/// reading a row, before there is an <see cref="IEvent"/>, so it may know only the sequence and the
+/// stored type alias. See <see cref="IEventFailureContext"/>.
+/// </para>
+///
+/// <para>
+/// <see cref="Sequence"/> plus the owning shard is also the correlation key to a
+/// <see cref="DeadLetterEvent"/> row: nothing is written to the dead-letter table when a shard PAUSES
+/// (a paused event was not skipped), but if the same event is later skipped —
+/// <see cref="ErrorHandlingOptions.SkipApplyErrors"/> and friends — its dead letter carries the same
+/// projection name, shard key, sequence and tenant, so a consumer can line the two up.
+/// </para>
+/// </summary>
+public record EventFailureDetails
+{
+    /// <summary>
+    /// Store-wide sequence number of the failing event.
+    /// </summary>
+    public required long Sequence { get; init; }
+
+    /// <summary>
+    /// The event store's type alias for the failing event (e.g. <c>trip_started</c>), if known.
+    /// </summary>
+    public string? EventTypeName { get; init; }
+
+    /// <summary>
+    /// Unique id of the failing event, if known.
+    /// </summary>
+    public Guid? EventId { get; init; }
+
+    /// <summary>
+    /// Stream id of the failing event for Guid-identified streams, if known.
+    /// </summary>
+    public Guid? StreamId { get; init; }
+
+    /// <summary>
+    /// Stream key of the failing event for string-identified streams, if known.
+    /// </summary>
+    public string? StreamKey { get; init; }
+
+    /// <summary>
+    /// Tenant of the failing event, if known.
+    /// </summary>
+    public string? TenantId { get; init; }
+
+    /// <summary>
+    /// Version of the failing event within its stream, if known.
+    /// </summary>
+    public long? Version { get; init; }
+
+    /// <summary>
+    /// Lift the failing event's identity off an exception that knows it.
+    /// </summary>
+    public static EventFailureDetails From(IEventFailureContext context)
+    {
+        return new EventFailureDetails
+        {
+            Sequence = context.Sequence,
+            EventTypeName = context.EventTypeName,
+            EventId = context.EventId,
+            StreamId = context.StreamId,
+            StreamKey = context.StreamKey,
+            TenantId = context.TenantId,
+            Version = context.Version
+        };
+    }
+
+    /// <summary>
+    /// Lift the identity off a fully materialized event.
+    /// </summary>
+    public static EventFailureDetails From(IEvent @event)
+    {
+        return new EventFailureDetails
+        {
+            Sequence = @event.Sequence,
+            EventTypeName = @event.EventTypeName,
+            EventId = @event.Id,
+            // Guid.Empty is how a string-keyed stream reports "no Guid id", and vice versa. Normalize
+            // both to null so a consumer never renders a meaningless Guid.Empty or an empty key.
+            StreamId = @event.StreamId == Guid.Empty ? null : @event.StreamId,
+            StreamKey = string.IsNullOrEmpty(@event.StreamKey) ? null : @event.StreamKey,
+            TenantId = @event.TenantId,
+            Version = @event.Version
+        };
+    }
+
+    public override string ToString()
+    {
+        var stream = StreamId?.ToString() ?? StreamKey;
+        return
+            $"event #{Sequence}{(EventTypeName == null ? "" : $" ({EventTypeName})")}{(stream == null ? "" : $" on stream {stream}")}{(TenantId == null ? "" : $" for tenant '{TenantId}'")}";
+    }
+}

src/JasperFx.Events/Daemon/IEventFailureContext.cs

@@ -0,0 +1,67 @@
+namespace JasperFx.Events.Daemon;
+
+/// <summary>
+/// jasperfx#565: implemented by any exception that can name the single event it failed on, so the daemon
+/// can classify a shard failure and report the offending event WITHOUT knowing the concrete exception
+/// types of the store underneath it.
+///
+/// <para>
+/// JasperFx.Events owns only one of these — <see cref="ApplyEventException"/>. The others live in the
+/// stores, because that is where events are read and deserialized: Marten's
+/// <c>EventDeserializationFailureException</c> / <c>UnknownEventTypeException</c> and Polecat's
+/// equivalents implement this interface and declare their own <see cref="Category"/>. That is
+/// deliberately the store's call rather than type-name sniffing in the daemon.
+/// </para>
+///
+/// <para>
+/// Only <see cref="Sequence"/> is guaranteed. A serialization failure is detected while reading a row,
+/// before there is an <see cref="IEvent"/> to inspect, so it may know nothing but the sequence and the
+/// stored type alias — every other member is nullable for exactly that reason. Whatever IS known lets a
+/// consumer correlate the failure with a <see cref="DeadLetterEvent"/> row for the same
+/// (projection, shard, sequence) if the event is later skipped.
+/// </para>
+/// </summary>
+public interface IEventFailureContext
+{
+    /// <summary>
+    /// How the daemon should classify a shard failure caused by this exception.
+    /// </summary>
+    ShardFailureCategory Category { get; }
+
+    /// <summary>
+    /// Store-wide sequence number of the failing event. The one member every implementation can supply,
+    /// and the key a consumer joins on to find a matching <see cref="DeadLetterEvent"/>.
+    /// </summary>
+    long Sequence { get; }
+
+    /// <summary>
+    /// The event store's type alias for the failing event (e.g. <c>trip_started</c>), when known.
+    /// </summary>
+    string? EventTypeName { get; }
+
+    /// <summary>
+    /// Unique id of the failing event, when the exception was raised late enough to have one.
+    /// </summary>
+    Guid? EventId { get; }
+
+    /// <summary>
+    /// Stream id of the failing event for Guid-identified streams, when known.
+    /// </summary>
+    Guid? StreamId { get; }
+
+    /// <summary>
+    /// Stream key of the failing event for string-identified streams, when known.
+    /// </summary>
+    string? StreamKey { get; }
+
+    /// <summary>
+    /// Tenant the failing event belongs to, when known. Part of the dead-letter correlation key on
+    /// tenant-partitioned stores, where one shard accumulates failures per tenant.
+    /// </summary>
+    string? TenantId { get; }
+
+    /// <summary>
+    /// Version of the failing event within its stream, when known.
+    /// </summary>
+    long? Version { get; }
+}

src/JasperFx.Events/Daemon/ISubscriptionAgent.cs

@@ -18,6 +18,22 @@ public interface ISubscriptionAgent : ISubscriptionController
     long HighWaterMark => 0;
 
     DateTimeOffset? PausedTime { get; }
+
+    /// <summary>
+    /// jasperfx#565: WHY this agent was paused or stopped, if it was. <see cref="Status"/> alone told an
+    /// external supervisor (Wolverine's <c>EventSubscriptionAgent</c>, which wraps a shard as a
+    /// distributed agent) that a shard had paused but never what to do about it, so progress could
+    /// flatline with no actionable alert. Set alongside <see cref="Status"/> when a failure is reported,
+    /// and cleared when the agent starts or replays.
+    ///
+    /// <para>
+    /// Defaulted to null so implementations that don't track failures — test doubles, wrappers that
+    /// delegate — are unaffected. A wrapper around a live inner agent should delegate this the same way
+    /// it delegates <see cref="Status"/>.
+    /// </para>
+    /// </summary>
+    ShardFailure? Failure => null;
+
     ISubscriptionMetrics Metrics { get; }
     void MarkHighWater(long sequence);