feat(reminders): operator visibility into reminder failures and skips (#1494)

feeds/skills/.system/files/netclaw-operations/SKILL.md

@@ -3,7 +3,7 @@ name: netclaw-operations
 description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance."
 metadata:
   author: netclaw
-  version: "2.19.0"
+  version: "2.20.0"
 ---
 
 # Netclaw Operations

feeds/skills/.system/files/netclaw-operations/references/scheduling.md

@@ -58,6 +58,28 @@ Reminders that hit 5 consecutive execution failures are auto-disabled with a
 `ReminderAutoDisabled` critical alert. The definition stays on disk so the
 operator can diagnose and re-enable after fixing the root cause.
 
+**Failure visibility.** When a reminder execution fails for any reason — including
+the 20-minute stall backstop that recovers a wedged run — the failure is posted
+as a plain-language notice to the reminder's **destination channel** (for
+`channel`-delivery reminders), so the operator sees it where they expect that
+reminder's output. This is bounded by the auto-disable threshold (at most a few
+notices plus the disabled notice), not the unbounded skip stream.
+
+A *skipped* fire (one that arrives while the prior execution is still running) is
+**not** posted to the channel — it would be too noisy — but it is counted and
+surfaced by the status command:
+
+```
+netclaw reminder status <id>
+```
+
+`status` shows, per reminder: whether it's enabled, whether an execution is in
+flight right now, when it next fires, the consecutive-failure count, the
+skipped-fire count (since daemon start), and recent run history. Reach for it
+when a reminder seems to have silently stopped doing its job — a high skip count
+means a prior run is wedged (it should self-recover within ~20 minutes), and a
+rising failure count points at a misconfigured or broken reminder.
+
 If `audience` is omitted during conversational scheduling, the reminder inherits
 the audience of the channel/session that created it. A reminder cannot be
 minted with broader audience than the creator currently holds; lowering the

src/Netclaw.Actors.Tests/Reminders/ReminderManagerActorTests.cs

@@ -73,7 +73,8 @@ protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IService
                     TimeProvider.System,
                     definitionStore,
                     historyStore,
-                    _notificationSink)),
+                    _notificationSink,
+                    NullReminderChannelNotifier.Instance)),
                 "reminder-manager-test");
 
             registry.Register<ReminderManagerActorKey>(reminderManager);
@@ -174,6 +175,44 @@ public async Task Health_query_on_empty_manager_returns_zeros()
         Assert.Equal(0, health.FailedCount);
     }
 
+    [Fact]
+    public async Task Status_query_returns_per_reminder_health()
+    {
+        var manager = await GetManagerAsync();
+
+        var definition = CreateDefinition("test-status", "Check status");
+        var authorization = new ReminderAudienceAuthorizationContext(TrustAudience.Team, "test");
+        await manager.Ask<ReminderSavedResponse>(
+            new SaveReminderCommand(definition, Authorization: authorization), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+
+        var status = await manager.Ask<ReminderStatusResponse>(
+            new GetReminderStatusQuery(definition.Id), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+
+        Assert.True(status.Found);
+        Assert.True(status.Enabled);
+        Assert.False(status.Executing);
+        Assert.Equal(0, status.ConsecutiveFailures);
+        Assert.Equal(0, status.SkippedDuplicates);
+        Assert.NotNull(status.NextFire);
+        Assert.Empty(status.RecentHistory);
+    }
+
+    [Fact]
+    public async Task Status_query_for_unknown_reminder_returns_not_found()
+    {
+        var manager = await GetManagerAsync();
+
+        var status = await manager.Ask<ReminderStatusResponse>(
+            new GetReminderStatusQuery(new ReminderId("does-not-exist")),
+            TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
+
+        Assert.False(status.Found);
+        Assert.False(status.Enabled);
+        Assert.Equal(0, status.ConsecutiveFailures);
+        Assert.Equal(0, status.SkippedDuplicates);
+        Assert.Empty(status.RecentHistory);
+    }
+
     [Fact]
     public async Task Reconcile_deletes_zombie_oneshot_reminders()
     {
@@ -411,7 +450,8 @@ public async Task Startup_emits_alert_for_legacy_reminder_missing_trust_fields()
                 TimeProvider.System,
                 store,
                 new ReminderHistoryStore(paths),
-                sink)),
+                sink,
+                NullReminderChannelNotifier.Instance)),
             "legacy-reminder-alert-manager");
 
         // The legacy-schema alert is emitted synchronously inside PreStart, and

src/Netclaw.Actors.Tests/Sessions/LlmSessionTestBase.cs

@@ -63,6 +63,7 @@ protected sealed override void ConfigureServices(HostBuilderContext context, ISe
         services.AddSingleton<ReminderDefinitionStore>();
         services.AddSingleton<ReminderHistoryStore>();
         services.AddSingleton<IOperationalNotificationSink>(NullNotificationSink.Instance);
+        services.AddSingleton<IReminderChannelNotifier>(NullReminderChannelNotifier.Instance);
         ConfigureSessionServices(services);
         services.AddLlmSessionCompositeRecords();
     }

src/Netclaw.Actors/Reminders/IReminderChannelNotifier.cs

@@ -0,0 +1,45 @@
+// -----------------------------------------------------------------------
+// <copyright file="IReminderChannelNotifier.cs" company="Petabridge, LLC">
+//      Copyright (C) 2026 - 2026 Petabridge, LLC <https://petabridge.com>
+// </copyright>
+// -----------------------------------------------------------------------
+using Netclaw.Tools;
+
+namespace Netclaw.Actors.Reminders;
+
+/// <summary>
+/// Posts an operator-facing notice to a reminder's destination channel when an
+/// execution fails. Implemented in the daemon over the channel outbound
+/// registry so the actor layer stays transport-agnostic (the manager hands over
+/// an already-resolved <see cref="ChannelDeliveryTargetInfo"/> and plain text).
+/// Fire-and-forget (<c>void</c>) — the manager must never block on channel
+/// delivery, mirroring <see cref="Configuration.IOperationalNotificationSink"/>.
+/// </summary>
+public interface IReminderChannelNotifier
+{
+    /// <summary>
+    /// Posts <paramref name="text"/> to <paramref name="target"/>. Must be
+    /// thread-safe and must not throw — delivery failures are the
+    /// implementation's problem to log, never the caller's to handle.
+    /// </summary>
+    void NotifyFailure(ChannelDeliveryTargetInfo target, string text);
+}
+
+/// <summary>
+/// No-op notifier for environments with no channel outbound path (e.g. tests, or
+/// a daemon with no channels configured). A real Null Object — explicit, not a
+/// silent fallback: callers still get a non-null required dependency.
+/// </summary>
+public sealed class NullReminderChannelNotifier : IReminderChannelNotifier
+{
+    public static readonly NullReminderChannelNotifier Instance = new();
+
+    private NullReminderChannelNotifier()
+    {
+    }
+
+    public void NotifyFailure(ChannelDeliveryTargetInfo target, string text)
+    {
+        // Intentionally does nothing.
+    }
+}

src/Netclaw.Actors/Reminders/ReminderExecutionActor.cs

@@ -487,7 +487,7 @@ private static string BuildChannelDeliveryGuidance(ReminderDefinition definition
             "Transport and address may be missing or invalid.");
     }
 
-    private static ChannelDeliveryTargetInfo? ResolveChannelDeliveryTarget(ReminderDefinition definition)
+    internal static ChannelDeliveryTargetInfo? ResolveChannelDeliveryTarget(ReminderDefinition definition)
     {
         if (definition.Delivery.Target is not null)
             return definition.Delivery.Target;

src/Netclaw.Actors/Reminders/ReminderManagerActor.cs

@@ -37,20 +37,25 @@ public sealed partial class ReminderManagerActor : ReceiveActor
     /// </summary>
     internal const int FailurePauseThreshold = 5;
 
+    /// <summary>Recent run records returned by the per-reminder status query.</summary>
+    internal const int RecentHistoryCount = 5;
+
     private readonly ISessionPipeline _pipeline;
     private readonly EffectivePolicyDefaults _defaults;
     private readonly SchedulingConfig _schedulingConfig;
     private readonly TimeProvider _timeProvider;
     private readonly ReminderDefinitionStore _definitionStore;
     private readonly ReminderHistoryStore _historyStore;
     private readonly IOperationalNotificationSink _notificationSink;
+    private readonly IReminderChannelNotifier _channelNotifier;
     private readonly ILoggingAdapter _log;
 
     private IReminderClient? _client;
 
     private readonly ActiveExecutionTracker _activeExecutions = new();
     private readonly Queue<ReminderId> _deferredQueue = new();
     private readonly Dictionary<ReminderId, int> _failureCounts = [];
+    private readonly Dictionary<ReminderId, int> _skipCounts = [];
 
     public ReminderManagerActor(
         ISessionPipeline pipeline,
@@ -59,7 +64,8 @@ public ReminderManagerActor(
         TimeProvider timeProvider,
         ReminderDefinitionStore definitionStore,
         ReminderHistoryStore historyStore,
-        IOperationalNotificationSink notificationSink)
+        IOperationalNotificationSink notificationSink,
+        IReminderChannelNotifier channelNotifier)
     {
         _pipeline = pipeline;
         _defaults = defaults;
@@ -68,6 +74,7 @@ public ReminderManagerActor(
         _definitionStore = definitionStore;
         _historyStore = historyStore;
         _notificationSink = notificationSink;
+        _channelNotifier = channelNotifier;
         _log = Context.GetLogger();
 
         ReceiveAsync<SaveReminderCommand>(HandleSaveAsync);
@@ -83,6 +90,7 @@ public ReminderManagerActor(
 
         ReceiveAsync<ReconcileReminders>(_ => HandleReconcileAsync());
         Receive<GetReminderHealthQuery>(_ => HandleGetHealth());
+        ReceiveAsync<GetReminderStatusQuery>(HandleGetStatusAsync);
     }
 
     protected override void PreStart()
@@ -417,6 +425,7 @@ private async Task<ReminderStateResponse> DisableReminderInternalAsync(ReminderI
         await CancelScheduleOnlyAsync(id);
 
         _failureCounts.Remove(id);
+        _skipCounts.Remove(id);
         RemoveFromDeferredQueue(id);
 
         _log.Info("Disabled reminder '{0}'", id.Value);
@@ -433,6 +442,7 @@ private async Task DeleteReminderInternalAsync(ReminderId id)
         _definitionStore.Delete(id);
         await CancelScheduleOnlyAsync(id);
         _failureCounts.Remove(id);
+        _skipCounts.Remove(id);
         RemoveFromDeferredQueue(id);
 
         try
@@ -576,8 +586,7 @@ private async Task HandleReminderFiredAsync(ReminderEnvelope<ReminderPayload> en
 
         if (_activeExecutions.IsExecuting(reminderId))
         {
-            _log.Warning("reminder_skipped_duplicate_execution reminder_id={0} title={1}",
-                reminderId.Value, definition.Title);
+            RecordSkippedDuplicate(reminderId, definition.Title, "scheduled");
             await _client!.AckAsync(envelope);
             return;
         }
@@ -621,6 +630,44 @@ private async Task HandleReminderFiredAsync(ReminderEnvelope<ReminderPayload> en
         }
     }
 
+    /// <summary>
+    /// Records a skipped fire (a fire that arrived while a prior execution of the
+    /// same reminder was still running). Counted in-memory since daemon start and
+    /// surfaced by <c>netclaw reminder status</c> so the silent-skip pattern from
+    /// #1492/#1494 (49 skips in a day, unnoticed) is now visible to operators.
+    /// </summary>
+    private void RecordSkippedDuplicate(ReminderId reminderId, string title, string source)
+    {
+        var count = _skipCounts.GetValueOrDefault(reminderId) + 1;
+        _skipCounts[reminderId] = count;
+        _log.Warning(
+            "reminder_skipped_duplicate_execution reminder_id={0} title={1} source={2} skip_count={3}",
+            reminderId.Value, title, source, count);
+    }
+
+    /// <summary>
+    /// Posts an operator-facing failure notice to a reminder's destination
+    /// channel. Only Channel-delivery reminders have such a channel; CurrentSession
+    /// and None failures are surfaced via the operational alert sink instead. The
+    /// notifier is fire-and-forget — never blocks or throws into the manager.
+    /// </summary>
+    private void PostFailureNoticeToChannel(ReminderDefinition? definition, string text)
+    {
+        if (definition is not { Delivery.Kind: DeliveryKind.Channel })
+            return;
+
+        var target = ReminderExecutionActor.ResolveChannelDeliveryTarget(definition);
+        if (target is null)
+        {
+            _log.Warning(
+                "Reminder '{0}' failed but its channel delivery target could not be resolved; no channel notice posted.",
+                definition.Id.Value);
+            return;
+        }
+
+        _channelNotifier.NotifyFailure(target, text);
+    }
+
     private async Task HandleExecutionCompletedAsync(ReminderExecutionCompleted completed)
     {
         if (!_activeExecutions.Remove(completed.Id))
@@ -659,6 +706,19 @@ private async Task HandleExecutionCompletedAsync(ReminderExecutionCompleted comp
                     ["error"] = completed.ErrorMessage ?? "unknown",
                 }));
 
+            // Surface the failure where the operator expects this reminder's
+            // output: its destination channel. Only below the threshold — on the
+            // threshold-hitting failure the disabled notice below already carries
+            // the last error, so posting both would double the noise on the most
+            // important event. Bounded overall by the threshold, never the
+            // unbounded skip stream that #1494 makes visible via status instead.
+            if (count < FailurePauseThreshold)
+            {
+                PostFailureNoticeToChannel(
+                    definition,
+                    $"Reminder \"{title}\" failed: {completed.ErrorMessage ?? "unknown error"}");
+            }
+
             if (count >= FailurePauseThreshold)
             {
                 _log.Warning("Reminder '{0}' hit failure threshold ({1}), disabling",
@@ -679,6 +739,11 @@ private async Task HandleExecutionCompletedAsync(ReminderExecutionCompleted comp
                         ["failureCount"] = count.ToString(),
                     }));
 
+                PostFailureNoticeToChannel(
+                    definition,
+                    $"Reminder \"{title}\" was automatically disabled after {count} consecutive failures. " +
+                    $"Last error: {completed.ErrorMessage ?? "unknown error"}");
+
                 await DisableReminderInternalAsync(completed.Id);
                 _failureCounts.Remove(completed.Id);
             }
@@ -806,8 +871,7 @@ private async Task ProcessDeferredQueueAsync()
 
             if (_activeExecutions.IsExecuting(nextId))
             {
-                _log.Warning("reminder_skipped_duplicate_execution reminder_id={0} title={1} source=deferred_queue",
-                    nextId.Value, definition.Title);
+                RecordSkippedDuplicate(nextId, definition.Title, "deferred_queue");
                 continue;
             }
 
@@ -989,6 +1053,50 @@ private void HandleGetHealth()
             _failureCounts.Count));
     }
 
+    private async Task HandleGetStatusAsync(GetReminderStatusQuery query)
+    {
+        var replyTo = Sender;
+        try
+        {
+            var definition = _definitionStore.Get(query.Id);
+            if (definition is null)
+            {
+                replyTo.Tell(new ReminderStatusResponse(
+                    query.Id, Found: false, Enabled: false, Executing: false,
+                    NextFire: null, ConsecutiveFailures: 0, SkippedDuplicates: 0,
+                    RecentHistory: []));
+                return;
+            }
+
+            // Two independent backend reads — run them concurrently so the
+            // query's latency is max(schedule, history) instead of their sum.
+            // Neither touches actor state until both complete.
+            var schedulesTask = ListScheduledRemindersAsync();
+            var historyTask = _historyStore.ReadAsync(query.Id, RecentHistoryCount);
+            await Task.WhenAll(schedulesTask, historyTask);
+
+            replyTo.Tell(new ReminderStatusResponse(
+                query.Id,
+                Found: true,
+                Enabled: definition.Enabled,
+                Executing: _activeExecutions.IsExecuting(query.Id),
+                NextFire: schedulesTask.Result.GetValueOrDefault(query.Id.Value),
+                ConsecutiveFailures: _failureCounts.GetValueOrDefault(query.Id),
+                SkippedDuplicates: _skipCounts.GetValueOrDefault(query.Id),
+                RecentHistory: historyTask.Result));
+        }
+        catch (Exception ex)
+        {
+            // The definition existed, so this is a transient read failure — NOT a
+            // missing reminder. Faulting the Ask surfaces a real error (the
+            // endpoint maps it to 5xx); replying not-found here would tell the
+            // operator a wedged reminder was deleted, the silent fallback this
+            // very feature exists to expose.
+            _log.Error(ex, "Error getting status for reminder '{0}'", query.Id.Value);
+            replyTo.Tell(new Status.Failure(ex));
+        }
+    }
+
     private sealed record ScheduleAttempt(bool IsSuccess, DateTimeOffset? NextFire, string? ErrorMessage) : INoSerializationVerificationNeeded
     {
         public static ScheduleAttempt Ok(DateTimeOffset? nextFire) => new(true, nextFire, null);