Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions src/Netclaw.Actors.Tests/Reminders/ReminderExecutionActorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,44 @@ public async Task Execution_pipeline_requests_streaming_and_tool_call_output()
Assert.True(pipeline.CapturedOptions!.Filter.HasFlag(OutputFilter.ToolCalls));
}

[Fact]
public async Task Mode_A_wedged_session_is_failed_by_stall_backstop_releasing_the_guard()
{
// #1492 regression: a Channel-delivery (Mode A) reminder whose session
// wedges — stops producing output without ever emitting TurnCompleted or
// Error — used to hold the duplicate-execution guard forever (the actor
// had no execution ceiling). The stall backstop must conclude it as a
// failure so the parent clears _activeExecutions and the next fire runs.
var originalTimeout = ReminderExecutionActor.ExecutionStallTimeout;
ReminderExecutionActor.ExecutionStallTimeout = TimeSpan.FromMilliseconds(250);
try
{
// Emits one non-terminal output, then goes silent forever — no
// TurnCompleted/Error ever arrives.
var pipeline = new ScriptedSessionPipeline(sessionId =>
[
new TextOutput("Working on it...") { SessionId = sessionId }
]);

var definition = CreateDefinition("mode-a-stall");
var probe = CreateTestProbe();
Sys.ActorOf(
Props.Create(() => new ParentProxy(probe.Ref, definition, pipeline, _historyStore)),
"exec-mode-a-stall");

var completed = await probe.ExpectMsgAsync<ReminderExecutionCompleted>(
TimeSpan.FromSeconds(5), cancellationToken: TestContext.Current.CancellationToken);

Assert.False(completed.Success);
Assert.Equal("mode-a-stall", completed.Id.Value);
Assert.Contains("stalled", completed.ErrorMessage!, StringComparison.OrdinalIgnoreCase);
}
finally
{
ReminderExecutionActor.ExecutionStallTimeout = originalTimeout;
}
}

private static ReminderDefinition CreateDefinition(string id)
{
var now = TimeProvider.System.GetUtcNow();
Expand Down
37 changes: 37 additions & 0 deletions src/Netclaw.Actors/Reminders/ReminderExecutionActor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,20 @@ internal sealed class ReminderExecutionActor : ReceiveActor
/// </summary>
internal static TimeSpan DeliveryObservedTimeout = TimeSpan.FromHours(1);

/// <summary>
/// Backstop inactivity ceiling for the Mode A (Channel / own-pipeline) path.
/// The actor runs its own session pipeline and concludes when it emits
/// <c>TurnCompleted</c>/<c>Error</c>; if the session wedges and stops
/// producing output without ever reaching a terminal signal, nothing else
/// stops this actor, so it would hold the duplicate-execution guard forever
/// (see #1492). A <see cref="ReceiveTimeout"/> reset by every session output
/// fires after this much silence and concludes the run as failed, releasing
/// the guard so the next fire can run. Reset by real output, so it never
/// preempts a live turn. Mode B (CurrentSession) does NOT arm this — its wait
/// is bounded separately by <see cref="DeliveryObservedTimeout"/>.
/// </summary>
internal static TimeSpan ExecutionStallTimeout = TimeSpan.FromMinutes(20);

private readonly Guid _executionId;
private readonly ReminderDefinition _definition;
private readonly ReminderHistoryStore _historyStore;
Expand Down Expand Up @@ -100,6 +114,7 @@ public ReminderExecutionActor(
Receive<ExecutionStarted>(_ => { });
Receive<ReminderDeliveryResult>(HandleDeliveryResult);
Receive<DeliveryBackstopTimeout>(HandleDeliveryBackstopTimeout);
Receive<ReceiveTimeout>(_ => HandleExecutionStall());
}

protected override void PreStart()
Expand Down Expand Up @@ -166,6 +181,13 @@ await inputQueue.OfferAsync(new ChannelInput
});

inputQueue.Complete();

// Arm the Mode A stall backstop: the pipeline now streams output to
// this actor, each ExecutionOutput resets the ReceiveTimeout, and a
// terminal TurnCompleted/Error stops us first. If the session wedges
// and goes silent without a terminal signal, this fires and releases
// the duplicate-execution guard instead of hanging forever (#1492).
Context.SetReceiveTimeout(ExecutionStallTimeout);

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LGTM, simple fix.

}
catch (Exception ex)
{
Expand Down Expand Up @@ -541,13 +563,28 @@ private void HandleOutput(ExecutionOutput wrapper)
}
}

private void HandleExecutionStall()
{
if (_completed)
return;

var elapsed = _timeProvider.GetUtcNow() - _dispatchedAt;
_log.Warning(
"ReminderExecution Stalled: execution_id={0} reminder_id={1} title={2} no session output for {3} (elapsed={4}); concluding as failed to release the execution guard.",
_executionId, _definition.Id, _definition.Title, ExecutionStallTimeout, elapsed);
ReportAndStop(false, $"Reminder execution stalled: no session output for {ExecutionStallTimeout}.");
}

private void ReportAndStop(bool success, string? errorMessage = null)
{
if (_completed)
return;

_completed = true;

// Disarm the Mode A stall backstop so it cannot fire during the drain.
Context.SetReceiveTimeout(null);

var durationMs = (long)(_timeProvider.GetUtcNow() - _dispatchedAt).TotalMilliseconds;
_pendingHistory = new HistoryRecord(
FiredAt: _dispatchedAt,
Expand Down
Loading