Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/Netclaw.Actors.Tests/SubAgents/RecordingSessionMetrics.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// -----------------------------------------------------------------------
// <copyright file="RecordingSessionMetrics.cs" company="Petabridge, LLC">
// Copyright (C) 2026 - 2026 Petabridge, LLC <https://petabridge.com>
// </copyright>
// -----------------------------------------------------------------------
using Netclaw.Actors.Telemetry;
using Netclaw.Configuration;

namespace Netclaw.Actors.Tests.SubAgents;

/// <summary>
/// Records every <see cref="ISessionMetrics.RecordTokenUsage"/> call so a test can
/// assert a sub-agent bills each LLM call to the daily-stats sink. Thread-safe: the
/// actor records on its mailbox thread while the test reads after the <c>Ask</c>
/// completes. Regression support for issue #1597.
/// </summary>
internal sealed class RecordingSessionMetrics : ISessionMetrics
{
private readonly object _gate = new();
private readonly List<(long Input, long Output)> _tokenUsageCalls = [];
private long _totalInput;
private long _totalOutput;

public IReadOnlyList<(long Input, long Output)> TokenUsageCalls
{
get { lock (_gate) { return _tokenUsageCalls.ToArray(); } }
}

public long TotalInputTokens { get { lock (_gate) { return _totalInput; } } }

public long TotalOutputTokens { get { lock (_gate) { return _totalOutput; } } }

public void RecordTokenUsage(long inputTokens, long outputTokens)
{
lock (_gate)
{
_tokenUsageCalls.Add((inputTokens, outputTokens));
_totalInput += inputTokens;
_totalOutput += outputTokens;
}
}

public void RecordTurnCompleted() { }
public void RecordSessionCreated() { }
public void RecordMemoriesFormed(int count) { }
public void RecordMemoriesRecalled(int count) { }
public void RecordSkillsLoaded(int count) { }
public void RecordSkillLoaded(string skillName, SkillLoadMethod method) { }
}
12 changes: 10 additions & 2 deletions src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1415,6 +1415,14 @@ internal sealed class FakeChatClient : IChatClient

public IReadOnlyList<string>? ResponseTextsByCall { get; set; }

/// <summary>
/// When set, every returned response carries these token counts as
/// <see cref="ChatResponse.Usage"/>. The streaming reader coalesces that back into
/// <c>response.Usage</c>, so a test can prove the sub-agent bills each LLM call's
/// tokens to <see cref="Netclaw.Actors.Telemetry.ISessionMetrics"/>.
/// </summary>
public UsageDetails? UsageOverride { get; set; }

public async Task<ChatResponse> GetResponseAsync(
IEnumerable<ChatMessage> messages,
ChatOptions? options = null,
Expand All @@ -1437,7 +1445,7 @@ public async Task<ChatResponse> GetResponseAsync(
var toolCallContents = new List<AIContent>(ToolCallsOnFirstCall);
var toolCallMessage = new ChatMessage(
ChatRole.Assistant, toolCallContents);
return new ChatResponse(toolCallMessage);
return new ChatResponse(toolCallMessage) { Usage = UsageOverride };
}
}

Expand All @@ -1448,7 +1456,7 @@ public async Task<ChatResponse> GetResponseAsync(
var responseMessage = new ChatMessage(
ChatRole.Assistant,
[new TextContent(responseText)]);
return new ChatResponse(responseMessage);
return new ChatResponse(responseMessage) { Usage = UsageOverride };
}

public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
Expand Down
75 changes: 75 additions & 0 deletions src/Netclaw.Actors.Tests/SubAgents/SubAgentObservabilityTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -107,4 +107,79 @@ await agent.Ask<SubAgentResult>(
NewRun("Greet the user"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
}, cancellationToken: TestContext.Current.CancellationToken);
}

// Regression coverage for issue #1597: sub-agent LLM calls used to discard
// ChatResponse.Usage entirely — the actor had no ISessionMetrics and never read
// response.Usage — so every sub-agent's token consumption was invisible to
// `netclaw stats`. These tests pin the sub-agent to the shared daily-stats sink.

[Fact]
public async Task Records_token_usage_to_session_metrics_on_text_response()
{
var metrics = new RecordingSessionMetrics();
var fakeClient = new FakeChatClient
{
UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 }
};
var agent = Sys.ActorOf(SubAgentActor.CreateProps(
CreateDefinition(), fakeClient, sessionMetrics: metrics));

var result = await agent.Ask<SubAgentResult>(
NewRun("Say hello"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);

Assert.True(result.Success);
// A single LLM call → exactly one usage record billed to the shared
// process-wide daily-stats sink (the same singleton the parent session uses).
var call = Assert.Single(metrics.TokenUsageCalls);
Assert.Equal((120L, 45L), call);
}

[Fact]
public async Task Records_token_usage_for_every_llm_call_across_the_turn_loop()
{
// A tool-call turn followed by a final-text turn = two LLM calls. Both must be
// billed. This is the crux of #1597: the sub-agent's INTERNAL calls (not just
// its single final output) have to reach `netclaw stats`, so the recorded total
// is the per-call usage summed — not one call's worth.
var metrics = new RecordingSessionMetrics();
var fakeTool = new FakeNetclawTool("greet", "Hello from tool!");
var fakeClient = new FakeChatClient
{
ToolCallsOnFirstCall =
[
new FunctionCallContent("call-1", "greet",
new Dictionary<string, object?> { ["name"] = "World" })
],
UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 }
};
var agent = Sys.ActorOf(SubAgentActor.CreateProps(
CreateDefinition([fakeTool]), fakeClient, sessionMetrics: metrics));

var result = await agent.Ask<SubAgentResult>(
NewRun("Greet the user"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);

Assert.True(result.Success);
Assert.Equal(2, fakeClient.CallCount);
Assert.Equal(2, metrics.TokenUsageCalls.Count);
Assert.Equal(240L, metrics.TotalInputTokens);
Assert.Equal(90L, metrics.TotalOutputTokens);
}

[Fact]
public async Task Completion_summary_reports_cumulative_token_totals()
{
var fakeClient = new FakeChatClient
{
UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 }
};
var agent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition(), fakeClient));

// The completion summary now carries token totals so sub-agent cost is visible
// in the logs (and Seq), not just tool/iteration/duration counts.
await EventFilter.Info(contains: "inputTokens=120, outputTokens=45").ExpectAsync(1, async () =>
{
await agent.Ask<SubAgentResult>(
NewRun("Say hello"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
}, cancellationToken: TestContext.Current.CancellationToken);
}
}
63 changes: 63 additions & 0 deletions src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,69 @@ public async Task Spawn_async_ignores_definition_tool_metadata_for_runtime_tool_
Assert.Equal(1, started.ToolCount);
}

[Fact]
public async Task Spawned_sub_agent_bills_its_llm_calls_to_session_metrics()
{
// Full-wiring regression guard for #1597: a sub-agent spawned through the real
// SubAgentSpawner must record its LLM-call tokens to the ISessionMetrics handed
// to the spawner. Unlike the actor-level tests, this exercises the
// spawner -> CreateProps -> actor pass-through, so dropping the metrics argument
// anywhere along that chain fails here. The SpawnChildActor factory materializes
// the spawner-built Props into a real SubAgentActor (a probe stand-in would
// bypass CreateProps entirely and hide a broken pass-through).
var toolRegistry = new ToolRegistry();
toolRegistry.Register(new FakeNetclawTool("inspect_context", "ok"));

var metrics = new RecordingSessionMetrics();
var chatClient = new FakeChatClient
{
UsageOverride = new UsageDetails { InputTokenCount = 175, OutputTokenCount = 60 }
};

var spawner = new SubAgentSpawner(
new SingleClientProvider(chatClient),
toolRegistry,
new ToolAccessPolicy(
new ToolConfig(),
new EffectivePolicyDefaults(
DeploymentPosture.Personal,
TrustAudience.Personal,
ShellExecutionMode.HostAllowed,
UsedStrictFallback: false),
new ShellCommandPolicy()),
approvalService: null,
new StaticSystemPromptProvider("You are a summarizer."),
NullLogger<SubAgentSpawner>.Instance,
sessionMetrics: metrics);

var context = new ToolExecutionContext("console/subagent-parent", "/tmp/netclaw/sessions/parent")
{
Audience = TrustAudience.Personal
};
context.SpawnChildActor = (props, name, _) => Task.FromResult<object>(Sys.ActorOf((Props)props, name));

var profile = new SubAgentProfile
{
Name = "summarizer",
Description = "Summarize content",
SystemPrompt = "You are a summarizer.",
ToolNames = ["inspect_context"],
Visibility = SubAgentVisibility.UserFacing
};

var result = await spawner.SpawnAsync(
profile,
"Summarize the repo.",
runtimeContext: null,
context,
TestContext.Current.CancellationToken);

Assert.True(result.Success, $"Expected success but got: {result.Output}");
// One text-only LLM call → exactly one usage record, carrying the fake's tokens.
var call = Assert.Single(metrics.TokenUsageCalls);
Assert.Equal((175L, 60L), call);
}

private sealed class NoOpChatClient : IChatClient
{
public Task<ChatResponse> GetResponseAsync(
Expand Down
63 changes: 55 additions & 8 deletions src/Netclaw.Actors/SubAgents/SubAgentActor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ [Subagent Execution Contract]
private readonly ToolAccessPolicy _toolAccessPolicy;
private readonly IToolApprovalService? _approvalService;
private readonly int _maxToolIterations;

// Process-wide daily-stats sink (the same singleton the parent session records
// to). Nullable because a hosting configuration without the daemon stats backend
// is a real runtime state — mirrors LlmSessionActor._sessionMetrics. When present,
// every LLM call this sub-agent makes is billed here so its tokens show up in
// `netclaw stats` instead of vanishing.
private readonly Telemetry.ISessionMetrics? _sessionMetrics;
private readonly ToolRegistry _toolRegistry;
private IReadOnlyList<AITool> _aiTools = [];
private ILoggingAdapter _log;
Expand All @@ -74,6 +81,12 @@ [Subagent Execution Contract]
// timer scheduler — it doesn't track elapsed time itself).
private readonly Stopwatch _runStopwatch = Stopwatch.StartNew();

// Cumulative token usage across every LLM call this sub-agent makes. Summed for
// the completion summary log; per-call usage is also recorded to _sessionMetrics
// as each call returns (see RecordUsage).
private long _runInputTokens;
private long _runOutputTokens;

// Conversation state (not persisted — ephemeral)
private readonly List<AiChatMessage> _history = [];
private long _llmCallId;
Expand Down Expand Up @@ -137,14 +150,16 @@ public SubAgentActor(
IChatClient chatClient,
ToolAccessPolicy? toolAccessPolicy = null,
IToolApprovalService? approvalService = null,
int maxToolIterations = DefaultMaxToolIterations)
int maxToolIterations = DefaultMaxToolIterations,
Telemetry.ISessionMetrics? sessionMetrics = null)
{
if (maxToolIterations <= 0)
throw new ArgumentOutOfRangeException(nameof(maxToolIterations), maxToolIterations,
"Sub-agent tool iteration budget must be greater than zero.");

_definition = definition;
_chatClient = chatClient;
_sessionMetrics = sessionMetrics;
_toolAccessPolicy = toolAccessPolicy ?? new ToolAccessPolicy(
new ToolConfig { ShellMode = ShellExecutionMode.HostAllowed },
new EffectivePolicyDefaults(
Expand Down Expand Up @@ -175,14 +190,16 @@ public static Props CreateProps(
IChatClient chatClient,
ToolAccessPolicy? toolAccessPolicy = null,
IToolApprovalService? approvalService = null,
int maxToolIterations = DefaultMaxToolIterations)
int maxToolIterations = DefaultMaxToolIterations,
Telemetry.ISessionMetrics? sessionMetrics = null)
{
return Props.Create(() => new SubAgentActor(
definition,
chatClient,
toolAccessPolicy,
approvalService,
maxToolIterations));
maxToolIterations,
sessionMetrics));
}

/// <summary>
Expand Down Expand Up @@ -320,6 +337,14 @@ private void Processing()
// the synchronous processing that follows (tool dispatch or completion).
RestartWatchdog(_interDeltaBudget);
var response = msg.Response;

// Record this call's token usage before branching so EVERY call is billed —
// tool-call turns, retries, the forced-no-tools final turn, and repair turns
// all flow through here exactly once. Mirrors the main session, which records
// its own per-call usage; without this the sub-agent's tokens never reach the
// daily-stats pipeline and `netclaw stats` under-counts by the whole sub-run.
RecordUsage(response.Usage);

var lastMessage = response.Messages[^1];
var analysis = LlmResponseClassifier.Analyze(lastMessage);

Expand Down Expand Up @@ -630,6 +655,24 @@ private void Processing()
});
}

// Bill one LLM call's token usage to the shared daily-stats sink and accumulate
// the run totals for the completion summary log. We record at the source (here in
// the child) rather than propagating totals up to the parent: the parent's
// ISessionMetrics is the SAME process-wide singleton, so re-recording there would
// double-count, and folding sub-agent tokens into the parent's UsageOutput would
// corrupt its context-window percentage (the sub-agent has its own context window).
private void RecordUsage(UsageDetails? usage)
{
if (usage is null)
return;

var input = usage.InputTokenCount ?? 0;
var output = usage.OutputTokenCount ?? 0;
_runInputTokens += input;
_runOutputTokens += output;
_sessionMetrics?.RecordTokenUsage(input, output);
}

private void HandleToolCalls(AiChatMessage assistantMessage, List<FunctionCallContent> toolCalls)
{
_turnState.ResetEmptyResponseGuards();
Expand Down Expand Up @@ -752,13 +795,17 @@ private void Complete(
_log.Info("SubAgent [{AgentName}] completed (success={Success}, outcome={Outcome}, reason={Reason}, output={OutputLength} chars, iterations={Iterations})",
_definition.Name, success, resolvedOutcome, outcomeReason?.Value ?? "-", output.Length, _turnState.ToolIterationCount);

// Log cumulative stats for observability — total LLM calls, tool usage, etc.
// This gives operators a single summary line for sub-agent duration analysis.
// Log cumulative stats for observability — total LLM calls, tool usage, tokens.
// This gives operators a single summary line for sub-agent cost/duration analysis.
// (success is already on the "completed" line above; omitted here to stay within
// ILoggingAdapter's 6-argument ceiling.)
_log.Info(
"SubAgent [{AgentName}] summary: success={Success}, totalToolCalls={TotalToolCalls}, "
+ "iterations={Iterations}, duration={Duration}s",
_definition.Name, success, _turnState.ToolCallCount,
"SubAgent [{AgentName}] summary: totalToolCalls={TotalToolCalls}, "
+ "iterations={Iterations}, inputTokens={InputTokens}, outputTokens={OutputTokens}, "
+ "duration={Duration}s",
_definition.Name, _turnState.ToolCallCount,
_turnState.ToolIterationCount,
_runInputTokens, _runOutputTokens,
_runStopwatch.Elapsed.TotalSeconds);

var findings = success && _definition.EmitStructuredFindings
Expand Down
Loading
Loading