diff --git a/src/Netclaw.Actors.Tests/SubAgents/RecordingSessionMetrics.cs b/src/Netclaw.Actors.Tests/SubAgents/RecordingSessionMetrics.cs new file mode 100644 index 000000000..8f2ff1c56 --- /dev/null +++ b/src/Netclaw.Actors.Tests/SubAgents/RecordingSessionMetrics.cs @@ -0,0 +1,49 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Netclaw.Actors.Telemetry; +using Netclaw.Configuration; + +namespace Netclaw.Actors.Tests.SubAgents; + +/// +/// Records every call so a test can +/// assert a sub-agent bills each LLM call to the daily-stats sink. Thread-safe: the +/// actor records on its mailbox thread while the test reads after the Ask +/// completes. Regression support for issue #1597. +/// +internal sealed class RecordingSessionMetrics : ISessionMetrics +{ + private readonly object _gate = new(); + private readonly List<(long Input, long Output)> _tokenUsageCalls = []; + private long _totalInput; + private long _totalOutput; + + public IReadOnlyList<(long Input, long Output)> TokenUsageCalls + { + get { lock (_gate) { return _tokenUsageCalls.ToArray(); } } + } + + public long TotalInputTokens { get { lock (_gate) { return _totalInput; } } } + + public long TotalOutputTokens { get { lock (_gate) { return _totalOutput; } } } + + public void RecordTokenUsage(long inputTokens, long outputTokens) + { + lock (_gate) + { + _tokenUsageCalls.Add((inputTokens, outputTokens)); + _totalInput += inputTokens; + _totalOutput += outputTokens; + } + } + + public void RecordTurnCompleted() { } + public void RecordSessionCreated() { } + public void RecordMemoriesFormed(int count) { } + public void RecordMemoriesRecalled(int count) { } + public void RecordSkillsLoaded(int count) { } + public void RecordSkillLoaded(string skillName, SkillLoadMethod method) { } +} diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs index 7b1f58b55..0a6ffd8f2 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentActorTests.cs @@ -1415,6 +1415,14 @@ internal sealed class FakeChatClient : IChatClient public IReadOnlyList? ResponseTextsByCall { get; set; } + /// + /// When set, every returned response carries these token counts as + /// . The streaming reader coalesces that back into + /// response.Usage, so a test can prove the sub-agent bills each LLM call's + /// tokens to . + /// + public UsageDetails? UsageOverride { get; set; } + public async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, @@ -1437,7 +1445,7 @@ public async Task GetResponseAsync( var toolCallContents = new List(ToolCallsOnFirstCall); var toolCallMessage = new ChatMessage( ChatRole.Assistant, toolCallContents); - return new ChatResponse(toolCallMessage); + return new ChatResponse(toolCallMessage) { Usage = UsageOverride }; } } @@ -1448,7 +1456,7 @@ public async Task GetResponseAsync( var responseMessage = new ChatMessage( ChatRole.Assistant, [new TextContent(responseText)]); - return new ChatResponse(responseMessage); + return new ChatResponse(responseMessage) { Usage = UsageOverride }; } public IAsyncEnumerable GetStreamingResponseAsync( diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentObservabilityTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentObservabilityTests.cs index c93cc824e..c37161dfc 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentObservabilityTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentObservabilityTests.cs @@ -107,4 +107,79 @@ await agent.Ask( NewRun("Greet the user"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); }, cancellationToken: TestContext.Current.CancellationToken); } + + // Regression coverage for issue #1597: sub-agent LLM calls used to discard + // ChatResponse.Usage entirely — the actor had no ISessionMetrics and never read + // response.Usage — so every sub-agent's token consumption was invisible to + // `netclaw stats`. These tests pin the sub-agent to the shared daily-stats sink. + + [Fact] + public async Task Records_token_usage_to_session_metrics_on_text_response() + { + var metrics = new RecordingSessionMetrics(); + var fakeClient = new FakeChatClient + { + UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 } + }; + var agent = Sys.ActorOf(SubAgentActor.CreateProps( + CreateDefinition(), fakeClient, sessionMetrics: metrics)); + + var result = await agent.Ask( + NewRun("Say hello"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(result.Success); + // A single LLM call → exactly one usage record billed to the shared + // process-wide daily-stats sink (the same singleton the parent session uses). + var call = Assert.Single(metrics.TokenUsageCalls); + Assert.Equal((120L, 45L), call); + } + + [Fact] + public async Task Records_token_usage_for_every_llm_call_across_the_turn_loop() + { + // A tool-call turn followed by a final-text turn = two LLM calls. Both must be + // billed. This is the crux of #1597: the sub-agent's INTERNAL calls (not just + // its single final output) have to reach `netclaw stats`, so the recorded total + // is the per-call usage summed — not one call's worth. + var metrics = new RecordingSessionMetrics(); + var fakeTool = new FakeNetclawTool("greet", "Hello from tool!"); + var fakeClient = new FakeChatClient + { + ToolCallsOnFirstCall = + [ + new FunctionCallContent("call-1", "greet", + new Dictionary { ["name"] = "World" }) + ], + UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 } + }; + var agent = Sys.ActorOf(SubAgentActor.CreateProps( + CreateDefinition([fakeTool]), fakeClient, sessionMetrics: metrics)); + + var result = await agent.Ask( + NewRun("Greet the user"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Assert.True(result.Success); + Assert.Equal(2, fakeClient.CallCount); + Assert.Equal(2, metrics.TokenUsageCalls.Count); + Assert.Equal(240L, metrics.TotalInputTokens); + Assert.Equal(90L, metrics.TotalOutputTokens); + } + + [Fact] + public async Task Completion_summary_reports_cumulative_token_totals() + { + var fakeClient = new FakeChatClient + { + UsageOverride = new UsageDetails { InputTokenCount = 120, OutputTokenCount = 45 } + }; + var agent = Sys.ActorOf(SubAgentActor.CreateProps(CreateDefinition(), fakeClient)); + + // The completion summary now carries token totals so sub-agent cost is visible + // in the logs (and Seq), not just tool/iteration/duration counts. + await EventFilter.Info(contains: "inputTokens=120, outputTokens=45").ExpectAsync(1, async () => + { + await agent.Ask( + NewRun("Say hello"), TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + }, cancellationToken: TestContext.Current.CancellationToken); + } } diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs index 21293c31c..cca1518b6 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnerTests.cs @@ -150,6 +150,69 @@ public async Task Spawn_async_ignores_definition_tool_metadata_for_runtime_tool_ Assert.Equal(1, started.ToolCount); } + [Fact] + public async Task Spawned_sub_agent_bills_its_llm_calls_to_session_metrics() + { + // Full-wiring regression guard for #1597: a sub-agent spawned through the real + // SubAgentSpawner must record its LLM-call tokens to the ISessionMetrics handed + // to the spawner. Unlike the actor-level tests, this exercises the + // spawner -> CreateProps -> actor pass-through, so dropping the metrics argument + // anywhere along that chain fails here. The SpawnChildActor factory materializes + // the spawner-built Props into a real SubAgentActor (a probe stand-in would + // bypass CreateProps entirely and hide a broken pass-through). + var toolRegistry = new ToolRegistry(); + toolRegistry.Register(new FakeNetclawTool("inspect_context", "ok")); + + var metrics = new RecordingSessionMetrics(); + var chatClient = new FakeChatClient + { + UsageOverride = new UsageDetails { InputTokenCount = 175, OutputTokenCount = 60 } + }; + + var spawner = new SubAgentSpawner( + new SingleClientProvider(chatClient), + toolRegistry, + new ToolAccessPolicy( + new ToolConfig(), + new EffectivePolicyDefaults( + DeploymentPosture.Personal, + TrustAudience.Personal, + ShellExecutionMode.HostAllowed, + UsedStrictFallback: false), + new ShellCommandPolicy()), + approvalService: null, + new StaticSystemPromptProvider("You are a summarizer."), + NullLogger.Instance, + sessionMetrics: metrics); + + var context = new ToolExecutionContext("console/subagent-parent", "/tmp/netclaw/sessions/parent") + { + Audience = TrustAudience.Personal + }; + context.SpawnChildActor = (props, name, _) => Task.FromResult(Sys.ActorOf((Props)props, name)); + + var profile = new SubAgentProfile + { + Name = "summarizer", + Description = "Summarize content", + SystemPrompt = "You are a summarizer.", + ToolNames = ["inspect_context"], + Visibility = SubAgentVisibility.UserFacing + }; + + var result = await spawner.SpawnAsync( + profile, + "Summarize the repo.", + runtimeContext: null, + context, + TestContext.Current.CancellationToken); + + Assert.True(result.Success, $"Expected success but got: {result.Output}"); + // One text-only LLM call → exactly one usage record, carrying the fake's tokens. + var call = Assert.Single(metrics.TokenUsageCalls); + Assert.Equal((175L, 60L), call); + } + private sealed class NoOpChatClient : IChatClient { public Task GetResponseAsync( diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index d28f62512..7dde15929 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -63,6 +63,13 @@ [Subagent Execution Contract] private readonly ToolAccessPolicy _toolAccessPolicy; private readonly IToolApprovalService? _approvalService; private readonly int _maxToolIterations; + + // Process-wide daily-stats sink (the same singleton the parent session records + // to). Nullable because a hosting configuration without the daemon stats backend + // is a real runtime state — mirrors LlmSessionActor._sessionMetrics. When present, + // every LLM call this sub-agent makes is billed here so its tokens show up in + // `netclaw stats` instead of vanishing. + private readonly Telemetry.ISessionMetrics? _sessionMetrics; private readonly ToolRegistry _toolRegistry; private IReadOnlyList _aiTools = []; private ILoggingAdapter _log; @@ -74,6 +81,12 @@ [Subagent Execution Contract] // timer scheduler — it doesn't track elapsed time itself). private readonly Stopwatch _runStopwatch = Stopwatch.StartNew(); + // Cumulative token usage across every LLM call this sub-agent makes. Summed for + // the completion summary log; per-call usage is also recorded to _sessionMetrics + // as each call returns (see RecordUsage). + private long _runInputTokens; + private long _runOutputTokens; + // Conversation state (not persisted — ephemeral) private readonly List _history = []; private long _llmCallId; @@ -137,7 +150,8 @@ public SubAgentActor( IChatClient chatClient, ToolAccessPolicy? toolAccessPolicy = null, IToolApprovalService? approvalService = null, - int maxToolIterations = DefaultMaxToolIterations) + int maxToolIterations = DefaultMaxToolIterations, + Telemetry.ISessionMetrics? sessionMetrics = null) { if (maxToolIterations <= 0) throw new ArgumentOutOfRangeException(nameof(maxToolIterations), maxToolIterations, @@ -145,6 +159,7 @@ public SubAgentActor( _definition = definition; _chatClient = chatClient; + _sessionMetrics = sessionMetrics; _toolAccessPolicy = toolAccessPolicy ?? new ToolAccessPolicy( new ToolConfig { ShellMode = ShellExecutionMode.HostAllowed }, new EffectivePolicyDefaults( @@ -175,14 +190,16 @@ public static Props CreateProps( IChatClient chatClient, ToolAccessPolicy? toolAccessPolicy = null, IToolApprovalService? approvalService = null, - int maxToolIterations = DefaultMaxToolIterations) + int maxToolIterations = DefaultMaxToolIterations, + Telemetry.ISessionMetrics? sessionMetrics = null) { return Props.Create(() => new SubAgentActor( definition, chatClient, toolAccessPolicy, approvalService, - maxToolIterations)); + maxToolIterations, + sessionMetrics)); } /// @@ -320,6 +337,14 @@ private void Processing() // the synchronous processing that follows (tool dispatch or completion). RestartWatchdog(_interDeltaBudget); var response = msg.Response; + + // Record this call's token usage before branching so EVERY call is billed — + // tool-call turns, retries, the forced-no-tools final turn, and repair turns + // all flow through here exactly once. Mirrors the main session, which records + // its own per-call usage; without this the sub-agent's tokens never reach the + // daily-stats pipeline and `netclaw stats` under-counts by the whole sub-run. + RecordUsage(response.Usage); + var lastMessage = response.Messages[^1]; var analysis = LlmResponseClassifier.Analyze(lastMessage); @@ -630,6 +655,24 @@ private void Processing() }); } + // Bill one LLM call's token usage to the shared daily-stats sink and accumulate + // the run totals for the completion summary log. We record at the source (here in + // the child) rather than propagating totals up to the parent: the parent's + // ISessionMetrics is the SAME process-wide singleton, so re-recording there would + // double-count, and folding sub-agent tokens into the parent's UsageOutput would + // corrupt its context-window percentage (the sub-agent has its own context window). + private void RecordUsage(UsageDetails? usage) + { + if (usage is null) + return; + + var input = usage.InputTokenCount ?? 0; + var output = usage.OutputTokenCount ?? 0; + _runInputTokens += input; + _runOutputTokens += output; + _sessionMetrics?.RecordTokenUsage(input, output); + } + private void HandleToolCalls(AiChatMessage assistantMessage, List toolCalls) { _turnState.ResetEmptyResponseGuards(); @@ -752,13 +795,17 @@ private void Complete( _log.Info("SubAgent [{AgentName}] completed (success={Success}, outcome={Outcome}, reason={Reason}, output={OutputLength} chars, iterations={Iterations})", _definition.Name, success, resolvedOutcome, outcomeReason?.Value ?? "-", output.Length, _turnState.ToolIterationCount); - // Log cumulative stats for observability — total LLM calls, tool usage, etc. - // This gives operators a single summary line for sub-agent duration analysis. + // Log cumulative stats for observability — total LLM calls, tool usage, tokens. + // This gives operators a single summary line for sub-agent cost/duration analysis. + // (success is already on the "completed" line above; omitted here to stay within + // ILoggingAdapter's 6-argument ceiling.) _log.Info( - "SubAgent [{AgentName}] summary: success={Success}, totalToolCalls={TotalToolCalls}, " - + "iterations={Iterations}, duration={Duration}s", - _definition.Name, success, _turnState.ToolCallCount, + "SubAgent [{AgentName}] summary: totalToolCalls={TotalToolCalls}, " + + "iterations={Iterations}, inputTokens={InputTokens}, outputTokens={OutputTokens}, " + + "duration={Duration}s", + _definition.Name, _turnState.ToolCallCount, _turnState.ToolIterationCount, + _runInputTokens, _runOutputTokens, _runStopwatch.Elapsed.TotalSeconds); var findings = success && _definition.EmitStructuredFindings diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index de2b673ea..530cda8b1 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -33,6 +33,12 @@ public sealed class SubAgentSpawner private readonly SubAgentConfig _subAgentConfig; private readonly ILogger _logger; + // The process-wide daily-stats sink, handed to each spawned SubAgentActor so its + // LLM calls are billed to `netclaw stats`. Nullable to match the rest of the stats + // wiring (a host without the daemon stats backend is a real runtime state); DI + // injects the registered singleton in production. + private readonly Telemetry.ISessionMetrics? _sessionMetrics; + public SubAgentSpawner( IChatClientProvider chatClientProvider, ToolRegistry toolRegistry, @@ -40,7 +46,8 @@ public SubAgentSpawner( IToolApprovalService? approvalService, ISystemPromptProvider promptProvider, ILogger logger, - SubAgentConfig? subAgentConfig = null) + SubAgentConfig? subAgentConfig = null, + Telemetry.ISessionMetrics? sessionMetrics = null) { _chatClientProvider = chatClientProvider; _toolRegistry = toolRegistry; @@ -49,6 +56,7 @@ public SubAgentSpawner( _promptProvider = promptProvider; _subAgentConfig = subAgentConfig ?? new SubAgentConfig(); _logger = logger; + _sessionMetrics = sessionMetrics; } /// @@ -139,7 +147,8 @@ public async Task SpawnAsync( chatClient, _toolAccessPolicy, _approvalService, - SubAgentMaxToolIterations); + SubAgentMaxToolIterations, + _sessionMetrics); var actorName = $"subagent-{definition.Name}-{runId}"; IActorRef subAgent; try