From b38ce13270a7723e663a57a82706ced00a25351c Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 25 Jun 2026 20:22:54 +0000 Subject: [PATCH 01/22] refactor(logging): route session logs off the log event, delete the AsyncLocal (#1472 step 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actor ILoggingAdapter lifecycle lines now reach per-session session.log reliably, and the SessionDiagnosticsContext AsyncLocal threading is gone. The session.log sink (RollingFileLoggerProvider) reads the session id off each log event's structured state instead of an ambient AsyncLocal. The Akka->MEL bridge passes the event's properties as the MEL state (AkkaLogState, carrying the actor's WithContext("SessionId", ...) tag), and MEL's own structured logging passes FormattedLogValues (a {SessionId} field). RollingFileLogger.Log reads "SessionId" from either and Enqueue(message, sessionId) routes to the existing per-session writer (SessionLogDiagnostic -> SessionLogDispatcher -> SessionLogActor). Because the id rides on the line, routing no longer depends on thread/async context — a line logged after an await still routes, which the AsyncLocal could not. Deleted (the context-threading workaround): - SessionDiagnosticsContext (the AsyncLocal) + its 9 Push() scope sites. - SessionLoggingScope (the MEL BeginScope helper) + its 3 chat-client call sites. - Obsolete tests/helpers: SidecarDiagnosticsContextTests, ScopeCapturingLogger, the SessionDiagnosticsContext test. NormalizeSessionId relocated to SubAgentSessionScope. Kept session-routed by carrying the id on the line: - The #1468 spawn breadcrumbs (SubAgentSpawner/SpawnAgentTool) now include a {SessionId} field, so the sink routes them without a scope. Behavior change: chat-client / LLM-pipeline INTERNAL diagnostics (retry, failover, request/response internals) were only ever session-tagged via the AsyncLocal — already lost after the first await — so they settle into daemon.log only. Each actor's own lifecycle log still reaches session.log via the sink, and OTLP/Seq correlation for actor logs is unaffected (it rides the same WithContext tag). Net -350 LOC. The bridge behavior (AkkaLogState carrying the WithContext properties) was verified by decompiling Akka.Hosting 1.5.69; provider-level + async-continuation tests cover the read path. Skill diagnostics guidance updated (netclaw-operations 2.19.0). Part of #1472. --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../references/diagnostics.md | 15 +- .../SidecarDiagnosticsContextTests.cs | 242 ------------------ .../SubAgents/SubAgentSessionScopeTests.cs} | 12 +- .../SubAgentSpawnObservabilityTests.cs | 27 +- .../Memory/MemoryCurationActor.cs | 1 - .../Protocol/SessionLogDiagnostic.cs | 11 +- .../Sessions/LlmSessionActor.cs | 1 - .../Pipelines/SessionCompactionPipeline.cs | 1 - .../Sessions/Pipelines/SessionLlmInvoker.cs | 1 - .../Pipelines/SessionTitleGenerator.cs | 1 - .../Sessions/SessionMemoryObserverActor.cs | 1 - .../SubAgents/SpawnAgentTool.cs | 17 +- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 12 +- .../SubAgents/SubAgentSessionScope.cs | 31 +++ .../SubAgents/SubAgentSpawner.cs | 38 ++- .../SessionDiagnosticsContext.cs | 62 ----- .../Configuration/LoggingChatClientTests.cs | 27 -- .../RollingFileLoggerProviderTests.cs | 104 +++++--- .../Configuration/ScopeCapturingLogger.cs | 45 ---- .../Configuration/LoggingChatClient.cs | 1 - .../Configuration/RetryingChatClient.cs | 11 +- .../RollingFileLoggerProvider.cs | 50 +++- .../Configuration/RoutingChatClient.cs | 15 +- .../Configuration/SessionLoggingScope.cs | 44 ---- 25 files changed, 211 insertions(+), 561 deletions(-) delete mode 100644 src/Netclaw.Actors.Tests/Sessions/SidecarDiagnosticsContextTests.cs rename src/{Netclaw.Configuration.Tests/SessionDiagnosticsContextTests.cs => Netclaw.Actors.Tests/SubAgents/SubAgentSessionScopeTests.cs} (66%) create mode 100644 src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs delete mode 100644 src/Netclaw.Configuration/SessionDiagnosticsContext.cs delete mode 100644 src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs delete mode 100644 src/Netclaw.Daemon/Configuration/SessionLoggingScope.cs diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index db762dd7d..7f00a4c59 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.18.0" + version: "2.19.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index d6ff9d05a..c0a137dc1 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -34,16 +34,19 @@ What to expect inside `session.log`: cross-referencing two files. - Two line shapes: session output audit lines (`User:`, `Assistant:`, `Thinking:`, `Tool call:`, `Tool result:`, `Usage:`, `Turn N completed`, - etc.) and `Diagnostic:` lines from MEL providers (LLM client, HTTP, - retry middleware) emitted under a session diagnostics scope. + etc.) and `Diagnostic:` lines. A diagnostic line reaches `session.log` + when its log event carries the session id; actor logs are tagged with it + automatically (`SessionId`), so session- and sub-agent-actor lifecycle + lines (startup, guards, timeouts, cancellation, completion) appear here — + routed by the tag on the line, not by any ambient context. - Best-effort observability. Individual lines may be dropped on transient IO errors and logged at Debug level in the daemon log; this is not a transactional audit trail. Cross-check the daemon log for warnings if a critical line appears missing. -- Sidecar paths (compaction, title generation, sub-agents, memory - distillation) currently bypass the session diagnostics scope, so their - internal diagnostics may not appear in `session.log` even though their - output audit lines do. Tracked in netclaw-dev/netclaw#920. +- Not everything is session-tagged: the LLM-client / HTTP / retry / failover + decorators log their internal diagnostics without a session id, so those + land in `daemon.log` only. For deep LLM-call internals, read `daemon.log` + (filter by the `SessionId` attribute where the line carries one). | Symptom | Check | |---------|-------| diff --git a/src/Netclaw.Actors.Tests/Sessions/SidecarDiagnosticsContextTests.cs b/src/Netclaw.Actors.Tests/Sessions/SidecarDiagnosticsContextTests.cs deleted file mode 100644 index 02101e129..000000000 --- a/src/Netclaw.Actors.Tests/Sessions/SidecarDiagnosticsContextTests.cs +++ /dev/null @@ -1,242 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -using System.Runtime.CompilerServices; -using Akka.Event; -using Akka.Hosting; -using Akka.Hosting.TestKit; -using Microsoft.Extensions.AI; -using Netclaw.Actors.Memory; -using Netclaw.Actors.Protocol; -using Netclaw.Actors.Sessions; -using Netclaw.Actors.Sessions.Pipelines; -using Netclaw.Actors.SubAgents; -using Netclaw.Configuration; -using Xunit; -using AiChatMessage = Microsoft.Extensions.AI.ChatMessage; -using AiChatRole = Microsoft.Extensions.AI.ChatRole; - -namespace Netclaw.Actors.Tests.Sessions; - -/// -/// Verifies that session-owned sidecar paths populate -/// around their IChatClient -/// calls so MEL provider diagnostics emitted during the call route into -/// the per-session log. One test per major sidecar covered by -/// netclaw-dev/netclaw#920. Test contract: a fake IChatClient -/// captures SessionDiagnosticsContext.SessionId at the moment the -/// chat client method is invoked. That is exactly the AsyncLocal value -/// any real provider plugin would see when emitting MEL log lines. -/// -public sealed class SidecarDiagnosticsContextTests : TestKit -{ - protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) - { - } - - [Fact] - public async Task TitleGenerator_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/title-thread"); - var captor = new SessionContextCapturingChatClient(); - var probe = CreateTestProbe(); - - await SessionTitleGenerator.GenerateAsync( - captor, - sessionId, - history: [], - self: probe.Ref, - log: NoLogger.Instance, - timeout: TimeSpan.FromSeconds(5)); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - [Fact] - public async Task CompactionPipeline_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/compaction-thread"); - var captor = new SessionContextCapturingChatClient(); - var history = new List - { - new() { Role = Netclaw.Actors.Protocol.ChatRole.User, Content = "hello" }, - new() { Role = Netclaw.Actors.Protocol.ChatRole.Assistant, Content = "hi" } - }; - - var observation = await SessionCompactionPipeline.GenerateObservationsAsync( - client: captor, - sessionId: sessionId, - history: history, - systemOffset: 0, - keepStartIndex: 1, - sidecarTimeout: TimeSpan.FromSeconds(5), - log: NoLogger.Instance, - cancellationToken: TestContext.Current.CancellationToken); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - Assert.NotNull(observation); - } - - [Fact] - public async Task MemoryExtraction_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/memory-extraction-thread"); - var captor = new SessionContextCapturingChatClient(); - var probe = CreateTestProbe(); - - await LlmSessionActor.InvokeMemoryExtractionCoreAsync( - captor, - sessionId, - history: [], - self: probe.Ref, - timeout: TimeSpan.FromSeconds(5)); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - [Fact] - public async Task SubAgent_InvokeLlm_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/subagent-thread"); - var captor = new SessionContextCapturingChatClient(); - var probe = CreateTestProbe(); - - await SubAgentActor.InvokeLlmAsync( - captor, - messages: [], - options: null, - sessionId: sessionId, - self: probe.Ref, - ct: TestContext.Current.CancellationToken); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - [Fact] - public async Task SubAgent_InvokeLlm_with_null_sessionId_pushes_null_scope() - { - // Sub-agents that run outside any session legitimately have no - // session id. The Push contract accepts null and the captor should - // see null inside the call. - var captor = new SessionContextCapturingChatClient(); - var probe = CreateTestProbe(); - - await SubAgentActor.InvokeLlmAsync( - captor, - messages: [], - options: null, - sessionId: null, - self: probe.Ref, - ct: TestContext.Current.CancellationToken); - - Assert.Null(captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - [Fact] - public async Task MemoryObserver_RunDistillation_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/distillation-thread"); - var captor = new SessionContextCapturingChatClient(); - var probe = CreateTestProbe(); - - await SessionMemoryObserverActor.RunDistillationAsync( - client: captor, - sessionId: sessionId, - turnCount: 5, - transcript: "user: hello\nassistant: hi", - existingProposals: [], - timeout: TimeSpan.FromSeconds(5), - self: probe.Ref, - runId: 1, - contentVersion: 1, - log: NoLogger.Instance); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - [Fact] - public async Task MemoryCuration_TryLlmEvaluation_populates_session_diagnostics_scope() - { - var sessionId = new SessionId("ch/curation-thread"); - var captor = new SessionContextCapturingChatClient(); - var operation = new SQLiteMemoryCurationOperation( - Kind: "document", - MemoryClass: "durable_fact", - MemoryId: null, - AnchorCanonicalName: "test-anchor", - AnchorType: "preference", - Title: "Test", - Content: "Test content for diagnostics scope verification.", - AliasesJson: "[]", - FacetsJson: "[]", - SlotsJson: null, - Relations: null, - UpdateSemantics: "merge-document", - Boundary: TrustBoundary.TrustedInstanceValue, - Audience: TrustAudience.Public, - Sensitivity: "normal", - RecallMode: "auto", - Confidence: 0.9, - FreshnessAtMs: TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds(), - ExpiresAtMs: null); - - await MemoryCurationActor.TryLlmEvaluationAsync( - captor, - sessionId, - operation, - candidates: [], - log: NoLogger.Instance); - - Assert.Equal(sessionId.Value, captor.CapturedSessionId); - Assert.Null(SessionDiagnosticsContext.SessionId); - } - - /// - /// IChatClient stub that captures - /// at the moment its methods are invoked. The captured value is the - /// AsyncLocal seen by the chat client — the same value any MEL provider - /// plugin emitting diagnostics inside the call would see. - /// - private sealed class SessionContextCapturingChatClient : IChatClient - { - public string? CapturedSessionId { get; private set; } - - public Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - CapturedSessionId = SessionDiagnosticsContext.SessionId; - var response = new ChatResponse(new AiChatMessage( - AiChatRole.Assistant, - (IList)[new TextContent("captured")])); - return Task.FromResult(response); - } - - public IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - => StreamAsync(cancellationToken); - - private async IAsyncEnumerable StreamAsync( - [EnumeratorCancellation] CancellationToken cancellationToken) - { - CapturedSessionId = SessionDiagnosticsContext.SessionId; - yield return new ChatResponseUpdate(AiChatRole.Assistant, "captured"); - await Task.CompletedTask; - } - - public object? GetService(Type serviceType, object? serviceKey = null) => null; - - public void Dispose() { } - } -} diff --git a/src/Netclaw.Configuration.Tests/SessionDiagnosticsContextTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSessionScopeTests.cs similarity index 66% rename from src/Netclaw.Configuration.Tests/SessionDiagnosticsContextTests.cs rename to src/Netclaw.Actors.Tests/SubAgents/SubAgentSessionScopeTests.cs index 6d023b24d..ed5afab08 100644 --- a/src/Netclaw.Configuration.Tests/SessionDiagnosticsContextTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSessionScopeTests.cs @@ -1,14 +1,14 @@ // ----------------------------------------------------------------------- -// +// // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using Netclaw.Configuration; +using Netclaw.Actors.SubAgents; using Xunit; -namespace Netclaw.Configuration.Tests; +namespace Netclaw.Actors.Tests.SubAgents; -public sealed class SessionDiagnosticsContextTests +public sealed class SubAgentSessionScopeTests { [Theory] [InlineData("ch/thr/subagent/skill", "ch/thr")] @@ -16,7 +16,7 @@ public sealed class SessionDiagnosticsContextTests [InlineData("ch/thr/subagent/skill/inner", "ch/thr")] public void NormalizeSessionId_collapses_subagent_marker_to_parent_id(string input, string expected) { - Assert.Equal(expected, SessionDiagnosticsContext.NormalizeSessionId(input)); + Assert.Equal(expected, SubAgentSessionScope.NormalizeSessionId(input)); } [Theory] @@ -25,6 +25,6 @@ public void NormalizeSessionId_collapses_subagent_marker_to_parent_id(string inp [InlineData(" ")] public void NormalizeSessionId_returns_null_for_blank(string? input) { - Assert.Null(SessionDiagnosticsContext.NormalizeSessionId(input)); + Assert.Null(SubAgentSessionScope.NormalizeSessionId(input)); } } diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs index 2bedfcb23..cb7e8d7c0 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs @@ -14,14 +14,12 @@ namespace Netclaw.Actors.Tests.SubAgents; /// -/// Regression coverage for sub-agent spawn observability. A sub-agent's own actor -/// logs go through Akka's async logger bridge, where the diagnostics AsyncLocal is -/// gone, so they never reach the per-session session.log. The spawn lifecycle -/// is instead recorded by parent-side breadcrumbs that must run while the parent -/// session scope is active — otherwise a refused or failed spawn is invisible in the -/// session transcript. These tests assert the scope is active at log time, which is -/// exactly the condition RollingFileLoggerProvider uses to route a line to -/// session.log. +/// Regression coverage for sub-agent spawn observability. The spawn lifecycle is +/// recorded by parent-side breadcrumbs that carry the session id as a structured log +/// field — exactly what RollingFileLoggerProvider reads off the log event to +/// route a line to the per-session session.log. These tests assert each +/// breadcrumb carries the session id; otherwise a refused or failed spawn would be +/// invisible in the session transcript. /// public sealed class SubAgentSpawnObservabilityTests : IDisposable { @@ -113,6 +111,17 @@ public void Log( TState state, Exception? exception, Func formatter) - => Entries.Add((logLevel, formatter(state, exception), SessionDiagnosticsContext.SessionId)); + => Entries.Add((logLevel, formatter(state, exception), ExtractSessionId(state))); + + // Mirror RollingFileLoggerProvider: read the session id off the structured log + // state (the {SessionId} field the breadcrumbs carry), not an ambient context. + private static string? ExtractSessionId(TState state) + { + if (state is IEnumerable> fields) + foreach (var field in fields) + if (field.Key == "SessionId" && field.Value is { } value) + return value.ToString(); + return null; + } } } diff --git a/src/Netclaw.Actors/Memory/MemoryCurationActor.cs b/src/Netclaw.Actors/Memory/MemoryCurationActor.cs index ae7b9fda9..457faaef5 100644 --- a/src/Netclaw.Actors/Memory/MemoryCurationActor.cs +++ b/src/Netclaw.Actors/Memory/MemoryCurationActor.cs @@ -298,7 +298,6 @@ private async Task EvaluateSingleAsync( try { using var cts = new CancellationTokenSource(LlmTimeout); - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); var messages = new List { diff --git a/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs b/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs index 5b6a551d4..698ba0f34 100644 --- a/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs +++ b/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs @@ -4,16 +4,15 @@ // // ----------------------------------------------------------------------- using Akka.Actor; -using Netclaw.Configuration; namespace Netclaw.Actors.Protocol; /// -/// Pre-formatted diagnostic line carried from the MEL logger provider -/// to the SessionLogDispatcher. The provider snapshots -/// SessionDiagnosticsContext.SessionId at log-emit time and embeds -/// it here so the dispatcher routes the line by message field, not by -/// any ambient context that would not flow across actor mailboxes. +/// Pre-formatted diagnostic line carried from the MEL logger provider to the +/// SessionLogDispatcher. The provider reads the session id off each log +/// event's structured state and embeds it here so the dispatcher routes the line +/// by message field, not by any ambient context that would not flow across actor +/// mailboxes. /// public sealed record SessionLogDiagnostic(SessionId SessionId, string Line) : IWithSessionId, INoSerializationVerificationNeeded; diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index b302d8f2f..ecbb9f183 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -1737,7 +1737,6 @@ internal static async Task InvokeMemoryExtractionCoreAsync( try { using var cts = new CancellationTokenSource(timeout); - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); var extractionMessages = new List { new(Microsoft.Extensions.AI.ChatRole.System, diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs index 660bd8cc7..386583693 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs @@ -173,7 +173,6 @@ public static async Task ExecuteAsync( { using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(sidecarTimeout); - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); var observerMessages = new List { new(Microsoft.Extensions.AI.ChatRole.System, diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionLlmInvoker.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionLlmInvoker.cs index a63dad637..76ce4e2ba 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionLlmInvoker.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionLlmInvoker.cs @@ -33,7 +33,6 @@ public static async Task InvokeAsync( // so sidecar calls (compaction, title gen) that bypass this invoker // naturally omit the header and round-robin across backends. SessionAffinityContext.SessionId = sessionId.Value; - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); try { var response = await StreamAsync(client, messages, options, self, callId, cancellationToken); diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs index 022e6e24f..e10a126d8 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs @@ -43,7 +43,6 @@ public static async Task GenerateAsync( try { using var cts = new CancellationTokenSource(timeout); - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); var messages = new List { new(Microsoft.Extensions.AI.ChatRole.User, diff --git a/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs b/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs index dd41d2013..e1f8cdbff 100644 --- a/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs @@ -348,7 +348,6 @@ internal static async Task RunDistillationAsync( try { using var cts = new CancellationTokenSource(timeout); - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId.Value); var messages = new List { new(Microsoft.Extensions.AI.ChatRole.System, DistillationSystemPrompt), diff --git a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs index e13a815e0..972f28d84 100644 --- a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs +++ b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs @@ -124,19 +124,18 @@ private static string FormatResult(string agent, SubAgentResult result) private (string? Error, SubAgentProfile? Profile) Resolve(Params args, ToolExecutionContext context) { // Rejections here return a (sometimes deliberately opaque) error string to - // the model. Mirror the real reason to the session transcript under the - // parent session scope so an operator can tell *why* a spawn was refused — - // the "This tool is not available." string hides the audience-vs-disabled - // distinction from the model on purpose. - using var diagnosticsScope = SessionDiagnosticsContext.Push(context.SessionId); + // the model. Mirror the real reason to the session transcript — each line + // carries the session id, so the log sink routes it to session.log — so an + // operator can tell *why* a spawn was refused; the "This tool is not + // available." string hides the audience-vs-disabled distinction on purpose. // Defense-in-depth: block subagent spawning for Public audience or when // the subagent subsystem is disabled. if (context.Audience == TrustAudience.Public || !_subAgentConfig.Enabled) { _logger?.LogWarning( - "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled})", - args.Agent, context.Audience, _subAgentConfig.Enabled); + "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled}, session={SessionId})", + args.Agent, context.Audience, _subAgentConfig.Enabled, context.SessionId); return ("Error: This tool is not available.", null); } @@ -153,8 +152,8 @@ private static string FormatResult(string agent, SubAgentResult result) { var available = _registry.GetUserFacing(); _logger?.LogWarning( - "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count})", - args.Agent, available.Count); + "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count}, session={SessionId})", + args.Agent, available.Count, context.SessionId); if (available.Count == 0) return ($"Error: No subagents are available. Agent '{args.Agent}' not found. Author one at {_paths.AgentsDirectory}/*.md or define a skill with metadata.subagent once #661 lands.", null); diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index b691eef95..779a23e45 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -258,10 +258,10 @@ private void Idle() // (SubSessionId), and is plainly attributable to the sub-agent. scopeId is // "{parentSessionId}/subagent/{name}/{runId}"; NormalizeSessionId strips the // "/subagent/..." suffix to recover the parent. SessionId matches the key the - // session/channel actors already use (see SessionLoggingScope), so sub-agent - // and parent logs share one filterable attribute; SubSessionId isolates a - // single run within that session. - var parentSessionId = SessionDiagnosticsContext.NormalizeSessionId(scopeId); + // session/channel actors already tag their loggers with, so sub-agent and + // parent logs share one filterable attribute (and route to the same + // session.log); SubSessionId isolates a single run within that session. + var parentSessionId = SubAgentSessionScope.NormalizeSessionId(scopeId); var enrichedLog = Context.GetLogger(); if (!string.IsNullOrWhiteSpace(parentSessionId)) enrichedLog = enrichedLog.WithContext("SessionId", parentSessionId); @@ -903,10 +903,6 @@ internal static async Task InvokeLlmAsync( { try { - // Sub-agents share the parent's diagnostics scope: SessionDiagnosticsContext - // strips the "/subagent/..." suffix back to the parent id. Null is intentional - // for sub-agents that run outside any session. - using var diagnosticsScope = SessionDiagnosticsContext.Push(sessionId?.Value); // Use streaming to match the main session path. The non-streaming // GetResponseAsync path drops reasoning content for some providers diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs new file mode 100644 index 000000000..a67421d56 --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs @@ -0,0 +1,31 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +namespace Netclaw.Actors.SubAgents; + +/// +/// Derives the owning (parent) session id from a sub-agent's composite scope id. +/// Sub-agents run as ephemeral children of a parent session and reuse its +/// session.log; their composite ids ({parentId}/subagent/{name}/{runId}) +/// must collapse back to the parent so their diagnostics route to the parent's log +/// rather than scatter into per-agent files operators do not monitor. This is the +/// only place that decision is made — keep it in sync with how sub-agent ids are +/// constructed in the spawner. +/// +internal static class SubAgentSessionScope +{ + public static string? NormalizeSessionId(string? sessionId) + { + if (string.IsNullOrWhiteSpace(sessionId)) + return null; + + var value = sessionId.Trim(); + var subAgentMarker = value.IndexOf("/subagent/", StringComparison.Ordinal); + if (subAgentMarker > 0) + value = value[..subAgentMarker]; + + return value; + } +} diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index 84c808514..9bc29f243 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -67,19 +67,17 @@ public async Task SpawnAsync( string? systemPromptOverlay = null, ChannelWriter? activitySink = null) { - // Session-scoped breadcrumbs: the sub-agent's own actor logs go through - // Akka's async logger bridge, where the diagnostics AsyncLocal is gone, so - // they never reach session.log. These parent-side lines run synchronously - // under the parent session scope, so the spawn lifecycle (request → outcome, - // including every early rejection) is always visible in the session transcript. - using var diagnosticsScope = SessionDiagnosticsContext.Push(context.SessionId); + // Session-scoped breadcrumbs: each parent-side line carries the session id as a + // structured field, so the log sink routes the spawn lifecycle (request → + // outcome, including early rejections that happen before the child actor even + // exists) into the parent's session.log alongside the sub-agent's own actor logs. _logger.LogInformation( - "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars})", - profile.Name, task.Length); + "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars}, session={SessionId})", + profile.Name, task.Length, context.SessionId); if (context.SpawnChildActor is null) { - _logger.LogWarning("SubAgent [{AgentName}] cannot spawn — no session context available", profile.Name); + _logger.LogWarning("SubAgent [{AgentName}] cannot spawn — no session context available (session={SessionId})", profile.Name, context.SessionId); activitySink?.TryComplete(); return new SubAgentResult { @@ -93,8 +91,8 @@ public async Task SpawnAsync( if (tools.Count == 0) { _logger.LogWarning( - "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn", - profile.Name); + "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn (session={SessionId})", + profile.Name, context.SessionId); activitySink?.TryComplete(); return new SubAgentResult { @@ -153,8 +151,8 @@ public async Task SpawnAsync( // spawn ask timed out). Record it to the session transcript before the // exception propagates to the tool pipeline. _logger.LogError( - ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId})", - profile.Name, runId); + ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId}, session={SessionId})", + profile.Name, runId, context.SessionId); // Balance the IsStarted=true notification above: the non-streaming path // (activitySink is null) relies solely on OnSubAgentActivity, so without // a terminal event the session UI shows a sub-agent stuck in "Started". @@ -170,8 +168,8 @@ public async Task SpawnAsync( } _logger.LogInformation( - "SubAgent [{AgentName}] child actor spawned (runId={RunId}); dispatching RunSubAgent", - profile.Name, runId); + "SubAgent [{AgentName}] child actor spawned (runId={RunId}, session={SessionId}); dispatching RunSubAgent", + profile.Name, runId, context.SessionId); var sw = Stopwatch.StartNew(); try @@ -229,8 +227,8 @@ public async Task SpawnAsync( }); _logger.LogInformation( - "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms)", - profile.Name, runId, result.Success, sw.ElapsedMilliseconds); + "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms, session={SessionId})", + profile.Name, runId, result.Success, sw.ElapsedMilliseconds, context.SessionId); return result; } @@ -249,7 +247,7 @@ public async Task SpawnAsync( Duration = sw.Elapsed }); - _logger.LogError(ex, "SubAgent [{AgentName}] run failed (runId={RunId})", profile.Name, runId); + _logger.LogError(ex, "SubAgent [{AgentName}] run failed (runId={RunId}, session={SessionId})", profile.Name, runId, context.SessionId); return new SubAgentResult { Success = false, @@ -283,8 +281,8 @@ private IReadOnlyList ResolveTools(SubAgentProfile profile, ToolEx // Sub-agents without certain tools may be unable to complete // their tasks, and this information is important for debugging. _logger.LogInformation( - "SubAgent [{AgentName}] tool '{ToolName}' denied by SubAgentToolPolicy", - profile.Name, tool.Name); + "SubAgent [{AgentName}] tool '{ToolName}' denied by SubAgentToolPolicy (session={SessionId})", + profile.Name, tool.Name, context.SessionId); } } diff --git a/src/Netclaw.Configuration/SessionDiagnosticsContext.cs b/src/Netclaw.Configuration/SessionDiagnosticsContext.cs deleted file mode 100644 index e5c2ec363..000000000 --- a/src/Netclaw.Configuration/SessionDiagnosticsContext.cs +++ /dev/null @@ -1,62 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -namespace Netclaw.Configuration; - -/// -/// Ambient session context used by diagnostics sinks that need to distinguish -/// daemon-global logs from session-owned logs. -/// -public static class SessionDiagnosticsContext -{ - private static readonly AsyncLocal Current = new(); - - public static string? SessionId - { - get => Current.Value; - set => Current.Value = NormalizeSessionId(value); - } - - public static IDisposable Push(string? sessionId) - { - var prior = Current.Value; - Current.Value = NormalizeSessionId(sessionId); - return new RestoreScope(prior); - } - - /// - /// Trims whitespace and collapses any nested sub-agent suffix back to the - /// owning session id. Sub-agents run as ephemeral children of a parent - /// session and reuse its session.log for the audit trail; treating - /// their composite ids ({parentId}/subagent/{agentName}) as - /// distinct sessions would scatter sub-agent diagnostics into per-agent - /// files that operators do not monitor. The split here is the only - /// place that decision is made — keep it in sync with how sub-agent - /// ids are constructed inside the sub-agent spawner. - /// - public static string? NormalizeSessionId(string? sessionId) - { - if (string.IsNullOrWhiteSpace(sessionId)) - return null; - - var value = sessionId.Trim(); - var subAgentMarker = value.IndexOf("/subagent/", StringComparison.Ordinal); - if (subAgentMarker > 0) - value = value[..subAgentMarker]; - - return value; - } - - private sealed class RestoreScope(string? prior) : IDisposable - { - private string? _prior = prior; - - public void Dispose() - { - Current.Value = _prior; - _prior = null; - } - } -} diff --git a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs index 9a7ec0c0e..d0a50a199 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs @@ -5,7 +5,6 @@ // ----------------------------------------------------------------------- using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; -using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; // Netclaw's LoggingChatClient collides with Microsoft.Extensions.AI.LoggingChatClient @@ -94,32 +93,6 @@ public async Task Streaming_LogsPromptDumpWhenTraceEnabled() Assert.Contains(logs, l => l.Contains("role=user")); } - [Fact] - public async Task Streaming_attaches_SessionId_scope_from_diagnostics_context() - { - var logger = new ScopeCapturingLogger(); - var client = new LoggingChatClient(new FakeChatClient(streaming: true), logger); - - using (SessionDiagnosticsContext.Push("ch/thread")) - { - await Drain(client); - } - - Assert.True(logger.HasSessionScope("ch/thread")); - } - - [Fact] - public async Task Streaming_without_session_context_attaches_no_scope() - { - Assert.Null(SessionDiagnosticsContext.SessionId); - var logger = new ScopeCapturingLogger(); - var client = new LoggingChatClient(new FakeChatClient(streaming: true), logger); - - await Drain(client); - - Assert.False(logger.HasAnySessionScope()); - } - private static async Task Drain(IChatClient client) { await foreach (var _ in client.GetStreamingResponseAsync( diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs index 8d54108be..d3b97e4b6 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs @@ -3,13 +3,13 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- +using System.Collections; using Akka.Actor; using Akka.Hosting; using Akka.Hosting.TestKit; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; using Netclaw.Actors.Protocol; -using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; @@ -24,22 +24,15 @@ protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IService private readonly string _basePath = Path.Combine(Path.GetTempPath(), $"netclaw-rolling-logger-tests-{Guid.NewGuid():N}"); [Fact] - public async Task Session_scoped_log_routes_diagnostic_to_dispatcher() + public async Task Mel_structured_log_with_session_id_routes_to_dispatcher() { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); - Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - var probe = CreateTestProbe(); - - using (var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider)) + // A direct MEL structured log: the {SessionId} placeholder lands in the + // FormattedLogValues state, which the provider reads to route per-session. + var (provider, probe) = NewProvider(); + using (provider) { - provider.AttachSessionDispatcher(Task.FromResult(probe.Ref)); var logger = provider.CreateLogger("Netclaw.Tests"); - - using (SessionDiagnosticsContext.Push("channel/thread")) - { - logger.LogInformation("session scoped message"); - } + logger.LogInformation("session scoped message {SessionId}", "channel/thread"); var diagnostic = await probe.ExpectMsgAsync( cancellationToken: TestContext.Current.CancellationToken); @@ -50,16 +43,36 @@ public async Task Session_scoped_log_routes_diagnostic_to_dispatcher() } [Fact] - public async Task Daemon_scoped_log_does_not_route_to_dispatcher() + public async Task Akka_bridged_log_state_with_session_id_routes_to_dispatcher() { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); - Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - var probe = CreateTestProbe(); + // The Akka→MEL bridge passes the log event's structured properties as the + // state (an AkkaLogState that enumerates as KeyValuePairs, including the + // "SessionId" tag from WithContext). The provider reads it off that state. + var (provider, probe) = NewProvider(); + using (provider) + { + var logger = provider.CreateLogger("Netclaw.Actors.SubAgents.SubAgentActor"); + var state = new FakeBridgedLogState( + [ + new("SessionId", "channel/thread"), + new("SubSessionId", "channel/thread/subagent/x/1"), + new("{OriginalFormat}", "actor lifecycle line"), + ]); + logger.Log(LogLevel.Information, default, state, exception: null, static (_, _) => "actor lifecycle line"); - using (var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider)) + var diagnostic = await probe.ExpectMsgAsync( + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("channel/thread", diagnostic.SessionId.Value); + Assert.Contains("actor lifecycle line", diagnostic.Line, StringComparison.Ordinal); + } + } + + [Fact] + public async Task Log_without_session_id_does_not_route_to_dispatcher() + { + var (provider, probe) = NewProvider(); + using (provider) { - provider.AttachSessionDispatcher(Task.FromResult(probe.Ref)); var logger = provider.CreateLogger("Netclaw.Tests"); logger.LogInformation("daemon message"); @@ -87,11 +100,8 @@ public async Task Pre_resolution_diagnostics_buffer_and_drain_to_dispatcher() provider.AttachSessionDispatcher(tcs.Task); var logger = provider.CreateLogger("Netclaw.Tests"); - using (SessionDiagnosticsContext.Push("channel/thread")) - { - logger.LogInformation("pre-resolution one"); - logger.LogInformation("pre-resolution two"); - } + logger.LogInformation("pre-resolution one {SessionId}", "channel/thread"); + logger.LogInformation("pre-resolution two {SessionId}", "channel/thread"); await probe.ExpectNoMsgAsync( TimeSpan.FromMilliseconds(100), @@ -109,23 +119,18 @@ await probe.ExpectNoMsgAsync( } [Fact] - public async Task Diagnostic_routes_correctly_when_logged_from_async_continuation() + public async Task Log_from_async_continuation_still_routes_via_state() { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); - Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - var probe = CreateTestProbe(); - - using (var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider)) + // The whole point of reading the id off the log state: routing no longer + // depends on ambient/async context, so a line logged after an await still + // routes (the old AsyncLocal could be lost across the await). + var (provider, probe) = NewProvider(); + using (provider) { - provider.AttachSessionDispatcher(Task.FromResult(probe.Ref)); var logger = provider.CreateLogger("Netclaw.Tests"); - using (SessionDiagnosticsContext.Push("channel/thread")) - { - await Task.Yield(); - logger.LogInformation("post-await"); - } + await Task.Yield(); + logger.LogInformation("post-await {SessionId}", "channel/thread"); var diagnostic = await probe.ExpectMsgAsync( cancellationToken: TestContext.Current.CancellationToken); @@ -134,6 +139,27 @@ public async Task Diagnostic_routes_correctly_when_logged_from_async_continuatio } } + private (RollingFileLoggerProvider Provider, Akka.TestKit.TestProbe Probe) NewProvider() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); + var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); + Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); + var probe = CreateTestProbe(); + var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider); + provider.AttachSessionDispatcher(Task.FromResult(probe.Ref)); + return (provider, probe); + } + + // Mimics Akka.Hosting's AkkaLogState: an MEL log state that enumerates as the + // event's structured properties (the bridge's surface the provider reads). + private sealed class FakeBridgedLogState(IReadOnlyList> fields) + : IEnumerable> + { + public IEnumerator> GetEnumerator() => fields.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + void IDisposable.Dispose() { try diff --git a/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs b/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs deleted file mode 100644 index 2b93e2bba..000000000 --- a/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs +++ /dev/null @@ -1,45 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -using Microsoft.Extensions.Logging; - -namespace Netclaw.Daemon.Tests.Configuration; - -/// -/// Test logger that records the state objects passed to -/// so tests can assert which scopes were opened around a logging call. Returns -/// null from BeginScope (a valid no-op for using var); only the -/// captured state matters. -/// -internal sealed class ScopeCapturingLogger : ILogger -{ - private const string SessionIdKey = "SessionId"; - - public List Scopes { get; } = []; - - public void Log( - LogLevel logLevel, EventId eventId, TState state, - Exception? exception, Func formatter) - { - } - - public bool IsEnabled(LogLevel logLevel) => true; - - public IDisposable? BeginScope(TState state) where TState : notnull - { - Scopes.Add(state); - return null; - } - - /// True if a scope tagging the given session id was opened. - public bool HasSessionScope(string expectedId) => - Scopes.Any(s => s is IEnumerable> kvps - && kvps.Any(kv => kv.Key == SessionIdKey && kv.Value is string v && v == expectedId)); - - /// True if any session-id scope (regardless of value) was opened. - public bool HasAnySessionScope() => - Scopes.Any(s => s is IEnumerable> kvps - && kvps.Any(kv => kv.Key == SessionIdKey)); -} diff --git a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs index ffa618b33..627271b64 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs @@ -39,7 +39,6 @@ public override async IAsyncEnumerable GetStreamingResponseA [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { - using var sessionScope = SessionLoggingScope.Begin(_logger); var messageList = messages as IReadOnlyList ?? messages.ToList(); LogPromptDiagnostics(messageList, options); diff --git a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs index 2a658e9bd..d710bfec4 100644 --- a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs @@ -122,14 +122,9 @@ public override async IAsyncEnumerable GetStreamingResponseA private async Task BackoffAsync(Exception ex, int attempt, CancellationToken cancellationToken) { var delay = _policy.GetDelay(attempt); - // Retry is the hottest failure path; tag the warning with the session id (like - // the other chat-client decorators) so retry storms correlate by session in Seq. - using (SessionLoggingScope.Begin(_logger)) - { - _logger.LogWarning(ex, - "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", - attempt + 1, _policy.MaxRetries, delay.TotalSeconds); - } + _logger.LogWarning(ex, + "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", + attempt + 1, _policy.MaxRetries, delay.TotalSeconds); await Task.Delay(delay, _timeProvider, cancellationToken); } diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index d24755418..b0c62704f 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -8,7 +8,6 @@ using Akka.Actor; using Microsoft.Extensions.Logging; using Netclaw.Actors.Protocol; -using Netclaw.Configuration; namespace Netclaw.Daemon.Configuration; @@ -16,12 +15,15 @@ namespace Netclaw.Daemon.Configuration; /// Simple file-based logger that writes to a daily rolling log file. /// Uses a background queue to avoid blocking callers. /// -/// Session-scoped lines (emitted under a populated -/// ) are mirrored to a per-session +/// Session-scoped lines — those whose log event carries a "SessionId" field +/// (actors via Context.GetLogger().WithContext("SessionId", ...), which +/// the Akka→MEL bridge surfaces as structured log state; or MEL callers via a +/// {SessionId} structured field) — are mirrored to a per-session /// session.log by routing through the SessionLogDispatcher /// actor. The dispatcher serializes all writes for a given session through /// a single mailbox, replacing the in-process file lock that previously -/// coordinated concurrent writers. +/// coordinated concurrent writers. The session id is read off each log event +/// at the sink, so no ambient/AsyncLocal context is threaded by producers. /// /// The dispatcher is wired in via /// post-construction (typically from an IHostedService that runs @@ -77,15 +79,11 @@ public void AttachSessionDispatcher(Task dispatcherTask) _ = ResolveSessionDispatcherAsync(dispatcherTask); } - internal void Enqueue(string message) + internal void Enqueue(string message, string? sessionId) { _queue.TryAdd(message); - if (_sessionRoutingEnabled == 0) - return; - - var sessionId = SessionDiagnosticsContext.SessionId; - if (string.IsNullOrWhiteSpace(sessionId)) + if (_sessionRoutingEnabled == 0 || string.IsNullOrWhiteSpace(sessionId)) return; var dispatcher = Volatile.Read(ref _sessionDispatcher); @@ -232,6 +230,36 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (exception is not null) line += Environment.NewLine + exception; - _provider.Enqueue(line); + _provider.Enqueue(line, ExtractSessionId(state)); + } + + // Read the session id the producer already put on the log event so the sink can + // route per-session with no ambient/AsyncLocal context. The Akka→MEL bridge passes + // the event's structured properties as the state (AkkaLogState, carrying + // WithContext("SessionId", ...)); MEL's own structured logging passes + // FormattedLogValues (carrying a {SessionId} field). Both expose their fields as + // KeyValuePair sequences — read via the public interface, no Akka internals. + private static string? ExtractSessionId(TState state) + { + if (state is IEnumerable> nullableFields) + { + foreach (var field in nullableFields) + if (string.Equals(field.Key, "SessionId", StringComparison.Ordinal) && field.Value is { } value) + return Normalize(value); + } + else if (state is IEnumerable> fields) + { + foreach (var field in fields) + if (string.Equals(field.Key, "SessionId", StringComparison.Ordinal) && field.Value is { } value) + return Normalize(value); + } + + return null; + + static string? Normalize(object value) + { + var id = value.ToString(); + return string.IsNullOrWhiteSpace(id) ? null : id; + } } } diff --git a/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs b/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs index bffd4dc1c..efe0e7eaa 100644 --- a/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs @@ -64,12 +64,12 @@ public async Task GetResponseAsync( catch (Exception ex) when (!cancellationToken.IsCancellationRequested && !isLast) { EmitFailover(ex); - LogWithSession(LogLevel.Warning, ex, "LLM provider failed, failing over to next candidate"); + _logger.Log(LogLevel.Warning, ex, "LLM provider failed, failing over to next candidate"); } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { EmitUnreachable(ex, candidates.Count); - LogWithSession(LogLevel.Error, ex, UnreachableMessage(candidates.Count)); + _logger.Log(LogLevel.Error, ex, UnreachableMessage(candidates.Count)); throw; } } @@ -137,12 +137,12 @@ public async IAsyncEnumerable GetStreamingResponseAsync( if (isLast) { EmitUnreachable(failure, candidates.Count); - LogWithSession(LogLevel.Error, failure, UnreachableMessage(candidates.Count)); + _logger.Log(LogLevel.Error, failure, UnreachableMessage(candidates.Count)); ExceptionDispatchInfo.Capture(failure).Throw(); } EmitFailover(failure); - LogWithSession(LogLevel.Warning, failure, "LLM provider failed, failing over to next candidate"); + _logger.Log(LogLevel.Warning, failure, "LLM provider failed, failing over to next candidate"); // outer loop advances to the next candidate } } @@ -180,13 +180,6 @@ private void EmitUnreachable(Exception ex, int candidateCount) => AlertSeverity.Critical, context: new Dictionary { ["error"] = ex.Message })); - // Failover/outage events are logged here, outside any per-pipeline LoggingChatClient - // scope, so attach the session id so they correlate by session in Seq. - private void LogWithSession(LogLevel level, Exception ex, string message) - { - using (SessionLoggingScope.Begin(_logger)) - _logger.Log(level, ex, message); - } } /// diff --git a/src/Netclaw.Daemon/Configuration/SessionLoggingScope.cs b/src/Netclaw.Daemon/Configuration/SessionLoggingScope.cs deleted file mode 100644 index 14e1da57c..000000000 --- a/src/Netclaw.Daemon/Configuration/SessionLoggingScope.cs +++ /dev/null @@ -1,44 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -using Microsoft.Extensions.Logging; -using Netclaw.Configuration; - -namespace Netclaw.Daemon.Configuration; - -/// -/// Shared helper for tagging chat-client log lines with the ambient session id. -/// -/// The id lives in (an AsyncLocal pushed by -/// every session-owned path). The OTLP log exporter has IncludeScopes enabled -/// but cannot read that AsyncLocal directly, so without a logging scope LLM logs reach -/// Seq with no session correlation. Opening a scope keyed SessionId surfaces it -/// as a filterable attribute — matching the WithContext("SessionId", …) key the -/// session/channel actors already use, so a single attribute correlates actor and -/// chat-client logs. -/// -/// MEL scopes are per-, so each chat-client decorator that emits -/// its own log lines (e.g. and the routing/failover -/// client) opens its own scope from this helper. -/// -internal static class SessionLoggingScope -{ - private const string SessionIdKey = "SessionId"; - - /// - /// Opens a scope tagging subsequent log lines on with the - /// ambient session id, or returns null when no session is in scope (a no-op - /// using). A single-entry array avoids a per-call dictionary allocation while - /// still presenting as IEnumerable<KeyValuePair>, which is what the OTLP - /// exporter projects into log attributes. - /// - public static IDisposable? Begin(ILogger logger) - { - var sessionId = SessionDiagnosticsContext.SessionId; - return sessionId is null - ? null - : logger.BeginScope(new[] { new KeyValuePair(SessionIdKey, sessionId) }); - } -} From 2bd56ea64e6653b977c86e4a96867a7c15b9ce0b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 25 Jun 2026 21:03:21 +0000 Subject: [PATCH 02/22] test(logging): end-to-end proof actor _log routes to session.log via the real bridge (#1472 step 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hand-built host (AddLogging(AddProvider) + AddAkka(ConfigureLoggers(AddLoggerFactory)) — production wiring) drives an actor's WithContext("SessionId", ...) log through the real Akka->MEL bridge into RollingFileLoggerProvider and asserts a SessionLogDiagnostic is routed for that session. Proves the essential goal through the live bridge, not a mocked state. Not Akka.Hosting.TestKit on purpose: its bridge (TestKitLoggerFactoryLogger : the production LoggerFactoryLogger) is hardwired to the TestKit's own xUnit test-output ILoggerFactory, so a TestKit-based test cannot observe the bridged log arriving at our provider. Relates to #1499, #1472. --- .../SessionLogBridgeEndToEndTests.cs | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs diff --git a/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs b/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs new file mode 100644 index 000000000..a1762fb3f --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs @@ -0,0 +1,121 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Event; +using Akka.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Protocol; +using Netclaw.Daemon.Configuration; +using Xunit; + +namespace Netclaw.Daemon.Tests.Configuration; + +/// +/// End-to-end proof of the essential #1472-step-6 goal: an actor logging through an +/// ILoggingAdapter tagged with WithContext("SessionId", ...) reaches the +/// per-session writer via the REAL Akka→MEL bridge + +/// — no AsyncLocal. The bridge packs the event's context properties into the MEL log state; +/// the provider reads the session id off that state and routes a +/// . +/// +/// This uses a hand-built host rather than Akka.Hosting.TestKit on purpose: the +/// TestKit hardwires its Akka→MEL bridge to its own xUnit test-output logger factory, so a +/// TestKit-based test can't observe the bridged log arriving at our provider. The host here +/// replicates production wiring exactly — AddLogging(AddProvider(...)) + +/// AddAkka(ConfigureLoggers(AddLoggerFactory)), as in LoggingRegistrationExtensions / +/// Program.cs — so the real bridge routes into our provider. +/// +public sealed class SessionLogBridgeEndToEndTests : IAsyncLifetime +{ + private readonly string _basePath = Path.Combine(Path.GetTempPath(), $"netclaw-bridge-e2e-{Guid.NewGuid():N}"); + private IHost _host = null!; + private ActorSystem _system = null!; + private RollingFileLoggerProvider _provider = null!; + + public async ValueTask InitializeAsync() + { + var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); + Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); + _provider = new RollingFileLoggerProvider(daemonLogPath, new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z"))); + + _host = new HostBuilder() + .ConfigureServices(services => + { + services.AddLogging(builder => + { + builder.SetMinimumLevel(Microsoft.Extensions.Logging.LogLevel.Information); + builder.AddProvider(_provider); + }); + services.AddAkka("test-system", builder => + { + builder.ConfigureLoggers(setup => + { + setup.ClearLoggers(); + setup.AddLoggerFactory(); + setup.LogLevel = Akka.Event.LogLevel.InfoLevel; + }); + }); + }) + .Build(); + + await _host.StartAsync(TestContext.Current.CancellationToken); + _system = _host.Services.GetRequiredService(); + } + + [Fact] + public async Task Actor_log_with_session_context_routes_to_session_log_through_the_real_bridge() + { + var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var captor = _system.ActorOf(Props.Create(() => new CaptureActor(captured)), "session-log-captor"); + _provider.AttachSessionDispatcher(Task.FromResult(captor)); + + // Exactly how the session/sub-agent actors tag their logger. + var log = Logging.GetLogger(_system, "Netclaw.Test.SubAgentActor") + .WithContext("SessionId", "channel/thread"); + log.Info("actor lifecycle line via the real bridge"); + + var diagnostic = await captured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + Assert.Equal("channel/thread", diagnostic.SessionId.Value); + Assert.Contains("actor lifecycle line via the real bridge", diagnostic.Line, StringComparison.Ordinal); + } + + public async ValueTask DisposeAsync() + { + try + { + await _host.StopAsync(TimeSpan.FromSeconds(5)); + _host.Dispose(); + } + finally + { + try + { + if (Directory.Exists(_basePath)) + Directory.Delete(_basePath, recursive: true); + } + catch (IOException ex) + { + Console.Error.WriteLine($"[SessionLogBridgeEndToEndTests] cleanup failed: {ex.Message}"); + } + catch (UnauthorizedAccessException ex) + { + Console.Error.WriteLine($"[SessionLogBridgeEndToEndTests] cleanup failed: {ex.Message}"); + } + } + } + + private sealed class CaptureActor : ReceiveActor + { + public CaptureActor(TaskCompletionSource captured) + { + Receive(d => captured.TrySetResult(d)); + } + } +} From c9f02e51240f9520f6542e66632cf4d10339f182 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 03:42:33 +0000 Subject: [PATCH 03/22] refactor(logging): publish session logs explicitly instead of routing in the sink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A max-effort review showed the previous step-6 approach (the file sink reading a "SessionId" property off each log event and routing it to session.log) was wrong: "{SessionId}" is a descriptive message-template placeholder in 86 places across the codebase (binding actors, gateway, webhooks, session catalog/registry, drain/recovery), each of which produces a structured "SessionId" property — so the sink would flood session.log with operational noise and create a self-feedback loop in SessionLogActor's write-failure path. SessionId is a correlation field, not a routing field. Replace sink-side routing with explicit publish through the existing actor-message contract (the same one that already carries the transcript): - RollingFileLoggerProvider reverts to a daemon.log-only writer — no ExtractSessionId, no dispatcher attach, no buffer. (Deletes the collision, the feedback loop, the per-line boxing, the dead branch, and the TOCTOU buffer race the review found.) - ToolExecutionContext.EmitSessionLogLine: LlmSessionActor wires it to _logActor.Tell(new SessionLogDiagnostic(parentSessionId, line)); the tool pipeline threads it to the spawn breadcrumbs, which publish their lifecycle explicitly. Using the parent id also fixes the composite-id normalization gap. - SessionId stays a pure Seq/OTLP correlation field on those lines, doing one job. - Removed the dead sessionId param on SubAgentActor.InvokeLlmAsync; fixed the stale LoggingChatClient doc; skill diagnostics guidance updated to the explicit-publish model. session.log scope = transcript + explicitly-published spawn lifecycle. Chat-client and sidecar internals are daemon.log-only (filter by the SessionId field). Sidecar diagnostic restore is a follow-up. Relates to #1499, #1472. --- .../references/diagnostics.md | 19 +-- .../SubAgentSpawnObservabilityTests.cs | 77 ++++----- .../Sessions/LlmSessionActor.cs | 7 + .../Pipelines/SessionToolExecutionPipeline.cs | 8 +- .../SubAgents/SpawnAgentTool.cs | 12 +- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 8 +- .../SubAgents/SubAgentSpawner.cs | 20 ++- .../RollingFileLoggerProviderTests.cs | 152 ++---------------- .../SessionLogBridgeEndToEndTests.cs | 121 -------------- .../Configuration/LoggingChatClient.cs | 9 +- .../LoggingRegistrationExtensions.cs | 42 +---- .../RollingFileLoggerProvider.cs | 134 +-------------- .../ToolExecutionContext.cs | 8 + 13 files changed, 117 insertions(+), 500 deletions(-) delete mode 100644 src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index c0a137dc1..364ece2f2 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -34,19 +34,20 @@ What to expect inside `session.log`: cross-referencing two files. - Two line shapes: session output audit lines (`User:`, `Assistant:`, `Thinking:`, `Tool call:`, `Tool result:`, `Usage:`, `Turn N completed`, - etc.) and `Diagnostic:` lines. A diagnostic line reaches `session.log` - when its log event carries the session id; actor logs are tagged with it - automatically (`SessionId`), so session- and sub-agent-actor lifecycle - lines (startup, guards, timeouts, cancellation, completion) appear here — - routed by the tag on the line, not by any ambient context. + etc.) and `Diagnostic:` lines. Diagnostic lines reach `session.log` only + when a producer *explicitly publishes* them — notably the sub-agent spawn + lifecycle (requested, child spawned, dispatched, completed/failed, and + guard rejections), so a failed or blocked spawn is visible here. It is an + explicit publish, not automatic routing of every log line. - Best-effort observability. Individual lines may be dropped on transient IO errors and logged at Debug level in the daemon log; this is not a transactional audit trail. Cross-check the daemon log for warnings if a critical line appears missing. -- Not everything is session-tagged: the LLM-client / HTTP / retry / failover - decorators log their internal diagnostics without a session id, so those - land in `daemon.log` only. For deep LLM-call internals, read `daemon.log` - (filter by the `SessionId` attribute where the line carries one). +- Most actor and operational logs are NOT in `session.log` — only explicitly + published lines are. An actor's own lifecycle logging, and the LLM-client / + HTTP / retry / failover decorator internals, go to `daemon.log` (filterable + by the `SessionId` structured field where the line carries one). For an + actor's full lifecycle or deep LLM-call internals, read `daemon.log`. | Symptom | Check | |---------|-------| diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs index cb7e8d7c0..9239de6e6 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs @@ -3,7 +3,7 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Netclaw.Actors.SubAgents; using Netclaw.Actors.Tools; using Netclaw.Configuration; @@ -15,11 +15,11 @@ namespace Netclaw.Actors.Tests.SubAgents; /// /// Regression coverage for sub-agent spawn observability. The spawn lifecycle is -/// recorded by parent-side breadcrumbs that carry the session id as a structured log -/// field — exactly what RollingFileLoggerProvider reads off the log event to -/// route a line to the per-session session.log. These tests assert each -/// breadcrumb carries the session id; otherwise a refused or failed spawn would be -/// invisible in the session transcript. +/// published to the parent's session.log explicitly via +/// (the session wires it to the +/// session-log dispatcher). These tests capture those publishes and assert each +/// lifecycle/rejection breadcrumb is emitted; otherwise a refused or failed spawn +/// would be invisible in the session transcript. /// public sealed class SubAgentSpawnObservabilityTests : IDisposable { @@ -37,9 +37,8 @@ public SubAgentSpawnObservabilityTests() public void Dispose() => _dir.Dispose(); [Fact] - public async Task Spawner_missing_session_context_records_breadcrumbs_under_session_scope() + public async Task Spawner_missing_session_context_publishes_lifecycle_to_session_log() { - var logger = new CapturingLogger(); // Only the parent-side breadcrumb path runs before the early return, so the // unused collaborators are never dereferenced. var spawner = new SubAgentSpawner( @@ -48,31 +47,40 @@ public async Task Spawner_missing_session_context_records_breadcrumbs_under_sess toolAccessPolicy: null!, approvalService: null, promptProvider: null!, - logger); + NullLogger.Instance); + var sessionLog = new List(); // A context with a session id but no SpawnChildActor factory — the // "subagent tried to spawn but never launched" failure shape. - var context = new ToolExecutionContext(SessionId, null) { Audience = TrustAudience.Personal }; + var context = new ToolExecutionContext(SessionId, null) + { + Audience = TrustAudience.Personal, + EmitSessionLogLine = sessionLog.Add + }; var result = await spawner.SpawnAsync( Profile("summarizer"), "do the work", null, context, TestContext.Current.CancellationToken); Assert.False(result.Success); - // The spawn attempt and its failure are both visible in the session transcript. - Assert.Contains(logger.Entries, e => e.SessionScope == SessionId && e.Message.Contains("spawn requested")); - Assert.Contains(logger.Entries, e => e.SessionScope == SessionId && e.Message.Contains("no session context available")); + // The spawn attempt and its failure are both published to the session transcript. + Assert.Contains(sessionLog, line => line.Contains("spawn requested", StringComparison.Ordinal)); + Assert.Contains(sessionLog, line => line.Contains("no session context available", StringComparison.Ordinal)); } [Fact] - public async Task Tool_refusal_records_real_reason_under_session_scope() + public async Task Tool_refusal_publishes_real_reason_to_session_log() { - var logger = new CapturingLogger(); var registry = new SubAgentDefinitionRegistry(); - var tool = new SpawnAgentTool(registry, spawner: null!, _paths, logger: logger); + var tool = new SpawnAgentTool(registry, spawner: null!, _paths, logger: NullLogger.Instance); + var sessionLog = new List(); // Public audience is refused with a deliberately opaque model-facing string; // the operator-facing breadcrumb must still record the real reason. - var context = new ToolExecutionContext(SessionId, null) { Audience = TrustAudience.Public }; + var context = new ToolExecutionContext(SessionId, null) + { + Audience = TrustAudience.Public, + EmitSessionLogLine = sessionLog.Add + }; var result = await tool.ExecuteAsync( new Dictionary { ["agent"] = "summarizer", ["task"] = "do the work" }, @@ -81,11 +89,8 @@ public async Task Tool_refusal_records_real_reason_under_session_scope() Assert.Equal("Error: This tool is not available.", result); Assert.Contains( - logger.Entries, - e => e.Level == LogLevel.Warning - && e.SessionScope == SessionId - && e.Message.Contains("refused") - && e.Message.Contains("Public")); + sessionLog, + line => line.Contains("refused", StringComparison.Ordinal) && line.Contains("Public", StringComparison.Ordinal)); } private static SubAgentProfile Profile(string name) => new() @@ -96,32 +101,4 @@ public async Task Tool_refusal_records_real_reason_under_session_scope() ToolNames = ["file_read"], Visibility = SubAgentVisibility.UserFacing }; - - private sealed class CapturingLogger : ILogger - { - public readonly List<(LogLevel Level, string Message, string? SessionScope)> Entries = new(); - - public IDisposable? BeginScope(TState state) where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log( - LogLevel logLevel, - EventId eventId, - TState state, - Exception? exception, - Func formatter) - => Entries.Add((logLevel, formatter(state, exception), ExtractSessionId(state))); - - // Mirror RollingFileLoggerProvider: read the session id off the structured log - // state (the {SessionId} field the breadcrumbs carry), not an ambient context. - private static string? ExtractSessionId(TState state) - { - if (state is IEnumerable> fields) - foreach (var field in fields) - if (field.Key == "SessionId" && field.Value is { } value) - return value.ToString(); - return null; - } - } } diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 273d714d1..1be1980a7 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -1971,6 +1971,12 @@ private void DispatchToolBatch( logActor?.Tell(output); }; + // Sub-agent / tool lifecycle lines are published explicitly into this session's + // session.log via the dispatcher (logActor). Routing is intentional here, not + // inferred from log metadata at the sink. + Action emitSessionLogLine = line => + logActor?.Tell(new SessionLogDiagnostic(sessionId, $"[{tp.GetUtcNow():o}] {line}")); + // Marshal child-actor spawning back onto the session actor thread. Func> spawnChildActor = async (props, name, ct) => await self.Ask( @@ -2005,6 +2011,7 @@ await self.Ask( oneTimeApprovalPreSeed: oneTimeApprovalPreSeed, decisionOverride: decisionOverride, turnContext: _currentTurnContext, + emitSessionLogLine: emitSessionLogLine, ct: toolExecutionCt); } diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs index 4d6b45cc6..ff1c5a58b 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs @@ -98,6 +98,7 @@ public static async Task ExecuteToolsAsync( IReadOnlyDictionary>? oneTimeApprovalPreSeed = null, IReadOnlyDictionary? decisionOverride = null, TurnContext? turnContext = null, + Action? emitSessionLogLine = null, CancellationToken ct = default) { try @@ -138,7 +139,8 @@ oneTimeApprovalPreSeed is not null ? overrideDecision : null, turnContext, - modelInputBudget); + modelInputBudget, + emitSessionLogLine); if (streamToolResults) self.Tell(new ToolExecutionSingleCompleted(result)); return result; @@ -209,7 +211,8 @@ public static async Task ExecuteSingleToolAsync( IReadOnlyList? oneTimeApprovalPreSeed = null, ApprovalDecision? decisionOverride = null, TurnContext? turnContext = null, - ModelInputBatchBudget? modelInputBudget = null) + ModelInputBatchBudget? modelInputBudget = null, + Action? emitSessionLogLine = null) { // Single execution-preflight seam, shared with the sub-agent path via // IToolExecutor.InterpretToolCall: validate the ORIGINAL arguments (parse @@ -285,6 +288,7 @@ public static async Task ExecuteSingleToolAsync( } var completedRuns = new List(); var acceptedFindings = new List(); + context.EmitSessionLogLine = emitSessionLogLine; context.OnSubAgentActivity = info => { if (info.IsStarted) diff --git a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs index 972f28d84..5fd11686a 100644 --- a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs +++ b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs @@ -124,10 +124,10 @@ private static string FormatResult(string agent, SubAgentResult result) private (string? Error, SubAgentProfile? Profile) Resolve(Params args, ToolExecutionContext context) { // Rejections here return a (sometimes deliberately opaque) error string to - // the model. Mirror the real reason to the session transcript — each line - // carries the session id, so the log sink routes it to session.log — so an - // operator can tell *why* a spawn was refused; the "This tool is not - // available." string hides the audience-vs-disabled distinction on purpose. + // the model. Mirror the real reason to the session transcript via + // EmitSessionLogLine (explicit publish) so an operator can tell *why* a spawn + // was refused; the "This tool is not available." string hides the + // audience-vs-disabled distinction on purpose. // Defense-in-depth: block subagent spawning for Public audience or when // the subagent subsystem is disabled. @@ -136,6 +136,8 @@ private static string FormatResult(string agent, SubAgentResult result) _logger?.LogWarning( "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled}, session={SessionId})", args.Agent, context.Audience, _subAgentConfig.Enabled, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"spawn_agent refused (agent={args.Agent}, audience={context.Audience}, subsystemEnabled={_subAgentConfig.Enabled})"); return ("Error: This tool is not available.", null); } @@ -154,6 +156,8 @@ private static string FormatResult(string agent, SubAgentResult result) _logger?.LogWarning( "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count}, session={SessionId})", args.Agent, available.Count, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"spawn_agent refused: agent '{args.Agent}' not found or not user-facing (availableCount={available.Count})"); if (available.Count == 0) return ($"Error: No subagents are available. Agent '{args.Agent}' not found. Author one at {_paths.AgentsDirectory}/*.md or define a skill with metadata.subagent once #661 lands.", null); diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index e8aa03691..c8c80b466 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -671,7 +671,6 @@ private void FireLlmCall(bool forceNoTools = false) }; } - var sessionId = _toolExecutionContext.SessionId is null ? (SessionId?)null : new SessionId(_toolExecutionContext.SessionId); _log.Info( "SubAgent [{AgentName}] LLM call start callId={CallId} iteration={Iteration} messages={MessageCount} toolsEnabled={ToolsEnabled} forceNoTools={ForceNoTools}", _definition.Name, @@ -680,7 +679,7 @@ private void FireLlmCall(bool forceNoTools = false) messages.Count, options?.Tools?.Count > 0, forceNoTools); - _ = InvokeLlmAsync(client, messages, options, sessionId, self, callId, _executionCts?.Token ?? CancellationToken.None); + _ = InvokeLlmAsync(client, messages, options, self, callId, _executionCts?.Token ?? CancellationToken.None); } private IReadOnlyList ResolveExposedAiTools() @@ -887,23 +886,20 @@ internal static Task InvokeLlmAsync( IChatClient client, List messages, ChatOptions? options, - SessionId? sessionId, IActorRef self, CancellationToken ct) - => InvokeLlmAsync(client, messages, options, sessionId, self, callId: 0, ct); + => InvokeLlmAsync(client, messages, options, self, callId: 0, ct); internal static async Task InvokeLlmAsync( IChatClient client, List messages, ChatOptions? options, - SessionId? sessionId, IActorRef self, long callId, CancellationToken ct) { try { - // Use streaming to match the main session path. The non-streaming // GetResponseAsync path drops reasoning content for some providers // (e.g., Qwen emits blocks that surface as TextReasoningContent diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index ace357b0b..4fc9b03f1 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -67,17 +67,20 @@ public async Task SpawnAsync( string? systemPromptOverlay = null, ChannelWriter? activitySink = null) { - // Session-scoped breadcrumbs: each parent-side line carries the session id as a - // structured field, so the log sink routes the spawn lifecycle (request → - // outcome, including early rejections that happen before the child actor even - // exists) into the parent's session.log alongside the sub-agent's own actor logs. + // Parent-side spawn breadcrumbs. _logger lines carry session={SessionId} for + // daemon.log/Seq correlation; EmitSessionLogLine publishes the same lifecycle + // (request → outcome, including early rejections that happen before the child + // actor even exists) explicitly into the parent's session.log. _logger.LogInformation( "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars}, session={SessionId})", profile.Name, task.Length, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{profile.Name}] spawn requested (taskChars={task.Length})"); if (context.SpawnChildActor is null) { _logger.LogWarning("SubAgent [{AgentName}] cannot spawn — no session context available (session={SessionId})", profile.Name, context.SessionId); + context.EmitSessionLogLine?.Invoke($"SubAgent [{profile.Name}] cannot spawn — no session context available"); activitySink?.TryComplete(); return new SubAgentResult { @@ -93,6 +96,8 @@ public async Task SpawnAsync( _logger.LogWarning( "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn (session={SessionId})", profile.Name, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{profile.Name}] has no tools available under the parent audience policy — cannot spawn"); activitySink?.TryComplete(); return new SubAgentResult { @@ -154,6 +159,8 @@ public async Task SpawnAsync( _logger.LogError( ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId}, session={SessionId})", profile.Name, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{profile.Name}] failed to spawn child actor (runId={runId}): {ex.Message}"); // Balance the IsStarted=true notification above: the non-streaming path // (activitySink is null) relies solely on OnSubAgentActivity, so without // a terminal event the session UI shows a sub-agent stuck in "Started". @@ -171,6 +178,8 @@ public async Task SpawnAsync( _logger.LogInformation( "SubAgent [{AgentName}] child actor spawned (runId={RunId}, session={SessionId}); dispatching RunSubAgent", profile.Name, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{profile.Name}] child actor spawned (runId={runId}); dispatching RunSubAgent"); var sw = Stopwatch.StartNew(); try @@ -230,6 +239,8 @@ public async Task SpawnAsync( _logger.LogInformation( "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms, session={SessionId})", profile.Name, runId, result.Success, sw.ElapsedMilliseconds, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{profile.Name}] completed (runId={runId}, success={result.Success}, duration={sw.ElapsedMilliseconds}ms)"); return result; } @@ -249,6 +260,7 @@ public async Task SpawnAsync( }); _logger.LogError(ex, "SubAgent [{AgentName}] run failed (runId={RunId}, session={SessionId})", profile.Name, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke($"SubAgent [{profile.Name}] run failed (runId={runId}): {ex.Message}"); return new SubAgentResult { Success = false, diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs index d3b97e4b6..ec85a13a5 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs @@ -3,164 +3,42 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using System.Collections; -using Akka.Actor; -using Akka.Hosting; -using Akka.Hosting.TestKit; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; -using Netclaw.Actors.Protocol; using Netclaw.Daemon.Configuration; using Xunit; namespace Netclaw.Daemon.Tests.Configuration; -public sealed class RollingFileLoggerProviderTests : TestKit, IDisposable +public sealed class RollingFileLoggerProviderTests : IDisposable { - protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) - { - } - private readonly string _basePath = Path.Combine(Path.GetTempPath(), $"netclaw-rolling-logger-tests-{Guid.NewGuid():N}"); [Fact] - public async Task Mel_structured_log_with_session_id_routes_to_dispatcher() - { - // A direct MEL structured log: the {SessionId} placeholder lands in the - // FormattedLogValues state, which the provider reads to route per-session. - var (provider, probe) = NewProvider(); - using (provider) - { - var logger = provider.CreateLogger("Netclaw.Tests"); - logger.LogInformation("session scoped message {SessionId}", "channel/thread"); - - var diagnostic = await probe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal("channel/thread", diagnostic.SessionId.Value); - Assert.Contains("session scoped message", diagnostic.Line, StringComparison.Ordinal); - Assert.Contains("Diagnostic:", diagnostic.Line, StringComparison.Ordinal); - } - } - - [Fact] - public async Task Akka_bridged_log_state_with_session_id_routes_to_dispatcher() - { - // The Akka→MEL bridge passes the log event's structured properties as the - // state (an AkkaLogState that enumerates as KeyValuePairs, including the - // "SessionId" tag from WithContext). The provider reads it off that state. - var (provider, probe) = NewProvider(); - using (provider) - { - var logger = provider.CreateLogger("Netclaw.Actors.SubAgents.SubAgentActor"); - var state = new FakeBridgedLogState( - [ - new("SessionId", "channel/thread"), - new("SubSessionId", "channel/thread/subagent/x/1"), - new("{OriginalFormat}", "actor lifecycle line"), - ]); - logger.Log(LogLevel.Information, default, state, exception: null, static (_, _) => "actor lifecycle line"); - - var diagnostic = await probe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal("channel/thread", diagnostic.SessionId.Value); - Assert.Contains("actor lifecycle line", diagnostic.Line, StringComparison.Ordinal); - } - } - - [Fact] - public async Task Log_without_session_id_does_not_route_to_dispatcher() - { - var (provider, probe) = NewProvider(); - using (provider) - { - var logger = provider.CreateLogger("Netclaw.Tests"); - logger.LogInformation("daemon message"); - - await probe.ExpectNoMsgAsync( - TimeSpan.FromMilliseconds(200), - cancellationToken: TestContext.Current.CancellationToken); - } - - var daemonLog = Directory.GetFiles(Path.Combine(_basePath, "logs"), "daemon-*.log", SearchOption.TopDirectoryOnly).Single(); - var daemonText = await File.ReadAllTextAsync(daemonLog, TestContext.Current.CancellationToken); - Assert.Contains("daemon message", daemonText, StringComparison.Ordinal); - } - - [Fact] - public async Task Pre_resolution_diagnostics_buffer_and_drain_to_dispatcher() + public async Task Writes_log_lines_to_the_daily_rolling_daemon_log() { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - var probe = CreateTestProbe(); - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - using (var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider)) - { - provider.AttachSessionDispatcher(tcs.Task); - var logger = provider.CreateLogger("Netclaw.Tests"); - - logger.LogInformation("pre-resolution one {SessionId}", "channel/thread"); - logger.LogInformation("pre-resolution two {SessionId}", "channel/thread"); - - await probe.ExpectNoMsgAsync( - TimeSpan.FromMilliseconds(100), - cancellationToken: TestContext.Current.CancellationToken); - - tcs.SetResult(probe.Ref); - - var first = await probe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); - var second = await probe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); - Assert.Contains("pre-resolution one", first.Line, StringComparison.Ordinal); - Assert.Contains("pre-resolution two", second.Line, StringComparison.Ordinal); - } - } - [Fact] - public async Task Log_from_async_continuation_still_routes_via_state() - { - // The whole point of reading the id off the log state: routing no longer - // depends on ambient/async context, so a line logged after an await still - // routes (the old AsyncLocal could be lost across the await). - var (provider, probe) = NewProvider(); - using (provider) + // Dispose drains the writer thread, so the line is flushed by the time we read. + using (var provider = new RollingFileLoggerProvider(daemonLogPath, time)) { var logger = provider.CreateLogger("Netclaw.Tests"); - - await Task.Yield(); - logger.LogInformation("post-await {SessionId}", "channel/thread"); - - var diagnostic = await probe.ExpectMsgAsync( - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal("channel/thread", diagnostic.SessionId.Value); - Assert.Contains("post-await", diagnostic.Line, StringComparison.Ordinal); + logger.LogInformation("hello daemon log {SessionId}", "channel/thread"); } - } - - private (RollingFileLoggerProvider Provider, Akka.TestKit.TestProbe Probe) NewProvider() - { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); - Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - var probe = CreateTestProbe(); - var provider = new RollingFileLoggerProvider(daemonLogPath, timeProvider); - provider.AttachSessionDispatcher(Task.FromResult(probe.Ref)); - return (provider, probe); - } - - // Mimics Akka.Hosting's AkkaLogState: an MEL log state that enumerates as the - // event's structured properties (the bridge's surface the provider reads). - private sealed class FakeBridgedLogState(IReadOnlyList> fields) - : IEnumerable> - { - public IEnumerator> GetEnumerator() => fields.GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + // The {SessionId} field is NOT special to this sink — it renders into the message + // text and the line goes to daemon.log only. Per-session routing is the producers' + // job (explicit SessionLogDiagnostic), not the logging sink's. + var daemonLog = Directory.GetFiles(Path.Combine(_basePath, "logs"), "daemon-*.log").Single(); + var text = await File.ReadAllTextAsync(daemonLog, TestContext.Current.CancellationToken); + Assert.Contains("hello daemon log", text, StringComparison.Ordinal); + Assert.Contains("Netclaw.Tests", text, StringComparison.Ordinal); + Assert.Contains("channel/thread", text, StringComparison.Ordinal); } - void IDisposable.Dispose() + public void Dispose() { try { diff --git a/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs b/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs deleted file mode 100644 index a1762fb3f..000000000 --- a/src/Netclaw.Daemon.Tests/Configuration/SessionLogBridgeEndToEndTests.cs +++ /dev/null @@ -1,121 +0,0 @@ -// ----------------------------------------------------------------------- -// -// Copyright (C) 2026 - 2026 Petabridge, LLC -// -// ----------------------------------------------------------------------- -using Akka.Actor; -using Akka.Event; -using Akka.Hosting; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Time.Testing; -using Netclaw.Actors.Protocol; -using Netclaw.Daemon.Configuration; -using Xunit; - -namespace Netclaw.Daemon.Tests.Configuration; - -/// -/// End-to-end proof of the essential #1472-step-6 goal: an actor logging through an -/// ILoggingAdapter tagged with WithContext("SessionId", ...) reaches the -/// per-session writer via the REAL Akka→MEL bridge + -/// — no AsyncLocal. The bridge packs the event's context properties into the MEL log state; -/// the provider reads the session id off that state and routes a -/// . -/// -/// This uses a hand-built host rather than Akka.Hosting.TestKit on purpose: the -/// TestKit hardwires its Akka→MEL bridge to its own xUnit test-output logger factory, so a -/// TestKit-based test can't observe the bridged log arriving at our provider. The host here -/// replicates production wiring exactly — AddLogging(AddProvider(...)) + -/// AddAkka(ConfigureLoggers(AddLoggerFactory)), as in LoggingRegistrationExtensions / -/// Program.cs — so the real bridge routes into our provider. -/// -public sealed class SessionLogBridgeEndToEndTests : IAsyncLifetime -{ - private readonly string _basePath = Path.Combine(Path.GetTempPath(), $"netclaw-bridge-e2e-{Guid.NewGuid():N}"); - private IHost _host = null!; - private ActorSystem _system = null!; - private RollingFileLoggerProvider _provider = null!; - - public async ValueTask InitializeAsync() - { - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); - Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); - _provider = new RollingFileLoggerProvider(daemonLogPath, new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z"))); - - _host = new HostBuilder() - .ConfigureServices(services => - { - services.AddLogging(builder => - { - builder.SetMinimumLevel(Microsoft.Extensions.Logging.LogLevel.Information); - builder.AddProvider(_provider); - }); - services.AddAkka("test-system", builder => - { - builder.ConfigureLoggers(setup => - { - setup.ClearLoggers(); - setup.AddLoggerFactory(); - setup.LogLevel = Akka.Event.LogLevel.InfoLevel; - }); - }); - }) - .Build(); - - await _host.StartAsync(TestContext.Current.CancellationToken); - _system = _host.Services.GetRequiredService(); - } - - [Fact] - public async Task Actor_log_with_session_context_routes_to_session_log_through_the_real_bridge() - { - var captured = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var captor = _system.ActorOf(Props.Create(() => new CaptureActor(captured)), "session-log-captor"); - _provider.AttachSessionDispatcher(Task.FromResult(captor)); - - // Exactly how the session/sub-agent actors tag their logger. - var log = Logging.GetLogger(_system, "Netclaw.Test.SubAgentActor") - .WithContext("SessionId", "channel/thread"); - log.Info("actor lifecycle line via the real bridge"); - - var diagnostic = await captured.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - - Assert.Equal("channel/thread", diagnostic.SessionId.Value); - Assert.Contains("actor lifecycle line via the real bridge", diagnostic.Line, StringComparison.Ordinal); - } - - public async ValueTask DisposeAsync() - { - try - { - await _host.StopAsync(TimeSpan.FromSeconds(5)); - _host.Dispose(); - } - finally - { - try - { - if (Directory.Exists(_basePath)) - Directory.Delete(_basePath, recursive: true); - } - catch (IOException ex) - { - Console.Error.WriteLine($"[SessionLogBridgeEndToEndTests] cleanup failed: {ex.Message}"); - } - catch (UnauthorizedAccessException ex) - { - Console.Error.WriteLine($"[SessionLogBridgeEndToEndTests] cleanup failed: {ex.Message}"); - } - } - } - - private sealed class CaptureActor : ReceiveActor - { - public CaptureActor(TaskCompletionSource captured) - { - Receive(d => captured.TrySetResult(d)); - } - } -} diff --git a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs index 627271b64..1f63c123c 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs @@ -13,10 +13,11 @@ namespace Netclaw.Daemon.Configuration; /// /// Decorates an with logging for elapsed time, token -/// usage, and errors, and tags log lines with the ambient session id. Stateless -/// and safe to share across sessions. Netclaw issues only streaming requests, so -/// only the streaming path is instrumented; the inherited non-streaming -/// pass-through is unused. +/// usage, and errors. These diagnostics go to daemon.log only (the decorator +/// is session-agnostic); they are not published to a session's session.log. +/// Stateless and safe to share across sessions. Netclaw issues only streaming +/// requests, so only the streaming path is instrumented; the inherited +/// non-streaming pass-through is unused. /// public sealed class LoggingChatClient : DelegatingChatClient { diff --git a/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs b/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs index d1ee4633f..e218f2fce 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs @@ -3,11 +3,8 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using Akka.Hosting; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Netclaw.Actors.Hosting; using Netclaw.Configuration; namespace Netclaw.Daemon.Configuration; @@ -25,18 +22,15 @@ public static LogLevel ConfigureNetclawLogging(this WebApplicationBuilder builde if (consoleEnabled) builder.Logging.AddSimpleConsole(options => options.SingleLine = true); - // Always write to a rolling log file in ~/.netclaw/logs/ + // Always write to a rolling daemon.log in ~/.netclaw/logs/. The provider must be + // constructed eagerly so MEL can see it via AddProvider — a + // Services.AddSingleton(factory) registration here is not picked + // up by the LoggerFactory in this hosting setup. This sink is daemon-global only; + // per-session lines are published explicitly to the session-log dispatcher. Directory.CreateDirectory(resolvedPaths.LogsDirectory); - - // Provider must be constructed eagerly so MEL can see it via AddProvider — - // a Services.AddSingleton(factory) registration here is - // not picked up by the LoggerFactory in this hosting setup. The session - // log dispatcher is wired in post-build by SessionLogDispatcherWiringService - // once Akka.Hosting has registered the actor system. var provider = new RollingFileLoggerProvider(resolvedPaths.DaemonLogPath); builder.Logging.AddProvider(provider); builder.Services.AddSingleton(provider); - builder.Services.AddHostedService(); builder.Logging.SetMinimumLevel(level); return level; @@ -51,29 +45,3 @@ private static LogLevel ResolveLogLevel(IConfiguration configuration) return LogLevel.Information; } } - -/// -/// Hooks the session log dispatcher into -/// once Akka.Hosting has registered SessionLogDispatcherActorKey. -/// -internal sealed class SessionLogDispatcherWiringService : IHostedService -{ - private readonly RollingFileLoggerProvider _provider; - private readonly IRequiredActor _dispatcher; - - public SessionLogDispatcherWiringService( - RollingFileLoggerProvider provider, - IRequiredActor dispatcher) - { - _provider = provider; - _dispatcher = dispatcher; - } - - public Task StartAsync(CancellationToken cancellationToken) - { - _provider.AttachSessionDispatcher(_dispatcher.GetAsync(cancellationToken)); - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; -} diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index b0c62704f..f6dc8e75d 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -5,45 +5,29 @@ // ----------------------------------------------------------------------- using System.Collections.Concurrent; using System.Globalization; -using Akka.Actor; using Microsoft.Extensions.Logging; -using Netclaw.Actors.Protocol; namespace Netclaw.Daemon.Configuration; /// -/// Simple file-based logger that writes to a daily rolling log file. +/// Simple file-based logger that writes to a daily rolling daemon.log file. /// Uses a background queue to avoid blocking callers. /// -/// Session-scoped lines — those whose log event carries a "SessionId" field -/// (actors via Context.GetLogger().WithContext("SessionId", ...), which -/// the Akka→MEL bridge surfaces as structured log state; or MEL callers via a -/// {SessionId} structured field) — are mirrored to a per-session -/// session.log by routing through the SessionLogDispatcher -/// actor. The dispatcher serializes all writes for a given session through -/// a single mailbox, replacing the in-process file lock that previously -/// coordinated concurrent writers. The session id is read off each log event -/// at the sink, so no ambient/AsyncLocal context is threaded by producers. -/// -/// The dispatcher is wired in via -/// post-construction (typically from an IHostedService that runs -/// after the actor system starts) because the provider is constructed -/// during host build, before Akka. +/// This sink is daemon-global ONLY. It does not route per-session: session-relevant +/// lines are published explicitly by their producers to the session-log dispatcher +/// (a SessionLogDiagnostic Tell), which serializes per-session writes through +/// SessionLogActor. Keeping routing out of the logging sink avoids inferring +/// intent from log metadata (e.g. a descriptive {SessionId} placeholder). /// internal sealed class RollingFileLoggerProvider : ILoggerProvider { private const long MaxFileSizeBytes = 10 * 1024 * 1024; // 10MB per file - private const int PreResolutionBufferLimit = 1000; private readonly string _basePath; private readonly TimeProvider _timeProvider; private readonly ConcurrentDictionary _loggers = new(); private readonly BlockingCollection _queue = new(1024); private readonly Thread _writerThread; - private ConcurrentQueue? _pendingDiagnostics; - private IActorRef? _sessionDispatcher; - private int _pendingCount; - private int _sessionRoutingEnabled; private StreamWriter? _writer; private string _currentDate = ""; @@ -62,79 +46,7 @@ public RollingFileLoggerProvider(string basePath, TimeProvider? timeProvider = n public ILogger CreateLogger(string categoryName) => _loggers.GetOrAdd(categoryName, name => new RollingFileLogger(name, this)); - /// - /// Enables session-scoped log routing through the dispatcher actor. - /// The provider buffers session-scoped diagnostic lines emitted between - /// this call and dispatcher resolution into a small bounded queue, then - /// drains them in order. Subsequent emits go straight to the dispatcher. - /// Failures during resolution surface a single ERR line in the daemon log - /// and disable session routing for the rest of the process. - /// - public void AttachSessionDispatcher(Task dispatcherTask) - { - if (Interlocked.Exchange(ref _sessionRoutingEnabled, 1) == 1) - return; - - _pendingDiagnostics = new ConcurrentQueue(); - _ = ResolveSessionDispatcherAsync(dispatcherTask); - } - - internal void Enqueue(string message, string? sessionId) - { - _queue.TryAdd(message); - - if (_sessionRoutingEnabled == 0 || string.IsNullOrWhiteSpace(sessionId)) - return; - - var dispatcher = Volatile.Read(ref _sessionDispatcher); - if (dispatcher is null && Volatile.Read(ref _pendingCount) >= PreResolutionBufferLimit) - return; - - var diagnostic = new SessionLogDiagnostic( - new SessionId(sessionId), - $"[{_timeProvider.GetUtcNow():o}] Diagnostic: {message}"); - - if (dispatcher is not null) - { - dispatcher.Tell(diagnostic); - return; - } - - Interlocked.Increment(ref _pendingCount); - _pendingDiagnostics!.Enqueue(diagnostic); - } - - private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) - { - IActorRef dispatcher; - try - { - dispatcher = await dispatcherTask.ConfigureAwait(false); - } - catch (Exception ex) - { - // Resolution failed permanently. Drop the buffer and switch the - // session-log path off — daemon-global logging continues normally, - // but session-scoped diagnostics will not appear in session.log - // for the remainder of this process. Surface a single loud line - // so operators see this in the daemon log instead of silently - // accumulating drops. - _queue.TryAdd($"{GetTimestamp()} [ERR] Netclaw.Logging: session log dispatcher resolution failed; session-scoped diagnostics disabled. {ex.Message}"); - _pendingDiagnostics = null; - return; - } - - // Publish the ref BEFORE draining so producers racing with the drainer - // see the dispatcher and Tell directly rather than enqueueing into a - // queue that we are about to abandon. - Volatile.Write(ref _sessionDispatcher, dispatcher); - - while (_pendingDiagnostics!.TryDequeue(out var pending)) - { - Interlocked.Decrement(ref _pendingCount); - dispatcher.Tell(pending); - } - } + internal void Enqueue(string message) => _queue.TryAdd(message); private void ProcessQueue() { @@ -230,36 +142,6 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (exception is not null) line += Environment.NewLine + exception; - _provider.Enqueue(line, ExtractSessionId(state)); - } - - // Read the session id the producer already put on the log event so the sink can - // route per-session with no ambient/AsyncLocal context. The Akka→MEL bridge passes - // the event's structured properties as the state (AkkaLogState, carrying - // WithContext("SessionId", ...)); MEL's own structured logging passes - // FormattedLogValues (carrying a {SessionId} field). Both expose their fields as - // KeyValuePair sequences — read via the public interface, no Akka internals. - private static string? ExtractSessionId(TState state) - { - if (state is IEnumerable> nullableFields) - { - foreach (var field in nullableFields) - if (string.Equals(field.Key, "SessionId", StringComparison.Ordinal) && field.Value is { } value) - return Normalize(value); - } - else if (state is IEnumerable> fields) - { - foreach (var field in fields) - if (string.Equals(field.Key, "SessionId", StringComparison.Ordinal) && field.Value is { } value) - return Normalize(value); - } - - return null; - - static string? Normalize(object value) - { - var id = value.ToString(); - return string.IsNullOrWhiteSpace(id) ? null : id; - } + _provider.Enqueue(line); } } diff --git a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs index 3339e1aa6..950b2d3ae 100644 --- a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs +++ b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs @@ -140,6 +140,14 @@ public ChannelDeliveryTargetInfo? EffectiveDeliveryTarget /// public Action? OnSubAgentActivity { get; set; } + /// + /// Publishes a pre-formatted diagnostic line to the owning session's + /// session.log (via the session-log dispatcher). The session wires this so + /// tools/sub-agents can record session-relevant lifecycle lines explicitly. Null + /// when no session-log sink is wired (e.g. console/test hosts). + /// + public Action? EmitSessionLogLine { get; set; } + /// /// Factory delegate for spawning subagent actors as children of the owning session. /// Wired by LlmSessionActor so subagents are supervised and lifecycle-managed From 95510670c0d5ca059ba71a87739866eb7dbd9791 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 15:31:38 +0000 Subject: [PATCH 04/22] fix(logging): publish routed-skill sub-agent spawn lifecycle to session.log (#1472 step 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The routed-skill spawn path (LlmSessionActor.ExecuteRoutedSkillAsync) built its own ToolExecutionContext and wired SpawnChildActor + OnSubAgentActivity but never set EmitSessionLogLine, so skill-routed sub-agent spawns — and critically their failure breadcrumbs — were invisible in session.log. That is the original #1467 gap still present on one of the three spawn paths. Wire EmitSessionLogLine to the session-log dispatcher, mirroring the tool-execution dispatch path, and fix the stale SessionLogDiagnostic doc comment that still described the deleted MEL-sink-routing model. Test: Routed_slash_command_publishes_spawn_lifecycle_to_session_log (verified failing without the fix). --- .../Sessions/SubAgentSpawnIntegrationTests.cs | 48 +++++++++++++++++++ .../Protocol/SessionLogDiagnostic.cs | 10 ++-- .../Sessions/LlmSessionActor.cs | 11 +++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index 196ec93bc..c6d2a823d 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -539,6 +539,54 @@ await sessionManager.Ask(new SendUserMessage && (m.Text?.Contains("Context:", StringComparison.Ordinal) ?? false)); } + [Fact] + public async Task Routed_slash_command_publishes_spawn_lifecycle_to_session_log() + { + // Regression: the routed-skill spawn path builds its own ToolExecutionContext + // and must wire EmitSessionLogLine, exactly like the tool-execution dispatch + // path. Without it the sub-agent spawn lifecycle (and, more importantly, spawn + // failures — the #1467 gap) never reach the parent's session.log. Register a + // probe as the session-log dispatcher before the session recovers (the actor + // resolves it lazily on RecoveryCompleted) and assert the breadcrumb arrives. + var logProbe = CreateTestProbe("routed-slash-session-log"); + ActorRegistry.For(Sys).Register(logProbe.Ref, overwrite: true); + + var sessionId = new SessionId("test-channel/routed-slash-session-log"); + var sessionManager = ActorRegistry.Get(); + var subscriber = CreateTestProbe("routed-slash-session-log-events"); + + await sessionManager.Ask(new JoinSession(subscriber) + { + SessionId = sessionId, + Filter = OutputFilter.Full + }, TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + + await sessionManager.Ask(new SendUserMessage + { + SessionId = sessionId, + Content = "/ops-route check daemon health", + Source = BuildPersonalSource() + }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); + + // The spawner publishes "spawn requested" first, then "completed" once the + // routed sub-agent run returns — both must land on the dispatcher. + var requested = (SessionLogDiagnostic)await logProbe.FishForMessageAsync( + m => m is SessionLogDiagnostic d + && d.SessionId == sessionId + && d.Line.Contains("SubAgent [summarizer] spawn requested", StringComparison.Ordinal), + TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal(sessionId, requested.SessionId); + + await logProbe.FishForMessageAsync( + m => m is SessionLogDiagnostic d + && d.SessionId == sessionId + && d.Line.Contains("SubAgent [summarizer] completed", StringComparison.Ordinal), + TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken); + } + [Fact] public async Task Reminder_sourced_slash_command_routes_like_normal_slash_dispatch() { diff --git a/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs b/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs index 698ba0f34..76e407edc 100644 --- a/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs +++ b/src/Netclaw.Actors/Protocol/SessionLogDiagnostic.cs @@ -8,11 +8,11 @@ namespace Netclaw.Actors.Protocol; /// -/// Pre-formatted diagnostic line carried from the MEL logger provider to the -/// SessionLogDispatcher. The provider reads the session id off each log -/// event's structured state and embeds it here so the dispatcher routes the line -/// by message field, not by any ambient context that would not flow across actor -/// mailboxes. +/// Pre-formatted diagnostic line published explicitly into a session's +/// session.log via the SessionLogDispatcher. The producer names the +/// owning on the message itself, so the dispatcher routes +/// the line by message field rather than by ambient context (which would not flow +/// across actor mailboxes) or by inferring intent from log metadata at the sink. /// public sealed record SessionLogDiagnostic(SessionId SessionId, string Line) : IWithSessionId, INoSerializationVerificationNeeded; diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 1be1980a7..977785baa 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -3048,6 +3048,17 @@ await self.Ask( timeout: _config.ToolExecutionTimeout, cancellationToken: ct); + // Publish the routed sub-agent's spawn lifecycle into this session's + // session.log, exactly as the tool-execution dispatch path does. Without + // it the routed-skill path drops the explicit breadcrumbs SubAgentSpawner + // emits — notably the spawn-failure reasons (#1467) that are otherwise + // invisible to an operator reading the transcript. + var logActor = _logActor; + var tp = _timeProvider; + var routedSessionId = _sessionId; + context.EmitSessionLogLine = line => + logActor?.Tell(new SessionLogDiagnostic(routedSessionId, $"[{tp.GetUtcNow():o}] {line}")); + context.OnSubAgentActivity = info => { self.Tell(new RoutedSkillSubAgentActivity( From 748b2bb9ff285e096644227d7a4c64313ee7e2bd Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 15:32:18 +0000 Subject: [PATCH 05/22] feat(logging): correlate LLM chat-client diagnostics to sessions via explicit ChatOptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting the SessionDiagnosticsContext AsyncLocal (#1472 step 6) removed the only way the session-agnostic chat-client decorators (logging / retry / routing) knew which session a call belonged to. LLM timing, retries, and — most importantly — provider failover/outage events lost all session correlation: gone from session.log (intentional) and carrying no SessionId field in Seq either. Restore the Seq/OTLP correlation explicitly, without resurrecting ambient state that does not flow across the actor mailbox boundary: - SessionScopedChatOptions carries the owning session id on the call's ChatOptions as a typed property — NOT in AdditionalProperties, which the self-hosted provider forwards verbatim onto the wire (a dictionary entry would leak the id into the LLM request body). LlmSessionActor and SubAgentActor (parent id) set it on every model call; sidecar calls stay session-agnostic. - ChatClientSessionScope re-opens the SessionId logging scope in all three decorators, sourced from the options. IncludeScopes surfaces it as a filterable Seq field; session.log stays narrow (no re-flood). Tests: decorator scope attach + negatives, a wire-leak guard proving the id never reaches the provider request body, and builder-side coverage that the session and sub-agent actors emit the carrier with the right id. Updates the netclaw-operations diagnostics skill (2.20.0) to state precisely where each line type is filterable (inline grep vs Seq scope). --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../references/diagnostics.md | 15 +++++-- .../Sessions/LlmSessionIntegrationTests.cs | 6 +++ .../Sessions/SubAgentSpawnIntegrationTests.cs | 9 ++++ .../Sessions/LlmSessionActor.cs | 13 +++--- .../Sessions/SessionScopedChatOptions.cs | 34 ++++++++++++++ src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 19 +++++--- .../Configuration/LoggingChatClientTests.cs | 34 ++++++++++++++ .../OpenAiCompatibleChatClientTests.cs | 29 ++++++++++++ .../Configuration/RetryingChatClientTests.cs | 25 +++++++++++ .../Configuration/RoutingChatClientTests.cs | 25 +++++++++++ .../Configuration/ScopeCapturingLogger.cs | 45 +++++++++++++++++++ .../Configuration/ChatClientSessionScope.cs | 35 +++++++++++++++ .../Configuration/LoggingChatClient.cs | 14 ++++-- .../Configuration/RetryingChatClient.cs | 19 +++++--- .../Configuration/RoutingChatClient.cs | 16 +++++-- 16 files changed, 309 insertions(+), 31 deletions(-) create mode 100644 src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs create mode 100644 src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs create mode 100644 src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 7f00a4c59..dec7f1351 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.19.0" + version: "2.20.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 364ece2f2..116e5bdf0 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -45,9 +45,18 @@ What to expect inside `session.log`: if a critical line appears missing. - Most actor and operational logs are NOT in `session.log` — only explicitly published lines are. An actor's own lifecycle logging, and the LLM-client / - HTTP / retry / failover decorator internals, go to `daemon.log` (filterable - by the `SessionId` structured field where the line carries one). For an - actor's full lifecycle or deep LLM-call internals, read `daemon.log`. + HTTP / retry / failover decorator internals, go to `daemon.log`. How to + correlate them to a session depends on the line: + - Actor lines carry the session id **inline** (e.g. `session=...`), so you + can `grep` the `daemon.log` text file by session id directly. + - LLM-client / retry / **provider failover & outage** decorator lines carry + the session id as a `SessionId` **logging-scope attribute** (filterable in + Seq/OTLP), not inline in the `daemon.log` text — the file sink does not + render scopes. Grepping the text file by session id will miss them; filter + on the `SessionId` field in Seq instead. + + For an actor's full lifecycle or deep LLM-call internals, read `daemon.log` + (and Seq, for per-session LLM-pipeline correlation). | Symptom | Check | |---------|-------| diff --git a/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs index 9da7e4c09..45607f873 100644 --- a/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/LlmSessionIntegrationTests.cs @@ -1678,6 +1678,11 @@ internal sealed class FakeChatClient : IChatClient public List> ReceivedMessages { get; } = []; public List> ReceivedToolNames { get; } = []; + /// The object passed on each call, so tests can + /// assert the session actor threads a SessionScopedChatOptions carrier through + /// for per-session log correlation. + public List ReceivedOptions { get; } = []; + public TimeSpan Delay { get; set; } = TimeSpan.Zero; /// @@ -1778,6 +1783,7 @@ public async Task GetResponseAsync( var messageList = messages.ToList(); ReceivedMessages.Add(messageList); + ReceivedOptions.Add(options); ReceivedToolNames.Add(options?.Tools? .Select(t => t is AIFunction f ? f.Name : t.GetType().Name) .ToList() diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index c6d2a823d..834d3eef4 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -265,6 +265,15 @@ await sessionManager.Ask(new SendUserMessage m.Role == Microsoft.Extensions.AI.ChatRole.System && (m.Text?.Contains("test assistant with subagent support", StringComparison.Ordinal) ?? false)); Assert.DoesNotContain(subagentCall, m => m.Role == Microsoft.Extensions.AI.ChatRole.User && (m.Text?.Contains("Use a subagent to summarize the file", StringComparison.Ordinal) ?? false)); + + // The session actor must thread a SessionScopedChatOptions carrier so the + // chat-client decorators can correlate LLM diagnostics to the session. The + // sub-agent call carries the *parent* session id (collapsing the scope suffix), + // so both the main turn and the sub-agent's LLM calls correlate to one session. + var mainOptions = Assert.IsType(_clientProvider.Main.ReceivedOptions[^1]); + Assert.Equal(sessionId.Value, mainOptions.SessionId); + var subagentOptions = Assert.IsType(_clientProvider.Compaction.ReceivedOptions[^1]); + Assert.Equal(sessionId.Value, subagentOptions.SessionId); } [Fact] diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 977785baa..403ff3698 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -2747,14 +2747,13 @@ private void FireLlmCall(string? recallQuery = null, bool forceNoTools = false) var client = _chatClient; var exposedTools = ResolveExposedToolsForCurrentTurn(); - ChatOptions? options = null; + // Always carry the session id so the session-agnostic chat-client decorators + // (logging/retry/routing) can correlate LLM diagnostics — including provider + // failover/outage — back to this session in Seq. Tools are attached only when + // the turn exposes them; an empty Tools list is wire-equivalent to no options. + var options = new SessionScopedChatOptions { SessionId = _sessionId.Value }; if (!forceNoTools && exposedTools.Count > 0) - { - options = new ChatOptions - { - Tools = [.. exposedTools] - }; - } + options.Tools = [.. exposedTools]; _watchdog.Start(ProcessingWatchdog.LlmCall, _config.PrefillTimeout, Timers, _config.NoProgressTimeout); diff --git a/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs b/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs new file mode 100644 index 000000000..5e316fbf5 --- /dev/null +++ b/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs @@ -0,0 +1,34 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.AI; + +namespace Netclaw.Actors.Sessions; + +/// +/// that also names the owning session, so the +/// session-agnostic chat-client decorators (logging / retry / routing, which live in +/// Netclaw.Daemon and are shared singletons across every session) can tag their +/// diagnostics with SessionId for per-session correlation in Seq/OTLP. +/// +/// This restores the correlation that the deleted SessionDiagnosticsContext +/// AsyncLocal used to provide, but through an explicit value carried on the call seam +/// rather than ambient context — ambient state does not flow across the actor mailbox +/// boundary between the session actor and the chat-client pipeline. +/// +/// The id rides as a typed CLR property, deliberately NOT in +/// : provider clients forward +/// AdditionalProperties verbatim onto the wire (see +/// OpenAiCompatibleChatClient), so a dictionary entry would leak the session id +/// into the LLM request body. A subclass property is invisible to the provider +/// serializers, which only read known fields plus AdditionalProperties. +/// +public sealed class SessionScopedChatOptions : ChatOptions +{ + /// The session whose turn this LLM call belongs to. For sub-agent calls + /// this is the parent session id, so a sub-agent's LLM diagnostics correlate to the + /// session that spawned it. + public required string SessionId { get; init; } +} diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index c8c80b466..a727410ed 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -98,6 +98,11 @@ [Subagent Execution Contract] private IParentApprovalBridge? _approvalBridge; private ChannelWriter? _activitySink; + // Parent session id (scopeId with the "/subagent/..." suffix stripped). Carried on + // the sub-agent's ChatOptions so its LLM diagnostics correlate to the spawning + // session in Seq, matching the SessionId its enriched logger already uses. + private string? _parentSessionId; + // Default wait-for-first-delta budget when the spawn message carries none // (direct/test callers). Mirrors SessionConfig.PrefillTimeout so an unset // prefill never collapses to the tighter inter-delta budget — that collapse @@ -262,6 +267,7 @@ private void Idle() // parent logs share one filterable attribute (and route to the same // session.log); SubSessionId isolates a single run within that session. var parentSessionId = SubAgentSessionScope.NormalizeSessionId(scopeId); + _parentSessionId = parentSessionId; var enrichedLog = Context.GetLogger(); if (!string.IsNullOrWhiteSpace(parentSessionId)) enrichedLog = enrichedLog.WithContext("SessionId", parentSessionId); @@ -662,13 +668,16 @@ private void FireLlmCall(bool forceNoTools = false) var messages = new List(_history); var callId = ++_llmCallId; - ChatOptions? options = null; + // Carry the parent session id (when known) so the chat-client decorators + // correlate this sub-agent's LLM diagnostics to the spawning session in Seq, + // just like the main session path. Direct/test callers leave it unset. + ChatOptions? options = _parentSessionId is { Length: > 0 } sid + ? new SessionScopedChatOptions { SessionId = sid } + : null; if (!forceNoTools && _aiTools.Count > 0) { - options = new ChatOptions - { - Tools = [.. _aiTools] - }; + options ??= new ChatOptions(); + options.Tools = [.. _aiTools]; } _log.Info( diff --git a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs index d0a50a199..8cce9d710 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; +using Netclaw.Actors.Sessions; using Netclaw.Daemon.Configuration; using Xunit; // Netclaw's LoggingChatClient collides with Microsoft.Extensions.AI.LoggingChatClient @@ -93,6 +94,39 @@ public async Task Streaming_LogsPromptDumpWhenTraceEnabled() Assert.Contains(logs, l => l.Contains("role=user")); } + [Fact] + public async Task Streaming_attaches_SessionId_scope_from_options() + { + // The decorator is session-agnostic; it learns the session from the call's + // SessionScopedChatOptions, replacing the deleted AsyncLocal diagnostics context. + var logger = new ScopeCapturingLogger(); + var client = new LoggingChatClient(new FakeChatClient(streaming: true), logger); + + await foreach (var _ in client.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken)) + { + } + + Assert.True(logger.HasSessionScope("ch/thread")); + } + + [Fact] + public async Task Streaming_with_plain_options_attaches_no_scope() + { + // A sidecar/session-agnostic call carries a plain ChatOptions: no session scope. + var logger = new ScopeCapturingLogger(); + var client = new LoggingChatClient(new FakeChatClient(streaming: true), logger); + + await foreach (var _ in client.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], new ChatOptions(), TestContext.Current.CancellationToken)) + { + } + + Assert.False(logger.HasAnySessionScope()); + } + private static async Task Drain(IChatClient client) { await foreach (var _ in client.GetStreamingResponseAsync( diff --git a/src/Netclaw.Daemon.Tests/Configuration/OpenAiCompatibleChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/OpenAiCompatibleChatClientTests.cs index ba41ca78a..8607e4896 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/OpenAiCompatibleChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/OpenAiCompatibleChatClientTests.cs @@ -98,6 +98,35 @@ [new ChatMessage(ChatRole.User, "hello")], Assert.Contains("\"required\":[\"query\"]", body, StringComparison.Ordinal); } + [Fact] + public async Task DoesNotLeakSessionId_FromSessionScopedChatOptions_ToWire() + { + // SessionScopedChatOptions carries the session id as a CLR property, NOT in + // AdditionalProperties — which this client forwards verbatim as top-level JSON. + // The session id must never appear in the outbound request body. + string? body = null; + using var handler = new RecordingHandler(req => + { + body = req.Content is null ? null : req.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"id\":\"1\",\"model\":\"test\",\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"hi\"}}]}", Encoding.UTF8, "application/json") + }; + }); + using var httpClient = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:8000") }; + var endpoint = OpenAiCompatibleEndpoint.FromBaseUrl("http://localhost:8000"); + var client = new OpenAiCompatibleChatClient(httpClient, endpoint, "test-model"); + + await client.GetResponseAsync( + [new ChatMessage(ChatRole.User, "hello")], + new Netclaw.Actors.Sessions.SessionScopedChatOptions { SessionId = "C123/167.42" }, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.NotNull(body); + Assert.DoesNotContain("SessionId", body, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("C123/167.42", body, StringComparison.Ordinal); + } + [Fact] public async Task CollapsesMultipleSystemMessages_IntoSingleLeadingSystemMessage() { diff --git a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs index 6441ff682..4d69755e4 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Sessions; using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; @@ -117,6 +118,30 @@ await Assert.ThrowsAsync(() => Assert.Equal(4, attempts); } + [Fact] + public async Task RetryWarning_carries_SessionId_scope_from_options() + { + // The retry warning is the hottest failure-correlation line. It must inherit the + // session id from the call's options so retry storms correlate by session in Seq. + var attempts = 0; + var fake = new FakeChatClient((_, _, _) => + { + attempts++; + if (attempts < 2) + throw new HttpRequestException("server error", null, HttpStatusCode.InternalServerError); + return Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "ok")])); + }); + var logger = new ScopeCapturingLogger(); + var client = new RetryingChatClient(fake, _policy, logger); + + await client.GetResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken); + + Assert.True(logger.HasSessionScope("ch/thread")); + } + [Fact] public async Task DoesNotRetryNonTransientErrors() { diff --git a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs index 4670c0904..e09fa1da5 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Sessions; using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; @@ -80,6 +81,30 @@ await Assert.ThrowsAsync(() => Assert.DoesNotContain(sink.Alerts, a => a.Category == AlertType.ProviderFailover); } + [Fact] + public async Task Failover_log_carries_SessionId_scope_from_options() + { + // Provider failover/outage is logged outside any per-pipeline LoggingChatClient + // scope, so RoutingChatClient must attach the session id (from the call options) + // itself — otherwise outages are uncorrelatable to the affected session in Seq. + var sink = new CapturingSink(); + var logger = new ScopeCapturingLogger(); + var primary = new FakeChatClient((_, _, _) => throw new HttpRequestException("primary down")); + var fallback = new FakeChatClient((_, _, _) => + Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "fallback")]))); + var client = new RoutingChatClient( + new StubRouter([primary, fallback]), + new ChatRoutingContext { Role = ModelRole.Main }, + sink, logger, TimeProvider.System); + + await client.GetResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken); + + Assert.True(logger.HasSessionScope("ch/thread")); + } + [Fact] public async Task DoesNotFailover_OnCancellation() { diff --git a/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs b/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs new file mode 100644 index 000000000..2b93e2bba --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs @@ -0,0 +1,45 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.Logging; + +namespace Netclaw.Daemon.Tests.Configuration; + +/// +/// Test logger that records the state objects passed to +/// so tests can assert which scopes were opened around a logging call. Returns +/// null from BeginScope (a valid no-op for using var); only the +/// captured state matters. +/// +internal sealed class ScopeCapturingLogger : ILogger +{ + private const string SessionIdKey = "SessionId"; + + public List Scopes { get; } = []; + + public void Log( + LogLevel logLevel, EventId eventId, TState state, + Exception? exception, Func formatter) + { + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) where TState : notnull + { + Scopes.Add(state); + return null; + } + + /// True if a scope tagging the given session id was opened. + public bool HasSessionScope(string expectedId) => + Scopes.Any(s => s is IEnumerable> kvps + && kvps.Any(kv => kv.Key == SessionIdKey && kv.Value is string v && v == expectedId)); + + /// True if any session-id scope (regardless of value) was opened. + public bool HasAnySessionScope() => + Scopes.Any(s => s is IEnumerable> kvps + && kvps.Any(kv => kv.Key == SessionIdKey)); +} diff --git a/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs new file mode 100644 index 000000000..553c6a5fa --- /dev/null +++ b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs @@ -0,0 +1,35 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Netclaw.Actors.Sessions; + +namespace Netclaw.Daemon.Configuration; + +/// +/// Opens a logging scope tagging a chat-client decorator's log lines with the owning +/// SessionId, read from on the call's +/// . The OTLP exporter has IncludeScopes enabled, so the +/// scope surfaces SessionId as a filterable attribute in Seq — matching the +/// WithContext("SessionId", …) key the session/channel actors already use, so +/// LLM-pipeline diagnostics (timing, retries, provider failover/outage) correlate to the +/// same session as the actor logs. +/// +/// The decorators are session-agnostic singletons, so the id must arrive on the call +/// seam: this reads it from the options object rather than ambient context (which would +/// not flow across the actor mailbox boundary). Returns null — a no-op +/// using — for sidecar/session-agnostic calls that carry a plain +/// . +/// +internal static class ChatClientSessionScope +{ + private const string SessionIdKey = "SessionId"; + + public static IDisposable? Begin(ILogger logger, ChatOptions? options) => + options is SessionScopedChatOptions { SessionId: { Length: > 0 } sessionId } + ? logger.BeginScope(new[] { new KeyValuePair(SessionIdKey, sessionId) }) + : null; +} diff --git a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs index 1f63c123c..ae6815681 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs @@ -8,15 +8,18 @@ using System.Text; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; +using Netclaw.Actors.Sessions; namespace Netclaw.Daemon.Configuration; /// /// Decorates an with logging for elapsed time, token -/// usage, and errors. These diagnostics go to daemon.log only (the decorator -/// is session-agnostic); they are not published to a session's session.log. -/// Stateless and safe to share across sessions. Netclaw issues only streaming -/// requests, so only the streaming path is instrumented; the inherited +/// usage, and errors. These diagnostics go to daemon.log (the decorator is +/// session-agnostic); they are not published to a session's session.log, but +/// they are tagged with the owning SessionId — read from +/// on the call — so they correlate to a session +/// in Seq/OTLP. Stateless and safe to share across sessions. Netclaw issues only +/// streaming requests, so only the streaming path is instrumented; the inherited /// non-streaming pass-through is unused. /// public sealed class LoggingChatClient : DelegatingChatClient @@ -40,6 +43,9 @@ public override async IAsyncEnumerable GetStreamingResponseA [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { + // Tag every line this call emits (prompt summary, timing, token usage, errors) + // with the owning session so they correlate in Seq. No-op for sidecar calls. + using var sessionScope = ChatClientSessionScope.Begin(_logger, options); var messageList = messages as IReadOnlyList ?? messages.ToList(); LogPromptDiagnostics(messageList, options); diff --git a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs index d710bfec4..6ad906f35 100644 --- a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs @@ -50,7 +50,7 @@ public override async Task GetResponseAsync( catch (Exception ex) when (!cancellationToken.IsCancellationRequested && _policy.ShouldRetry(ex, attempt)) { - await BackoffAsync(ex, attempt, cancellationToken); + await BackoffAsync(ex, attempt, options, cancellationToken); attempt++; } } @@ -79,7 +79,7 @@ public override async IAsyncEnumerable GetStreamingResponseA catch (Exception ex) when (!cancellationToken.IsCancellationRequested && _policy.ShouldRetry(ex, attempt)) { - await BackoffAsync(ex, attempt, cancellationToken); + await BackoffAsync(ex, attempt, options, cancellationToken); attempt++; continue; } @@ -113,18 +113,23 @@ public override async IAsyncEnumerable GetStreamingResponseA if (preFirstChunkFailure is null) yield break; // clean completion (or a post-first-chunk throw already unwound) - await BackoffAsync(preFirstChunkFailure, attempt, cancellationToken); + await BackoffAsync(preFirstChunkFailure, attempt, options, cancellationToken); attempt++; // outer loop re-initiates the stream } } - private async Task BackoffAsync(Exception ex, int attempt, CancellationToken cancellationToken) + private async Task BackoffAsync(Exception ex, int attempt, ChatOptions? options, CancellationToken cancellationToken) { var delay = _policy.GetDelay(attempt); - _logger.LogWarning(ex, - "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", - attempt + 1, _policy.MaxRetries, delay.TotalSeconds); + // Retry is the hottest failure path; tag the warning with the session id (from + // the call's options) so retry storms correlate by session in Seq. + using (ChatClientSessionScope.Begin(_logger, options)) + { + _logger.LogWarning(ex, + "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", + attempt + 1, _policy.MaxRetries, delay.TotalSeconds); + } await Task.Delay(delay, _timeProvider, cancellationToken); } diff --git a/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs b/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs index efe0e7eaa..337e9e71d 100644 --- a/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RoutingChatClient.cs @@ -64,12 +64,12 @@ public async Task GetResponseAsync( catch (Exception ex) when (!cancellationToken.IsCancellationRequested && !isLast) { EmitFailover(ex); - _logger.Log(LogLevel.Warning, ex, "LLM provider failed, failing over to next candidate"); + LogWithSession(LogLevel.Warning, ex, "LLM provider failed, failing over to next candidate", options); } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { EmitUnreachable(ex, candidates.Count); - _logger.Log(LogLevel.Error, ex, UnreachableMessage(candidates.Count)); + LogWithSession(LogLevel.Error, ex, UnreachableMessage(candidates.Count), options); throw; } } @@ -137,12 +137,12 @@ public async IAsyncEnumerable GetStreamingResponseAsync( if (isLast) { EmitUnreachable(failure, candidates.Count); - _logger.Log(LogLevel.Error, failure, UnreachableMessage(candidates.Count)); + LogWithSession(LogLevel.Error, failure, UnreachableMessage(candidates.Count), options); ExceptionDispatchInfo.Capture(failure).Throw(); } EmitFailover(failure); - _logger.Log(LogLevel.Warning, failure, "LLM provider failed, failing over to next candidate"); + LogWithSession(LogLevel.Warning, failure, "LLM provider failed, failing over to next candidate", options); // outer loop advances to the next candidate } } @@ -180,6 +180,14 @@ private void EmitUnreachable(Exception ex, int candidateCount) => AlertSeverity.Critical, context: new Dictionary { ["error"] = ex.Message })); + // Failover/outage events are logged outside any per-pipeline LoggingChatClient + // scope, so attach the session id (carried on the call's options) here so they + // correlate by session in Seq. + private void LogWithSession(LogLevel level, Exception ex, string message, ChatOptions? options) + { + using (ChatClientSessionScope.Begin(_logger, options)) + _logger.Log(level, ex, message); + } } /// From 7dace131e2f8bb06a06036aa133c2c28e662a9ed Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 16:09:03 +0000 Subject: [PATCH 06/22] refactor(logging): address code-review findings on session-log correlation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the two preceding commits, from a multi-agent review pass: - Dedupe the explicit session-log emitter: the routed-skill path and the tool-execution dispatch path built the identical logActor?.Tell(SessionLogDiagnostic(...)) closure. Collapse both onto one private CreateSessionLogEmitter() so their timestamp format and routing cannot drift. - Drop a now-dead null-conditional (options?.Tools) — options is unconditionally non-null since it became the SessionScopedChatOptions carrier. - Cover the production (streaming) path: the chat-client scope tests only exercised the non-streaming GetResponseAsync branch, but StreamingResponseReader only ever calls GetStreamingResponseAsync. Add streaming retry-backoff and provider-failover scope tests, plus a composed-pipeline test proving the SessionScopedChatOptions subclass survives Logging->Retry by reference (guards the correlation against a future cloning decorator). - Correct the diagnostics skill: session id is reliably filterable in Seq for both actor and decorator lines; daemon.log *text* grep only catches lines that template the id inline. The prior wording over-claimed inline greppability for context-enriched actor logs. --- .../references/diagnostics.md | 25 +++++++----- .../Sessions/LlmSessionActor.cs | 40 ++++++++++++------- .../PipelineChatClientFactoryTests.cs | 30 ++++++++++++++ .../Configuration/RetryingChatClientTests.cs | 25 ++++++++++++ .../Configuration/RoutingChatClientTests.cs | 25 ++++++++++++ 5 files changed, 121 insertions(+), 24 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 116e5bdf0..58181f39b 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -45,18 +45,23 @@ What to expect inside `session.log`: if a critical line appears missing. - Most actor and operational logs are NOT in `session.log` — only explicitly published lines are. An actor's own lifecycle logging, and the LLM-client / - HTTP / retry / failover decorator internals, go to `daemon.log`. How to - correlate them to a session depends on the line: - - Actor lines carry the session id **inline** (e.g. `session=...`), so you - can `grep` the `daemon.log` text file by session id directly. - - LLM-client / retry / **provider failover & outage** decorator lines carry - the session id as a `SessionId` **logging-scope attribute** (filterable in - Seq/OTLP), not inline in the `daemon.log` text — the file sink does not - render scopes. Grepping the text file by session id will miss them; filter - on the `SessionId` field in Seq instead. + HTTP / retry / failover decorator internals, go to `daemon.log`. How the + session id is carried — and so how you filter by it — depends on the line: + - **Seq/OTLP is the dependable filter.** Both the actor logs (session id + attached via `WithContext("SessionId", …)`) and the LLM-client / retry / + **provider failover & outage** decorator logs (attached via a `SessionId` + logging scope) surface `SessionId` as a structured field. Filtering on it + in Seq catches every one of these lines for a session. + - **`grep` of the `daemon.log` text file is partial.** It finds only lines + that template the id straight into the message (`session=…` / `{SessionId}` + — most service, binding, gateway, and spawn-breadcrumb lines). Lines that + carry the id only as a context/scope attribute (the session/sub-agent + actor's own lifecycle logs and the chat-client decorator internals) may not + render it in the text file, so a text grep can miss them — use Seq for + those. For an actor's full lifecycle or deep LLM-call internals, read `daemon.log` - (and Seq, for per-session LLM-pipeline correlation). + (and Seq, for reliable per-session correlation). | Symptom | Check | |---------|-------| diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 403ff3698..d94dfd68a 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -1972,10 +1972,9 @@ private void DispatchToolBatch( }; // Sub-agent / tool lifecycle lines are published explicitly into this session's - // session.log via the dispatcher (logActor). Routing is intentional here, not - // inferred from log metadata at the sink. - Action emitSessionLogLine = line => - logActor?.Tell(new SessionLogDiagnostic(sessionId, $"[{tp.GetUtcNow():o}] {line}")); + // session.log via the dispatcher. Routing is intentional here, not inferred from + // log metadata at the sink. Same emitter as the routed-skill path (one definition). + Action emitSessionLogLine = CreateSessionLogEmitter(); // Marshal child-actor spawning back onto the session actor thread. Func> spawnChildActor = async (props, name, ct) => @@ -2759,7 +2758,7 @@ private void FireLlmCall(string? recallQuery = null, bool forceNoTools = false) TurnLog().Info("turn_llm_call_start messages={MessageCount} toolsEnabled={ToolsEnabled} forceNoTools={ForceNoTools} callId={CallId}", messages.Count, - options?.Tools?.Count > 0, + options.Tools?.Count > 0, forceNoTools, _activeCallId); @@ -3048,15 +3047,11 @@ await self.Ask( cancellationToken: ct); // Publish the routed sub-agent's spawn lifecycle into this session's - // session.log, exactly as the tool-execution dispatch path does. Without - // it the routed-skill path drops the explicit breadcrumbs SubAgentSpawner - // emits — notably the spawn-failure reasons (#1467) that are otherwise - // invisible to an operator reading the transcript. - var logActor = _logActor; - var tp = _timeProvider; - var routedSessionId = _sessionId; - context.EmitSessionLogLine = line => - logActor?.Tell(new SessionLogDiagnostic(routedSessionId, $"[{tp.GetUtcNow():o}] {line}")); + // session.log, exactly as the tool-execution dispatch path does (one shared + // emitter). Without it the routed-skill path drops the explicit breadcrumbs + // SubAgentSpawner emits — notably the spawn-failure reasons (#1467) that are + // otherwise invisible to an operator reading the transcript. + context.EmitSessionLogLine = CreateSessionLogEmitter(); context.OnSubAgentActivity = info => { @@ -4261,6 +4256,23 @@ private void EmitOutput(SessionOutput output, OutputFilter requiredFlag = Output _observerActor?.Tell(output); } + /// + /// Builds the explicit session-log emitter wired into + /// . Both the tool-execution + /// dispatch path and the routed-skill path publish lifecycle/breadcrumb lines through + /// this single definition so their timestamp format and dispatcher routing never + /// diverge. _logActor is null only in unit-test scenarios that skip the hosting + /// wiring; in production it is resolved on recovery, so the line is delivered. Snapshots + /// the fields so the returned closure is safe to invoke off the actor thread. + /// + private Action CreateSessionLogEmitter() + { + var logActor = _logActor; + var timeProvider = _timeProvider; + var sessionId = _sessionId; + return line => logActor?.Tell(new SessionLogDiagnostic(sessionId, $"[{timeProvider.GetUtcNow():o}] {line}")); + } + private async Task PersistApprovalCandidatesAsync( PendingToolInteraction pending, ApprovalDecision decision, diff --git a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs index b92516be7..c9ea25bce 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Netclaw.Actors.Sessions; using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; @@ -52,6 +53,35 @@ [new ChatMessage(ChatRole.User, "hi")], cancellationToken: TestContext.Current.C Assert.Contains(logs, l => l.Contains("LLM streaming call completed")); // Logging middleware wired } + [Fact] + public async Task Compose_streaming_tags_SessionId_scope_through_pipeline() + { + // Cross-cutting invariant: SessionScopedChatOptions must survive *by reference* + // through the composed Logging -> Retry pipeline (no decorator clones it down to a + // base ChatOptions), so the streaming production path still surfaces SessionId as a + // Seq scope. A future decorator that rebuilt options would break this test, not just + // a unit decorator tested in isolation. + var logger = new ScopeCapturingLogger(); + var pipeline = PipelineChatClientFactory.Compose( + new FakeChatClient(streaming: true), _policy, new SingleLoggerFactory(logger), TimeProvider.System); + + await foreach (var _ in pipeline.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken)) + { + } + + Assert.True(logger.HasSessionScope("ch/thread")); + } + + private sealed class SingleLoggerFactory(ILogger logger) : ILoggerFactory + { + public ILogger CreateLogger(string categoryName) => logger; + public void AddProvider(ILoggerProvider provider) { } + public void Dispose() { } + } + private sealed class ListLoggerFactory : ILoggerFactory { private readonly List _logs; diff --git a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs index 4d69755e4..63bb6c587 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs @@ -142,6 +142,31 @@ [new ChatMessage(ChatRole.User, "hi")], Assert.True(logger.HasSessionScope("ch/thread")); } + [Fact] + public async Task StreamingRetryWarning_carries_SessionId_scope_from_options() + { + // Production only ever calls GetStreamingResponseAsync (StreamingResponseReader), + // so the streaming backoff path — not the non-streaming one above — is the path + // that must tag retry warnings with the session id. + var attempts = 0; + var fake = new FakeChatClient(streamHandler: (_, _, ct) => + { + attempts++; + return ThrowBeforeChunkThenYield(attempts, failUntil: 2, ct); + }); + var logger = new ScopeCapturingLogger(); + var client = new RetryingChatClient(fake, _policy, logger); + + await foreach (var _ in client.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken)) + { + } + + Assert.True(logger.HasSessionScope("ch/thread")); + } + [Fact] public async Task DoesNotRetryNonTransientErrors() { diff --git a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs index e09fa1da5..fb4a5dd84 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs @@ -105,6 +105,31 @@ [new ChatMessage(ChatRole.User, "hi")], Assert.True(logger.HasSessionScope("ch/thread")); } + [Fact] + public async Task Streaming_failover_log_carries_SessionId_scope_from_options() + { + // Production takes the streaming failover path (StreamingResponseReader), so the + // streaming branch — not the non-streaming one above — is the one that must tag + // provider-failover/outage warnings with the session id. + var sink = new CapturingSink(); + var logger = new ScopeCapturingLogger(); + var primary = new FakeChatClient(streamHandler: (_, _, ct) => ThrowBeforeFirstChunkAsync(true, ct)); + var fallback = new FakeChatClient(streamHandler: (_, _, ct) => SingleTextUpdateAsync("fallback", ct)); + var client = new RoutingChatClient( + new StubRouter([primary, fallback]), + new ChatRoutingContext { Role = ModelRole.Main }, + sink, logger, TimeProvider.System); + + await foreach (var _ in client.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken)) + { + } + + Assert.True(logger.HasSessionScope("ch/thread")); + } + [Fact] public async Task DoesNotFailover_OnCancellation() { From dad8fb41c40e89bf0db089acbb889ace3ca1eff7 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 16:51:28 +0000 Subject: [PATCH 07/22] refactor(logging): centralize the "SessionId"/"SubSessionId" log-attribute keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structured-logging key "SessionId" was hardcoded as a raw literal at ~9 WithContext sites across the actor system and every channel adapter, plus two private SessionIdKey consts in the chat-client scope helper and its test double. A rename to align OTLP attributes would have had to find every copy, and the producer could silently drift from the test assertion. Introduce Netclaw.Actors.Protocol.NetclawLogProperties (SessionId, SubSessionId) — Netclaw.Actors is already referenced by every consumer (Daemon, all Channels.*, Daemon.Tests), so no new project references — and route all logging-key usages through it. Type references, proto field names, DTO/JSON keys, and doc-comment prose are untouched; only WithContext/BeginScope keys changed. --- .../Protocol/NetclawLogProperties.cs | 32 +++++++++++++++++++ .../Sessions/LlmSessionActor.cs | 2 +- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 4 +-- .../DiscordSessionBindingActor.cs | 2 +- .../MattermostSessionBindingActor.cs | 2 +- .../SlackConversationActor.cs | 2 +- .../SlackThreadBindingActor.cs | 2 +- .../ChannelConversationActor.cs | 2 +- .../Configuration/ScopeCapturingLogger.cs | 7 ++-- .../Configuration/ChatClientSessionScope.cs | 5 ++- .../Gateway/SignalRSessionActor.cs | 2 +- 11 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 src/Netclaw.Actors/Protocol/NetclawLogProperties.cs diff --git a/src/Netclaw.Actors/Protocol/NetclawLogProperties.cs b/src/Netclaw.Actors/Protocol/NetclawLogProperties.cs new file mode 100644 index 000000000..682708f15 --- /dev/null +++ b/src/Netclaw.Actors/Protocol/NetclawLogProperties.cs @@ -0,0 +1,32 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- + +namespace Netclaw.Actors.Protocol; + +/// +/// Shared structured-logging attribute keys used across the actor system, +/// channel adapters, and chat-client decorators. Centralising these constants +/// ensures that every log producer emits the same filterable field name so log +/// aggregators (Seq, OTLP) can correlate entries across subsystem boundaries. +/// +public static class NetclawLogProperties +{ + /// + /// Structured-logging attribute key correlating a log line to a session + /// ({channelId}/{threadTs}). Used as the WithContext / BeginScope key so + /// actor logs and chat-client decorator logs share one filterable field in + /// Seq/OTLP. + /// + public const string SessionId = "SessionId"; + + /// + /// Structured-logging attribute key correlating a log line to a specific + /// sub-agent run within its parent session. Used alongside + /// so sub-agent and parent logs share a parent + /// filter while remaining independently queryable by run. + /// + public const string SubSessionId = "SubSessionId"; +} diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index d94dfd68a..bcb59cc92 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -246,7 +246,7 @@ public LlmSessionActor( PersistenceId = $"session-{entityId}"; // Enrich logger with session context — all log messages automatically include SessionId - _log = Context.GetLogger().WithContext("SessionId", _sessionId.Value); + _log = Context.GetLogger().WithContext(NetclawLogProperties.SessionId, _sessionId.Value); // Load all non-MCP tools for initial LLM calls. // MCP tools are loaded dynamically via search_tools and can be retained for a diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index a727410ed..410f19f5f 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -270,9 +270,9 @@ private void Idle() _parentSessionId = parentSessionId; var enrichedLog = Context.GetLogger(); if (!string.IsNullOrWhiteSpace(parentSessionId)) - enrichedLog = enrichedLog.WithContext("SessionId", parentSessionId); + enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SessionId, parentSessionId); if (!string.IsNullOrWhiteSpace(scopeId)) - enrichedLog = enrichedLog.WithContext("SubSessionId", scopeId); + enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SubSessionId, scopeId); _log = enrichedLog; // The run is bounded by a two-phase inactivity watchdog re-armed on diff --git a/src/Netclaw.Channels.Discord/DiscordSessionBindingActor.cs b/src/Netclaw.Channels.Discord/DiscordSessionBindingActor.cs index 918bd42c0..4c9bb6a6b 100644 --- a/src/Netclaw.Channels.Discord/DiscordSessionBindingActor.cs +++ b/src/Netclaw.Channels.Discord/DiscordSessionBindingActor.cs @@ -116,7 +116,7 @@ public DiscordSessionBindingActor( _log = Context.GetLogger() .WithContext("Adapter", "discord") - .WithContext("SessionId", _sessionId.Value) + .WithContext(NetclawLogProperties.SessionId, _sessionId.Value) .WithContext("DiscordChannelId", _channelId.Value) .WithContext("DiscordThreadOrMessageId", _threadOrMessageId.Value); diff --git a/src/Netclaw.Channels.Mattermost/MattermostSessionBindingActor.cs b/src/Netclaw.Channels.Mattermost/MattermostSessionBindingActor.cs index d372126e8..e2b4c5f19 100644 --- a/src/Netclaw.Channels.Mattermost/MattermostSessionBindingActor.cs +++ b/src/Netclaw.Channels.Mattermost/MattermostSessionBindingActor.cs @@ -110,7 +110,7 @@ public MattermostSessionBindingActor( _log = Context.GetLogger() .WithContext("Adapter", "mattermost") - .WithContext("SessionId", _sessionId.Value) + .WithContext(NetclawLogProperties.SessionId, _sessionId.Value) .WithContext("MattermostChannelId", _channelId.Value) .WithContext("MattermostRootPostId", _rootPostId.Value); diff --git a/src/Netclaw.Channels.Slack/SlackConversationActor.cs b/src/Netclaw.Channels.Slack/SlackConversationActor.cs index bf676ee56..1afdd5d91 100644 --- a/src/Netclaw.Channels.Slack/SlackConversationActor.cs +++ b/src/Netclaw.Channels.Slack/SlackConversationActor.cs @@ -117,7 +117,7 @@ public SlackConversationActor(SlackChannelId conversationId, SlackGatewayDepende : message.EventId.Value; var log = _log .WithContext("SlackThreadTs", threadTs.Value) - .WithContext("SessionId", sessionId.Value) + .WithContext(NetclawLogProperties.SessionId, sessionId.Value) .WithContext("TurnId", turnId) .WithContext("SlackEventId", message.EventId.Value); diff --git a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs index c69384f07..435dd1466 100644 --- a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs +++ b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs @@ -99,7 +99,7 @@ public SlackThreadBindingActor( _handle = new SessionPipelineHandle(dependencies.Pipeline, Context.GetLogger(), "slack-thread"); _log = Context.GetLogger() .WithContext("Adapter", "slack") - .WithContext("SessionId", _sessionId.Value) + .WithContext(NetclawLogProperties.SessionId, _sessionId.Value) .WithContext("SlackChannelId", _channelId) .WithContext("SlackThreadTs", _threadTs); diff --git a/src/Netclaw.Channels/ChannelConversationActor.cs b/src/Netclaw.Channels/ChannelConversationActor.cs index 678703785..6dc0783da 100644 --- a/src/Netclaw.Channels/ChannelConversationActor.cs +++ b/src/Netclaw.Channels/ChannelConversationActor.cs @@ -208,7 +208,7 @@ private void HandleGatewayMessage(TMessage message) var log = Log .WithContext(ThreadLogContextKey, threadKey) - .WithContext("SessionId", sessionId.Value) + .WithContext(NetclawLogProperties.SessionId, sessionId.Value) .WithContext("TurnId", turnId) .WithContext(EventLogContextKey, eventId); diff --git a/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs b/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs index 2b93e2bba..075e71bf1 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/ScopeCapturingLogger.cs @@ -4,6 +4,7 @@ // // ----------------------------------------------------------------------- using Microsoft.Extensions.Logging; +using Netclaw.Actors.Protocol; namespace Netclaw.Daemon.Tests.Configuration; @@ -15,8 +16,6 @@ namespace Netclaw.Daemon.Tests.Configuration; /// internal sealed class ScopeCapturingLogger : ILogger { - private const string SessionIdKey = "SessionId"; - public List Scopes { get; } = []; public void Log( @@ -36,10 +35,10 @@ public void Log( /// True if a scope tagging the given session id was opened. public bool HasSessionScope(string expectedId) => Scopes.Any(s => s is IEnumerable> kvps - && kvps.Any(kv => kv.Key == SessionIdKey && kv.Value is string v && v == expectedId)); + && kvps.Any(kv => kv.Key == NetclawLogProperties.SessionId && kv.Value is string v && v == expectedId)); /// True if any session-id scope (regardless of value) was opened. public bool HasAnySessionScope() => Scopes.Any(s => s is IEnumerable> kvps - && kvps.Any(kv => kv.Key == SessionIdKey)); + && kvps.Any(kv => kv.Key == NetclawLogProperties.SessionId)); } diff --git a/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs index 553c6a5fa..c039b8dc2 100644 --- a/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs +++ b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; +using Netclaw.Actors.Protocol; using Netclaw.Actors.Sessions; namespace Netclaw.Daemon.Configuration; @@ -26,10 +27,8 @@ namespace Netclaw.Daemon.Configuration; /// internal static class ChatClientSessionScope { - private const string SessionIdKey = "SessionId"; - public static IDisposable? Begin(ILogger logger, ChatOptions? options) => options is SessionScopedChatOptions { SessionId: { Length: > 0 } sessionId } - ? logger.BeginScope(new[] { new KeyValuePair(SessionIdKey, sessionId) }) + ? logger.BeginScope(new[] { new KeyValuePair(NetclawLogProperties.SessionId, sessionId) }) : null; } diff --git a/src/Netclaw.Daemon/Gateway/SignalRSessionActor.cs b/src/Netclaw.Daemon/Gateway/SignalRSessionActor.cs index 2c9a635cc..20a206d20 100644 --- a/src/Netclaw.Daemon/Gateway/SignalRSessionActor.cs +++ b/src/Netclaw.Daemon/Gateway/SignalRSessionActor.cs @@ -55,7 +55,7 @@ public SignalRSessionActor( _hubContext = hubContext; _log = Context.GetLogger() .WithContext("Adapter", "signalr") - .WithContext("SessionId", _sessionId.Value); + .WithContext(NetclawLogProperties.SessionId, _sessionId.Value); _handle = new SessionPipelineHandle(pipeline, _log, "signalr"); Initializing(); From fc10a1a530bcd8f06eb2810834d85a1c6c1e0b6f Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 16:58:11 +0000 Subject: [PATCH 08/22] refactor(logging): drop RetryingChatClient's redundant SessionId scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review follow-up. In the composed pipeline RetryingChatClient always runs inside LoggingChatClient's streaming scope (guarded by Compose_puts_Logging_ outermost), which stays open for the whole enumeration that drives the retry loop — so BackoffAsync re-opening an identical SessionId scope was pure duplication. Drop it and document the inherited-scope contract; only RoutingChatClient, which logs outside any LoggingChatClient scope, keeps its own. Move the retry-warning correlation coverage from two isolated-decorator tests (which asserted the now-removed self-scope) to one composed-pipeline test that fires a real retry and asserts, at the message level, that the warning line is emitted while the SessionId scope is active — i.e. it tests the production wiring (LoggingChatClient enclosing Retry) rather than a decorator in isolation. --- .../PipelineChatClientFactoryTests.cs | 82 +++++++++++++++++++ .../Configuration/RetryingChatClientTests.cs | 54 ++---------- .../Configuration/RetryingChatClient.cs | 24 +++--- 3 files changed, 99 insertions(+), 61 deletions(-) diff --git a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs index c9ea25bce..8275fc413 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs @@ -3,6 +3,8 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- +using System.Net; +using System.Runtime.CompilerServices; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -75,6 +77,86 @@ [new ChatMessage(ChatRole.User, "hi")], Assert.True(logger.HasSessionScope("ch/thread")); } + [Fact] + public async Task Compose_streaming_retry_warning_inherits_SessionId_scope() + { + // RetryingChatClient deliberately opens no SessionId scope of its own — the retry + // warning must still be correlated, inheriting the enclosing LoggingChatClient + // streaming scope. Prove it at the message level: the retry-warning line is emitted + // while the session id is the active scope. + var logger = new MessageScopeLogger(); + var attempts = 0; + var leaf = new FakeChatClient(streamHandler: (_, _, ct) => + { + attempts++; + return ThrowFirstThenYield(shouldThrow: attempts < 2, ct); + }); + var pipeline = PipelineChatClientFactory.Compose( + leaf, _policy, new SingleLoggerFactory(logger), TimeProvider.System); + + await foreach (var _ in pipeline.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "hi")], + new SessionScopedChatOptions { SessionId = "ch/thread" }, + TestContext.Current.CancellationToken)) + { + } + + var retryWarning = Assert.Single( + logger.Entries, e => e.Message.Contains("LLM call failed (attempt", StringComparison.Ordinal)); + Assert.Equal("ch/thread", retryWarning.SessionId); + } + + private static async IAsyncEnumerable ThrowFirstThenYield( + bool shouldThrow, [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + cancellationToken.ThrowIfCancellationRequested(); + if (shouldThrow) + throw new HttpRequestException("transient", null, HttpStatusCode.TooManyRequests); + + yield return new ChatResponseUpdate { Role = ChatRole.Assistant, Contents = [new TextContent("ok")] }; + } + + // Records, for each log call, the session id on the active scope stack — so a test can + // assert that a specific line (not just the call as a whole) was emitted in scope. + private sealed class MessageScopeLogger : ILogger + { + private readonly List _scopes = []; + public List<(string Message, string? SessionId)> Entries { get; } = []; + + public IDisposable BeginScope(TState state) where TState : notnull + { + _scopes.Add(state); + return new Pop(_scopes); + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, EventId eventId, TState state, + Exception? exception, Func formatter) + => Entries.Add((formatter(state, exception), ActiveSessionId())); + + private string? ActiveSessionId() + { + for (var i = _scopes.Count - 1; i >= 0; i--) + if (_scopes[i] is IEnumerable> kvps) + foreach (var kv in kvps) + if (kv.Key == Netclaw.Actors.Protocol.NetclawLogProperties.SessionId && kv.Value is string s) + return s; + return null; + } + + private sealed class Pop(List scopes) : IDisposable + { + public void Dispose() + { + if (scopes.Count > 0) + scopes.RemoveAt(scopes.Count - 1); + } + } + } + private sealed class SingleLoggerFactory(ILogger logger) : ILoggerFactory { public ILogger CreateLogger(string categoryName) => logger; diff --git a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs index 63bb6c587..9035e013c 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RetryingChatClientTests.cs @@ -7,7 +7,6 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging.Abstractions; -using Netclaw.Actors.Sessions; using Netclaw.Configuration; using Netclaw.Daemon.Configuration; using Xunit; @@ -118,54 +117,11 @@ await Assert.ThrowsAsync(() => Assert.Equal(4, attempts); } - [Fact] - public async Task RetryWarning_carries_SessionId_scope_from_options() - { - // The retry warning is the hottest failure-correlation line. It must inherit the - // session id from the call's options so retry storms correlate by session in Seq. - var attempts = 0; - var fake = new FakeChatClient((_, _, _) => - { - attempts++; - if (attempts < 2) - throw new HttpRequestException("server error", null, HttpStatusCode.InternalServerError); - return Task.FromResult(new ChatResponse([new ChatMessage(ChatRole.Assistant, "ok")])); - }); - var logger = new ScopeCapturingLogger(); - var client = new RetryingChatClient(fake, _policy, logger); - - await client.GetResponseAsync( - [new ChatMessage(ChatRole.User, "hi")], - new SessionScopedChatOptions { SessionId = "ch/thread" }, - TestContext.Current.CancellationToken); - - Assert.True(logger.HasSessionScope("ch/thread")); - } - - [Fact] - public async Task StreamingRetryWarning_carries_SessionId_scope_from_options() - { - // Production only ever calls GetStreamingResponseAsync (StreamingResponseReader), - // so the streaming backoff path — not the non-streaming one above — is the path - // that must tag retry warnings with the session id. - var attempts = 0; - var fake = new FakeChatClient(streamHandler: (_, _, ct) => - { - attempts++; - return ThrowBeforeChunkThenYield(attempts, failUntil: 2, ct); - }); - var logger = new ScopeCapturingLogger(); - var client = new RetryingChatClient(fake, _policy, logger); - - await foreach (var _ in client.GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "hi")], - new SessionScopedChatOptions { SessionId = "ch/thread" }, - TestContext.Current.CancellationToken)) - { - } - - Assert.True(logger.HasSessionScope("ch/thread")); - } + // NOTE: RetryingChatClient deliberately does NOT open its own SessionId scope — it + // inherits the enclosing LoggingChatClient scope in the composed pipeline. The retry + // warning's session correlation is therefore covered by + // PipelineChatClientFactoryTests.Compose_streaming_retry_warning_inherits_SessionId_scope, + // which exercises the real composition, not this decorator in isolation. [Fact] public async Task DoesNotRetryNonTransientErrors() diff --git a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs index 6ad906f35..e320b400e 100644 --- a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs @@ -50,7 +50,7 @@ public override async Task GetResponseAsync( catch (Exception ex) when (!cancellationToken.IsCancellationRequested && _policy.ShouldRetry(ex, attempt)) { - await BackoffAsync(ex, attempt, options, cancellationToken); + await BackoffAsync(ex, attempt, cancellationToken); attempt++; } } @@ -79,7 +79,7 @@ public override async IAsyncEnumerable GetStreamingResponseA catch (Exception ex) when (!cancellationToken.IsCancellationRequested && _policy.ShouldRetry(ex, attempt)) { - await BackoffAsync(ex, attempt, options, cancellationToken); + await BackoffAsync(ex, attempt, cancellationToken); attempt++; continue; } @@ -113,23 +113,23 @@ public override async IAsyncEnumerable GetStreamingResponseA if (preFirstChunkFailure is null) yield break; // clean completion (or a post-first-chunk throw already unwound) - await BackoffAsync(preFirstChunkFailure, attempt, options, cancellationToken); + await BackoffAsync(preFirstChunkFailure, attempt, cancellationToken); attempt++; // outer loop re-initiates the stream } } - private async Task BackoffAsync(Exception ex, int attempt, ChatOptions? options, CancellationToken cancellationToken) + private async Task BackoffAsync(Exception ex, int attempt, CancellationToken cancellationToken) { var delay = _policy.GetDelay(attempt); - // Retry is the hottest failure path; tag the warning with the session id (from - // the call's options) so retry storms correlate by session in Seq. - using (ChatClientSessionScope.Begin(_logger, options)) - { - _logger.LogWarning(ex, - "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", - attempt + 1, _policy.MaxRetries, delay.TotalSeconds); - } + // No SessionId scope is opened here: PipelineChatClientFactory.Compose always + // wraps this decorator inside LoggingChatClient (guarded by + // Compose_puts_Logging_outermost), whose streaming scope stays open for the whole + // enumeration that drives this retry loop — so the warning already inherits the + // session id. Re-opening an identical scope would be pure duplication. + _logger.LogWarning(ex, + "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", + attempt + 1, _policy.MaxRetries, delay.TotalSeconds); await Task.Delay(delay, _timeProvider, cancellationToken); } From 96408abf6607733c084e773710f0d062a1771e9e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Fri, 26 Jun 2026 18:02:13 +0000 Subject: [PATCH 09/22] refactor(logging): collapse sub-agent spawn breadcrumbs to one fan-out emitter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every spawn-lifecycle breadcrumb hand-drove two sinks at the call site — a structured _logger call for daemon.log/Seq and a parallel EmitSessionLogLine string for the session.log transcript — with two message strings to keep in sync. The sinks are genuinely different artifacts (operational diagnostics vs the per-session audit transcript; unifying them at the sink was tried and reverted), but that seam belongs behind one emit point, not duplicated at every site. Add SubAgentSpawnBreadcrumbs: one method per lifecycle event that fans out to both sinks from a single source of truth. SubAgentSpawner (7 sites) and SpawnAgentTool (2 refusal sites) now state each event once; the two renderings live together and can't drift. Message text is byte-for-byte preserved, so the existing SubAgentSpawnObservabilityTests and the routed-slash session-log test cover it unchanged. --- .../SubAgents/SpawnAgentTool.cs | 12 +- .../SubAgents/SubAgentSpawnBreadcrumbs.cs | 112 ++++++++++++++++++ .../SubAgents/SubAgentSpawner.cs | 43 ++----- 3 files changed, 124 insertions(+), 43 deletions(-) create mode 100644 src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs diff --git a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs index 5fd11686a..9cd6c70bf 100644 --- a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs +++ b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs @@ -133,11 +133,7 @@ private static string FormatResult(string agent, SubAgentResult result) // the subagent subsystem is disabled. if (context.Audience == TrustAudience.Public || !_subAgentConfig.Enabled) { - _logger?.LogWarning( - "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled}, session={SessionId})", - args.Agent, context.Audience, _subAgentConfig.Enabled, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"spawn_agent refused (agent={args.Agent}, audience={context.Audience}, subsystemEnabled={_subAgentConfig.Enabled})"); + SubAgentSpawnBreadcrumbs.SpawnRefused(_logger, context, args.Agent, context.Audience, _subAgentConfig.Enabled); return ("Error: This tool is not available.", null); } @@ -153,11 +149,7 @@ private static string FormatResult(string agent, SubAgentResult result) if (profile is null || profile.Visibility != SubAgentVisibility.UserFacing) { var available = _registry.GetUserFacing(); - _logger?.LogWarning( - "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count}, session={SessionId})", - args.Agent, available.Count, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"spawn_agent refused: agent '{args.Agent}' not found or not user-facing (availableCount={available.Count})"); + SubAgentSpawnBreadcrumbs.UnknownAgentRefused(_logger, context, args.Agent, available.Count); if (available.Count == 0) return ($"Error: No subagents are available. Agent '{args.Agent}' not found. Author one at {_paths.AgentsDirectory}/*.md or define a skill with metadata.subagent once #661 lands.", null); diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs new file mode 100644 index 000000000..a35bbf8a2 --- /dev/null +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs @@ -0,0 +1,112 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Microsoft.Extensions.Logging; +using Netclaw.Configuration; +using Netclaw.Tools; + +namespace Netclaw.Actors.SubAgents; + +/// +/// Single emit point for the sub-agent spawn lifecycle. Each method fans one event +/// out to BOTH sinks from one source of truth, so callers state the event once and +/// the two renderings cannot drift: +/// +/// daemon.log / Seq — the structured diagnostic line (queryable +/// {AgentName}/{RunId}/{SessionId} fields, plus the exception +/// object on failure paths). +/// session.log — the flat audit line for the parent's transcript, via +/// . The session=… suffix +/// is omitted because that file is already per-session. +/// +/// These are two different artifacts (operational diagnostics vs the per-session +/// audit transcript), not two ways of writing the same file — unifying them at the +/// sink was tried and reverted (routing by SessionId floods the transcript). +/// The is nullable so tool call sites with an optional +/// logger can share these emitters. +/// +internal static class SubAgentSpawnBreadcrumbs +{ + public static void SpawnRequested(ILogger? logger, ToolExecutionContext context, string agentName, int taskChars) + { + logger?.LogInformation( + "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars}, session={SessionId})", + agentName, taskChars, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] spawn requested (taskChars={taskChars})"); + } + + public static void NoSessionContext(ILogger? logger, ToolExecutionContext context, string agentName) + { + logger?.LogWarning( + "SubAgent [{AgentName}] cannot spawn — no session context available (session={SessionId})", + agentName, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] cannot spawn — no session context available"); + } + + public static void NoToolsAvailable(ILogger? logger, ToolExecutionContext context, string agentName) + { + logger?.LogWarning( + "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn (session={SessionId})", + agentName, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] has no tools available under the parent audience policy — cannot spawn"); + } + + public static void ChildSpawnFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) + { + logger?.LogError( + ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId}, session={SessionId})", + agentName, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] failed to spawn child actor (runId={runId}): {ex.Message}"); + } + + public static void ChildSpawned(ILogger? logger, ToolExecutionContext context, string agentName, string runId) + { + logger?.LogInformation( + "SubAgent [{AgentName}] child actor spawned (runId={RunId}, session={SessionId}); dispatching RunSubAgent", + agentName, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] child actor spawned (runId={runId}); dispatching RunSubAgent"); + } + + public static void Completed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, bool success, long durationMs) + { + logger?.LogInformation( + "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms, session={SessionId})", + agentName, runId, success, durationMs, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] completed (runId={runId}, success={success}, duration={durationMs}ms)"); + } + + public static void RunFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) + { + logger?.LogError( + ex, "SubAgent [{AgentName}] run failed (runId={RunId}, session={SessionId})", + agentName, runId, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"SubAgent [{agentName}] run failed (runId={runId}): {ex.Message}"); + } + + public static void SpawnRefused(ILogger? logger, ToolExecutionContext context, string agentName, TrustAudience audience, bool subsystemEnabled) + { + logger?.LogWarning( + "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled}, session={SessionId})", + agentName, audience, subsystemEnabled, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"spawn_agent refused (agent={agentName}, audience={audience}, subsystemEnabled={subsystemEnabled})"); + } + + public static void UnknownAgentRefused(ILogger? logger, ToolExecutionContext context, string agentName, int availableCount) + { + logger?.LogWarning( + "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count}, session={SessionId})", + agentName, availableCount, context.SessionId); + context.EmitSessionLogLine?.Invoke( + $"spawn_agent refused: agent '{agentName}' not found or not user-facing (availableCount={availableCount})"); + } +} diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index 4fc9b03f1..d90fa3ecf 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -67,20 +67,14 @@ public async Task SpawnAsync( string? systemPromptOverlay = null, ChannelWriter? activitySink = null) { - // Parent-side spawn breadcrumbs. _logger lines carry session={SessionId} for - // daemon.log/Seq correlation; EmitSessionLogLine publishes the same lifecycle - // (request → outcome, including early rejections that happen before the child - // actor even exists) explicitly into the parent's session.log. - _logger.LogInformation( - "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars}, session={SessionId})", - profile.Name, task.Length, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{profile.Name}] spawn requested (taskChars={task.Length})"); + // Parent-side spawn breadcrumbs — each event is fanned out to daemon.log/Seq and + // the parent's session.log from one place (see SubAgentSpawnBreadcrumbs), covering + // request → outcome plus early rejections that happen before the child even exists. + SubAgentSpawnBreadcrumbs.SpawnRequested(_logger, context, profile.Name, task.Length); if (context.SpawnChildActor is null) { - _logger.LogWarning("SubAgent [{AgentName}] cannot spawn — no session context available (session={SessionId})", profile.Name, context.SessionId); - context.EmitSessionLogLine?.Invoke($"SubAgent [{profile.Name}] cannot spawn — no session context available"); + SubAgentSpawnBreadcrumbs.NoSessionContext(_logger, context, profile.Name); activitySink?.TryComplete(); return new SubAgentResult { @@ -93,11 +87,7 @@ public async Task SpawnAsync( var tools = ResolveTools(profile, context); if (tools.Count == 0) { - _logger.LogWarning( - "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn (session={SessionId})", - profile.Name, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{profile.Name}] has no tools available under the parent audience policy — cannot spawn"); + SubAgentSpawnBreadcrumbs.NoToolsAvailable(_logger, context, profile.Name); activitySink?.TryComplete(); return new SubAgentResult { @@ -156,11 +146,7 @@ public async Task SpawnAsync( // The child actor was never created (session actor ActorOf failed or the // spawn ask timed out). Record it to the session transcript before the // exception propagates to the tool pipeline. - _logger.LogError( - ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId}, session={SessionId})", - profile.Name, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{profile.Name}] failed to spawn child actor (runId={runId}): {ex.Message}"); + SubAgentSpawnBreadcrumbs.ChildSpawnFailed(_logger, context, profile.Name, runId, ex); // Balance the IsStarted=true notification above: the non-streaming path // (activitySink is null) relies solely on OnSubAgentActivity, so without // a terminal event the session UI shows a sub-agent stuck in "Started". @@ -175,11 +161,7 @@ public async Task SpawnAsync( throw; } - _logger.LogInformation( - "SubAgent [{AgentName}] child actor spawned (runId={RunId}, session={SessionId}); dispatching RunSubAgent", - profile.Name, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{profile.Name}] child actor spawned (runId={runId}); dispatching RunSubAgent"); + SubAgentSpawnBreadcrumbs.ChildSpawned(_logger, context, profile.Name, runId); var sw = Stopwatch.StartNew(); try @@ -236,11 +218,7 @@ public async Task SpawnAsync( Findings = result.Findings }); - _logger.LogInformation( - "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms, session={SessionId})", - profile.Name, runId, result.Success, sw.ElapsedMilliseconds, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{profile.Name}] completed (runId={runId}, success={result.Success}, duration={sw.ElapsedMilliseconds}ms)"); + SubAgentSpawnBreadcrumbs.Completed(_logger, context, profile.Name, runId, result.Success, sw.ElapsedMilliseconds); return result; } @@ -259,8 +237,7 @@ public async Task SpawnAsync( Duration = sw.Elapsed }); - _logger.LogError(ex, "SubAgent [{AgentName}] run failed (runId={RunId}, session={SessionId})", profile.Name, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke($"SubAgent [{profile.Name}] run failed (runId={runId}): {ex.Message}"); + SubAgentSpawnBreadcrumbs.RunFailed(_logger, context, profile.Name, runId, ex); return new SubAgentResult { Success = false, From 33a708a14602d8816686bb746dd0e0d0d6558c6e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 29 Jun 2026 20:17:38 +0000 Subject: [PATCH 10/22] feat(logging): partition the log stream by session instead of explicit-publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the bolted-on EmitSessionLogLine side-channel with a single partitioning logging sink. The routing rule is now simply "does this line carry a session id?": - RollingFileLoggerProvider implements ISupportExternalScope and, per line, resolves the session id from the event's structured state (an actor's WithContext("SessionId", ...) bridged by Akka->MEL, or a {SessionId} message field) or from the active scopes (the chat-client decorators' BeginScope). A session-tagged line is Tell'd to the SessionLogDispatcher as a SessionLogDiagnostic and is NOT written to daemon.log; everything else goes to daemon.log. The full stream still exports to OTEL with SessionId as an attribute — the receiver does the global slicing. - The session-log writer's own lines (identified by its Akka LogSource/ActorPath) are forced to daemon.log so a write-failure log can never recurse into the file that just failed. - session.log is now the session's full local slice (transcript + every session-scoped operational line), so SessionLogActor switches from per-line AutoFlush to a batched/idle flush (~1s or 256 writes). FlushTick is INotInfluenceReceiveTimeout so the cadence doesn't keep an idle session alive. Deletes ToolExecutionContext.EmitSessionLogLine, LlmSessionActor.CreateSessionLogEmitter, and the pipeline threading. SubAgentSpawnBreadcrumbs now logs each event under a SessionId scope (one call, no fan-out); because routing is no longer per-path-wired, the original routed-skill gap (commit 95510670) is structurally impossible. Retains the chat-client SessionId-scope work, which is now what routes LLM-pipeline lines. Restores the SessionLogDispatcherWiringService IHostedService to attach the dispatcher post-start. Tests: RollingFileLoggerPartitionTests (routing, scope, feedback carve-out, pre-attach buffering); SubAgentSpawnObservabilityTests reworked to assert the SessionId scope; diagnostics skill updated to 2.21.0. --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../references/diagnostics.md | 80 +++---- .../Sessions/SubAgentSpawnIntegrationTests.cs | 51 +---- .../SubAgentSpawnObservabilityTests.cs | 92 +++++--- .../Sessions/LlmSessionActor.cs | 30 --- .../Pipelines/SessionToolExecutionPipeline.cs | 8 +- .../Sessions/SessionLogActor.cs | 64 +++++- .../SubAgents/SpawnAgentTool.cs | 8 +- .../SubAgents/SubAgentSpawnBreadcrumbs.cs | 111 ++++----- .../RollingFileLoggerPartitionTests.cs | 166 ++++++++++++++ .../RollingFileLoggerProviderTests.cs | 6 +- .../LoggingRegistrationExtensions.cs | 43 +++- .../RollingFileLoggerProvider.cs | 216 ++++++++++++++++-- .../ToolExecutionContext.cs | 8 - 14 files changed, 625 insertions(+), 260 deletions(-) create mode 100644 src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index dec7f1351..352cbe69f 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.20.0" + version: "2.21.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 58181f39b..40b71ec81 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -14,54 +14,44 @@ When something seems wrong with Netclaw itself: 3. Check daemon logs at `~/.netclaw/logs/daemon-{yyyy-MM-dd}.log` 4. Check session logs at `~/.netclaw/logs/sessions/{sanitized-session-id}/session.log` -Log split: - -- Daemon-global diagnostics stay in the daemon log (rolled daily, capped - at 10 MB per file). -- Session-owned diagnostics and session output audit trails append to - `~/.netclaw/logs/sessions/{sanitized-session-id}/session.log` — - one file per session, no rotation today (see netclaw-dev/netclaw#919). -- Session log directories use the sanitized session ID (`/`, `.`, spaces, - etc. replaced with `_`). Sub-agent diagnostics roll up into the parent - session's `session.log`; you will not find a separate file for a - sub-agent run. +Log split — one stream, partitioned locally by session: + +- A log line that carries a session id (an actor's `WithContext("SessionId", …)`, + a `{SessionId}` message field, or a `SessionId` logging scope) is written to that + session's `session.log` and **not** to `daemon.log`. The partition is by session + id — nothing is duplicated locally. +- `daemon.log` holds only genuinely daemon-wide lines: startup/config, session + start/stop, and global errors (e.g. an inference provider becoming unreachable). + Rolled daily, capped at 10 MB per file. +- The **full** stream (daemon and session lines alike) is also exported to OTEL/Seq + with the session id as an attribute; do the global slicing/distilling on the OTEL + receiver side. +- Session log directories use the sanitized session ID (`/`, `.`, spaces, etc. + replaced with `_`). Sub-agent activity rolls up into the parent session's + `session.log`; there is no separate file per sub-agent run. No rotation today + (see netclaw-dev/netclaw#919). What to expect inside `session.log`: -- A single chronological timeline. One actor (`SessionLogActor`) is the - only writer per file, so audit lines and diagnostic lines interleave in - wall-clock order — useful for reading "what happened, in order" without - cross-referencing two files. -- Two line shapes: session output audit lines (`User:`, `Assistant:`, - `Thinking:`, `Tool call:`, `Tool result:`, `Usage:`, `Turn N completed`, - etc.) and `Diagnostic:` lines. Diagnostic lines reach `session.log` only - when a producer *explicitly publishes* them — notably the sub-agent spawn - lifecycle (requested, child spawned, dispatched, completed/failed, and - guard rejections), so a failed or blocked spawn is visible here. It is an - explicit publish, not automatic routing of every log line. -- Best-effort observability. Individual lines may be dropped on transient - IO errors and logged at Debug level in the daemon log; this is not a - transactional audit trail. Cross-check the daemon log for warnings - if a critical line appears missing. -- Most actor and operational logs are NOT in `session.log` — only explicitly - published lines are. An actor's own lifecycle logging, and the LLM-client / - HTTP / retry / failover decorator internals, go to `daemon.log`. How the - session id is carried — and so how you filter by it — depends on the line: - - **Seq/OTLP is the dependable filter.** Both the actor logs (session id - attached via `WithContext("SessionId", …)`) and the LLM-client / retry / - **provider failover & outage** decorator logs (attached via a `SessionId` - logging scope) surface `SessionId` as a structured field. Filtering on it - in Seq catches every one of these lines for a session. - - **`grep` of the `daemon.log` text file is partial.** It finds only lines - that template the id straight into the message (`session=…` / `{SessionId}` - — most service, binding, gateway, and spawn-breadcrumb lines). Lines that - carry the id only as a context/scope attribute (the session/sub-agent - actor's own lifecycle logs and the chat-client decorator internals) may not - render it in the text file, so a text grep can miss them — use Seq for - those. - - For an actor's full lifecycle or deep LLM-call internals, read `daemon.log` - (and Seq, for reliable per-session correlation). +- The session's **full local slice** of the log stream, in wall-clock order: the + conversation audit (`User:`, `Assistant:`, `Thinking:`, `Tool call:`, + `Tool result:`, `Usage:`, `Turn N completed`) interleaved with every operational + line scoped to that session — the LLM pipeline, retries, provider failover, tool + and sub-agent activity (spawn requested → child spawned → completed/failed, plus + guard rejections), memory, etc. One actor (`SessionLogActor`) is the only writer + per file. +- Because the partition is by session id, you usually do **not** need to grep — open + the one file for the session and read top to bottom. To correlate across sessions + or globally, use Seq/OTLP (every line is there with `SessionId` as a field). +- The session-log writer's own failure lines are the one exception: they go to + `daemon.log`, never routed back into the file that just failed. +- Best-effort, **batched** writes (flushed on a ~1s cadence): a recent line may lag + by up to a second, and individual lines may be dropped on transient IO errors (a + warning lands in `daemon.log`). Not a transactional audit trail. + +What stays in `daemon.log`: only sessionless lines — daemon startup/config, session +lifecycle, and global errors. Debugging one session → read its `session.log`; +debugging a daemon-wide problem → read `daemon.log`. | Symptom | Check | |---------|-------| diff --git a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs index 834d3eef4..26e8aaab7 100644 --- a/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Sessions/SubAgentSpawnIntegrationTests.cs @@ -548,53 +548,10 @@ await sessionManager.Ask(new SendUserMessage && (m.Text?.Contains("Context:", StringComparison.Ordinal) ?? false)); } - [Fact] - public async Task Routed_slash_command_publishes_spawn_lifecycle_to_session_log() - { - // Regression: the routed-skill spawn path builds its own ToolExecutionContext - // and must wire EmitSessionLogLine, exactly like the tool-execution dispatch - // path. Without it the sub-agent spawn lifecycle (and, more importantly, spawn - // failures — the #1467 gap) never reach the parent's session.log. Register a - // probe as the session-log dispatcher before the session recovers (the actor - // resolves it lazily on RecoveryCompleted) and assert the breadcrumb arrives. - var logProbe = CreateTestProbe("routed-slash-session-log"); - ActorRegistry.For(Sys).Register(logProbe.Ref, overwrite: true); - - var sessionId = new SessionId("test-channel/routed-slash-session-log"); - var sessionManager = ActorRegistry.Get(); - var subscriber = CreateTestProbe("routed-slash-session-log-events"); - - await sessionManager.Ask(new JoinSession(subscriber) - { - SessionId = sessionId, - Filter = OutputFilter.Full - }, TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); - await subscriber.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - - await sessionManager.Ask(new SendUserMessage - { - SessionId = sessionId, - Content = "/ops-route check daemon health", - Source = BuildPersonalSource() - }, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken); - - // The spawner publishes "spawn requested" first, then "completed" once the - // routed sub-agent run returns — both must land on the dispatcher. - var requested = (SessionLogDiagnostic)await logProbe.FishForMessageAsync( - m => m is SessionLogDiagnostic d - && d.SessionId == sessionId - && d.Line.Contains("SubAgent [summarizer] spawn requested", StringComparison.Ordinal), - TimeSpan.FromSeconds(5), - cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(sessionId, requested.SessionId); - - await logProbe.FishForMessageAsync( - m => m is SessionLogDiagnostic d - && d.SessionId == sessionId - && d.Line.Contains("SubAgent [summarizer] completed", StringComparison.Ordinal), - TimeSpan.FromSeconds(5), - cancellationToken: TestContext.Current.CancellationToken); - } + // NOTE: routing the spawn lifecycle to session.log is no longer per-path-wired — the + // breadcrumbs log under a SessionId scope and the file-logger partitions them regardless of + // which path (tool-execution or routed-skill) drove the spawn. The producer side is covered + // by SubAgentSpawnObservabilityTests; the routing by RollingFileLoggerPartitionTests. [Fact] public async Task Reminder_sourced_slash_command_routes_like_normal_slash_dispatch() diff --git a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs index 9239de6e6..afea66548 100644 --- a/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs +++ b/src/Netclaw.Actors.Tests/SubAgents/SubAgentSpawnObservabilityTests.cs @@ -3,7 +3,8 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; +using Netclaw.Actors.Protocol; using Netclaw.Actors.SubAgents; using Netclaw.Actors.Tools; using Netclaw.Configuration; @@ -14,12 +15,11 @@ namespace Netclaw.Actors.Tests.SubAgents; /// -/// Regression coverage for sub-agent spawn observability. The spawn lifecycle is -/// published to the parent's session.log explicitly via -/// (the session wires it to the -/// session-log dispatcher). These tests capture those publishes and assert each -/// lifecycle/rejection breadcrumb is emitted; otherwise a refused or failed spawn -/// would be invisible in the session transcript. +/// Regression coverage for sub-agent spawn observability. Each lifecycle/rejection breadcrumb is +/// an ordinary structured log call wrapped in a SessionId scope; the file-logger +/// partitions scoped lines into the spawning session's session.log (routing itself is +/// covered by RollingFileLoggerPartitionTests). These tests assert the producer side: a +/// refused or failed spawn still logs its real reason under the session's id, so it is not lost. /// public sealed class SubAgentSpawnObservabilityTests : IDisposable { @@ -37,8 +37,9 @@ public SubAgentSpawnObservabilityTests() public void Dispose() => _dir.Dispose(); [Fact] - public async Task Spawner_missing_session_context_publishes_lifecycle_to_session_log() + public async Task Spawner_missing_session_context_logs_lifecycle_under_session_scope() { + var logger = new RecordingLogger(); // Only the parent-side breadcrumb path runs before the early return, so the // unused collaborators are never dereferenced. var spawner = new SubAgentSpawner( @@ -47,40 +48,34 @@ public async Task Spawner_missing_session_context_publishes_lifecycle_to_session toolAccessPolicy: null!, approvalService: null, promptProvider: null!, - NullLogger.Instance); + logger); - var sessionLog = new List(); // A context with a session id but no SpawnChildActor factory — the // "subagent tried to spawn but never launched" failure shape. - var context = new ToolExecutionContext(SessionId, null) - { - Audience = TrustAudience.Personal, - EmitSessionLogLine = sessionLog.Add - }; + var context = new ToolExecutionContext(SessionId, null) { Audience = TrustAudience.Personal }; var result = await spawner.SpawnAsync( Profile("summarizer"), "do the work", null, context, TestContext.Current.CancellationToken); Assert.False(result.Success); - // The spawn attempt and its failure are both published to the session transcript. - Assert.Contains(sessionLog, line => line.Contains("spawn requested", StringComparison.Ordinal)); - Assert.Contains(sessionLog, line => line.Contains("no session context available", StringComparison.Ordinal)); + // The spawn attempt and its failure are both logged under the session's id, so the + // file-logger routes them to that session's session.log. + var requested = Assert.Single(logger.Entries, e => e.Message.Contains("spawn requested", StringComparison.Ordinal)); + Assert.Equal(SessionId, requested.SessionId); + var noContext = Assert.Single(logger.Entries, e => e.Message.Contains("no session context available", StringComparison.Ordinal)); + Assert.Equal(SessionId, noContext.SessionId); } [Fact] - public async Task Tool_refusal_publishes_real_reason_to_session_log() + public async Task Tool_refusal_logs_real_reason_under_session_scope() { var registry = new SubAgentDefinitionRegistry(); - var tool = new SpawnAgentTool(registry, spawner: null!, _paths, logger: NullLogger.Instance); + var logger = new RecordingLogger(); + var tool = new SpawnAgentTool(registry, spawner: null!, _paths, logger: logger); - var sessionLog = new List(); // Public audience is refused with a deliberately opaque model-facing string; // the operator-facing breadcrumb must still record the real reason. - var context = new ToolExecutionContext(SessionId, null) - { - Audience = TrustAudience.Public, - EmitSessionLogLine = sessionLog.Add - }; + var context = new ToolExecutionContext(SessionId, null) { Audience = TrustAudience.Public }; var result = await tool.ExecuteAsync( new Dictionary { ["agent"] = "summarizer", ["task"] = "do the work" }, @@ -88,9 +83,10 @@ public async Task Tool_refusal_publishes_real_reason_to_session_log() TestContext.Current.CancellationToken); Assert.Equal("Error: This tool is not available.", result); - Assert.Contains( - sessionLog, - line => line.Contains("refused", StringComparison.Ordinal) && line.Contains("Public", StringComparison.Ordinal)); + var refused = Assert.Single( + logger.Entries, + e => e.Message.Contains("refused", StringComparison.Ordinal) && e.Message.Contains("Public", StringComparison.Ordinal)); + Assert.Equal(SessionId, refused.SessionId); } private static SubAgentProfile Profile(string name) => new() @@ -101,4 +97,42 @@ public async Task Tool_refusal_publishes_real_reason_to_session_log() ToolNames = ["file_read"], Visibility = SubAgentVisibility.UserFacing }; + + // Records each log line with the SessionId on the scope active at emit time, so a test can + // assert a specific breadcrumb was logged under the right session. + private sealed class RecordingLogger : ILogger + { + private readonly List _scopes = []; + public List<(string Message, string? SessionId)> Entries { get; } = []; + + public IDisposable BeginScope(TState state) where TState : notnull + { + _scopes.Add(state); + return new Pop(_scopes); + } + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => Entries.Add((formatter(state, exception), ActiveSessionId())); + + private string? ActiveSessionId() + { + for (var i = _scopes.Count - 1; i >= 0; i--) + if (_scopes[i] is IEnumerable> kvps) + foreach (var kv in kvps) + if (kv.Key == NetclawLogProperties.SessionId && kv.Value is string s) + return s; + return null; + } + + private sealed class Pop(List scopes) : IDisposable + { + public void Dispose() + { + if (scopes.Count > 0) + scopes.RemoveAt(scopes.Count - 1); + } + } + } } diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index bcb59cc92..7da917b3c 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -1971,11 +1971,6 @@ private void DispatchToolBatch( logActor?.Tell(output); }; - // Sub-agent / tool lifecycle lines are published explicitly into this session's - // session.log via the dispatcher. Routing is intentional here, not inferred from - // log metadata at the sink. Same emitter as the routed-skill path (one definition). - Action emitSessionLogLine = CreateSessionLogEmitter(); - // Marshal child-actor spawning back onto the session actor thread. Func> spawnChildActor = async (props, name, ct) => await self.Ask( @@ -2010,7 +2005,6 @@ await self.Ask( oneTimeApprovalPreSeed: oneTimeApprovalPreSeed, decisionOverride: decisionOverride, turnContext: _currentTurnContext, - emitSessionLogLine: emitSessionLogLine, ct: toolExecutionCt); } @@ -3046,13 +3040,6 @@ await self.Ask( timeout: _config.ToolExecutionTimeout, cancellationToken: ct); - // Publish the routed sub-agent's spawn lifecycle into this session's - // session.log, exactly as the tool-execution dispatch path does (one shared - // emitter). Without it the routed-skill path drops the explicit breadcrumbs - // SubAgentSpawner emits — notably the spawn-failure reasons (#1467) that are - // otherwise invisible to an operator reading the transcript. - context.EmitSessionLogLine = CreateSessionLogEmitter(); - context.OnSubAgentActivity = info => { self.Tell(new RoutedSkillSubAgentActivity( @@ -4256,23 +4243,6 @@ private void EmitOutput(SessionOutput output, OutputFilter requiredFlag = Output _observerActor?.Tell(output); } - /// - /// Builds the explicit session-log emitter wired into - /// . Both the tool-execution - /// dispatch path and the routed-skill path publish lifecycle/breadcrumb lines through - /// this single definition so their timestamp format and dispatcher routing never - /// diverge. _logActor is null only in unit-test scenarios that skip the hosting - /// wiring; in production it is resolved on recovery, so the line is delivered. Snapshots - /// the fields so the returned closure is safe to invoke off the actor thread. - /// - private Action CreateSessionLogEmitter() - { - var logActor = _logActor; - var timeProvider = _timeProvider; - var sessionId = _sessionId; - return line => logActor?.Tell(new SessionLogDiagnostic(sessionId, $"[{timeProvider.GetUtcNow():o}] {line}")); - } - private async Task PersistApprovalCandidatesAsync( PendingToolInteraction pending, ApprovalDecision decision, diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs index ff1c5a58b..4d6b45cc6 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionToolExecutionPipeline.cs @@ -98,7 +98,6 @@ public static async Task ExecuteToolsAsync( IReadOnlyDictionary>? oneTimeApprovalPreSeed = null, IReadOnlyDictionary? decisionOverride = null, TurnContext? turnContext = null, - Action? emitSessionLogLine = null, CancellationToken ct = default) { try @@ -139,8 +138,7 @@ oneTimeApprovalPreSeed is not null ? overrideDecision : null, turnContext, - modelInputBudget, - emitSessionLogLine); + modelInputBudget); if (streamToolResults) self.Tell(new ToolExecutionSingleCompleted(result)); return result; @@ -211,8 +209,7 @@ public static async Task ExecuteSingleToolAsync( IReadOnlyList? oneTimeApprovalPreSeed = null, ApprovalDecision? decisionOverride = null, TurnContext? turnContext = null, - ModelInputBatchBudget? modelInputBudget = null, - Action? emitSessionLogLine = null) + ModelInputBatchBudget? modelInputBudget = null) { // Single execution-preflight seam, shared with the sub-agent path via // IToolExecutor.InterpretToolCall: validate the ORIGINAL arguments (parse @@ -288,7 +285,6 @@ public static async Task ExecuteSingleToolAsync( } var completedRuns = new List(); var acceptedFindings = new List(); - context.EmitSessionLogLine = emitSessionLogLine; context.OnSubAgentActivity = info => { if (info.IsStarted) diff --git a/src/Netclaw.Actors/Sessions/SessionLogActor.cs b/src/Netclaw.Actors/Sessions/SessionLogActor.cs index 5a7bbcb56..49079934a 100644 --- a/src/Netclaw.Actors/Sessions/SessionLogActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionLogActor.cs @@ -27,18 +27,29 @@ namespace Netclaw.Actors.Sessions; /// /// File handle lifecycle: /// - Open once in with append mode + read-share. -/// - AutoFlush enabled — each WriteLine flushes immediately. -/// - Close/dispose in — no retry loop needed since the -/// handle is kept open and single-writer is enforced by the actor mailbox. +/// - Buffered writes flushed in batches (after a write burst or on a periodic tick), +/// not per line: now that the whole per-session log stream lands here, an fsync per +/// line would dominate. is INotInfluenceReceiveTimeout +/// so the flush cadence does not keep an idle session alive. +/// - Close/dispose in (which flushes) — no retry loop needed since +/// the handle is kept open and single-writer is enforced by the actor mailbox. /// -public sealed class SessionLogActor : ReceiveActor +public sealed class SessionLogActor : ReceiveActor, IWithTimers { + // Flush after this many buffered writes (bounds the buffer under a burst) or on the + // periodic tick (bounds tail latency for a trickle), whichever comes first. + private const int FlushAfterWrites = 256; + private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(1); + private readonly SessionId _sessionId; private readonly string _sessionLogsBasePath; private readonly TimeProvider _timeProvider; private readonly TimeSpan _idleTimeout; private readonly ILoggingAdapter _log = Context.GetLogger(); private StreamWriter? _writer; + private int _unflushedWrites; + + public ITimerScheduler Timers { get; set; } = null!; public static Props CreateProps(SessionId sessionId, string sessionLogsBasePath, TimeProvider timeProvider, TimeSpan? idleTimeout = null) => Props.Create(() => new SessionLogActor(sessionId, sessionLogsBasePath, timeProvider, idleTimeout ?? TimeSpan.FromMinutes(10))); @@ -53,9 +64,18 @@ public SessionLogActor(SessionId sessionId, string sessionLogsBasePath, TimeProv Receive(OnUserMessage); Receive(OnOutput); Receive(OnDiagnostic); + Receive(_ => Flush()); Receive(_ => Context.Stop(Self)); } + // Periodic flush signal. INotInfluenceReceiveTimeout so the 1s cadence never resets the + // session's idle-stop timer. + private sealed class FlushTick : INotInfluenceReceiveTimeout + { + public static readonly FlushTick Instance = new(); + private FlushTick() { } + } + protected override void PreStart() { base.PreStart(); @@ -75,15 +95,43 @@ protected override void PreStart() bufferSize: 4096, useAsync: false); - _writer = new StreamWriter(stream) { AutoFlush = true }; + _writer = new StreamWriter(stream) { AutoFlush = false }; + Timers.StartPeriodicTimer("session-log-flush", FlushTick.Instance, FlushInterval); } protected override void PostStop() { + // Dispose flushes, but flush explicitly first so a write failure here is logged + // rather than swallowed by Dispose. + Flush(); _writer?.Dispose(); base.PostStop(); } + // Buffered write: lines accumulate in the StreamWriter buffer and are flushed in batches. + private void Write(string line) + { + _writer?.WriteLine(line); + if (++_unflushedWrites >= FlushAfterWrites) + Flush(); + } + + private void Flush() + { + if (_unflushedWrites == 0) + return; + + try + { + _writer?.Flush(); + _unflushedWrites = 0; + } + catch (Exception ex) + { + _log.Warning(ex, "Failed to flush session.log for {SessionId}", _sessionId.Value); + } + } + private void OnUserMessage(SendUserMessage msg) { try @@ -93,7 +141,7 @@ private void OnUserMessage(SendUserMessage msg) : string.Empty; var line = $"[{_timeProvider.GetUtcNow():o}] User: {TextTruncation.EllipsisAppend(msg.Content, 1000)}{mediaNote}"; - _writer?.WriteLine(line); + Write(line); } catch (Exception ex) { @@ -129,7 +177,7 @@ private void OnOutput(SessionOutput output) if (line is not null) { - _writer?.WriteLine($"[{_timeProvider.GetUtcNow():o}] {line}"); + Write($"[{_timeProvider.GetUtcNow():o}] {line}"); } } catch (Exception ex) @@ -142,7 +190,7 @@ private void OnDiagnostic(SessionLogDiagnostic diagnostic) { try { - _writer?.WriteLine(diagnostic.Line); + Write(diagnostic.Line); } catch (Exception ex) { diff --git a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs index 9cd6c70bf..954ed00f6 100644 --- a/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs +++ b/src/Netclaw.Actors/SubAgents/SpawnAgentTool.cs @@ -124,10 +124,10 @@ private static string FormatResult(string agent, SubAgentResult result) private (string? Error, SubAgentProfile? Profile) Resolve(Params args, ToolExecutionContext context) { // Rejections here return a (sometimes deliberately opaque) error string to - // the model. Mirror the real reason to the session transcript via - // EmitSessionLogLine (explicit publish) so an operator can tell *why* a spawn - // was refused; the "This tool is not available." string hides the - // audience-vs-disabled distinction on purpose. + // the model. The breadcrumb records the real reason under a SessionId scope (so it + // lands in the session.log) so an operator can tell *why* a spawn was refused; the + // "This tool is not available." string hides the audience-vs-disabled distinction on + // purpose. // Defense-in-depth: block subagent spawning for Public audience or when // the subagent subsystem is disabled. diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs index a35bbf8a2..0667d4efa 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs @@ -4,109 +4,98 @@ // // ----------------------------------------------------------------------- using Microsoft.Extensions.Logging; +using Netclaw.Actors.Protocol; using Netclaw.Configuration; using Netclaw.Tools; namespace Netclaw.Actors.SubAgents; /// -/// Single emit point for the sub-agent spawn lifecycle. Each method fans one event -/// out to BOTH sinks from one source of truth, so callers state the event once and -/// the two renderings cannot drift: -/// -/// daemon.log / Seq — the structured diagnostic line (queryable -/// {AgentName}/{RunId}/{SessionId} fields, plus the exception -/// object on failure paths). -/// session.log — the flat audit line for the parent's transcript, via -/// . The session=… suffix -/// is omitted because that file is already per-session. -/// -/// These are two different artifacts (operational diagnostics vs the per-session -/// audit transcript), not two ways of writing the same file — unifying them at the -/// sink was tried and reverted (routing by SessionId floods the transcript). -/// The is nullable so tool call sites with an optional -/// logger can share these emitters. +/// Centralizes the sub-agent spawn-lifecycle log lines. Each is an ordinary structured log call +/// wrapped in a SessionId scope, so the file-logger partitions it into the spawning +/// session's session.log (and the OTLP exporter sees the session id as an attribute). +/// and are plain classes — no actor +/// WithContext — so the scope is what carries the id. The is +/// nullable so tool call sites with an optional logger can share these emitters. /// internal static class SubAgentSpawnBreadcrumbs { public static void SpawnRequested(ILogger? logger, ToolExecutionContext context, string agentName, int taskChars) { - logger?.LogInformation( - "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars}, session={SessionId})", - agentName, taskChars, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] spawn requested (taskChars={taskChars})"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogInformation( + "SubAgent [{AgentName}] spawn requested (taskChars={TaskChars})", + agentName, taskChars); } public static void NoSessionContext(ILogger? logger, ToolExecutionContext context, string agentName) { - logger?.LogWarning( - "SubAgent [{AgentName}] cannot spawn — no session context available (session={SessionId})", - agentName, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] cannot spawn — no session context available"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogWarning( + "SubAgent [{AgentName}] cannot spawn — no session context available", + agentName); } public static void NoToolsAvailable(ILogger? logger, ToolExecutionContext context, string agentName) { - logger?.LogWarning( - "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn (session={SessionId})", - agentName, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] has no tools available under the parent audience policy — cannot spawn"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogWarning( + "SubAgent [{AgentName}] has no tools available under the parent audience policy — cannot spawn", + agentName); } public static void ChildSpawnFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) { - logger?.LogError( - ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId}, session={SessionId})", - agentName, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] failed to spawn child actor (runId={runId}): {ex.Message}"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogError( + ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId})", + agentName, runId); } public static void ChildSpawned(ILogger? logger, ToolExecutionContext context, string agentName, string runId) { - logger?.LogInformation( - "SubAgent [{AgentName}] child actor spawned (runId={RunId}, session={SessionId}); dispatching RunSubAgent", - agentName, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] child actor spawned (runId={runId}); dispatching RunSubAgent"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogInformation( + "SubAgent [{AgentName}] child actor spawned (runId={RunId}); dispatching RunSubAgent", + agentName, runId); } public static void Completed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, bool success, long durationMs) { - logger?.LogInformation( - "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms, session={SessionId})", - agentName, runId, success, durationMs, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] completed (runId={runId}, success={success}, duration={durationMs}ms)"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogInformation( + "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms)", + agentName, runId, success, durationMs); } public static void RunFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) { - logger?.LogError( - ex, "SubAgent [{AgentName}] run failed (runId={RunId}, session={SessionId})", - agentName, runId, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"SubAgent [{agentName}] run failed (runId={runId}): {ex.Message}"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogError( + ex, "SubAgent [{AgentName}] run failed (runId={RunId})", + agentName, runId); } public static void SpawnRefused(ILogger? logger, ToolExecutionContext context, string agentName, TrustAudience audience, bool subsystemEnabled) { - logger?.LogWarning( - "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled}, session={SessionId})", - agentName, audience, subsystemEnabled, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"spawn_agent refused (agent={agentName}, audience={audience}, subsystemEnabled={subsystemEnabled})"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogWarning( + "spawn_agent refused (agent={Agent}, audience={Audience}, subsystemEnabled={Enabled})", + agentName, audience, subsystemEnabled); } public static void UnknownAgentRefused(ILogger? logger, ToolExecutionContext context, string agentName, int availableCount) { - logger?.LogWarning( - "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count}, session={SessionId})", - agentName, availableCount, context.SessionId); - context.EmitSessionLogLine?.Invoke( - $"spawn_agent refused: agent '{agentName}' not found or not user-facing (availableCount={availableCount})"); + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogWarning( + "spawn_agent refused: agent '{Agent}' not found or not user-facing (availableCount={Count})", + agentName, availableCount); } + + // Opens the SessionId scope the file-logger routes on. No-op (null using) when there is no + // logger or no session — the line then falls through to daemon.log as a sessionless line. + private static IDisposable? BeginSessionScope(ILogger? logger, string? sessionId) => + logger is not null && !string.IsNullOrWhiteSpace(sessionId) + ? logger.BeginScope(new[] { new KeyValuePair(NetclawLogProperties.SessionId, sessionId) }) + : null; } diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs new file mode 100644 index 000000000..b8bc44730 --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -0,0 +1,166 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Protocol; +using Netclaw.Daemon.Configuration; +using Xunit; + +namespace Netclaw.Daemon.Tests.Configuration; + +/// +/// The provider owns the LOCAL partition of the log stream: a line carrying a session id goes +/// to that session's session.log (Tell'd as a to the +/// dispatcher) and NOT to daemon.log; everything else goes to daemon.log. The session-log +/// writer's own lines are excluded so a write-failure log cannot recurse. +/// +public sealed class RollingFileLoggerPartitionTests : TestKit +{ + private static readonly DateTimeOffset FixedNow = DateTimeOffset.Parse("2026-05-07T12:00:00Z"); + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + { + } + + [Fact] + public async Task Session_tagged_line_routes_to_session_log_not_daemon_log() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + provider.CreateLogger("Netclaw.Tools").LogInformation("spawn requested {SessionId}", "C1/T1"); + + var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("C1/T1", diag.SessionId.Value); + Assert.Contains("spawn requested", diag.Line, StringComparison.Ordinal); + } + + // Partition: the session-tagged line is NOT also in daemon.log. + Assert.DoesNotContain("spawn requested", ReadDaemonLog(dir), StringComparison.Ordinal); + Cleanup(dir); + } + + [Fact] + public async Task Session_id_carried_in_a_scope_routes() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); + + // A real LoggerFactory wires the external scope provider into the sink, mirroring the + // chat-client decorators that carry the session id via BeginScope rather than the message. + var factory = LoggerFactory.Create(b => b.AddProvider(provider)); + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + + var logger = factory.CreateLogger("Netclaw.LlmPipeline"); + using (logger.BeginScope(new[] { new KeyValuePair(NetclawLogProperties.SessionId, "C2/T2") })) + logger.LogInformation("LLM streaming call completed"); + + var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("C2/T2", diag.SessionId.Value); + Assert.Contains("LLM streaming call completed", diag.Line, StringComparison.Ordinal); + + factory.Dispose(); // disposes the provider, draining daemon.log + Assert.DoesNotContain("LLM streaming call completed", ReadDaemonLog(dir), StringComparison.Ordinal); + Cleanup(dir); + } + + [Fact] + public async Task Sessionless_line_goes_to_daemon_log() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + provider.CreateLogger("Netclaw.Daemon").LogInformation("daemon listening on port 8080"); + + await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + } + + Assert.Contains("daemon listening on port 8080", ReadDaemonLog(dir), StringComparison.Ordinal); + Cleanup(dir); + } + + [Fact] + public async Task Session_log_writers_own_line_is_not_routed_back() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + + // Shape of a bridged Akka log from the SessionLogActor: it carries a SessionId but its + // ActorPath is under the session-log dispatcher. It must NOT route (feedback guard). + var state = new List> + { + new(NetclawLogProperties.SessionId, "C3/T3"), + new("ActorPath", "akka://netclaw/user/session-log-dispatcher/C3%2FT3"), + }; + provider.CreateLogger("Akka.Actor.ActorSystem") + .Log(LogLevel.Warning, new EventId(0), state, null, (_, _) => "Failed to flush session.log"); + + await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + } + + Assert.Contains("Failed to flush session.log", ReadDaemonLog(dir), StringComparison.Ordinal); + Cleanup(dir); + } + + [Fact] + public async Task Session_lines_emitted_before_dispatcher_resolves_buffer_then_drain() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + var pending = new TaskCompletionSource(); + + using var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); + provider.AttachSessionDispatcher(pending.Task); // routing on, dispatcher not yet resolved + provider.CreateLogger("Netclaw.Tools").LogInformation("buffered op {SessionId}", "C4/T4"); + + await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); + + pending.SetResult(dispatcher.Ref); // resolution drains the buffer in order + var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("C4/T4", diag.SessionId.Value); + Assert.Contains("buffered op", diag.Line, StringComparison.Ordinal); + Cleanup(dir); + } + + private static (string Dir, string DaemonPath) TempPaths() + { + var dir = Path.Combine(Path.GetTempPath(), $"netclaw-partition-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + return (dir, Path.Combine(dir, "daemon.log")); + } + + private static string ReadDaemonLog(string dir) + { + var files = Directory.GetFiles(dir, "daemon-*.log"); + return files.Length == 0 ? string.Empty : File.ReadAllText(files[0]); + } + + private static void Cleanup(string dir) + { + try + { + if (Directory.Exists(dir)) + Directory.Delete(dir, recursive: true); + } + catch (IOException ex) + { + Console.Error.WriteLine($"[RollingFileLoggerPartitionTests] cleanup failed: {ex.Message}"); + } + } +} diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs index ec85a13a5..f253826e8 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs @@ -28,9 +28,9 @@ public async Task Writes_log_lines_to_the_daily_rolling_daemon_log() logger.LogInformation("hello daemon log {SessionId}", "channel/thread"); } - // The {SessionId} field is NOT special to this sink — it renders into the message - // text and the line goes to daemon.log only. Per-session routing is the producers' - // job (explicit SessionLogDiagnostic), not the logging sink's. + // No dispatcher was attached, so per-session routing is off and even a session-tagged + // line falls back to daemon.log. (When routing IS attached the {SessionId} line is + // partitioned to session.log — see RollingFileLoggerPartitionTests.) var daemonLog = Directory.GetFiles(Path.Combine(_basePath, "logs"), "daemon-*.log").Single(); var text = await File.ReadAllTextAsync(daemonLog, TestContext.Current.CancellationToken); Assert.Contains("hello daemon log", text, StringComparison.Ordinal); diff --git a/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs b/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs index e218f2fce..baef41848 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingRegistrationExtensions.cs @@ -3,8 +3,11 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- +using Akka.Hosting; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Netclaw.Actors.Hosting; using Netclaw.Configuration; namespace Netclaw.Daemon.Configuration; @@ -22,15 +25,18 @@ public static LogLevel ConfigureNetclawLogging(this WebApplicationBuilder builde if (consoleEnabled) builder.Logging.AddSimpleConsole(options => options.SingleLine = true); - // Always write to a rolling daemon.log in ~/.netclaw/logs/. The provider must be - // constructed eagerly so MEL can see it via AddProvider — a - // Services.AddSingleton(factory) registration here is not picked - // up by the LoggerFactory in this hosting setup. This sink is daemon-global only; - // per-session lines are published explicitly to the session-log dispatcher. + // This provider owns the local partition of the log stream: session-tagged lines go + // to per-session session.log files, everything else to daemon.log (see + // RollingFileLoggerProvider). It must be constructed eagerly so MEL sees it via + // AddProvider — a Services.AddSingleton(factory) registration here is + // not picked up by the LoggerFactory in this hosting setup. The session-log dispatcher + // is wired in post-build by SessionLogDispatcherWiringService once Akka.Hosting has + // registered the actor system. Directory.CreateDirectory(resolvedPaths.LogsDirectory); var provider = new RollingFileLoggerProvider(resolvedPaths.DaemonLogPath); builder.Logging.AddProvider(provider); builder.Services.AddSingleton(provider); + builder.Services.AddHostedService(); builder.Logging.SetMinimumLevel(level); return level; @@ -45,3 +51,30 @@ private static LogLevel ResolveLogLevel(IConfiguration configuration) return LogLevel.Information; } } + +/// +/// Hooks the session-log dispatcher into once +/// Akka.Hosting has registered SessionLogDispatcherActorKey. The provider is built during +/// host construction (before Akka), so the dispatcher can only be attached post-start. +/// +internal sealed class SessionLogDispatcherWiringService : IHostedService +{ + private readonly RollingFileLoggerProvider _provider; + private readonly IRequiredActor _dispatcher; + + public SessionLogDispatcherWiringService( + RollingFileLoggerProvider provider, + IRequiredActor dispatcher) + { + _provider = provider; + _dispatcher = dispatcher; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _provider.AttachSessionDispatcher(_dispatcher.GetAsync(cancellationToken)); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index f6dc8e75d..aa9cd772b 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -5,29 +5,46 @@ // ----------------------------------------------------------------------- using System.Collections.Concurrent; using System.Globalization; +using Akka.Actor; using Microsoft.Extensions.Logging; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; namespace Netclaw.Daemon.Configuration; /// -/// Simple file-based logger that writes to a daily rolling daemon.log file. -/// Uses a background queue to avoid blocking callers. -/// -/// This sink is daemon-global ONLY. It does not route per-session: session-relevant -/// lines are published explicitly by their producers to the session-log dispatcher -/// (a SessionLogDiagnostic Tell), which serializes per-session writes through -/// SessionLogActor. Keeping routing out of the logging sink avoids inferring -/// intent from log metadata (e.g. a descriptive {SessionId} placeholder). +/// File-based logger that owns the LOCAL partition of the one log stream. Every line is +/// written to exactly one place on disk: +/// +/// A line that carries a session id (an actor's WithContext("SessionId", …), +/// which the Akka→MEL bridge surfaces as structured state; a MEL {SessionId} message +/// field; or a BeginScope carrying it) is routed to that session's session.log +/// via the SessionLogDispatcher and is NOT written to daemon.log. +/// Everything else — genuinely daemon-wide lines (startup, config, session lifecycle, +/// global errors) — goes to daemon.log. +/// +/// The full stream still reaches OTEL via the separate OTLP exporter (with the session id as +/// an attribute); the OTEL receiver does the global slicing/distilling. The dispatcher is +/// attached post-construction via (the provider is built +/// during host build, before Akka). Lines emitted by the itself +/// are forced to daemon.log so a write-failure log can never recurse into the file that +/// just failed. /// -internal sealed class RollingFileLoggerProvider : ILoggerProvider +internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExternalScope { private const long MaxFileSizeBytes = 10 * 1024 * 1024; // 10MB per file + private const int PreResolutionBufferLimit = 1000; private readonly string _basePath; private readonly TimeProvider _timeProvider; private readonly ConcurrentDictionary _loggers = new(); private readonly BlockingCollection _queue = new(1024); private readonly Thread _writerThread; + private IExternalScopeProvider? _scopeProvider; + private ConcurrentQueue? _pendingDiagnostics; + private IActorRef? _sessionDispatcher; + private int _pendingCount; + private int _sessionRoutingEnabled; private StreamWriter? _writer; private string _currentDate = ""; @@ -46,7 +63,124 @@ public RollingFileLoggerProvider(string basePath, TimeProvider? timeProvider = n public ILogger CreateLogger(string categoryName) => _loggers.GetOrAdd(categoryName, name => new RollingFileLogger(name, this)); - internal void Enqueue(string message) => _queue.TryAdd(message); + // MEL hands every scope-aware provider the same shared scope provider; the chat-client + // decorators carry the session id via BeginScope, so we read it from here at emit time. + public void SetScopeProvider(IExternalScopeProvider scopeProvider) => _scopeProvider = scopeProvider; + + /// + /// Enables per-session routing once the actor system is up. Session-tagged lines emitted + /// between this call and dispatcher resolution are held in a small bounded buffer and then + /// drained in order; afterwards they Tell the dispatcher directly. If resolution fails, a + /// single ERR line is written to daemon.log and routing is disabled (session-tagged + /// lines fall back to daemon.log) for the rest of the process. + /// + public void AttachSessionDispatcher(Task dispatcherTask) + { + if (Interlocked.Exchange(ref _sessionRoutingEnabled, 1) == 1) + return; + + _pendingDiagnostics = new ConcurrentQueue(); + _ = ResolveSessionDispatcherAsync(dispatcherTask); + } + + /// + /// Partitions one already-formatted line: routes it to its session's session.log + /// when it carries a session id (and is not from the session-log writer itself), otherwise + /// writes it to daemon.log. is the id found on the + /// log event's state; the active scopes are consulted only as a fallback. + /// + internal void Route(string line, string? stateSessionId, bool fromSessionLogActor) + { + // Carve-out: the session-log writer's own lines go to daemon.log so a failed write that + // logs an error cannot route back into the same (failing) session.log — an infinite loop. + var sessionId = fromSessionLogActor ? null : (stateSessionId ?? FindSessionIdInScopes()); + + if (string.IsNullOrWhiteSpace(sessionId) || _sessionRoutingEnabled == 0) + { + _queue.TryAdd(line); + return; + } + + var diagnostic = new SessionLogDiagnostic(new SessionId(sessionId), line); + + var dispatcher = Volatile.Read(ref _sessionDispatcher); + if (dispatcher is not null) + { + dispatcher.Tell(diagnostic); + return; + } + + // Dispatcher not resolved yet. Buffer up to a bound; once full, fall back to daemon.log + // rather than drop the line. + if (Volatile.Read(ref _pendingCount) >= PreResolutionBufferLimit) + { + _queue.TryAdd(line); + return; + } + + Interlocked.Increment(ref _pendingCount); + _pendingDiagnostics!.Enqueue(diagnostic); + } + + private string? FindSessionIdInScopes() + { + var scopeProvider = _scopeProvider; + if (scopeProvider is null) + return null; + + string? found = null; + scopeProvider.ForEachScope( + (scope, _) => + { + if (found is not null) + return; + + if (scope is IEnumerable> kvps) + { + foreach (var kv in kvps) + { + if (kv.Key == NetclawLogProperties.SessionId + && kv.Value?.ToString() is { Length: > 0 } id) + { + found = id; + return; + } + } + } + }, + (object?)null); + + return found; + } + + private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) + { + IActorRef dispatcher; + try + { + dispatcher = await dispatcherTask.ConfigureAwait(false); + } + catch (Exception ex) + { + // Resolution failed permanently. Drop the buffer and switch routing off — daemon + // logging continues, but session-scoped lines fall back to daemon.log for the rest + // of the process. Surface one loud line so this is visible, not silently degraded. + _queue.TryAdd($"{GetTimestamp()} [ERR] Netclaw.Logging: session log dispatcher resolution failed; per-session routing disabled. {ex.Message}"); + Volatile.Write(ref _sessionRoutingEnabled, 0); + _pendingDiagnostics = null; + return; + } + + // Publish the ref BEFORE draining so producers racing with the drainer Tell directly + // rather than enqueueing into a buffer we are about to abandon. + Volatile.Write(ref _sessionDispatcher, dispatcher); + + while (_pendingDiagnostics!.TryDequeue(out var pending)) + { + Interlocked.Decrement(ref _pendingCount); + dispatcher.Tell(pending); + } + } private void ProcessQueue() { @@ -60,7 +194,7 @@ private void ProcessQueue() } catch (Exception ex) { - // Last-resort: write to stderr to avoid silent swallow + // Last-resort: write to stderr to avoid silent swallow. Console.Error.WriteLine($"[NetclawLogWriter] Failed to write log: {ex.Message}"); } } @@ -127,6 +261,12 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (!IsEnabled(logLevel)) return; + // Single pass over the event's structured state: pick up the session id (for routing), + // the Akka log source (for a useful per-line label — every actor shares the generic MEL + // category "Akka.Actor.ActorSystem"), and whether this line is the session-log writer's + // own (so we never route it back into the file it writes). + ScanState(state, out var sessionId, out var logSource, out var fromSessionLogActor); + var timestamp = _provider.GetTimestamp(); var level = logLevel switch { @@ -137,11 +277,61 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except _ => "DBG" }; + var source = logSource ?? _category; var message = formatter(state, exception); - var line = $"{timestamp} [{level}] {_category}: {message}"; + var line = $"{timestamp} [{level}] {source}: {message}"; if (exception is not null) line += Environment.NewLine + exception; - _provider.Enqueue(line); + _provider.Route(line, sessionId, fromSessionLogActor); + } + + // Read the fields the producer already put on the log event. The Akka→MEL bridge passes an + // AkkaLogState carrying WithContext("SessionId", …) plus "LogSource"/"ActorPath"; MEL's own + // structured logging passes FormattedLogValues carrying a {SessionId} field. Both surface + // their fields as KeyValuePair sequences (the nullable-annotated and + // unannotated forms are the same runtime type), so one branch reads both — no Akka internals. + private static void ScanState(TState state, out string? sessionId, out string? logSource, out bool fromSessionLogActor) + { + sessionId = null; + logSource = null; + fromSessionLogActor = false; + + if (state is IEnumerable> fields) + { + foreach (var field in fields) + Apply(field.Key, field.Value, ref sessionId, ref logSource, ref fromSessionLogActor); + } } + + private static void Apply(string key, object? value, ref string? sessionId, ref string? logSource, ref bool fromSessionLogActor) + { + if (value is null) + return; + + if (key == NetclawLogProperties.SessionId) + { + if (value.ToString() is { Length: > 0 } id) + sessionId = id; + } + else if (key == "LogSource") + { + logSource = value.ToString(); + if (IsSessionLogActorSource(logSource)) + fromSessionLogActor = true; + } + else if (key == "ActorPath") + { + if (IsSessionLogActorSource(value.ToString())) + fromSessionLogActor = true; + } + } + + // The session-log writer (and its dispatcher) must never have its own lines routed back to + // session.log. All actors share one MEL category, so identify it by Akka log source instead: + // the actor type (SessionLogActor) or the dispatcher path ("session-log-dispatcher"). + private static bool IsSessionLogActorSource(string? source) => + source is not null + && (source.Contains(nameof(SessionLogActor), StringComparison.Ordinal) + || source.Contains("session-log-dispatcher", StringComparison.Ordinal)); } diff --git a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs index 950b2d3ae..3339e1aa6 100644 --- a/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs +++ b/src/Netclaw.Tools.Abstractions/ToolExecutionContext.cs @@ -140,14 +140,6 @@ public ChannelDeliveryTargetInfo? EffectiveDeliveryTarget /// public Action? OnSubAgentActivity { get; set; } - /// - /// Publishes a pre-formatted diagnostic line to the owning session's - /// session.log (via the session-log dispatcher). The session wires this so - /// tools/sub-agents can record session-relevant lifecycle lines explicitly. Null - /// when no session-log sink is wired (e.g. console/test hosts). - /// - public Action? EmitSessionLogLine { get; set; } - /// /// Factory delegate for spawning subagent actors as children of the owning session. /// Wired by LlmSessionActor so subagents are supervised and lifecycle-managed From 38460e3745fae88d7804be46c859744e9c54cd6e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 29 Jun 2026 22:36:49 +0000 Subject: [PATCH 11/22] test(logging): end-to-end partition proof through the real Akka->MEL bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unit tests simulate the log-event state shape; this drives the real thing. A test actor tags its logger with WithContext("SessionId", ...) and logs; the line flows through ConfigureLoggers(AddLoggerFactory) -> the host ILoggerFactory -> RollingFileLoggerProvider, which extracts the id from the live AkkaLogState and routes it to the real SessionLogDispatcher/SessionLogActor. Asserts the line lands in that session's session.log file and NOT in daemon.log — confirming the bridge produces the structured state the provider's ScanState reads. --- .../SessionLogPartitionIntegrationTests.cs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs diff --git a/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs b/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs new file mode 100644 index 000000000..f1e942718 --- /dev/null +++ b/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs @@ -0,0 +1,122 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using Akka.Actor; +using Akka.Event; +using Akka.Hosting; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Netclaw.Actors.Hosting; +using Netclaw.Actors.Protocol; +using Netclaw.Daemon.Configuration; +using Xunit; + +namespace Netclaw.Daemon.Tests.Configuration; + +/// +/// End-to-end proof through the REAL Akka→MEL bridge: an actor that tags its logger with +/// WithContext("SessionId", …) has its line partitioned into that session's +/// session.log (not daemon.log) by . This +/// exercises the live AkkaLogState shape that the unit tests can only simulate, plus the +/// real SessionLogDispatcher and SessionLogActor file write. +/// +public sealed class SessionLogPartitionIntegrationTests : TestKit +{ + private readonly string _logDir = Path.Combine(Path.GetTempPath(), $"netclaw-logpart-int-{Guid.NewGuid():N}"); + private readonly FakeTimeProvider _time = new(DateTimeOffset.Parse("2026-05-07T12:00:00Z")); + private RollingFileLoggerProvider _provider = null!; + + private string DaemonPath => Path.Combine(_logDir, "daemon.log"); + private string SessionsDir => Path.Combine(_logDir, "sessions"); + + protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) + { + builder + .ConfigureLoggers(setup => + { + setup.ClearLoggers(); + setup.AddLoggerFactory(); + setup.LogLevel = Akka.Event.LogLevel.DebugLevel; + }) + .WithSessionLogDispatcher(SessionsDir, _time); + } + + protected override void ConfigureServices(HostBuilderContext context, IServiceCollection services) + { + Directory.CreateDirectory(_logDir); + _provider = new RollingFileLoggerProvider(DaemonPath, _time); + // Wire our provider into the host's ILoggerFactory so AddLoggerFactory bridges every + // Akka actor log to it — the same path production uses. + services.AddSingleton(_provider); + } + + [Fact] + public async Task Actor_log_tagged_with_session_context_lands_in_session_log_not_daemon_log() + { + // Attach the provider to the real dispatcher, as SessionLogDispatcherWiringService does. + var dispatcher = ActorRegistry.Get(); + _provider.AttachSessionDispatcher(Task.FromResult(dispatcher)); + + var sessionId = new SessionId("intgr-channel/intgr-thread"); + const string marker = "hello from a real actor (partition integration)"; + + var probe = Sys.ActorOf(Props.Create(() => new SessionTaggedLogger()), "session-tagged-logger"); + probe.Tell(new LogIt(sessionId.Value, marker)); + + var sessionLogPath = SessionLogFile.GetLogPath(sessionId, SessionsDir); + await AwaitAssertAsync( + async () => + { + Assert.True(File.Exists(sessionLogPath), "session.log was not created for the tagged actor log"); + var text = await ReadSharedAsync(sessionLogPath, TestContext.Current.CancellationToken); + Assert.Contains(marker, text, StringComparison.Ordinal); + }, + TimeSpan.FromSeconds(10), + cancellationToken: TestContext.Current.CancellationToken); + + // Partition: the session-tagged line is NOT in daemon.log (daemon writes flush per line). + var daemonFiles = Directory.GetFiles(_logDir, "daemon-*.log"); + var daemonText = daemonFiles.Length == 0 + ? string.Empty + : await ReadSharedAsync(daemonFiles[0], TestContext.Current.CancellationToken); + Assert.DoesNotContain(marker, daemonText, StringComparison.Ordinal); + + TryCleanup(); + } + + private static async Task ReadSharedAsync(string path, CancellationToken ct) + { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + using var reader = new StreamReader(stream); + return await reader.ReadToEndAsync(ct); + } + + private void TryCleanup() + { + try + { + if (Directory.Exists(_logDir)) + Directory.Delete(_logDir, recursive: true); + } + catch (IOException ex) + { + Console.Error.WriteLine($"[SessionLogPartitionIntegrationTests] cleanup failed: {ex.Message}"); + } + } + + private sealed record LogIt(string SessionId, string Message); + + private sealed class SessionTaggedLogger : ReceiveActor + { + public SessionTaggedLogger() + { + Receive(m => + Context.GetLogger().WithContext("SessionId", m.SessionId).Info(m.Message)); + } + } +} From d95ba18a06a88f4f9e0bedba4fefd57835e16936 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 00:37:23 +0000 Subject: [PATCH 12/22] fix(logging): harden the session-partition sink (code-review follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xhigh review fixes on the partition rework: - Concurrency in RollingFileLoggerProvider's pre-resolution buffer had a null-deref (NRE thrown into a logging call) and a lost-line race against dispatcher resolution. Replaced the Interlocked/ConcurrentQueue dance with a lock-guarded slow path; the steady state (dispatcher resolved) stays lock-free via a Volatile read. On resolution FAILURE the buffered lines now drain to daemon.log instead of being dropped, and the one-shot failure beacon has a stderr fallback if the daemon queue is saturated. - SessionLogActor flush failure: Flush() now resets _unflushedWrites and rate-limits the warning to onset+recovery, so a persistent disk error no longer floods daemon.log every tick. Its own self-logs use a {Session} key (not {SessionId}) so they can't arm routing back into the file that just failed — defense-in-depth on the feedback carve-out. - FindSessionIdInScopes: static lambda + StrongBox state to drop the per-line closure allocation on the chat-client hot path. - Docs: fixed the LoggingChatClient comment (its lines DO route to session.log now) and the diagnostics runbook (a daemon-wide provider outage surfaces as the sessionless provider.unreachable alert in daemon.log; per-call failover detail is in each session's session.log). Test: Dispatcher_resolution_failure_drains_buffer_and_beacons_to_daemon_log. --- .../references/diagnostics.md | 10 +- .../Sessions/SessionLogActor.cs | 22 +++- .../RollingFileLoggerPartitionTests.cs | 43 ++++++++ .../Configuration/LoggingChatClient.cs | 16 +-- .../RollingFileLoggerProvider.cs | 100 ++++++++++++------ 5 files changed, 146 insertions(+), 45 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 40b71ec81..602132ccd 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -20,9 +20,13 @@ Log split — one stream, partitioned locally by session: a `{SessionId}` message field, or a `SessionId` logging scope) is written to that session's `session.log` and **not** to `daemon.log`. The partition is by session id — nothing is duplicated locally. -- `daemon.log` holds only genuinely daemon-wide lines: startup/config, session - start/stop, and global errors (e.g. an inference provider becoming unreachable). - Rolled daily, capped at 10 MB per file. +- `daemon.log` holds only sessionless, daemon-wide lines: startup/config, session + start/stop, and operational **alerts** (e.g. the `provider.unreachable` / + `provider.failover` alert raised when an inference provider goes down — surfaced + here, and to webhooks, by the notification sink). Note the *per-call* failover/retry + log lines emitted while serving a specific session carry that session's id, so they + partition into its `session.log`; the daemon-wide outage signal is the alert in + `daemon.log`. Rolled daily, capped at 10 MB per file. - The **full** stream (daemon and session lines alike) is also exported to OTEL/Seq with the session id as an attribute; do the global slicing/distilling on the OTEL receiver side. diff --git a/src/Netclaw.Actors/Sessions/SessionLogActor.cs b/src/Netclaw.Actors/Sessions/SessionLogActor.cs index 49079934a..c78bc4aa5 100644 --- a/src/Netclaw.Actors/Sessions/SessionLogActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionLogActor.cs @@ -48,6 +48,7 @@ public sealed class SessionLogActor : ReceiveActor, IWithTimers private readonly ILoggingAdapter _log = Context.GetLogger(); private StreamWriter? _writer; private int _unflushedWrites; + private bool _flushFailing; public ITimerScheduler Timers { get; set; } = null!; @@ -125,10 +126,23 @@ private void Flush() { _writer?.Flush(); _unflushedWrites = 0; + if (_flushFailing) + { + _flushFailing = false; + _log.Info("session.log flushing recovered for {Session}", _sessionId.Value); + } } catch (Exception ex) { - _log.Warning(ex, "Failed to flush session.log for {SessionId}", _sessionId.Value); + // Reset so a persistent failure (full disk, locked file) does not busy-flush-and-warn + // on every 1s tick; the unflushed bytes stay in the StreamWriter buffer and are retried + // on the next write batch (and on dispose). Warn once on onset, once on recovery. + _unflushedWrites = 0; + if (!_flushFailing) + { + _flushFailing = true; + _log.Warning(ex, "Failed to flush session.log for {Session}; further flush failures suppressed until recovery", _sessionId.Value); + } } } @@ -145,7 +159,7 @@ private void OnUserMessage(SendUserMessage msg) } catch (Exception ex) { - _log.Warning(ex, "Dropped user message audit line for {SessionId}", _sessionId.Value); + _log.Warning(ex, "Dropped user message audit line for {Session}", _sessionId.Value); } } @@ -182,7 +196,7 @@ private void OnOutput(SessionOutput output) } catch (Exception ex) { - _log.Warning(ex, "Dropped session log audit line for {SessionId}", _sessionId.Value); + _log.Warning(ex, "Dropped session log audit line for {Session}", _sessionId.Value); } } @@ -194,7 +208,7 @@ private void OnDiagnostic(SessionLogDiagnostic diagnostic) } catch (Exception ex) { - _log.Warning(ex, "Dropped diagnostic audit line for {SessionId}", _sessionId.Value); + _log.Warning(ex, "Dropped diagnostic audit line for {Session}", _sessionId.Value); } } diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs index b8bc44730..a9121188c 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -138,6 +138,49 @@ public async Task Session_lines_emitted_before_dispatcher_resolves_buffer_then_d Cleanup(dir); } + [Fact] + public async Task Dispatcher_resolution_failure_drains_buffer_and_beacons_to_daemon_log() + { + var (dir, daemonPath) = TempPaths(); + var pending = new TaskCompletionSource(); + var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); + try + { + provider.AttachSessionDispatcher(pending.Task); // routing on, dispatcher not resolved + provider.CreateLogger("Netclaw.Tools").LogInformation("buffered before failure {SessionId}", "C9/T9"); + + pending.SetException(new InvalidOperationException("dispatcher never registered")); + + // On failure: the buffered session line is NOT dropped — it falls back to daemon.log, + // and a single beacon records that routing was disabled. + await AwaitAssertAsync( + async () => + { + var text = await ReadDaemonSharedAsync(dir, TestContext.Current.CancellationToken); + Assert.Contains("buffered before failure", text, StringComparison.Ordinal); + Assert.Contains("per-session routing disabled", text, StringComparison.Ordinal); + }, + TimeSpan.FromSeconds(5), + cancellationToken: TestContext.Current.CancellationToken); + } + finally + { + provider.Dispose(); + } + + Cleanup(dir); + } + + private static async Task ReadDaemonSharedAsync(string dir, CancellationToken ct) + { + var files = Directory.GetFiles(dir, "daemon-*.log"); + if (files.Length == 0) + return string.Empty; + await using var stream = new FileStream(files[0], FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + using var reader = new StreamReader(stream); + return await reader.ReadToEndAsync(ct); + } + private static (string Dir, string DaemonPath) TempPaths() { var dir = Path.Combine(Path.GetTempPath(), $"netclaw-partition-{Guid.NewGuid():N}"); diff --git a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs index ae6815681..dcf6c75e9 100644 --- a/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/LoggingChatClient.cs @@ -13,14 +13,14 @@ namespace Netclaw.Daemon.Configuration; /// -/// Decorates an with logging for elapsed time, token -/// usage, and errors. These diagnostics go to daemon.log (the decorator is -/// session-agnostic); they are not published to a session's session.log, but -/// they are tagged with the owning SessionId — read from -/// on the call — so they correlate to a session -/// in Seq/OTLP. Stateless and safe to share across sessions. Netclaw issues only -/// streaming requests, so only the streaming path is instrumented; the inherited -/// non-streaming pass-through is unused. +/// Decorates an with logging for elapsed time, token usage, and +/// errors. The decorator is session-agnostic but tags each line with the owning +/// SessionId via a logging scope — read from on +/// the call (). The file-logger partitions scoped lines into +/// that session's session.log (and they carry SessionId as a Seq/OTLP field); a +/// call with no session scope falls through to daemon.log. Stateless and safe to share +/// across sessions. Netclaw issues only streaming requests, so only the streaming path is +/// instrumented; the inherited non-streaming pass-through is unused. /// public sealed class LoggingChatClient : DelegatingChatClient { diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index aa9cd772b..bc0b52f8f 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -5,6 +5,7 @@ // ----------------------------------------------------------------------- using System.Collections.Concurrent; using System.Globalization; +using System.Runtime.CompilerServices; using Akka.Actor; using Microsoft.Extensions.Logging; using Netclaw.Actors.Protocol; @@ -41,9 +42,14 @@ internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExter private readonly BlockingCollection _queue = new(1024); private readonly Thread _writerThread; private IExternalScopeProvider? _scopeProvider; - private ConcurrentQueue? _pendingDiagnostics; + + // The startup-window buffer and its dispatcher/failed transition are all guarded by + // _routeGate. The lock is only ever taken on the slow path (before _sessionDispatcher is + // published); once resolved, Route's fast path reads _sessionDispatcher lock-free. + private readonly object _routeGate = new(); + private Queue? _pendingDiagnostics; private IActorRef? _sessionDispatcher; - private int _pendingCount; + private bool _routingFailed; private int _sessionRoutingEnabled; private StreamWriter? _writer; private string _currentDate = ""; @@ -79,7 +85,9 @@ public void AttachSessionDispatcher(Task dispatcherTask) if (Interlocked.Exchange(ref _sessionRoutingEnabled, 1) == 1) return; - _pendingDiagnostics = new ConcurrentQueue(); + lock (_routeGate) + _pendingDiagnostics = new Queue(); + _ = ResolveSessionDispatcherAsync(dispatcherTask); } @@ -103,6 +111,8 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo var diagnostic = new SessionLogDiagnostic(new SessionId(sessionId), line); + // Fast path: once resolved, _sessionDispatcher is non-null for the rest of the process, + // so the steady state never takes the lock. var dispatcher = Volatile.Read(ref _sessionDispatcher); if (dispatcher is not null) { @@ -110,16 +120,27 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo return; } - // Dispatcher not resolved yet. Buffer up to a bound; once full, fall back to daemon.log - // rather than drop the line. - if (Volatile.Read(ref _pendingCount) >= PreResolutionBufferLimit) + // Slow path — only during the brief window before resolution (or after a failure). + // Serialize with ResolveSessionDispatcherAsync so the buffer is never enqueued-into after + // it has been drained/abandoned (no lost lines, no null-deref on a torn-down buffer). + lock (_routeGate) { - _queue.TryAdd(line); - return; - } + if (_sessionDispatcher is { } resolved) + { + resolved.Tell(diagnostic); + return; + } - Interlocked.Increment(ref _pendingCount); - _pendingDiagnostics!.Enqueue(diagnostic); + // Routing failed, the buffer is gone, or it is full → fall back to daemon.log rather + // than drop the line. + if (_routingFailed || _pendingDiagnostics is null || _pendingDiagnostics.Count >= PreResolutionBufferLimit) + { + _queue.TryAdd(line); + return; + } + + _pendingDiagnostics.Enqueue(diagnostic); + } } private string? FindSessionIdInScopes() @@ -128,11 +149,13 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo if (scopeProvider is null) return null; - string? found = null; + // static lambda + StrongBox state: the delegate is cached and nothing is captured, so the + // chat-client diagnostic hot path doesn't allocate a closure per logged line. + var box = new StrongBox(null); scopeProvider.ForEachScope( - (scope, _) => + static (scope, state) => { - if (found is not null) + if (state.Value is not null) return; if (scope is IEnumerable> kvps) @@ -142,15 +165,15 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo if (kv.Key == NetclawLogProperties.SessionId && kv.Value?.ToString() is { Length: > 0 } id) { - found = id; + state.Value = id; return; } } } }, - (object?)null); + box); - return found; + return box.Value; } private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) @@ -162,23 +185,40 @@ private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) } catch (Exception ex) { - // Resolution failed permanently. Drop the buffer and switch routing off — daemon - // logging continues, but session-scoped lines fall back to daemon.log for the rest - // of the process. Surface one loud line so this is visible, not silently degraded. - _queue.TryAdd($"{GetTimestamp()} [ERR] Netclaw.Logging: session log dispatcher resolution failed; per-session routing disabled. {ex.Message}"); - Volatile.Write(ref _sessionRoutingEnabled, 0); - _pendingDiagnostics = null; + // Resolution failed permanently. Mark routing failed and FLUSH the buffered lines to + // daemon.log (rather than drop them) so the diagnostics that preceded the failure + // aren't lost; session-scoped lines then fall back to daemon.log for the rest of the + // process. + lock (_routeGate) + { + _routingFailed = true; + if (_pendingDiagnostics is not null) + { + while (_pendingDiagnostics.TryDequeue(out var pending)) + _queue.TryAdd(pending.Line); + _pendingDiagnostics = null; + } + } + + // One loud beacon, with a stderr fallback so the single failure signal is never lost + // even if the daemon-log queue is saturated. + var beacon = $"{GetTimestamp()} [ERR] Netclaw.Logging: session log dispatcher resolution failed; per-session routing disabled. {ex.Message}"; + if (!_queue.TryAdd(beacon)) + Console.Error.WriteLine(beacon); return; } - // Publish the ref BEFORE draining so producers racing with the drainer Tell directly - // rather than enqueueing into a buffer we are about to abandon. - Volatile.Write(ref _sessionDispatcher, dispatcher); - - while (_pendingDiagnostics!.TryDequeue(out var pending)) + // Publish the ref and drain the buffer under the same lock that Route's slow path uses, + // so a line is never enqueued into the buffer after this drain has run. + lock (_routeGate) { - Interlocked.Decrement(ref _pendingCount); - dispatcher.Tell(pending); + Volatile.Write(ref _sessionDispatcher, dispatcher); + if (_pendingDiagnostics is not null) + { + while (_pendingDiagnostics.TryDequeue(out var pending)) + dispatcher.Tell(pending); + _pendingDiagnostics = null; + } } } From fb629a38a9e70ed015059fe607ccb1269ac0314c Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 01:02:50 +0000 Subject: [PATCH 13/22] docs(logging): document the dormant non-streaming retry session-scope gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RetryingChatClient inherits its SessionId scope from the enclosing LoggingChatClient, which instruments only the streaming path. The non-streaming GetResponseAsync retry path therefore opens no scope — dormant today (Netclaw issues only streaming requests), but if a non-streaming path is ever added the retry warnings would lose session correlation and route to daemon.log. Leave a caveat at BackoffAsync pointing the future fix at ChatClientSessionScope.Begin / LoggingChatClient.GetResponseAsync. --- src/Netclaw.Daemon/Configuration/RetryingChatClient.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs index e320b400e..2ea0d8c4e 100644 --- a/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs +++ b/src/Netclaw.Daemon/Configuration/RetryingChatClient.cs @@ -127,6 +127,15 @@ private async Task BackoffAsync(Exception ex, int attempt, CancellationToken can // Compose_puts_Logging_outermost), whose streaming scope stays open for the whole // enumeration that drives this retry loop — so the warning already inherits the // session id. Re-opening an identical scope would be pure duplication. + // + // CAVEAT — this inheritance holds ONLY on the streaming path. LoggingChatClient + // instruments streaming only; its inherited non-streaming GetResponseAsync opens no + // scope. Netclaw issues only streaming requests today, so the non-streaming retry + // path above is unreachable — but if that ever changes, these retry warnings would + // carry no SessionId and the file-logger would route them to daemon.log instead of + // the owning session.log (and they'd be uncorrelated in Seq). The fix at that point + // is to open a SessionId scope here from `options` (ChatClientSessionScope.Begin), + // or to instrument LoggingChatClient.GetResponseAsync. See RetryingChatClientTests. _logger.LogWarning(ex, "LLM call failed (attempt {Attempt}/{Max}), retrying in {Delay:F1}s", attempt + 1, _policy.MaxRetries, delay.TotalSeconds); From 70d8bf42c82e8697c6ae76264bf616077888b6f3 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 15:03:37 +0000 Subject: [PATCH 14/22] refactor(logging): drop the startup buffer + lock for a single volatile dispatcher ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-resolution buffer (and the lock guarding it, where the review found a null-deref and a lost-line race) only existed to route session-tagged lines emitted in the window between AttachSessionDispatcher and the dispatcher resolving. But the things that log with a SessionId — session actors, chat-client calls — don't exist until a session starts processing a turn, well after startup, so that window carries no session traffic. Collapse it to one volatile _sessionDispatcher reference that IRequiredActor.GetAsync resolves in the background; a line that finds it still null (startup window or resolution failure) falls back to daemon.log, which is already the documented routing-off behavior. Removes _pendingDiagnostics, the _routeGate lock, _routingFailed, _sessionRoutingEnabled, and the whole concurrency surface the review flagged (~60 lines). Resolution stays in the background — not a blocking await in the wiring service's StartAsync, which runs before Akka and would deadlock. Tests updated: a pre-resolution session line now falls back to daemon.log rather than buffer-and-drain. --- .../RollingFileLoggerPartitionTests.cs | 31 ++--- .../RollingFileLoggerProvider.cs | 114 ++++-------------- 2 files changed, 42 insertions(+), 103 deletions(-) diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs index a9121188c..b90093d86 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -119,45 +119,46 @@ public async Task Session_log_writers_own_line_is_not_routed_back() } [Fact] - public async Task Session_lines_emitted_before_dispatcher_resolves_buffer_then_drain() + public async Task Session_line_before_dispatcher_resolves_falls_back_to_daemon_log() { var (dir, daemonPath) = TempPaths(); var dispatcher = CreateTestProbe("dispatcher"); var pending = new TaskCompletionSource(); - using var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); - provider.AttachSessionDispatcher(pending.Task); // routing on, dispatcher not yet resolved - provider.CreateLogger("Netclaw.Tools").LogInformation("buffered op {SessionId}", "C4/T4"); + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(pending.Task); // never resolves during this test + provider.CreateLogger("Netclaw.Tools").LogInformation("before resolve {SessionId}", "C4/T4"); - await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); + // No buffering: a line logged before the dispatcher resolves is not held for it; it + // goes straight to daemon.log (the dispatcher never sees it). + await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + } - pending.SetResult(dispatcher.Ref); // resolution drains the buffer in order - var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal("C4/T4", diag.SessionId.Value); - Assert.Contains("buffered op", diag.Line, StringComparison.Ordinal); + Assert.Contains("before resolve", ReadDaemonLog(dir), StringComparison.Ordinal); Cleanup(dir); } [Fact] - public async Task Dispatcher_resolution_failure_drains_buffer_and_beacons_to_daemon_log() + public async Task Dispatcher_resolution_failure_falls_back_and_beacons_to_daemon_log() { var (dir, daemonPath) = TempPaths(); var pending = new TaskCompletionSource(); var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); try { - provider.AttachSessionDispatcher(pending.Task); // routing on, dispatcher not resolved - provider.CreateLogger("Netclaw.Tools").LogInformation("buffered before failure {SessionId}", "C9/T9"); + provider.AttachSessionDispatcher(pending.Task); // dispatcher not resolved + provider.CreateLogger("Netclaw.Tools").LogInformation("before failure {SessionId}", "C9/T9"); pending.SetException(new InvalidOperationException("dispatcher never registered")); - // On failure: the buffered session line is NOT dropped — it falls back to daemon.log, - // and a single beacon records that routing was disabled. + // The session line fell back to daemon.log at log time (dispatcher still null); on + // failure a single beacon records that per-session routing is disabled. await AwaitAssertAsync( async () => { var text = await ReadDaemonSharedAsync(dir, TestContext.Current.CancellationToken); - Assert.Contains("buffered before failure", text, StringComparison.Ordinal); + Assert.Contains("before failure", text, StringComparison.Ordinal); Assert.Contains("per-session routing disabled", text, StringComparison.Ordinal); }, TimeSpan.FromSeconds(5), diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index bc0b52f8f..05fa84ac8 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -34,7 +34,6 @@ namespace Netclaw.Daemon.Configuration; internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExternalScope { private const long MaxFileSizeBytes = 10 * 1024 * 1024; // 10MB per file - private const int PreResolutionBufferLimit = 1000; private readonly string _basePath; private readonly TimeProvider _timeProvider; @@ -43,14 +42,12 @@ internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExter private readonly Thread _writerThread; private IExternalScopeProvider? _scopeProvider; - // The startup-window buffer and its dispatcher/failed transition are all guarded by - // _routeGate. The lock is only ever taken on the slow path (before _sessionDispatcher is - // published); once resolved, Route's fast path reads _sessionDispatcher lock-free. - private readonly object _routeGate = new(); - private Queue? _pendingDiagnostics; + // The dispatcher reference is the only shared state on the routing path: null until + // IRequiredActor.GetAsync resolves it, then written once (Volatile). No buffer/lock — a line + // that finds it still null (the brief startup window, or a resolution failure) falls back to + // daemon.log, which is the documented routing-off behavior. private IActorRef? _sessionDispatcher; - private bool _routingFailed; - private int _sessionRoutingEnabled; + private int _attached; private StreamWriter? _writer; private string _currentDate = ""; @@ -74,28 +71,26 @@ public ILogger CreateLogger(string categoryName) => public void SetScopeProvider(IExternalScopeProvider scopeProvider) => _scopeProvider = scopeProvider; /// - /// Enables per-session routing once the actor system is up. Session-tagged lines emitted - /// between this call and dispatcher resolution are held in a small bounded buffer and then - /// drained in order; afterwards they Tell the dispatcher directly. If resolution fails, a - /// single ERR line is written to daemon.log and routing is disabled (session-tagged - /// lines fall back to daemon.log) for the rest of the process. + /// Resolves the per-session log dispatcher in the background (the caller passes + /// IRequiredActor<SessionLogDispatcherActorKey>.GetAsync()) and publishes it. + /// Resolving in the background — rather than blocking — keeps this safe to call from a hosted + /// service that starts before Akka. Session-tagged lines logged before it resolves (a brief + /// startup window in which session actors do not yet exist) fall back to daemon.log, as + /// do all session-tagged lines if resolution fails (a single ERR beacon records that). /// public void AttachSessionDispatcher(Task dispatcherTask) { - if (Interlocked.Exchange(ref _sessionRoutingEnabled, 1) == 1) + if (Interlocked.Exchange(ref _attached, 1) == 1) return; - lock (_routeGate) - _pendingDiagnostics = new Queue(); - _ = ResolveSessionDispatcherAsync(dispatcherTask); } /// - /// Partitions one already-formatted line: routes it to its session's session.log - /// when it carries a session id (and is not from the session-log writer itself), otherwise - /// writes it to daemon.log. is the id found on the - /// log event's state; the active scopes are consulted only as a fallback. + /// Partitions one already-formatted line: routes it to its session's session.log when + /// it carries a session id (and is not the session-log writer's own line) and the dispatcher + /// is resolved; otherwise writes it to daemon.log. is + /// the id found on the log event's state; the active scopes are consulted only as a fallback. /// internal void Route(string line, string? stateSessionId, bool fromSessionLogActor) { @@ -103,44 +98,15 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo // logs an error cannot route back into the same (failing) session.log — an infinite loop. var sessionId = fromSessionLogActor ? null : (stateSessionId ?? FindSessionIdInScopes()); - if (string.IsNullOrWhiteSpace(sessionId) || _sessionRoutingEnabled == 0) + if (!string.IsNullOrWhiteSpace(sessionId) && Volatile.Read(ref _sessionDispatcher) is { } dispatcher) { - _queue.TryAdd(line); + dispatcher.Tell(new SessionLogDiagnostic(new SessionId(sessionId), line)); return; } - var diagnostic = new SessionLogDiagnostic(new SessionId(sessionId), line); - - // Fast path: once resolved, _sessionDispatcher is non-null for the rest of the process, - // so the steady state never takes the lock. - var dispatcher = Volatile.Read(ref _sessionDispatcher); - if (dispatcher is not null) - { - dispatcher.Tell(diagnostic); - return; - } - - // Slow path — only during the brief window before resolution (or after a failure). - // Serialize with ResolveSessionDispatcherAsync so the buffer is never enqueued-into after - // it has been drained/abandoned (no lost lines, no null-deref on a torn-down buffer). - lock (_routeGate) - { - if (_sessionDispatcher is { } resolved) - { - resolved.Tell(diagnostic); - return; - } - - // Routing failed, the buffer is gone, or it is full → fall back to daemon.log rather - // than drop the line. - if (_routingFailed || _pendingDiagnostics is null || _pendingDiagnostics.Count >= PreResolutionBufferLimit) - { - _queue.TryAdd(line); - return; - } - - _pendingDiagnostics.Enqueue(diagnostic); - } + // No session id, the dispatcher hasn't resolved yet (startup window), or resolution + // failed → daemon.log. + _queue.TryAdd(line); } private string? FindSessionIdInScopes() @@ -178,47 +144,19 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) { - IActorRef dispatcher; try { - dispatcher = await dispatcherTask.ConfigureAwait(false); + // Publish once. After this, every Route fast-path reads it and Tells directly. + Volatile.Write(ref _sessionDispatcher, await dispatcherTask.ConfigureAwait(false)); } catch (Exception ex) { - // Resolution failed permanently. Mark routing failed and FLUSH the buffered lines to - // daemon.log (rather than drop them) so the diagnostics that preceded the failure - // aren't lost; session-scoped lines then fall back to daemon.log for the rest of the - // process. - lock (_routeGate) - { - _routingFailed = true; - if (_pendingDiagnostics is not null) - { - while (_pendingDiagnostics.TryDequeue(out var pending)) - _queue.TryAdd(pending.Line); - _pendingDiagnostics = null; - } - } - - // One loud beacon, with a stderr fallback so the single failure signal is never lost - // even if the daemon-log queue is saturated. + // Resolution failed: _sessionDispatcher stays null, so session-tagged lines keep + // falling back to daemon.log. One loud beacon, with a stderr fallback so the single + // failure signal survives even a saturated daemon-log queue. var beacon = $"{GetTimestamp()} [ERR] Netclaw.Logging: session log dispatcher resolution failed; per-session routing disabled. {ex.Message}"; if (!_queue.TryAdd(beacon)) Console.Error.WriteLine(beacon); - return; - } - - // Publish the ref and drain the buffer under the same lock that Route's slow path uses, - // so a line is never enqueued into the buffer after this drain has run. - lock (_routeGate) - { - Volatile.Write(ref _sessionDispatcher, dispatcher); - if (_pendingDiagnostics is not null) - { - while (_pendingDiagnostics.TryDequeue(out var pending)) - dispatcher.Tell(pending); - _pendingDiagnostics = null; - } } } From f49a87b46cc0715cf37de5ee430d377bcf3ef259 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 17:59:13 +0000 Subject: [PATCH 15/22] fix(logging): breadcrumb emitters take SubAgentRunId after the merged run-id refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merged SubAgentRunId/SubAgentScopeId refactor changed SubAgentSpawner's runId from a string to a SubAgentRunId value object, but the spawn breadcrumb emitters still took `string runId` — a CI compile failure (CS1503). Take SubAgentRunId and log runId.Value. --- .../SubAgents/SubAgentSpawnBreadcrumbs.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs index 0667d4efa..73e223c9c 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs @@ -44,36 +44,36 @@ public static void NoToolsAvailable(ILogger? logger, ToolExecutionContext contex agentName); } - public static void ChildSpawnFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) + public static void ChildSpawnFailed(ILogger? logger, ToolExecutionContext context, string agentName, SubAgentRunId runId, Exception ex) { using (BeginSessionScope(logger, context.SessionId)) logger?.LogError( ex, "SubAgent [{AgentName}] failed to spawn child actor (runId={RunId})", - agentName, runId); + agentName, runId.Value); } - public static void ChildSpawned(ILogger? logger, ToolExecutionContext context, string agentName, string runId) + public static void ChildSpawned(ILogger? logger, ToolExecutionContext context, string agentName, SubAgentRunId runId) { using (BeginSessionScope(logger, context.SessionId)) logger?.LogInformation( "SubAgent [{AgentName}] child actor spawned (runId={RunId}); dispatching RunSubAgent", - agentName, runId); + agentName, runId.Value); } - public static void Completed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, bool success, long durationMs) + public static void Completed(ILogger? logger, ToolExecutionContext context, string agentName, SubAgentRunId runId, bool success, long durationMs) { using (BeginSessionScope(logger, context.SessionId)) logger?.LogInformation( "SubAgent [{AgentName}] completed (runId={RunId}, success={Success}, duration={Duration}ms)", - agentName, runId, success, durationMs); + agentName, runId.Value, success, durationMs); } - public static void RunFailed(ILogger? logger, ToolExecutionContext context, string agentName, string runId, Exception ex) + public static void RunFailed(ILogger? logger, ToolExecutionContext context, string agentName, SubAgentRunId runId, Exception ex) { using (BeginSessionScope(logger, context.SessionId)) logger?.LogError( ex, "SubAgent [{AgentName}] run failed (runId={RunId})", - agentName, runId); + agentName, runId.Value); } public static void SpawnRefused(ILogger? logger, ToolExecutionContext context, string agentName, TrustAudience audience, bool subsystemEnabled) From a9b1ea06a3ac7a759fc84c8b4be50ae46b59368f Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 18:11:17 +0000 Subject: [PATCH 16/22] feat(logging): give each sub-agent run its own session.log, keyed by SubSessionId Per the design call: a parent investigating a sub-agent reviews that run's own log, and the parent's session.log stays clean (transcript + spawn breadcrumbs only). - RollingFileLoggerProvider reads SubSessionId (from log state or scope) and partitions the LOCAL file by SubSessionId when present, else SessionId. The line still carries the parent SessionId, so OTEL groups sub-agent lines under the parent AND slices by SubSessionId. - SessionScopedChatOptions gains SubSessionId; ChatClientSessionScope emits it in the scope; SubAgentActor sets it (its scopeId {parent}/subagent/{name}/{runId}) on the LLM-call options. The sub-agent's own actor logs already carried both ids. - Spawn breadcrumbs stay in the PARENT's log (parent context, no SubSessionId) as the pointer. Tests: provider routes a state- or scope-carried SubSessionId line to the sub-run file (not the parent's). diagnostics skill updated (2.22.0). --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../references/diagnostics.md | 9 +- .../Sessions/SessionScopedChatOptions.cs | 12 ++- src/Netclaw.Actors/SubAgents/SubAgentActor.cs | 23 +++-- .../RollingFileLoggerPartitionTests.cs | 53 +++++++++++ .../Configuration/ChatClientSessionScope.cs | 22 ++++- .../RollingFileLoggerProvider.cs | 90 +++++++++++++------ 7 files changed, 165 insertions(+), 46 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 352cbe69f..f5a57c96b 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.21.0" + version: "2.22.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 602132ccd..9d9afe5bc 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -31,8 +31,13 @@ Log split — one stream, partitioned locally by session: with the session id as an attribute; do the global slicing/distilling on the OTEL receiver side. - Session log directories use the sanitized session ID (`/`, `.`, spaces, etc. - replaced with `_`). Sub-agent activity rolls up into the parent session's - `session.log`; there is no separate file per sub-agent run. No rotation today + replaced with `_`). Each **sub-agent run** writes to its **own** `session.log`, + keyed by its sub-session id (`{parentId}/subagent/{name}/{runId}`, sanitized) — + so a sub-agent's detail stays out of the parent's log and you review it in that + run's own file. The parent's `session.log` keeps the spawn breadcrumbs (requested + → spawned → completed/failed) as the pointer to each run. In OTEL the sub-agent + lines still carry the parent `SessionId` (so they group under the parent) plus + `SubSessionId` (so they slice by run). No rotation today (see netclaw-dev/netclaw#919). What to expect inside `session.log`: diff --git a/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs b/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs index 5e316fbf5..ecf8f6610 100644 --- a/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs +++ b/src/Netclaw.Actors/Sessions/SessionScopedChatOptions.cs @@ -27,8 +27,14 @@ namespace Netclaw.Actors.Sessions; /// public sealed class SessionScopedChatOptions : ChatOptions { - /// The session whose turn this LLM call belongs to. For sub-agent calls - /// this is the parent session id, so a sub-agent's LLM diagnostics correlate to the - /// session that spawned it. + /// The session whose turn this LLM call belongs to. For sub-agent calls this is + /// the parent session id, so OTEL/Seq can still group the call under the spawning session. public required string SessionId { get; init; } + + /// For a sub-agent's LLM call, the sub-session id (its run scope + /// {parent}/subagent/{name}/{runId}). When set, the file-logger partitions these + /// lines into the sub-agent's own session.log (the local file is keyed by the + /// sub-session, while keeps OTEL grouping under the parent). Null + /// for top-level session calls. + public string? SubSessionId { get; init; } } diff --git a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs index 3fd8919af..d28f62512 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentActor.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentActor.cs @@ -98,10 +98,12 @@ [Subagent Execution Contract] private IParentApprovalBridge? _approvalBridge; private ChannelWriter? _activitySink; - // Parent session id (scopeId with the "/subagent/..." suffix stripped). Carried on - // the sub-agent's ChatOptions so its LLM diagnostics correlate to the spawning - // session in Seq, matching the SessionId its enriched logger already uses. + // Parent session id (scopeId with the "/subagent/..." suffix stripped) and the full + // sub-session scope id. Carried on the sub-agent's ChatOptions so its LLM-pipeline lines + // group under the parent in OTEL (SessionId) while the file-logger partitions them into the + // sub-agent's own session.log (SubSessionId) — matching the enriched logger's own context. private string? _parentSessionId; + private string? _subSessionId; // Default wait-for-first-delta budget when the spawn message carries none // (direct/test callers). Mirrors SessionConfig.PrefillTimeout so an unset @@ -269,10 +271,12 @@ private void Idle() // "{parentSessionId}/subagent/{name}/{runId}"; NormalizeSessionId strips the // "/subagent/..." suffix to recover the parent. SessionId matches the key the // session/channel actors already tag their loggers with, so sub-agent and - // parent logs share one filterable attribute (and route to the same - // session.log); SubSessionId isolates a single run within that session. + // parent logs share one filterable attribute (so OTEL groups them under the parent); + // SubSessionId isolates a single run and is what the file-logger partitions on, so the + // sub-agent's lines land in its OWN session.log rather than the parent's. var parentSessionId = SubAgentSessionScope.NormalizeSessionId(scopeId); _parentSessionId = parentSessionId; + _subSessionId = scopeId; var enrichedLog = Context.GetLogger(); if (!string.IsNullOrWhiteSpace(parentSessionId)) enrichedLog = enrichedLog.WithContext(NetclawLogProperties.SessionId, parentSessionId); @@ -682,11 +686,12 @@ private void FireLlmCall(bool forceNoTools = false) var messages = new List(_history); var callId = ++_llmCallId; - // Carry the parent session id (when known) so the chat-client decorators - // correlate this sub-agent's LLM diagnostics to the spawning session in Seq, - // just like the main session path. Direct/test callers leave it unset. + // Carry the parent session id (when known) so the chat-client decorators group this + // sub-agent's LLM-pipeline lines under the spawning session in OTEL, plus the sub-session + // id so the file-logger partitions them into the sub-agent's OWN session.log. Direct/test + // callers leave both unset. ChatOptions? options = _parentSessionId is { Length: > 0 } sid - ? new SessionScopedChatOptions { SessionId = sid } + ? new SessionScopedChatOptions { SessionId = sid, SubSessionId = _subSessionId } : null; if (!forceNoTools && _aiTools.Count > 0) { diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs index b90093d86..79d16c373 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -73,6 +73,59 @@ public async Task Session_id_carried_in_a_scope_routes() Cleanup(dir); } + [Fact] + public async Task Sub_agent_line_routes_to_its_own_file_keyed_by_sub_session_id() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + + // A bridged sub-agent line carries the parent SessionId (for OTEL grouping) AND the + // sub-session id; the LOCAL file is partitioned by the sub-session. + var state = new List> + { + new(NetclawLogProperties.SessionId, "C1/T1"), + new(NetclawLogProperties.SubSessionId, "C1/T1/subagent/summarizer/ab12"), + }; + provider.CreateLogger("Akka.Actor.ActorSystem") + .Log(LogLevel.Information, new EventId(0), state, null, (_, _) => "sub-agent did work"); + + var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("C1/T1/subagent/summarizer/ab12", diag.SessionId.Value); // sub file, not the parent + Assert.Contains("sub-agent did work", diag.Line, StringComparison.Ordinal); + } + + Cleanup(dir); + } + + [Fact] + public async Task Sub_session_id_carried_in_a_scope_routes_to_the_sub_agent_file() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow)); + var factory = LoggerFactory.Create(b => b.AddProvider(provider)); + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + + var logger = factory.CreateLogger("Netclaw.LlmPipeline"); + using (logger.BeginScope(new[] + { + new KeyValuePair(NetclawLogProperties.SessionId, "C2/T2"), + new KeyValuePair(NetclawLogProperties.SubSessionId, "C2/T2/subagent/coder/cd34"), + })) + logger.LogInformation("sub-agent LLM streaming call completed"); + + var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); + Assert.Equal("C2/T2/subagent/coder/cd34", diag.SessionId.Value); + Assert.Contains("sub-agent LLM streaming call completed", diag.Line, StringComparison.Ordinal); + + factory.Dispose(); + Cleanup(dir); + } + [Fact] public async Task Sessionless_line_goes_to_daemon_log() { diff --git a/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs index c039b8dc2..baa97bb43 100644 --- a/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs +++ b/src/Netclaw.Daemon/Configuration/ChatClientSessionScope.cs @@ -27,8 +27,22 @@ namespace Netclaw.Daemon.Configuration; /// internal static class ChatClientSessionScope { - public static IDisposable? Begin(ILogger logger, ChatOptions? options) => - options is SessionScopedChatOptions { SessionId: { Length: > 0 } sessionId } - ? logger.BeginScope(new[] { new KeyValuePair(NetclawLogProperties.SessionId, sessionId) }) - : null; + public static IDisposable? Begin(ILogger logger, ChatOptions? options) + { + if (options is not SessionScopedChatOptions { SessionId: { Length: > 0 } sessionId } scoped) + return null; + + // A sub-agent's call also carries its SubSessionId, so the file-logger routes the line + // into the sub-agent's own session.log while SessionId keeps OTEL grouping under the parent. + return string.IsNullOrWhiteSpace(scoped.SubSessionId) + ? logger.BeginScope(new[] + { + new KeyValuePair(NetclawLogProperties.SessionId, sessionId), + }) + : logger.BeginScope(new[] + { + new KeyValuePair(NetclawLogProperties.SessionId, sessionId), + new KeyValuePair(NetclawLogProperties.SubSessionId, scoped.SubSessionId), + }); + } } diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index 05fa84ac8..3c1824e02 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -5,7 +5,6 @@ // ----------------------------------------------------------------------- using System.Collections.Concurrent; using System.Globalization; -using System.Runtime.CompilerServices; using Akka.Actor; using Microsoft.Extensions.Logging; using Netclaw.Actors.Protocol; @@ -92,15 +91,33 @@ public void AttachSessionDispatcher(Task dispatcherTask) /// is resolved; otherwise writes it to daemon.log. is /// the id found on the log event's state; the active scopes are consulted only as a fallback. /// - internal void Route(string line, string? stateSessionId, bool fromSessionLogActor) + internal void Route(string line, string? stateSessionId, string? stateSubSessionId, bool fromSessionLogActor) { // Carve-out: the session-log writer's own lines go to daemon.log so a failed write that // logs an error cannot route back into the same (failing) session.log — an infinite loop. - var sessionId = fromSessionLogActor ? null : (stateSessionId ?? FindSessionIdInScopes()); + if (fromSessionLogActor) + { + _queue.TryAdd(line); + return; + } - if (!string.IsNullOrWhiteSpace(sessionId) && Volatile.Read(ref _sessionDispatcher) is { } dispatcher) + var sessionId = stateSessionId; + var subSessionId = stateSubSessionId; + if (sessionId is null || subSessionId is null) { - dispatcher.Tell(new SessionLogDiagnostic(new SessionId(sessionId), line)); + // Chat-client lines carry the ids via BeginScope rather than message state. + FindScopeIds(out var scopeSessionId, out var scopeSubSessionId); + sessionId ??= scopeSessionId; + subSessionId ??= scopeSubSessionId; + } + + // Partition the LOCAL file: a sub-agent's lines (which carry a SubSessionId) get their own + // session.log keyed by the sub-session; everything else goes to its session's file. The + // line still carries the parent SessionId, so OTEL groups it under the parent regardless. + var routingId = subSessionId ?? sessionId; + if (!string.IsNullOrWhiteSpace(routingId) && Volatile.Read(ref _sessionDispatcher) is { } dispatcher) + { + dispatcher.Tell(new SessionLogDiagnostic(new SessionId(routingId), line)); return; } @@ -109,37 +126,49 @@ internal void Route(string line, string? stateSessionId, bool fromSessionLogActo _queue.TryAdd(line); } - private string? FindSessionIdInScopes() + private void FindScopeIds(out string? sessionId, out string? subSessionId) { + sessionId = null; + subSessionId = null; + var scopeProvider = _scopeProvider; if (scopeProvider is null) - return null; + return; - // static lambda + StrongBox state: the delegate is cached and nothing is captured, so the - // chat-client diagnostic hot path doesn't allocate a closure per logged line. - var box = new StrongBox(null); + // static lambda + a single mutable holder: the delegate is cached and nothing is + // captured, so the chat-client diagnostic hot path doesn't allocate a closure per line. + var ids = new ScopeIds(); scopeProvider.ForEachScope( static (scope, state) => { - if (state.Value is not null) + if (state.SessionId is not null && state.SubSessionId is not null) return; if (scope is IEnumerable> kvps) { foreach (var kv in kvps) { - if (kv.Key == NetclawLogProperties.SessionId + if (state.SessionId is null + && kv.Key == NetclawLogProperties.SessionId && kv.Value?.ToString() is { Length: > 0 } id) - { - state.Value = id; - return; - } + state.SessionId = id; + else if (state.SubSessionId is null + && kv.Key == NetclawLogProperties.SubSessionId + && kv.Value?.ToString() is { Length: > 0 } subId) + state.SubSessionId = subId; } } }, - box); + ids); - return box.Value; + sessionId = ids.SessionId; + subSessionId = ids.SubSessionId; + } + + private sealed class ScopeIds + { + public string? SessionId; + public string? SubSessionId; } private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) @@ -239,11 +268,12 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (!IsEnabled(logLevel)) return; - // Single pass over the event's structured state: pick up the session id (for routing), - // the Akka log source (for a useful per-line label — every actor shares the generic MEL - // category "Akka.Actor.ActorSystem"), and whether this line is the session-log writer's - // own (so we never route it back into the file it writes). - ScanState(state, out var sessionId, out var logSource, out var fromSessionLogActor); + // Single pass over the event's structured state: pick up the session id and (for a + // sub-agent's lines) the sub-session id (for routing), the Akka log source (for a useful + // per-line label — every actor shares the generic MEL category "Akka.Actor.ActorSystem"), + // and whether this line is the session-log writer's own (so we never route it back into + // the file it writes). + ScanState(state, out var sessionId, out var subSessionId, out var logSource, out var fromSessionLogActor); var timestamp = _provider.GetTimestamp(); var level = logLevel switch @@ -261,7 +291,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (exception is not null) line += Environment.NewLine + exception; - _provider.Route(line, sessionId, fromSessionLogActor); + _provider.Route(line, sessionId, subSessionId, fromSessionLogActor); } // Read the fields the producer already put on the log event. The Akka→MEL bridge passes an @@ -269,20 +299,21 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except // structured logging passes FormattedLogValues carrying a {SessionId} field. Both surface // their fields as KeyValuePair sequences (the nullable-annotated and // unannotated forms are the same runtime type), so one branch reads both — no Akka internals. - private static void ScanState(TState state, out string? sessionId, out string? logSource, out bool fromSessionLogActor) + private static void ScanState(TState state, out string? sessionId, out string? subSessionId, out string? logSource, out bool fromSessionLogActor) { sessionId = null; + subSessionId = null; logSource = null; fromSessionLogActor = false; if (state is IEnumerable> fields) { foreach (var field in fields) - Apply(field.Key, field.Value, ref sessionId, ref logSource, ref fromSessionLogActor); + Apply(field.Key, field.Value, ref sessionId, ref subSessionId, ref logSource, ref fromSessionLogActor); } } - private static void Apply(string key, object? value, ref string? sessionId, ref string? logSource, ref bool fromSessionLogActor) + private static void Apply(string key, object? value, ref string? sessionId, ref string? subSessionId, ref string? logSource, ref bool fromSessionLogActor) { if (value is null) return; @@ -292,6 +323,11 @@ private static void Apply(string key, object? value, ref string? sessionId, ref if (value.ToString() is { Length: > 0 } id) sessionId = id; } + else if (key == NetclawLogProperties.SubSessionId) + { + if (value.ToString() is { Length: > 0 } subId) + subSessionId = subId; + } else if (key == "LogSource") { logSource = value.ToString(); From 841dc7ce4d1d7208af3181617fea63ec261ceced Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 18:47:09 +0000 Subject: [PATCH 17/22] fix(logging): restore session correlation for sidecar LLM paths (code-review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AsyncLocal removal stripped SessionDiagnosticsContext from five session-owned sidecar chat calls but never gave them the replacement SessionScopedChatOptions carrier, so their chat-client diagnostics (timing, retries, provider failover) silently regressed to daemon.log uncorrelated — and the #920 guard test was deleted in the same commit. Restore correlation by threading SessionScopedChatOptions through all five (each already had the session id in scope): SessionTitleGenerator, SessionCompactionPipeline observer, SessionMemoryObserverActor distillation, MemoryCurationActor, and LlmSessionActor memory-extraction. Replace the deleted guard with SidecarSessionCorrelationTests (one case per sidecar, asserting the carrier). The diagnostics doc's claim that session.log holds the session's LLM-pipeline/memory lines is now true rather than aspirational. Also from the review: - Harden RollingFileLoggerProvider.Route: only consult scopes when there is no state session id, so an ambient SubSessionId scope cannot hijack an actor line's routing. - SessionLogActor.Flush: keep retrying on the 1s tick while failing, so buffered audit bytes are not stranded through an idle period after the disk recovers. - Correct SubAgentSessionScope doc: it claimed sub-agents reuse the parent's log and were the "only place" the /subagent/ split happens — both now false (own file; ToolApprovalActor too). --- .../SidecarSessionCorrelationTests.cs | 173 ++++++++++++++++++ .../Memory/MemoryCurationActor.cs | 7 +- .../Sessions/LlmSessionActor.cs | 5 +- .../Pipelines/SessionCompactionPipeline.cs | 5 +- .../Pipelines/SessionTitleGenerator.cs | 6 +- .../Sessions/SessionLogActor.cs | 11 +- .../Sessions/SessionMemoryObserverActor.cs | 5 +- .../SubAgents/SubAgentSessionScope.cs | 15 +- .../RollingFileLoggerProvider.cs | 8 +- 9 files changed, 216 insertions(+), 19 deletions(-) create mode 100644 src/Netclaw.Actors.Tests/Sessions/SidecarSessionCorrelationTests.cs diff --git a/src/Netclaw.Actors.Tests/Sessions/SidecarSessionCorrelationTests.cs b/src/Netclaw.Actors.Tests/Sessions/SidecarSessionCorrelationTests.cs new file mode 100644 index 000000000..4425ed83c --- /dev/null +++ b/src/Netclaw.Actors.Tests/Sessions/SidecarSessionCorrelationTests.cs @@ -0,0 +1,173 @@ +// ----------------------------------------------------------------------- +// +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- +using System.Runtime.CompilerServices; +using Akka.Event; +using Akka.Hosting.TestKit; +using Microsoft.Extensions.AI; +using Netclaw.Actors.Memory; +using Netclaw.Actors.Protocol; +using Netclaw.Actors.Sessions; +using Netclaw.Actors.Sessions.Pipelines; +using Netclaw.Configuration; +using Xunit; +using AiChatMessage = Microsoft.Extensions.AI.ChatMessage; +using AiChatRole = Microsoft.Extensions.AI.ChatRole; + +namespace Netclaw.Actors.Tests.Sessions; + +/// +/// Regression guard (replaces the deleted SidecarDiagnosticsContextTests, netclaw-dev/netclaw#920) +/// for the session-owned sidecar LLM paths — title generation, compaction observation, memory +/// extraction/distillation, and memory curation. After the SessionDiagnosticsContext AsyncLocal +/// was removed, each must carry its owning session id explicitly via +/// on the chat-call's , so the +/// file-logger routes the call's chat-client diagnostics into that session's session.log (and +/// Seq/OTLP correlate them). Test contract: a fake IChatClient captures the ChatOptions it is +/// invoked with; the assertion is that it is a SessionScopedChatOptions naming the session. +/// Without the carrier these calls silently regress to daemon.log, uncorrelated. +/// +public sealed class SidecarSessionCorrelationTests : TestKit +{ + protected override void ConfigureAkka(Akka.Hosting.AkkaConfigurationBuilder builder, IServiceProvider provider) + { + } + + [Fact] + public async Task TitleGenerator_carries_session_scoped_options() + { + var sessionId = new SessionId("ch/title-thread"); + var captor = new OptionsCapturingChatClient(); + + await SessionTitleGenerator.GenerateAsync( + captor, sessionId, history: [], self: CreateTestProbe().Ref, + log: NoLogger.Instance, timeout: TimeSpan.FromSeconds(5)); + + AssertScopedTo(sessionId, captor); + } + + [Fact] + public async Task CompactionObserver_carries_session_scoped_options() + { + var sessionId = new SessionId("ch/compaction-thread"); + var captor = new OptionsCapturingChatClient(); + var history = new List + { + new() { Role = Netclaw.Actors.Protocol.ChatRole.User, Content = "hello" }, + new() { Role = Netclaw.Actors.Protocol.ChatRole.Assistant, Content = "hi" } + }; + + await SessionCompactionPipeline.GenerateObservationsAsync( + client: captor, sessionId: sessionId, history: history, systemOffset: 0, + keepStartIndex: 1, sidecarTimeout: TimeSpan.FromSeconds(5), log: NoLogger.Instance, + cancellationToken: TestContext.Current.CancellationToken); + + AssertScopedTo(sessionId, captor); + } + + [Fact] + public async Task MemoryExtraction_carries_session_scoped_options() + { + var sessionId = new SessionId("ch/memory-extraction-thread"); + var captor = new OptionsCapturingChatClient(); + + await LlmSessionActor.InvokeMemoryExtractionCoreAsync( + captor, sessionId, history: [], self: CreateTestProbe().Ref, timeout: TimeSpan.FromSeconds(5)); + + AssertScopedTo(sessionId, captor); + } + + [Fact] + public async Task MemoryDistillation_carries_session_scoped_options() + { + var sessionId = new SessionId("ch/distillation-thread"); + var captor = new OptionsCapturingChatClient(); + + await SessionMemoryObserverActor.RunDistillationAsync( + client: captor, sessionId: sessionId, turnCount: 5, + transcript: "user: hello\nassistant: hi", existingProposals: [], + timeout: TimeSpan.FromSeconds(5), self: CreateTestProbe().Ref, runId: 1, + contentVersion: 1, log: NoLogger.Instance); + + AssertScopedTo(sessionId, captor); + } + + [Fact] + public async Task MemoryCuration_carries_session_scoped_options() + { + var sessionId = new SessionId("ch/curation-thread"); + var captor = new OptionsCapturingChatClient(); + var operation = new SQLiteMemoryCurationOperation( + Kind: "document", + MemoryClass: "durable_fact", + MemoryId: null, + AnchorCanonicalName: "test-anchor", + AnchorType: "preference", + Title: "Test", + Content: "Test content for session correlation verification.", + AliasesJson: "[]", + FacetsJson: "[]", + SlotsJson: null, + Relations: null, + UpdateSemantics: "merge-document", + Boundary: TrustBoundary.TrustedInstanceValue, + Audience: TrustAudience.Public, + Sensitivity: "normal", + RecallMode: "auto", + Confidence: 0.9, + FreshnessAtMs: TimeProvider.System.GetUtcNow().ToUnixTimeMilliseconds(), + ExpiresAtMs: null); + + await MemoryCurationActor.TryLlmEvaluationAsync( + captor, sessionId, operation, candidates: [], log: NoLogger.Instance); + + AssertScopedTo(sessionId, captor); + } + + private static void AssertScopedTo(SessionId sessionId, OptionsCapturingChatClient captor) + { + var scoped = Assert.IsType(captor.CapturedOptions); + Assert.Equal(sessionId.Value, scoped.SessionId); + } + + /// + /// IChatClient stub that records the it is invoked with — the + /// object the chat-client decorators read the session id from to open the routing scope. + /// + private sealed class OptionsCapturingChatClient : IChatClient + { + public ChatOptions? CapturedOptions { get; private set; } + + public Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + CapturedOptions = options; + return Task.FromResult(new ChatResponse(new AiChatMessage( + AiChatRole.Assistant, (IList)[new TextContent("captured")]))); + } + + public IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + CapturedOptions = options; + return StreamAsync(cancellationToken); + } + + private static async IAsyncEnumerable StreamAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + yield return new ChatResponseUpdate(AiChatRole.Assistant, "captured"); + await Task.CompletedTask; + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() { } + } +} diff --git a/src/Netclaw.Actors/Memory/MemoryCurationActor.cs b/src/Netclaw.Actors/Memory/MemoryCurationActor.cs index 457faaef5..1983196b1 100644 --- a/src/Netclaw.Actors/Memory/MemoryCurationActor.cs +++ b/src/Netclaw.Actors/Memory/MemoryCurationActor.cs @@ -6,6 +6,7 @@ using Akka.Actor; using Akka.Event; using Microsoft.Extensions.AI; +using Netclaw.Actors.Sessions; using Netclaw.Actors.Sessions.Pipelines; using Netclaw.Configuration; using SessionId = Netclaw.Actors.Protocol.SessionId; @@ -305,8 +306,12 @@ private async Task EvaluateSingleAsync( new(ChatRole.User, CurationPromptBuilder.BuildUserMessage(operation, candidates)) }; - var options = new ChatOptions + // SessionScopedChatOptions carries the session id so this sidecar's chat-client + // diagnostics route to the session's session.log and correlate in Seq/OTLP (replaces + // the deleted SessionDiagnosticsContext AsyncLocal). + var options = new SessionScopedChatOptions { + SessionId = sessionId.Value, // Headroom for reasoning models: with a tight cap a thinking model // spends the whole budget on hidden reasoning and returns empty // content, silently disabling this LLM tier. Non-reasoning models diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs index 691753f1d..9673100a0 100644 --- a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs +++ b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs @@ -1745,8 +1745,11 @@ internal static async Task InvokeMemoryExtractionCoreAsync( new(Microsoft.Extensions.AI.ChatRole.User, CompactionPromptBuilder.BuildMemoryExtractionUserPrompt(history)) }; + // Carry the session id so memory-extraction chat-client diagnostics route to the + // session's session.log and correlate in Seq/OTLP (replaces the deleted AsyncLocal). + var options = new SessionScopedChatOptions { SessionId = sessionId.Value }; var extractionResult = await StreamingResponseReader.ReadAsync( - client, extractionMessages, options: null, cts.Token); + client, extractionMessages, options, cts.Token); var extractedText = extractionResult.Response.Text ?? string.Empty; self.Tell(new MemoryExtractionCompleted { ExtractedMemories = extractedText }); } diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs index 386583693..b0835e73b 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionCompactionPipeline.cs @@ -181,8 +181,11 @@ public static async Task ExecuteAsync( ObservationPromptBuilder.BuildObservationUserPrompt(remainingDiscarded)) }; + // Carry the session id so the observer's chat-client diagnostics route to the + // session's session.log and correlate in Seq/OTLP (replaces the deleted AsyncLocal). + var observerOptions = new SessionScopedChatOptions { SessionId = sessionId.Value }; var result = await StreamingResponseReader.ReadAsync( - client, observerMessages, options: null, cts.Token); + client, observerMessages, observerOptions, cts.Token); var text = result.Response.Text; if (string.IsNullOrWhiteSpace(text)) diff --git a/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs b/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs index e10a126d8..c7f650b04 100644 --- a/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs +++ b/src/Netclaw.Actors/Sessions/Pipelines/SessionTitleGenerator.cs @@ -48,7 +48,11 @@ public static async Task GenerateAsync( new(Microsoft.Extensions.AI.ChatRole.User, CompactionPromptBuilder.BuildTitleGenerationPrompt(history)) }; - var result = await StreamingResponseReader.ReadAsync(client, messages, options: null, cts.Token); + // Carry the session id so this sidecar's chat-client diagnostics (timing, retries, + // provider failover) route to the session's session.log and correlate in Seq/OTLP — + // the explicit carrier that replaced the deleted SessionDiagnosticsContext AsyncLocal. + var options = new SessionScopedChatOptions { SessionId = sessionId.Value }; + var result = await StreamingResponseReader.ReadAsync(client, messages, options, cts.Token); var title = result.Response.Text ?? string.Empty; if (string.IsNullOrWhiteSpace(title)) diff --git a/src/Netclaw.Actors/Sessions/SessionLogActor.cs b/src/Netclaw.Actors/Sessions/SessionLogActor.cs index 8f30bd02d..fb7110d4f 100644 --- a/src/Netclaw.Actors/Sessions/SessionLogActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionLogActor.cs @@ -119,7 +119,10 @@ private void Write(string line) private void Flush() { - if (_unflushedWrites == 0) + // Nothing buffered AND not in a failing state → skip. While failing, fall through so the + // periodic 1s tick keeps retrying the flush (the bytes are still in the StreamWriter + // buffer) and becomes durable as soon as the disk recovers — even with no new writes. + if (_unflushedWrites == 0 && !_flushFailing) return; try @@ -134,9 +137,9 @@ private void Flush() } catch (Exception ex) { - // Reset so a persistent failure (full disk, locked file) does not busy-flush-and-warn - // on every 1s tick; the unflushed bytes stay in the StreamWriter buffer and are retried - // on the next write batch (and on dispose). Warn once on onset, once on recovery. + // Reset the write-batch counter so writes don't trigger a per-line flush storm during a + // persistent failure (full disk, locked file); _flushFailing stays set, so the 1s tick + // keeps retrying until recovery. Warn once on onset, once on recovery. _unflushedWrites = 0; if (!_flushFailing) { diff --git a/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs b/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs index 14f82b762..c4ff443fc 100644 --- a/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionMemoryObserverActor.cs @@ -355,7 +355,10 @@ internal static async Task RunDistillationAsync( sessionId, turnCount, transcript, existingProposals)) }; - var response = (await StreamingResponseReader.ReadAsync(client, messages, options: null, cts.Token)).Response; + // Carry the session id so memory-distillation chat-client diagnostics route to the + // session's session.log and correlate in Seq/OTLP (replaces the deleted AsyncLocal). + var options = new SessionScopedChatOptions { SessionId = sessionId.Value }; + var response = (await StreamingResponseReader.ReadAsync(client, messages, options, cts.Token)).Response; var text = response.Text ?? string.Empty; inputTokens = response.Usage?.InputTokenCount; diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs index a67421d56..96e0f89d7 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs @@ -6,13 +6,14 @@ namespace Netclaw.Actors.SubAgents; /// -/// Derives the owning (parent) session id from a sub-agent's composite scope id. -/// Sub-agents run as ephemeral children of a parent session and reuse its -/// session.log; their composite ids ({parentId}/subagent/{name}/{runId}) -/// must collapse back to the parent so their diagnostics route to the parent's log -/// rather than scatter into per-agent files operators do not monitor. This is the -/// only place that decision is made — keep it in sync with how sub-agent ids are -/// constructed in the spawner. +/// Derives the owning (parent) session id from a sub-agent's composite scope id +/// ({parentId}/subagent/{name}/{runId}) by stripping the /subagent/… suffix. +/// Sub-agent log lines are tagged with this parent id as their SessionId so OTEL/Seq +/// groups a sub-agent's activity under the session that spawned it; the full composite id is +/// carried separately as SubSessionId and is what partitions the local +/// session.log into the sub-agent's OWN file. Keep this in sync with how sub-agent ids +/// are constructed in the spawner. Note: ToolApprovalActor performs the same +/// /subagent/ split for approval inheritance — if the id format changes, update both. /// internal static class SubAgentSessionScope { diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index 3c1824e02..23b823cac 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -103,11 +103,13 @@ internal void Route(string line, string? stateSessionId, string? stateSubSession var sessionId = stateSessionId; var subSessionId = stateSubSessionId; - if (sessionId is null || subSessionId is null) + if (sessionId is null) { - // Chat-client lines carry the ids via BeginScope rather than message state. + // Only chat-client lines lack a state session id; they carry both ids via BeginScope. + // An actor line already names its ids in state, so we never consult scopes for it — + // otherwise an unrelated ambient SubSessionId scope could hijack its routing. FindScopeIds(out var scopeSessionId, out var scopeSubSessionId); - sessionId ??= scopeSessionId; + sessionId = scopeSessionId; subSessionId ??= scopeSubSessionId; } From cfcff8f8d2de9e80ea47a342d394a50f6f749b98 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 20:39:49 +0000 Subject: [PATCH 18/22] feat(logging): flush the audit transcript immediately, keep diagnostics batched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the durability decision: SessionLogActor has two inputs — the audit transcript (user/assistant/tool/usage via SendUserMessage/SessionOutput, low volume, durability- critical) and the high-volume diagnostic lines (SessionLogDiagnostic). The audit handlers now WriteDurable (write + immediate flush) so a hard process death (SIGKILL/OOM) cannot drop the audit record's tail, while diagnostics stay on the batched Write() path (an fsync per line would dominate now that the whole per-session stream lands here). An audit line also flushes any diagnostics buffered before it, so audit lines are natural flush points. --- .../Sessions/SessionLogActor.cs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/Netclaw.Actors/Sessions/SessionLogActor.cs b/src/Netclaw.Actors/Sessions/SessionLogActor.cs index fb7110d4f..9fbb38ec0 100644 --- a/src/Netclaw.Actors/Sessions/SessionLogActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionLogActor.cs @@ -27,10 +27,12 @@ namespace Netclaw.Actors.Sessions; /// /// File handle lifecycle: /// - Open once in with append mode + read-share. -/// - Buffered writes flushed in batches (after a write burst or on a periodic tick), -/// not per line: now that the whole per-session log stream lands here, an fsync per -/// line would dominate. is INotInfluenceReceiveTimeout -/// so the flush cadence does not keep an idle session alive. +/// - The high-volume diagnostic lines are flushed in batches (after a write burst or on a +/// periodic tick), not per line: now that the whole per-session log stream lands here, an +/// fsync per line would dominate. The audit transcript (user/assistant/tool/usage) is +/// flushed immediately instead, so a hard process death cannot drop the audit record's tail. +/// is INotInfluenceReceiveTimeout so the flush cadence does +/// not keep an idle session alive. /// - Close/dispose in (which flushes) — no retry loop needed since /// the handle is kept open and single-writer is enforced by the actor mailbox. /// @@ -110,6 +112,7 @@ protected override void PostStop() } // Buffered write: lines accumulate in the StreamWriter buffer and are flushed in batches. + // Used for the high-volume diagnostic lines, where an fsync per line would dominate. private void Write(string line) { _writer?.WriteLine(line); @@ -117,6 +120,16 @@ private void Write(string line) Flush(); } + // Durable write for the audit transcript (user/assistant/tool/usage lines): flush immediately + // so a hard process death (SIGKILL/OOM) cannot drop the security/audit record's tail. The + // flush also drains any diagnostics buffered before this line, so audit lines are natural + // flush points. Diagnostics themselves stay on the batched Write() path above. + private void WriteDurable(string line) + { + Write(line); + Flush(); + } + private void Flush() { // Nothing buffered AND not in a failing state → skip. While failing, fall through so the @@ -158,7 +171,7 @@ private void OnUserMessage(SendUserMessage msg) : string.Empty; var line = $"[{_timeProvider.GetUtcNow():o}] User: {TextTruncation.EllipsisAppend(msg.Content, 1000)}{mediaNote}"; - Write(line); + WriteDurable(line); } catch (Exception ex) { @@ -194,7 +207,7 @@ private void OnOutput(SessionOutput output) if (line is not null) { - Write($"[{_timeProvider.GetUtcNow():o}] {line}"); + WriteDurable($"[{_timeProvider.GetUtcNow():o}] {line}"); } } catch (Exception ex) From bebffdec1639e0993df1f1da40797498e7930a56 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 20:58:18 +0000 Subject: [PATCH 19/22] refactor(logging): batch daemon.log flushes + consolidate the /subagent/ split (code-review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two minor code-review follow-ups: - daemon.log writer no longer fsyncs per line. It flushes when the queue drains (so sparse daemon.log lines — startup, config, lifecycle, alerts — stay immediately durable) or after a 256-line burst cap, matching the cadence the sibling SessionLogActor already uses. The tail is flushed by Dispose() when the writer thread exits. - ToolApprovalActor's approval-inheritance walk now calls SubAgentSessionScope.NormalizeSessionId instead of re-implementing the "/subagent/" split inline, so there is a single owner of that parse (behavior preserved — IndexOf-first already walked leaf -> root parent). Doc updated. --- .../SubAgents/SubAgentSessionScope.cs | 5 +++-- src/Netclaw.Actors/Tools/ToolApprovalActor.cs | 10 +++++++--- .../Configuration/RollingFileLoggerProvider.cs | 13 ++++++++++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs index 96e0f89d7..0f0cc7caf 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSessionScope.cs @@ -12,8 +12,9 @@ namespace Netclaw.Actors.SubAgents; /// groups a sub-agent's activity under the session that spawned it; the full composite id is /// carried separately as SubSessionId and is what partitions the local /// session.log into the sub-agent's OWN file. Keep this in sync with how sub-agent ids -/// are constructed in the spawner. Note: ToolApprovalActor performs the same -/// /subagent/ split for approval inheritance — if the id format changes, update both. +/// are constructed in the spawner. ToolApprovalActor also calls this to walk a +/// sub-agent's approvals up to its parent, so this is the single owner of the /subagent/ +/// split. /// internal static class SubAgentSessionScope { diff --git a/src/Netclaw.Actors/Tools/ToolApprovalActor.cs b/src/Netclaw.Actors/Tools/ToolApprovalActor.cs index b150968a6..8fccdc6f1 100644 --- a/src/Netclaw.Actors/Tools/ToolApprovalActor.cs +++ b/src/Netclaw.Actors/Tools/ToolApprovalActor.cs @@ -6,6 +6,7 @@ using Akka.Actor; using Akka.Event; using Netclaw.Actors.Protocol; +using Netclaw.Actors.SubAgents; using Netclaw.Configuration; using Netclaw.Security; using Netclaw.Tools; @@ -104,11 +105,14 @@ private bool IsSessionApproved(SessionId sessionId, TrustAudience audience, Tool return true; } - var subagentMarker = scopeId.IndexOf("/subagent/", StringComparison.Ordinal); - if (subagentMarker <= 0) + // Walk to the parent session so a sub-agent inherits its parent's approvals. + // SubAgentSessionScope.NormalizeSessionId owns the "/subagent/" split (one + // implementation shared with log routing); break once there is nothing left to strip. + var parent = SubAgentSessionScope.NormalizeSessionId(scopeId); + if (string.IsNullOrEmpty(parent) || parent == scopeId) break; - scopeId = scopeId[..subagentMarker]; + scopeId = parent; } return false; diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index 23b823cac..3971c9c3b 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -33,6 +33,7 @@ namespace Netclaw.Daemon.Configuration; internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExternalScope { private const long MaxFileSizeBytes = 10 * 1024 * 1024; // 10MB per file + private const int DaemonLogFlushBatch = 256; // flush cap under a burst; idle flushes every line private readonly string _basePath; private readonly TimeProvider _timeProvider; @@ -193,13 +194,23 @@ private async Task ResolveSessionDispatcherAsync(Task dispatcherTask) private void ProcessQueue() { + var batched = 0; foreach (var message in _queue.GetConsumingEnumerable()) { try { EnsureWriter(); _writer!.WriteLine(message); - _writer.Flush(); + + // Flush when the queue has drained — so sparse daemon.log lines (startup, config, + // lifecycle, alerts) stay immediately durable — or after a burst batch, so a + // sustained burst doesn't pay an fsync per line. The final tail is flushed by + // Dispose() when the writer thread exits. + if (_queue.Count == 0 || ++batched >= DaemonLogFlushBatch) + { + _writer.Flush(); + batched = 0; + } } catch (Exception ex) { From 3ddd796f937a441d331b432e894a19e738df817e Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 21:17:52 +0000 Subject: [PATCH 20/22] fix(logging): guard PostStop dispose, fix audit-durability doc, scope the tool-denial line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review follow-ups (clear-cut findings): - SessionLogActor.PostStop: guard _writer.Dispose() in try/catch. On a disk-full stop the explicit Flush() already reports via its rate-limited warning; the Dispose flush could throw IOException out of PostStop and bury the real cause under Akka's PostStop-failed noise. - diagnostics skill: the flush description claimed session.log is uniformly batched (~1s lag). It now documents the split — the conversation audit is flushed immediately (durable), only the diagnostics batch. (skill 2.23.0; System Skills Sync Rule.) - SubAgentSpawner tool-denial line: route via SubAgentSpawnBreadcrumbs.ToolDenied (a SessionId scope) instead of a hand-rolled "(session={SessionId})" message field, so it can't drift from the centralized NetclawLogProperties key like every other breadcrumb. --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../netclaw-operations/references/diagnostics.md | 8 +++++--- src/Netclaw.Actors/Sessions/SessionLogActor.cs | 14 +++++++++++--- .../SubAgents/SubAgentSpawnBreadcrumbs.cs | 11 +++++++++++ src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs | 7 +------ 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index f5a57c96b..63feec59e 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.22.0" + version: "2.23.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index 9d9afe5bc..9343dedb6 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -54,9 +54,11 @@ What to expect inside `session.log`: or globally, use Seq/OTLP (every line is there with `SessionId` as a field). - The session-log writer's own failure lines are the one exception: they go to `daemon.log`, never routed back into the file that just failed. -- Best-effort, **batched** writes (flushed on a ~1s cadence): a recent line may lag - by up to a second, and individual lines may be dropped on transient IO errors (a - warning lands in `daemon.log`). Not a transactional audit trail. +- Writes are split by kind. The **conversation audit** (User/Assistant/Tool/Usage + lines) is flushed **immediately**, so a hard process death cannot drop the audit + tail. The higher-volume **diagnostics** are **batched** (flushed on a ~1s cadence), + so a recent diagnostic line may lag by up to a second. Individual lines may be + dropped on transient IO errors (a warning lands in `daemon.log`). What stays in `daemon.log`: only sessionless lines — daemon startup/config, session lifecycle, and global errors. Debugging one session → read its `session.log`; diff --git a/src/Netclaw.Actors/Sessions/SessionLogActor.cs b/src/Netclaw.Actors/Sessions/SessionLogActor.cs index 9fbb38ec0..cb6105d30 100644 --- a/src/Netclaw.Actors/Sessions/SessionLogActor.cs +++ b/src/Netclaw.Actors/Sessions/SessionLogActor.cs @@ -104,10 +104,18 @@ protected override void PreStart() protected override void PostStop() { - // Dispose flushes, but flush explicitly first so a write failure here is logged - // rather than swallowed by Dispose. + // Flush explicitly first so a write failure is reported via Flush's rate-limited warning. + // Dispose flushes again; guard it so a still-failing disk (the _flushFailing state) cannot + // throw IOException out of PostStop and bury the real cause under Akka's PostStop noise. Flush(); - _writer?.Dispose(); + try + { + _writer?.Dispose(); + } + catch (Exception ex) + { + _log.Warning(ex, "Failed to close session.log for {Session}", _sessionId.Value); + } base.PostStop(); } diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs index 73e223c9c..6ec48cec0 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawnBreadcrumbs.cs @@ -76,6 +76,17 @@ public static void RunFailed(ILogger? logger, ToolExecutionContext context, stri agentName, runId.Value); } + public static void ToolDenied(ILogger? logger, ToolExecutionContext context, string agentName, string toolName) + { + // INFO so tool denials are visible in production: a sub-agent missing a tool may be unable + // to finish its task. Routed via the SessionId scope like every other breadcrumb (no + // hand-rolled (session=…) field), so it lands in the session.log without string drift. + using (BeginSessionScope(logger, context.SessionId)) + logger?.LogInformation( + "SubAgent [{AgentName}] tool '{ToolName}' denied by SubAgentToolPolicy", + agentName, toolName); + } + public static void SpawnRefused(ILogger? logger, ToolExecutionContext context, string agentName, TrustAudience audience, bool subsystemEnabled) { using (BeginSessionScope(logger, context.SessionId)) diff --git a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs index 95ffcaf43..de2b673ea 100644 --- a/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs +++ b/src/Netclaw.Actors/SubAgents/SubAgentSpawner.cs @@ -286,12 +286,7 @@ private IReadOnlyList ResolveTools(SubAgentProfile profile, ToolEx } else { - // Log at INFO so tool denials are visible in production logs. - // Sub-agents without certain tools may be unable to complete - // their tasks, and this information is important for debugging. - _logger.LogInformation( - "SubAgent [{AgentName}] tool '{ToolName}' denied by SubAgentToolPolicy (session={SessionId})", - profile.Name, tool.Name, context.SessionId); + SubAgentSpawnBreadcrumbs.ToolDenied(_logger, context, profile.Name, tool.Name); } } From 63958634a69dd9298918b2e922699668466b4d9b Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 30 Jun 2026 21:51:03 +0000 Subject: [PATCH 21/22] fix(logging): keep daemon-infra session-named lines in daemon.log; cap-accurate roll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review #1 (the dominant finding) + #3, per the daemon.log decision. The field-based router diverted ANY line carrying a {SessionId} into that session's session.log — including daemon-infrastructure errors that merely NAME a session (gateway "failed to mark session X active", drain "failed to drain session X before shutdown", restart-recovery). Those are daemon functionality, and the class contract promises them in daemon.log. On dev they went there because the old AsyncLocal was only set inside session-serving code. Restore that structurally: a state SessionId routes only when it rode in on an Akka actor context (the bridge's AkkaLogState carries a LogSource) or a BeginScope (chat-client decorators, spawn breadcrumbs). A bare {SessionId} message field on a plain daemon-service ILogger has no LogSource → stays in daemon.log. No hand-maintained category list. Swept all {SessionId} message templates: only daemon/channel/infra lines lack an actor context (consistent with dev); the one session-serving exception (tool-denial) was already moved to a scope. Also: daemon.log size-roll flushes within a batch of the 10MB cap so AutoFlush=off buffered bytes can't overshoot it (#3). Tests updated to model actor-context vs message-field lines, plus a new case locking in the daemon-service → daemon.log behavior. Skill bumped once for the PR (2.22.0). --- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../RollingFileLoggerPartitionTests.cs | 68 +++++++++++++++---- .../RollingFileLoggerProvider.cs | 36 ++++++++-- 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 63feec59e..f5a57c96b 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.23.0" + version: "2.22.0" --- # Netclaw Operations diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs index 79d16c373..a817185d0 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -14,10 +14,13 @@ namespace Netclaw.Daemon.Tests.Configuration; /// -/// The provider owns the LOCAL partition of the log stream: a line carrying a session id goes -/// to that session's session.log (Tell'd as a to the -/// dispatcher) and NOT to daemon.log; everything else goes to daemon.log. The session-log -/// writer's own lines are excluded so a write-failure log cannot recurse. +/// The provider owns the LOCAL partition of the log stream: a line tagged with a session id by +/// session-SERVING code (an actor's WithContext, which the Akka bridge surfaces alongside a +/// LogSource; or a BeginScope) goes to that session's session.log (Tell'd as a +/// to the dispatcher) and NOT to daemon.log. A daemon-service +/// line that merely NAMES a session in its message template (a bare {SessionId} field with no +/// LogSource) stays in daemon.log, as does everything sessionless. The session-log writer's own +/// lines are excluded so a write-failure log cannot recurse. /// public sealed class RollingFileLoggerPartitionTests : TestKit { @@ -36,7 +39,8 @@ public async Task Session_tagged_line_routes_to_session_log_not_daemon_log() using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) { provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); - provider.CreateLogger("Netclaw.Tools").LogInformation("spawn requested {SessionId}", "C1/T1"); + provider.CreateLogger("Akka.Actor.ActorSystem") + .Log(LogLevel.Information, new EventId(0), ActorState("C1/T1"), null, (_, _) => "spawn requested"); var diag = await dispatcher.ExpectMsgAsync(cancellationToken: TestContext.Current.CancellationToken); Assert.Equal("C1/T1", diag.SessionId.Value); @@ -85,11 +89,7 @@ public async Task Sub_agent_line_routes_to_its_own_file_keyed_by_sub_session_id( // A bridged sub-agent line carries the parent SessionId (for OTEL grouping) AND the // sub-session id; the LOCAL file is partitioned by the sub-session. - var state = new List> - { - new(NetclawLogProperties.SessionId, "C1/T1"), - new(NetclawLogProperties.SubSessionId, "C1/T1/subagent/summarizer/ab12"), - }; + var state = ActorState("C1/T1", "C1/T1/subagent/summarizer/ab12"); provider.CreateLogger("Akka.Actor.ActorSystem") .Log(LogLevel.Information, new EventId(0), state, null, (_, _) => "sub-agent did work"); @@ -144,6 +144,29 @@ public async Task Sessionless_line_goes_to_daemon_log() Cleanup(dir); } + [Fact] + public async Task Daemon_service_line_naming_a_session_in_its_message_stays_in_daemon_log() + { + var (dir, daemonPath) = TempPaths(); + var dispatcher = CreateTestProbe("dispatcher"); + + using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) + { + provider.AttachSessionDispatcher(Task.FromResult(dispatcher.Ref)); + + // A plain daemon-service ILogger line that merely names a session in its message + // template — no actor LogSource, no scope. It is daemon infrastructure, not the + // session's own work, so it must NOT be diverted into session.log. + provider.CreateLogger("Netclaw.Daemon.Gateway.SessionCatalogService") + .LogWarning("Failed to mark session {SessionId} active", "C7/T7"); + + await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); + } + + Assert.Contains("Failed to mark session", ReadDaemonLog(dir), StringComparison.Ordinal); + Cleanup(dir); + } + [Fact] public async Task Session_log_writers_own_line_is_not_routed_back() { @@ -181,10 +204,11 @@ public async Task Session_line_before_dispatcher_resolves_falls_back_to_daemon_l using (var provider = new RollingFileLoggerProvider(daemonPath, new FakeTimeProvider(FixedNow))) { provider.AttachSessionDispatcher(pending.Task); // never resolves during this test - provider.CreateLogger("Netclaw.Tools").LogInformation("before resolve {SessionId}", "C4/T4"); + provider.CreateLogger("Akka.Actor.ActorSystem") + .Log(LogLevel.Information, new EventId(0), ActorState("C4/T4"), null, (_, _) => "before resolve"); - // No buffering: a line logged before the dispatcher resolves is not held for it; it - // goes straight to daemon.log (the dispatcher never sees it). + // No buffering: a routable line logged before the dispatcher resolves is not held for + // it; it goes straight to daemon.log (the dispatcher never sees it). await dispatcher.ExpectNoMsgAsync(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); } @@ -201,7 +225,8 @@ public async Task Dispatcher_resolution_failure_falls_back_and_beacons_to_daemon try { provider.AttachSessionDispatcher(pending.Task); // dispatcher not resolved - provider.CreateLogger("Netclaw.Tools").LogInformation("before failure {SessionId}", "C9/T9"); + provider.CreateLogger("Akka.Actor.ActorSystem") + .Log(LogLevel.Warning, new EventId(0), ActorState("C9/T9"), null, (_, _) => "before failure"); pending.SetException(new InvalidOperationException("dispatcher never registered")); @@ -235,6 +260,21 @@ private static async Task ReadDaemonSharedAsync(string dir, Cancellation return await reader.ReadToEndAsync(ct); } + // Shape of a bridged Akka actor log: WithContext("SessionId") surfaces alongside the actor's + // LogSource. The LogSource is what marks the line as session-SERVING (vs a daemon service that + // merely names a session in its message), so the router treats the session id as routable. + private static List> ActorState(string sessionId, string? subSessionId = null) + { + var state = new List> + { + new(NetclawLogProperties.SessionId, sessionId), + new("LogSource", "[akka://netclaw/user/session-manager#1]"), + }; + if (subSessionId is not null) + state.Add(new(NetclawLogProperties.SubSessionId, subSessionId)); + return state; + } + private static (string Dir, string DaemonPath) TempPaths() { var dir = Path.Combine(Path.GetTempPath(), $"netclaw-partition-{Guid.NewGuid():N}"); diff --git a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs index 3971c9c3b..80b75e17a 100644 --- a/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs +++ b/src/Netclaw.Daemon/Configuration/RollingFileLoggerProvider.cs @@ -16,12 +16,16 @@ namespace Netclaw.Daemon.Configuration; /// File-based logger that owns the LOCAL partition of the one log stream. Every line is /// written to exactly one place on disk: /// -/// A line that carries a session id (an actor's WithContext("SessionId", …), -/// which the Akka→MEL bridge surfaces as structured state; a MEL {SessionId} message -/// field; or a BeginScope carrying it) is routed to that session's session.log -/// via the SessionLogDispatcher and is NOT written to daemon.log. -/// Everything else — genuinely daemon-wide lines (startup, config, session lifecycle, -/// global errors) — goes to daemon.log. +/// A line tagged with a session id by session-serving code is routed to that +/// session's session.log via the SessionLogDispatcher and is NOT written to +/// daemon.log. "Serving" is the key: the id must ride in on an actor context — an +/// actor's WithContext("SessionId", …), which the Akka→MEL bridge surfaces alongside a +/// LogSource — or a BeginScope (the chat-client decorators and spawn breadcrumbs). +/// Everything else goes to daemon.log: genuinely daemon-wide lines (startup, +/// config, session lifecycle, global errors) AND a daemon-infrastructure line that merely +/// names a session in its message template (a bare {SessionId} field with no actor +/// context — e.g. the gateway/catalog/drain "failed to … session X" warnings). Those are daemon +/// functionality, not the session's own work, so they stay where an operator triages the daemon. /// /// The full stream still reaches OTEL via the separate OTLP exporter (with the session id as /// an attribute); the OTEL receiver does the global slicing/distilling. The dispatcher is @@ -34,6 +38,7 @@ internal sealed class RollingFileLoggerProvider : ILoggerProvider, ISupportExter { private const long MaxFileSizeBytes = 10 * 1024 * 1024; // 10MB per file private const int DaemonLogFlushBatch = 256; // flush cap under a burst; idle flushes every line + private const long RollFlushMarginBytes = 128 * 1024; // flush near the size cap so rolls aren't late private readonly string _basePath; private readonly TimeProvider _timeProvider; @@ -225,6 +230,13 @@ private void EnsureWriter() var today = _timeProvider.GetUtcNow().ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); if (_writer is not null && _currentDate == today) { + // With AutoFlush off (batched writes), BaseStream.Length lags the buffered bytes. Flush + // once we're within a batch of the cap so the roll decision sees the true size — paying + // the fsync only near the threshold, not per line — then the file can't overshoot 10MB + // by a batch's worth of buffered data. + if (_writer.BaseStream.Length >= MaxFileSizeBytes - RollFlushMarginBytes) + _writer.Flush(); + // Roll if file exceeds size limit if (_writer.BaseStream.Length >= MaxFileSizeBytes) { @@ -304,7 +316,17 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (exception is not null) line += Environment.NewLine + exception; - _provider.Route(line, sessionId, subSessionId, fromSessionLogActor); + // A session id found in the message STATE is a routing trigger only when it rode in on an + // Akka actor context: the Akka→MEL bridge's AkkaLogState carries a LogSource alongside the + // WithContext("SessionId", …). A bare {SessionId} message field on a plain daemon-service + // ILogger (gateway/catalog/drain) has no LogSource — that is daemon infrastructure + // merely naming a session, so it stays in daemon.log. Session-serving non-actor producers + // (chat-client decorators, spawn breadcrumbs) tag via BeginScope instead, which Route still + // consults as a fallback. + var stateSessionId = logSource is not null ? sessionId : null; + var stateSubSessionId = logSource is not null ? subSessionId : null; + + _provider.Route(line, stateSessionId, stateSubSessionId, fromSessionLogActor); } // Read the fields the producer already put on the log event. The Akka→MEL bridge passes an From 28c400a178fdadf58fb4fe6ff35444f81ca4a634 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 1 Jul 2026 00:04:40 +0000 Subject: [PATCH 22/22] test(logging): address code-quality bot findings (Path.Join, drain-loop comments) GitHub code-quality (Copilot) review follow-ups on this PR's test files: - Path.Combine -> Path.Join in the logging test temp-path helpers: the CodeQL "may drop earlier arguments" rule can't prove the second segment is non-rooted; Join is the correct idiom for plain concatenation and clears the recurring finding (identical result for these inputs). - Intent comments on the empty `await foreach (var _ in stream) { }` drain loops in the chat-client tests, so the enumerate-and-discard reads as intentional (it drives the pipeline's scope/retry/logging side effects, which the test then asserts). Declined the "use .Where()" suggestions: on the provider hot path (FindScopeIds) a LINQ Where allocates a per-line iterator/closure that the manual loop + static lambda deliberately avoids (documented); the mirroring test helpers stay foreach for consistency with it. --- .../Configuration/LoggingChatClientTests.cs | 4 ++++ .../Configuration/PipelineChatClientFactoryTests.cs | 2 ++ .../Configuration/RollingFileLoggerPartitionTests.cs | 4 ++-- .../Configuration/RollingFileLoggerProviderTests.cs | 6 +++--- .../Configuration/RoutingChatClientTests.cs | 3 ++- .../Configuration/SessionLogPartitionIntegrationTests.cs | 6 +++--- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs index 8cce9d710..11549c5b7 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/LoggingChatClientTests.cs @@ -75,6 +75,7 @@ public async Task Streaming_LogsPromptSummaryInDebugMode() ] }, TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } Assert.Contains(logs, l => l.Contains("LLM prompt summary")); @@ -107,6 +108,7 @@ [new ChatMessage(ChatRole.User, "hi")], new SessionScopedChatOptions { SessionId = "ch/thread" }, TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } Assert.True(logger.HasSessionScope("ch/thread")); @@ -122,6 +124,7 @@ public async Task Streaming_with_plain_options_attaches_no_scope() await foreach (var _ in client.GetStreamingResponseAsync( [new ChatMessage(ChatRole.User, "hi")], new ChatOptions(), TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } Assert.False(logger.HasAnySessionScope()); @@ -132,6 +135,7 @@ private static async Task Drain(IChatClient client) await foreach (var _ in client.GetStreamingResponseAsync( [new ChatMessage(ChatRole.User, "hi")], cancellationToken: TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } } diff --git a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs index 8275fc413..a2491d4a6 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/PipelineChatClientFactoryTests.cs @@ -72,6 +72,7 @@ [new ChatMessage(ChatRole.User, "hi")], new SessionScopedChatOptions { SessionId = "ch/thread" }, TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } Assert.True(logger.HasSessionScope("ch/thread")); @@ -99,6 +100,7 @@ [new ChatMessage(ChatRole.User, "hi")], new SessionScopedChatOptions { SessionId = "ch/thread" }, TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } var retryWarning = Assert.Single( diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs index a817185d0..396e2e807 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerPartitionTests.cs @@ -277,9 +277,9 @@ private static List> ActorState(string sessionId, s private static (string Dir, string DaemonPath) TempPaths() { - var dir = Path.Combine(Path.GetTempPath(), $"netclaw-partition-{Guid.NewGuid():N}"); + var dir = Path.Join(Path.GetTempPath(), $"netclaw-partition-{Guid.NewGuid():N}"); Directory.CreateDirectory(dir); - return (dir, Path.Combine(dir, "daemon.log")); + return (dir, Path.Join(dir, "daemon.log")); } private static string ReadDaemonLog(string dir) diff --git a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs index f253826e8..f25c6507d 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RollingFileLoggerProviderTests.cs @@ -12,13 +12,13 @@ namespace Netclaw.Daemon.Tests.Configuration; public sealed class RollingFileLoggerProviderTests : IDisposable { - private readonly string _basePath = Path.Combine(Path.GetTempPath(), $"netclaw-rolling-logger-tests-{Guid.NewGuid():N}"); + private readonly string _basePath = Path.Join(Path.GetTempPath(), $"netclaw-rolling-logger-tests-{Guid.NewGuid():N}"); [Fact] public async Task Writes_log_lines_to_the_daily_rolling_daemon_log() { var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-05-07T12:34:56Z")); - var daemonLogPath = Path.Combine(_basePath, "logs", "daemon.log"); + var daemonLogPath = Path.Join(_basePath, "logs", "daemon.log"); Directory.CreateDirectory(Path.GetDirectoryName(daemonLogPath)!); // Dispose drains the writer thread, so the line is flushed by the time we read. @@ -31,7 +31,7 @@ public async Task Writes_log_lines_to_the_daily_rolling_daemon_log() // No dispatcher was attached, so per-session routing is off and even a session-tagged // line falls back to daemon.log. (When routing IS attached the {SessionId} line is // partitioned to session.log — see RollingFileLoggerPartitionTests.) - var daemonLog = Directory.GetFiles(Path.Combine(_basePath, "logs"), "daemon-*.log").Single(); + var daemonLog = Directory.GetFiles(Path.Join(_basePath, "logs"), "daemon-*.log").Single(); var text = await File.ReadAllTextAsync(daemonLog, TestContext.Current.CancellationToken); Assert.Contains("hello daemon log", text, StringComparison.Ordinal); Assert.Contains("Netclaw.Tests", text, StringComparison.Ordinal); diff --git a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs index fb4a5dd84..00e3db65f 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/RoutingChatClientTests.cs @@ -125,6 +125,7 @@ [new ChatMessage(ChatRole.User, "hi")], new SessionScopedChatOptions { SessionId = "ch/thread" }, TestContext.Current.CancellationToken)) { + // Drain the stream to completion so the pipeline (scope/retry/logging) runs; updates aren't asserted here. } Assert.True(logger.HasSessionScope("ch/thread")); @@ -254,7 +255,7 @@ public async Task Streaming_DoesNotFailover_OnCancellation() await Assert.ThrowsAnyAsync(async () => { await foreach (var _ in Client(sink, primary, fallback).GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "hi")], cancellationToken: cts.Token)) { } + [new ChatMessage(ChatRole.User, "hi")], cancellationToken: cts.Token)) { /* drain the stream so the pipeline runs */ } }); Assert.Equal(0, fallbackCalls); diff --git a/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs b/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs index f1e942718..4323ef9a8 100644 --- a/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs +++ b/src/Netclaw.Daemon.Tests/Configuration/SessionLogPartitionIntegrationTests.cs @@ -27,12 +27,12 @@ namespace Netclaw.Daemon.Tests.Configuration; /// public sealed class SessionLogPartitionIntegrationTests : TestKit { - private readonly string _logDir = Path.Combine(Path.GetTempPath(), $"netclaw-logpart-int-{Guid.NewGuid():N}"); + private readonly string _logDir = Path.Join(Path.GetTempPath(), $"netclaw-logpart-int-{Guid.NewGuid():N}"); private readonly FakeTimeProvider _time = new(DateTimeOffset.Parse("2026-05-07T12:00:00Z")); private RollingFileLoggerProvider _provider = null!; - private string DaemonPath => Path.Combine(_logDir, "daemon.log"); - private string SessionsDir => Path.Combine(_logDir, "sessions"); + private string DaemonPath => Path.Join(_logDir, "daemon.log"); + private string SessionsDir => Path.Join(_logDir, "sessions"); protected override void ConfigureAkka(AkkaConfigurationBuilder builder, IServiceProvider provider) {