From 8e5a29048d89c6bc482b1da1eae3031aa5a93ac6 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 2 Sep 2025 11:23:36 -0500 Subject: [PATCH 1/2] Fix IIS/Windows Service console race condition (#7691) - Detect when running in IIS/Windows Service environments where Console.Out and Console.Error are redirected to the same StreamWriter.Null singleton - Skip console output entirely in these environments to prevent race conditions that cause IndexOutOfRangeException and cascade failures - Improve DefaultLogger error handling to prevent feedback loops - Add unit tests for non-console scenarios The race condition occurs because: 1. IIS/Services redirect both Console.Out and Console.Error to StreamWriter.Null 2. StreamWriter.Null is a singleton, not thread-safe for concurrent access 3. Multiple threads writing to both streams cause IndexOutOfRangeException 4. Console output goes nowhere in these environments anyway Fixes #7691 --- .../Loggers/StandardOutWriterSpec.cs | 94 +++++++++++++++++++ src/core/Akka/Event/DefaultLogger.cs | 19 +++- src/core/Akka/Util/StandardOutWriter.cs | 42 +++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 src/core/Akka.Tests/Loggers/StandardOutWriterSpec.cs diff --git a/src/core/Akka.Tests/Loggers/StandardOutWriterSpec.cs b/src/core/Akka.Tests/Loggers/StandardOutWriterSpec.cs new file mode 100644 index 00000000000..9475bdb466c --- /dev/null +++ b/src/core/Akka.Tests/Loggers/StandardOutWriterSpec.cs @@ -0,0 +1,94 @@ +//----------------------------------------------------------------------- +// +// Copyright (C) 2009-2022 Lightbend Inc. +// Copyright (C) 2013-2025 .NET Foundation +// +//----------------------------------------------------------------------- + +using System; +using System.IO; +using System.Threading.Tasks; +using Akka.TestKit; +using Akka.Util; +using Xunit; +using Xunit.Abstractions; + +namespace Akka.Tests.Loggers +{ + /// + /// Tests for StandardOutWriter to ensure it handles IIS/Windows Service environments correctly + /// where Console.Out and Console.Error may be redirected to StreamWriter.Null + /// + public class StandardOutWriterSpec : AkkaSpec + { + public StandardOutWriterSpec(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void StandardOutWriter_should_handle_concurrent_writes_without_race_conditions() + { + // This test simulates the concurrent access pattern that causes issues in IIS + // In normal test environments this won't reproduce the issue, but it ensures + // our fix doesn't break normal console operation + + var tasks = new Task[100]; + + for (int i = 0; i < tasks.Length; i++) + { + var taskId = i; + tasks[i] = Task.Run(() => + { + for (int j = 0; j < 10; j++) + { + // These calls should not throw even under concurrent access + StandardOutWriter.WriteLine($"Task {taskId} - Line {j}"); + StandardOutWriter.Write($"Task {taskId} - Write {j} "); + } + }); + } + + // Should complete without throwing IndexOutOfRangeException + Assert.True(Task.WaitAll(tasks, TimeSpan.FromSeconds(5))); + } + + [Fact] + public void StandardOutWriter_should_not_throw_when_console_is_redirected() + { + // Save original streams + var originalOut = Console.Out; + var originalError = Console.Error; + + try + { + // Simulate IIS/Windows Service environment by redirecting to null + Console.SetOut(StreamWriter.Null); + Console.SetError(StreamWriter.Null); + + // These should not throw even when console is redirected to null + StandardOutWriter.WriteLine("This should not throw"); + StandardOutWriter.Write("Neither should this"); + + // Test with colors (which would normally fail in IIS) + StandardOutWriter.WriteLine("Colored output", ConsoleColor.Red); + StandardOutWriter.Write("Colored write", ConsoleColor.Blue, ConsoleColor.Yellow); + } + finally + { + // Restore original streams + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + [Fact] + public void StandardOutWriter_should_handle_null_and_empty_messages() + { + // Should not throw + StandardOutWriter.WriteLine(null); + StandardOutWriter.WriteLine(""); + StandardOutWriter.Write(null); + StandardOutWriter.Write(""); + } + } +} \ No newline at end of file diff --git a/src/core/Akka/Event/DefaultLogger.cs b/src/core/Akka/Event/DefaultLogger.cs index 129b844deae..f4b8d799fb5 100644 --- a/src/core/Akka/Event/DefaultLogger.cs +++ b/src/core/Akka/Event/DefaultLogger.cs @@ -46,9 +46,24 @@ protected override bool Receive(object message) protected virtual void Print(LogEvent logEvent) { if (_stdoutLogger == null) - throw new Exception("Logger has not been initialized yet."); + { + // Include context about the failed log event to help with debugging + // but avoid creating a cascade of logging failures + var logDetails = $"[{logEvent.LogLevel()}] {logEvent.LogSource}: {logEvent.Message}"; + throw new Exception($"Logger has not been initialized yet. Failed to log: {logDetails}"); + } - _stdoutLogger.Tell(logEvent); + try + { + _stdoutLogger.Tell(logEvent); + } + catch (Exception ex) + { + // Prevent cascade failures by not attempting to log the logging failure + // In IIS/Windows Service environments, this prevents the feedback loop + // that causes millions of log messages and memory exhaustion + System.Diagnostics.Debug.WriteLine($"Failed to write to stdout logger: {ex.Message}"); + } } } } diff --git a/src/core/Akka/Util/StandardOutWriter.cs b/src/core/Akka/Util/StandardOutWriter.cs index a049c308bbe..e006374bdab 100644 --- a/src/core/Akka/Util/StandardOutWriter.cs +++ b/src/core/Akka/Util/StandardOutWriter.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System; +using System.IO; namespace Akka.Util { @@ -16,6 +17,37 @@ namespace Akka.Util public static class StandardOutWriter { private static readonly object _lock = new(); + private static readonly bool _isConsoleAvailable = DetectConsoleAvailability(); + + /// + /// Detects whether a real console is available for output. + /// In environments like IIS and Windows Services, console output is redirected to StreamWriter.Null, + /// which is a singleton. When multiple threads write to both Console.Out and Console.Error + /// (which point to the same StreamWriter.Null instance), it causes race conditions. + /// + /// Since console output goes nowhere in these environments anyway, we skip it entirely + /// to prevent the race condition and improve performance. + /// + private static bool DetectConsoleAvailability() + { + // Check if console streams are redirected (common in IIS, Windows Services, etc.) + // When all streams are redirected, we're likely in a non-interactive environment + // where console output goes to StreamWriter.Null + if (Console.IsOutputRedirected && Console.IsErrorRedirected) + { + // Additional check: In IIS/Windows Services, Console.Out is StreamWriter.Null + // This is the specific condition that causes the race condition + if (Console.Out == StreamWriter.Null || Console.Error == StreamWriter.Null) + return false; + } + + // For .NET Framework compatibility, also check Environment.UserInteractive + // This returns false for Windows Services and IIS (though not reliable in .NET Core) + if (!Environment.UserInteractive) + return false; + + return true; + } /// /// Writes the specified value to the standard output stream. Optionally @@ -46,6 +78,16 @@ public static void WriteLine(string message, ConsoleColor? foregroundColor = nul private static void WriteToConsole(string message, ConsoleColor? foregroundColor = null, ConsoleColor? backgroundColor = null, bool line = true) { + // Skip console output in IIS, Windows Services, and other non-console environments. + // In these environments: + // 1. Console output is redirected to StreamWriter.Null (goes nowhere anyway) + // 2. Both Console.Out and Console.Error point to the same StreamWriter.Null singleton + // 3. Concurrent writes to both streams cause race conditions and IndexOutOfRangeException + // 4. Skipping output entirely prevents the race condition and improves performance + // See: https://github.com/akkadotnet/akka.net/issues/7691 + if (!_isConsoleAvailable) + return; + lock (_lock) { ConsoleColor? fg = null; From f043289d79c4dcbe23b8a6fdd38b47ab99b361f5 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 2 Sep 2025 11:36:19 -0500 Subject: [PATCH 2/2] Refine console detection and simplify error handling - Make console detection more precise: only skip output when both Console.Out AND Console.Error point to StreamWriter.Null (the exact race condition scenario) - Remove unnecessary try-catch in DefaultLogger.Print() since Tell() is unlikely to throw - Keep improved error message for debugging when logger is not initialized --- src/core/Akka/Event/DefaultLogger.cs | 13 +------------ src/core/Akka/Util/StandardOutWriter.cs | 21 +++++++++------------ 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/core/Akka/Event/DefaultLogger.cs b/src/core/Akka/Event/DefaultLogger.cs index f4b8d799fb5..3d87cec932e 100644 --- a/src/core/Akka/Event/DefaultLogger.cs +++ b/src/core/Akka/Event/DefaultLogger.cs @@ -48,22 +48,11 @@ protected virtual void Print(LogEvent logEvent) if (_stdoutLogger == null) { // Include context about the failed log event to help with debugging - // but avoid creating a cascade of logging failures var logDetails = $"[{logEvent.LogLevel()}] {logEvent.LogSource}: {logEvent.Message}"; throw new Exception($"Logger has not been initialized yet. Failed to log: {logDetails}"); } - try - { - _stdoutLogger.Tell(logEvent); - } - catch (Exception ex) - { - // Prevent cascade failures by not attempting to log the logging failure - // In IIS/Windows Service environments, this prevents the feedback loop - // that causes millions of log messages and memory exhaustion - System.Diagnostics.Debug.WriteLine($"Failed to write to stdout logger: {ex.Message}"); - } + _stdoutLogger.Tell(logEvent); } } } diff --git a/src/core/Akka/Util/StandardOutWriter.cs b/src/core/Akka/Util/StandardOutWriter.cs index e006374bdab..d0495dd9685 100644 --- a/src/core/Akka/Util/StandardOutWriter.cs +++ b/src/core/Akka/Util/StandardOutWriter.cs @@ -30,19 +30,16 @@ public static class StandardOutWriter /// private static bool DetectConsoleAvailability() { - // Check if console streams are redirected (common in IIS, Windows Services, etc.) - // When all streams are redirected, we're likely in a non-interactive environment - // where console output goes to StreamWriter.Null - if (Console.IsOutputRedirected && Console.IsErrorRedirected) - { - // Additional check: In IIS/Windows Services, Console.Out is StreamWriter.Null - // This is the specific condition that causes the race condition - if (Console.Out == StreamWriter.Null || Console.Error == StreamWriter.Null) - return false; - } + // Specifically detect the IIS/Windows Service scenario where both Console.Out + // and Console.Error point to the SAME StreamWriter.Null singleton instance. + // This is the exact condition that causes the race condition. + // Note: We check both because in these environments, both are always set to the same instance + if (Console.Out == StreamWriter.Null && Console.Error == StreamWriter.Null) + return false; - // For .NET Framework compatibility, also check Environment.UserInteractive - // This returns false for Windows Services and IIS (though not reliable in .NET Core) + // Also check Environment.UserInteractive for additional safety + // This returns false for Windows Services and IIS in .NET Framework + // (though less reliable in .NET Core, the StreamWriter.Null check above is the key) if (!Environment.UserInteractive) return false;