feat(engine): raise thread pool minimum when blocking tests starve the pool

docs/docs/execution/parallelism.md

@@ -18,6 +18,10 @@ With no attributes, every test is eligible to run concurrently. The .NET thread
 :::note Async tests and the thread pool
 TUnit uses the standard .NET thread pool and, like most .NET applications, is susceptible to thread pool starvation. Blocking calls or heavy CPU-bound `Task.Run` work can delay async continuations in other tests and cause timed waits to expire.
 
+By default, the pool adds threads beyond its minimum only about once or twice per second, so a burst of blocking tests can delay timers and `[Timeout]` handling in unrelated tests. To reduce this, TUnit watches the thread pool while tests run. If queued work waits for a thread while most CPUs are idle, TUnit raises the pool's minimum worker thread count to the maximum number of parallel tests (plus one per processor) for the rest of the run. Suites that do not block never trigger this, so they keep the default thread count. TUnit never lowers a minimum that you configured yourself. Set the `TUNIT_DISABLE_THREADPOOL_TUNING=1` environment variable to turn this off.
+
+When a test times out, the error message includes a thread pool snapshot. If the pool looked saturated, the message says the timeout may come from blocking calls in other tests.
+
 Use proper async throughout your tests: `await` asynchronous APIs instead of blocking with `.Wait()`, `.Result`, or `.GetAwaiter().GetResult()`. Await asynchronous I/O directly rather than wrapping it in `Task.Run`; `Task.Run` still uses the same thread pool.
 
 For heavy CPU-bound tests, use a shared [`ParallelLimiter<T>`](#parallellimitert--limiting-concurrent-test-count) to reduce concurrency for those tests. The limiter caps concurrent tests that share it, not the number of tasks each test creates.

docs/docs/reference/environment-variables.md

@@ -132,6 +132,14 @@ export TUNIT_MAX_PARALLEL_TESTS=0    # Unlimited parallelism
 
 **Note:** Command-line arguments take precedence over environment variables.
 
+### TUNIT_DISABLE_THREADPOOL_TUNING
+
+By default, TUnit watches the thread pool while tests run. If queued work waits for a thread while most CPUs are idle (a sign that tests are blocking), TUnit raises the pool's minimum worker thread count to the maximum number of parallel tests (plus one per processor). This stops blocking tests from delaying timers and timeouts in other tests while the pool slowly adds threads. TUnit never lowers a minimum that you configured yourself. Set this variable to turn the behavior off.
+
+```bash
+export TUNIT_DISABLE_THREADPOOL_TUNING=1
+```
+
 ### TUNIT_EXECUTION_MODE
 
 Controls whether tests run in source-generation (AOT-optimized) mode or reflection mode.
@@ -305,6 +313,7 @@ When the same setting is configured in multiple places, TUnit follows this prior
 | `TUNIT_DISABLE_ARTIFACT_UPLOAD` | - | Keeps the HTML report file but skips the GitHub Actions artifact upload (`context.Settings.Reporting.ArtifactUploadEnabled = false`) |
 | `JUNIT_XML_OUTPUT_PATH` | - | JUnit output path |
 | `TUNIT_MAX_PARALLEL_TESTS` | `--maximum-parallel-tests` | Max parallel tests |
+| `TUNIT_DISABLE_THREADPOOL_TUNING` | - | Disables the thread pool starvation monitor (`1` to disable) |
 | `TUNIT_EXECUTION_MODE` | `--reflection` | Selects source-generation (`sourcegeneration`/`aot`) or `reflection` execution mode |
 | `TUNIT_DISCOVERY_DIAGNOSTICS` | - | Enables discovery-time diagnostics (`1` to enable) |
 | `TUNIT_DIAGNOSTIC_CAST` | - | Enables `CastHelper` conversion diagnostics (`true` to enable) |

src/TUnit.Engine/Configuration/EnvironmentConstants.cs

@@ -29,6 +29,8 @@ internal static class EnvironmentConstants
     // TUnit-specific: Execution
     public const string ExecutionMode = "TUNIT_EXECUTION_MODE";
     public const string MaxParallelTests = "TUNIT_MAX_PARALLEL_TESTS";
+    // Opts out of the thread pool starvation monitor (raises min worker threads when tests block).
+    public const string DisableThreadPoolTuning = "TUNIT_DISABLE_THREADPOOL_TUNING";
 
     // TUnit-specific: Display and diagnostics
     public const string DisableLogo = "TUNIT_DISABLE_LOGO";

src/TUnit.Engine/Helpers/TimeoutDiagnostics.cs

@@ -41,6 +41,9 @@ public static string BuildTimeoutDiagnosticsMessage(
         var sb = new StringBuilder(baseMessage);
 
         AppendTaskStatus(sb, executionTask, executionException);
+#if NET
+        AppendThreadPoolDiagnostics(sb, ThreadPool.ThreadCount, ThreadPool.PendingWorkItemCount);
+#endif
         AppendStackTraceDiagnostics(sb);
 
         return sb.ToString();
@@ -89,6 +92,43 @@ private static void AppendTaskException(StringBuilder sb, Exception exception)
         sb.Append(exception.Message);
     }
 
+#if NET
+    /// <summary>
+    /// Appends a thread pool snapshot. A backlog of queued work means continuations and timer
+    /// callbacks were waiting for a thread, so the timeout may be caused by other tests blocking.
+    /// </summary>
+    internal static void AppendThreadPoolDiagnostics(StringBuilder sb, int threadCount, long pendingWorkItems)
+    {
+        ThreadPool.GetMinThreads(out var minWorkerThreads, out _);
+        ThreadPool.GetMaxThreads(out var maxWorkerThreads, out _);
+
+        sb.AppendLine();
+        sb.AppendLine();
+        sb.Append("--- Thread Pool ---");
+        sb.AppendLine();
+        sb.Append("Threads: ");
+        sb.Append(threadCount);
+        sb.Append(", pending work items: ");
+        sb.Append(pendingWorkItems);
+        sb.Append(", min worker threads: ");
+        sb.Append(minWorkerThreads);
+        sb.Append(", max worker threads: ");
+        sb.Append(maxWorkerThreads);
+
+        if (IsThreadPoolSaturated(threadCount, pendingWorkItems))
+        {
+            sb.AppendLine();
+            sb.Append("The thread pool appears saturated. This timeout may be caused by blocking calls ");
+            sb.Append("(.Result, .Wait(), Thread.Sleep, synchronous I/O) in other concurrently running tests ");
+            sb.Append("rather than by this test. Use async APIs throughout, or limit concurrency for heavy tests ");
+            sb.Append("with [ParallelLimiter<T>] or --maximum-parallel-tests.");
+        }
+    }
+
+    internal static bool IsThreadPoolSaturated(int threadCount, long pendingWorkItems)
+        => pendingWorkItems > 0 && pendingWorkItems >= threadCount;
+#endif
+
     private static void AppendStackTraceDiagnostics(StringBuilder sb)
     {
         string stackTrace;

src/TUnit.Engine/Scheduling/TestScheduler.cs

@@ -103,6 +103,10 @@ public async Task<bool> ScheduleAndExecuteAsync(
             return true;
         }
 
+        // Covers every phase that runs user code (static property initializers, tests,
+        // session hooks) so blocking tests do not delay timers in other tests.
+        using var starvationMonitor = ThreadPoolStarvationMonitor.Start(_maxParallelism.Value, _logger);
+
         // Initialize static properties before tests run
         await _staticPropertyHandler.InitializeStaticPropertiesAsync(cancellationToken).ConfigureAwait(false);
 

src/TUnit.Engine/Scheduling/ThreadPoolStarvationMonitor.cs

@@ -0,0 +1,216 @@
+using TUnit.Core.Logging;
+#if NET
+using System.Diagnostics;
+using System.Runtime.Versioning;
+using TUnit.Engine.Configuration;
+#endif
+
+namespace TUnit.Engine.Scheduling;
+
+/// <summary>
+/// Raises the thread pool's minimum worker thread count while tests run, but only once the pool
+/// is starved by blocked threads.
+/// <para>
+/// Test bodies run inline on pool threads. When concurrently running tests block (Thread.Sleep,
+/// synchronous I/O, sync-over-async), the pool adds threads beyond its minimum (ProcessorCount by
+/// default) at only one or two per second. Timer callbacks — Task.Delay, CancelAfter and TUnit's own
+/// [Timeout] — also need a pool thread, so starvation makes timing-sensitive tests in other classes
+/// overshoot their deadlines.
+/// </para>
+/// <para>
+/// A static minimum is not free: extra pool threads spin while searching for work and slow down
+/// suites of many short tests. The monitor therefore raises the minimum only after a probe work
+/// item waits a whole sample interval for a thread while the process leaves most CPUs idle — the
+/// signature of blocked, not busy, threads. Pending-item counts alone are not enough: the engine's
+/// own short work items keep a small queue at low CPU without any test blocking.
+/// </para>
+/// </summary>
+internal sealed class ThreadPoolStarvationMonitor : IDisposable
+{
+#if NET
+    private static readonly TimeSpan SampleInterval = TimeSpan.FromMilliseconds(50);
+
+    // Below this share of total CPU capacity, queued work is waiting on blocked threads rather
+    // than on busy ones, so adding threads helps instead of oversubscribing the cores.
+    internal const double StarvedCpuUtilization = 0.5;
+
+    // Never disposed: the sampling thread may still be waiting on it after Dispose returns.
+    private readonly ManualResetEventSlim _stopSignal = new(false);
+    private readonly Lock _lock = new();
+    private readonly ILogger _logger;
+    private readonly int _originalMinWorkerThreads;
+    private readonly int _maxWorkerThreads;
+    private int _raisedMinWorkerThreads;
+    private bool _stopped;
+    // Timestamp of the queued probe work item, or 0 once a pool thread has run it.
+    private long _probeQueuedAt;
+
+    private ThreadPoolStarvationMonitor(int originalMinWorkerThreads, int maxWorkerThreads, ILogger logger)
+    {
+        _originalMinWorkerThreads = originalMinWorkerThreads;
+        _maxWorkerThreads = maxWorkerThreads;
+        _logger = logger;
+    }
+#endif
+
+    /// <summary>
+    /// Starts monitoring, or returns <c>null</c> when monitoring cannot help or is disabled.
+    /// </summary>
+    public static ThreadPoolStarvationMonitor? Start(int maxParallelism, ILogger logger)
+    {
+#if NET
+        if (OperatingSystem.IsBrowser())
+        {
+            return null;
+        }
+
+        if (IsTruthyEnv(Environment.GetEnvironmentVariable(EnvironmentConstants.DisableThreadPoolTuning)))
+        {
+            logger.LogDebug($"Thread pool starvation monitor disabled by {EnvironmentConstants.DisableThreadPoolTuning}");
+            return null;
+        }
+
+        ThreadPool.GetMinThreads(out var originalMinWorkerThreads, out _);
+
+        // Headroom above the test limit for timer callbacks, the dynamic test queue and engine work.
+        var maxWorkerThreads = (int)Math.Min((long)maxParallelism + Environment.ProcessorCount, short.MaxValue);
+
+        if (originalMinWorkerThreads >= maxWorkerThreads)
+        {
+            // The user already configured enough threads.
+            return null;
+        }
+
+        var monitor = new ThreadPoolStarvationMonitor(originalMinWorkerThreads, maxWorkerThreads, logger);
+
+        // A dedicated thread: the monitor must keep sampling while every pool thread is blocked.
+        var thread = new Thread(monitor.Run)
+        {
+            IsBackground = true,
+            Name = "TUnit thread pool starvation monitor",
+        };
+        thread.Start();
+        return monitor;
+#else
+        // ThreadPool.PendingWorkItemCount is unavailable; the runtime's own thread injection applies.
+        return null;
+#endif
+    }
+
+#if NET
+    [UnsupportedOSPlatform("browser")]
+    private void Run()
+    {
+        var processorCount = Environment.ProcessorCount;
+        var previousCpuTime = GetProcessCpuTime();
+        var previousTimestamp = Stopwatch.GetTimestamp();
+
+        while (!_stopSignal.Wait(SampleInterval))
+        {
+            var cpuTime = GetProcessCpuTime();
+            var timestamp = Stopwatch.GetTimestamp();
+
+            var elapsedMs = Stopwatch.GetElapsedTime(previousTimestamp, timestamp).TotalMilliseconds;
+            var cpuUtilization = elapsedMs <= 0
+                ? 1
+                : (cpuTime - previousCpuTime).TotalMilliseconds / (elapsedMs * processorCount);
+
+            var probeQueuedAt = Volatile.Read(ref _probeQueuedAt);
+
+            if (probeQueuedAt == 0)
+            {
+                // The global queue is where timer callbacks wait for a thread, so a probe queued
+                // there measures how late a Task.Delay or [Timeout] continuation would run.
+                Volatile.Write(ref _probeQueuedAt, timestamp);
+                ThreadPool.UnsafeQueueUserWorkItem(static monitor => Volatile.Write(ref monitor._probeQueuedAt, 0), this, preferLocal: false);
+            }
+            else if (IsStarved(Stopwatch.GetElapsedTime(probeQueuedAt, timestamp), cpuUtilization))
+            {
+                // One raise to the cap is all the monitor does, so it stops sampling afterwards.
+                Raise();
+                return;
+            }
+
+            previousCpuTime = cpuTime;
+            previousTimestamp = timestamp;
+        }
+    }
+
+    private void Raise()
+    {
+        lock (_lock)
+        {
+            if (_stopped)
+            {
+                return;
+            }
+
+            // Raise straight to the cap in one step. When threads are blocked in Task.Wait or .Result,
+            // the runtime's cooperative-blocking logic takes over the raise and adds threads only
+            // under narrow conditions. Repeated small raises then left the pool stuck below its new
+            // minimum for seconds, but a single raise reliably adds the threads.
+            ThreadPool.GetMinThreads(out var currentMinWorkerThreads, out var currentMinIoThreads);
+
+            if (currentMinWorkerThreads < _maxWorkerThreads
+                && ThreadPool.SetMinThreads(_maxWorkerThreads, currentMinIoThreads))
+            {
+                _raisedMinWorkerThreads = _maxWorkerThreads;
+            }
+        }
+    }
+
+    [UnsupportedOSPlatform("browser")]
+    private static TimeSpan GetProcessCpuTime()
+    {
+#if NET9_0_OR_GREATER
+        return Environment.CpuUsage.TotalTime;
+#else
+        using var process = Process.GetCurrentProcess();
+        return process.TotalProcessorTime;
+#endif
+    }
+
+    /// <summary>
+    /// Queued work that waits a whole sample interval for a thread while most CPUs are idle means
+    /// pool threads are blocked rather than busy.
+    /// </summary>
+    internal static bool IsStarved(TimeSpan probeWait, double cpuUtilization)
+        => probeWait >= SampleInterval && cpuUtilization < StarvedCpuUtilization;
+
+    private static bool IsTruthyEnv(string? value)
+        => value is not null
+           && (value == "1"
+               || value.Equals("true", StringComparison.OrdinalIgnoreCase)
+               || value.Equals("yes", StringComparison.OrdinalIgnoreCase));
+#endif
+
+    public void Dispose()
+    {
+#if NET
+        lock (_lock)
+        {
+            if (_stopped)
+            {
+                return;
+            }
+
+            _stopped = true;
+            _stopSignal.Set();
+
+            if (_raisedMinWorkerThreads == 0)
+            {
+                return;
+            }
+
+            // Leave the pool as we found it unless user code changed the minimum in the meantime.
+            ThreadPool.GetMinThreads(out var currentMinWorkerThreads, out var currentMinIoThreads);
+            if (currentMinWorkerThreads == _raisedMinWorkerThreads)
+            {
+                ThreadPool.SetMinThreads(_originalMinWorkerThreads, currentMinIoThreads);
+            }
+        }
+
+        _logger.LogDebug($"Thread pool starvation detected while tests ran; minimum worker threads were raised from {_originalMinWorkerThreads} to {_raisedMinWorkerThreads}");
+#endif
+    }
+}

tests/TUnit.UnitTests/ThreadPoolStarvationMonitorTests.cs

@@ -0,0 +1,52 @@
+#if NET
+using System.Text;
+using TUnit.Engine.Helpers;
+using TUnit.Engine.Scheduling;
+
+namespace TUnit.UnitTests;
+
+public class ThreadPoolStarvationMonitorTests
+{
+    [Test]
+    [Arguments(60, 0.1, true)]
+    [Arguments(50, 0.49, true)]
+    [Arguments(10, 0.1, false)]
+    [Arguments(60, 0.9, false)]
+    public async Task Starved_Only_When_Probe_Waits_And_Cpus_Are_Idle(int probeWaitMs, double cpuUtilization, bool expected)
+    {
+        await Assert.That(ThreadPoolStarvationMonitor.IsStarved(TimeSpan.FromMilliseconds(probeWaitMs), cpuUtilization))
+            .IsEqualTo(expected);
+    }
+
+    [Test]
+    [Arguments(10, 0, false)]
+    [Arguments(10, 9, false)]
+    [Arguments(10, 10, true)]
+    [Arguments(10, 500, true)]
+    public async Task Detects_Saturated_Thread_Pool(int threadCount, long pendingWorkItems, bool expected)
+    {
+        await Assert.That(TimeoutDiagnostics.IsThreadPoolSaturated(threadCount, pendingWorkItems)).IsEqualTo(expected);
+    }
+
+    [Test]
+    public async Task Timeout_Diagnostics_Include_Thread_Pool_Snapshot_And_Saturation_Hint()
+    {
+        var sb = new StringBuilder();
+        TimeoutDiagnostics.AppendThreadPoolDiagnostics(sb, threadCount: 4, pendingWorkItems: 50);
+        var message = sb.ToString();
+
+        await Assert.That(message).Contains("--- Thread Pool ---");
+        await Assert.That(message).Contains("Threads: 4, pending work items: 50");
+        await Assert.That(message).Contains("thread pool appears saturated");
+    }
+
+    [Test]
+    public async Task Timeout_Diagnostics_Omit_Saturation_Hint_When_Pool_Is_Idle()
+    {
+        var sb = new StringBuilder();
+        TimeoutDiagnostics.AppendThreadPoolDiagnostics(sb, threadCount: 4, pendingWorkItems: 0);
+
+        await Assert.That(sb.ToString()).DoesNotContain("saturated");
+    }
+}
+#endif