diff --git a/TUnit.Engine/CommandLineProviders/AdaptiveMetricsCommandProvider.cs b/TUnit.Engine/CommandLineProviders/AdaptiveMetricsCommandProvider.cs new file mode 100644 index 0000000000..f7d1cdd903 --- /dev/null +++ b/TUnit.Engine/CommandLineProviders/AdaptiveMetricsCommandProvider.cs @@ -0,0 +1,41 @@ +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace TUnit.Engine.CommandLineProviders; + +internal class AdaptiveMetricsCommandProvider(IExtension extension) : ICommandLineOptionsProvider +{ + public const string AdaptiveMetrics = "adaptive-metrics"; + + public Task IsEnabledAsync() + { + return extension.IsEnabledAsync(); + } + + public string Uid => extension.Uid; + + public string Version => extension.Version; + + public string DisplayName => extension.DisplayName; + + public string Description => extension.Description; + + public IReadOnlyCollection GetCommandLineOptions() + { + return + [ + new CommandLineOption(AdaptiveMetrics, "Enable detailed metrics logging for adaptive parallelism", ArgumentArity.Zero, false) + ]; + } + + public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + return ValidationResult.ValidTask; + } + + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + { + return ValidationResult.ValidTask; + } +} \ No newline at end of file diff --git a/TUnit.Engine/CommandLineProviders/ParallelismStrategyCommandProvider.cs b/TUnit.Engine/CommandLineProviders/ParallelismStrategyCommandProvider.cs new file mode 100644 index 0000000000..b77752a003 --- /dev/null +++ b/TUnit.Engine/CommandLineProviders/ParallelismStrategyCommandProvider.cs @@ -0,0 +1,55 @@ +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace TUnit.Engine.CommandLineProviders; + +internal class ParallelismStrategyCommandProvider(IExtension extension) : ICommandLineOptionsProvider +{ + public const string ParallelismStrategy = "parallelism-strategy"; + + public Task IsEnabledAsync() + { + return extension.IsEnabledAsync(); + } + + public string Uid => extension.Uid; + + public string Version => extension.Version; + + public string DisplayName => extension.DisplayName; + + public string Description => extension.Description; + + public IReadOnlyCollection GetCommandLineOptions() + { + return + [ + new CommandLineOption(ParallelismStrategy, "Parallelism strategy: fixed or adaptive (default: adaptive)", ArgumentArity.ExactlyOne, false) + ]; + } + + public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + { + if (commandOption.Name == ParallelismStrategy && arguments.Length != 1) + { + return ValidationResult.InvalidTask("A single value must be provided for parallelism strategy"); + } + + if (commandOption.Name == ParallelismStrategy) + { + var strategy = arguments[0].ToLowerInvariant(); + if (strategy != "fixed" && strategy != "adaptive") + { + return ValidationResult.InvalidTask("Parallelism strategy must be 'fixed' or 'adaptive'"); + } + } + + return ValidationResult.ValidTask; + } + + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) + { + return ValidationResult.ValidTask; + } +} \ No newline at end of file diff --git a/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs b/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs index b70d738946..78f8cead2d 100644 --- a/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs +++ b/TUnit.Engine/Extensions/TestApplicationBuilderExtensions.cs @@ -33,6 +33,10 @@ public static void AddTUnit(this ITestApplicationBuilder testApplicationBuilder) testApplicationBuilder.CommandLine.AddProvider(() => new ReflectionModeCommandProvider(extension)); testApplicationBuilder.CommandLine.AddProvider(() => new DisableLogoCommandProvider(extension)); + // Adaptive parallelism command providers + testApplicationBuilder.CommandLine.AddProvider(() => new ParallelismStrategyCommandProvider(extension)); + testApplicationBuilder.CommandLine.AddProvider(() => new AdaptiveMetricsCommandProvider(extension)); + // Unified verbosity control (replaces HideTestOutput, DisableLogo, DetailedStacktrace) testApplicationBuilder.CommandLine.AddProvider(() => new VerbosityCommandProvider(extension)); diff --git a/TUnit.Engine/Scheduling/AdaptiveParallelismController.cs b/TUnit.Engine/Scheduling/AdaptiveParallelismController.cs new file mode 100644 index 0000000000..e49e0a50bd --- /dev/null +++ b/TUnit.Engine/Scheduling/AdaptiveParallelismController.cs @@ -0,0 +1,243 @@ +using TUnit.Core.Logging; +using TUnit.Engine.Logging; +using TUnit.Engine.Services; + +namespace TUnit.Engine.Scheduling; + +/// +/// Controller that runs in the background and adjusts parallelism based on system metrics +/// +internal sealed class AdaptiveParallelismController : IDisposable +{ + private readonly AdaptiveSemaphore _semaphore; + private readonly SystemMetricsCollector _metricsCollector; + private readonly ParallelismAdjustmentStrategy _adjustmentStrategy; + private readonly TUnitFrameworkLogger _logger; + private readonly bool _enableMetricsLogging; + private readonly CancellationTokenSource _cancellationSource; + private readonly Task _adjustmentTask; + private readonly Task? _metricsLoggingTask; + private int _currentParallelism; + private bool _disposed; + + public AdaptiveParallelismController( + AdaptiveSemaphore semaphore, + TUnitFrameworkLogger logger, + int minParallelism, + int maxParallelism, + int initialParallelism, + bool enableMetricsLogging) + { + _semaphore = semaphore ?? throw new ArgumentNullException(nameof(semaphore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _enableMetricsLogging = enableMetricsLogging; + _currentParallelism = initialParallelism; + + _metricsCollector = new SystemMetricsCollector(); + _adjustmentStrategy = new ParallelismAdjustmentStrategy(minParallelism, maxParallelism); + _cancellationSource = new CancellationTokenSource(); + + // Start background tasks + _adjustmentTask = RunAdjustmentLoopAsync(_cancellationSource.Token); + + if (_enableMetricsLogging) + { + _metricsLoggingTask = RunMetricsLoggingLoopAsync(_cancellationSource.Token); + } + } + + /// + /// Gets the current parallelism level + /// + public int CurrentParallelism => _currentParallelism; + + /// + /// Records a test completion for metrics + /// + public void RecordTestCompletion(TimeSpan executionTime) + { + _adjustmentStrategy.RecordTestCompletion(executionTime); + } + + private async Task RunAdjustmentLoopAsync(CancellationToken cancellationToken) + { +#if NET6_0_OR_GREATER + // Use PeriodicTimer for cleaner async timing (500ms intervals) + using var timer = new PeriodicTimer(TimeSpan.FromMilliseconds(500)); + + while (!cancellationToken.IsCancellationRequested) + { + try + { + await timer.WaitForNextTickAsync(cancellationToken); + await AdjustParallelismAsync(); + } + catch (OperationCanceledException) + { + // Expected when cancellation is requested + break; + } + catch (Exception ex) + { + // Log error but don't crash the adjustment loop + await _logger.LogErrorAsync($"Error in adaptive parallelism adjustment: {ex.Message}"); + } + } +#else + // Fallback for netstandard2.0 + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(500, cancellationToken); + await AdjustParallelismAsync(); + } + catch (OperationCanceledException) + { + // Expected when cancellation is requested + break; + } + catch (Exception ex) + { + // Log error but don't crash the adjustment loop + await _logger.LogErrorAsync($"Error in adaptive parallelism adjustment: {ex.Message}"); + } + } +#endif + } + + private async Task RunMetricsLoggingLoopAsync(CancellationToken cancellationToken) + { + // Initial delay to let tests start + await Task.Delay(1000, cancellationToken); + +#if NET6_0_OR_GREATER + // Use PeriodicTimer for metrics logging (3 second intervals) + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(3)); + + while (!cancellationToken.IsCancellationRequested) + { + try + { + await timer.WaitForNextTickAsync(cancellationToken); + + // Get current metrics + var metrics = _metricsCollector.GetMetrics(); + await LogMetrics(metrics); + } + catch (OperationCanceledException) + { + // Expected when cancellation is requested + break; + } + catch (Exception ex) + { + // Log error but continue + await _logger.LogErrorAsync($"Error logging adaptive metrics: {ex.Message}"); + } + } +#else + // Fallback for netstandard2.0 + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(3000, cancellationToken); + + // Get current metrics + var metrics = _metricsCollector.GetMetrics(); + await LogMetrics(metrics); + } + catch (OperationCanceledException) + { + // Expected when cancellation is requested + break; + } + catch (Exception ex) + { + // Log error but continue + await _logger.LogErrorAsync($"Error logging adaptive metrics: {ex.Message}"); + } + } +#endif + } + + private async Task AdjustParallelismAsync() + { + // Collect metrics + var metrics = _metricsCollector.GetMetrics(); + + // Calculate adjustment + var recommendation = _adjustmentStrategy.CalculateAdjustment(metrics, _currentParallelism); + + // Apply adjustment if needed + if (recommendation.NewParallelism != _currentParallelism) + { + _semaphore.AdjustMaxCount(recommendation.NewParallelism); + var oldParallelism = _currentParallelism; + _currentParallelism = recommendation.NewParallelism; + + if (_enableMetricsLogging) + { + await LogAdjustment(oldParallelism, recommendation, metrics); + } + } + } + + private async Task LogAdjustment(int oldParallelism, AdjustmentRecommendation recommendation, SystemMetrics metrics) + { + var direction = recommendation.Direction == AdjustmentDirection.Increase ? "↑" : "↓"; + await _logger.LogDebugAsync( + $"[Adaptive] Parallelism adjusted: {oldParallelism} {direction} {recommendation.NewParallelism} | " + + $"Reason: {recommendation.Reason} | " + + $"CPU: {metrics.SystemCpuUsagePercent:F1}% | " + + $"Threads: {metrics.AvailableWorkerThreads}/{metrics.MaxWorkerThreads} | " + + $"Memory: {metrics.TotalMemoryBytes / 1_000_000}MB"); + } + + private async Task LogMetrics(SystemMetrics metrics) + { + var semaphoreAvailable = _semaphore.CurrentCount; + var activeTests = _currentParallelism - semaphoreAvailable; + + await _logger.LogDebugAsync( + $"[Adaptive] Metrics | " + + $"Parallelism: {_currentParallelism} (Active: {activeTests}, Available: {semaphoreAvailable}) | " + + $"CPU: {metrics.SystemCpuUsagePercent:F1}% | " + + $"Threads: {metrics.AvailableWorkerThreads}/{metrics.MaxWorkerThreads} | " + + $"Pending: {metrics.PendingWorkItems} | " + + $"Memory: {metrics.TotalMemoryBytes / 1_000_000}MB"); + } + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + + // Cancel background tasks + _cancellationSource.Cancel(); + + // Wait for tasks to complete (with timeout) + try + { + var tasksToWait = new[] { _adjustmentTask, _metricsLoggingTask } + .Where(t => t != null) + .Cast() + .ToArray(); + + if (tasksToWait.Length > 0) + { + Task.WaitAll(tasksToWait, TimeSpan.FromSeconds(5)); + } + } + catch (AggregateException) + { + // Tasks may have been cancelled, which is expected + } + + _cancellationSource.Dispose(); + _metricsCollector?.Dispose(); + } +} \ No newline at end of file diff --git a/TUnit.Engine/Scheduling/AdaptiveSemaphore.cs b/TUnit.Engine/Scheduling/AdaptiveSemaphore.cs new file mode 100644 index 0000000000..eb9d53c19d --- /dev/null +++ b/TUnit.Engine/Scheduling/AdaptiveSemaphore.cs @@ -0,0 +1,184 @@ +using System.Collections.Concurrent; + +namespace TUnit.Engine.Scheduling; + +/// +/// A semaphore that supports dynamic adjustment of maximum count +/// +internal sealed class AdaptiveSemaphore : IDisposable +{ + private readonly object _lock = new(); + private readonly ConcurrentQueue> _waiters = new(); + private int _currentCount; + private int _maxCount; + private bool _disposed; + + public AdaptiveSemaphore(int initialCount, int maxCount) + { + if (initialCount < 0) throw new ArgumentOutOfRangeException(nameof(initialCount)); + if (maxCount < 1) throw new ArgumentOutOfRangeException(nameof(maxCount)); + if (initialCount > maxCount) throw new ArgumentException("Initial count cannot exceed max count"); + + _currentCount = initialCount; + _maxCount = maxCount; + } + + /// + /// Gets the current available count + /// + public int CurrentCount + { + get + { + lock (_lock) + { + return _currentCount; + } + } + } + + /// + /// Gets the current maximum count + /// + public int MaxCount + { + get + { + lock (_lock) + { + return _maxCount; + } + } + } + + /// + /// Waits to enter the semaphore + /// + public async Task WaitAsync(CancellationToken cancellationToken = default) + { + TaskCompletionSource? waiter = null; + + lock (_lock) + { + if (_disposed) + throw new ObjectDisposedException(nameof(AdaptiveSemaphore)); + + if (_currentCount > 0) + { + _currentCount--; + return; + } + + waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _waiters.Enqueue(waiter); + } + + using (cancellationToken.Register(() => waiter.TrySetCanceled())) + { + await waiter.Task.ConfigureAwait(false); + } + } + + /// + /// Releases one count back to the semaphore + /// + public void Release() + { + TaskCompletionSource? waiterToRelease = null; + + lock (_lock) + { + if (_disposed) + throw new ObjectDisposedException(nameof(AdaptiveSemaphore)); + + if (_waiters.TryDequeue(out waiterToRelease)) + { + // Release directly to a waiter without incrementing count + } + else + { + // Don't throw if we're at max - this can happen when max count is reduced + // while tests are still running. Just silently ignore the release. + if (_currentCount < _maxCount) + { + _currentCount++; + } + } + } + + waiterToRelease?.TrySetResult(true); + } + + /// + /// Adjusts the maximum count of the semaphore + /// + public void AdjustMaxCount(int newMaxCount) + { + if (newMaxCount < 1) + throw new ArgumentOutOfRangeException(nameof(newMaxCount)); + + var waitersToRelease = new List>(); + + lock (_lock) + { + if (_disposed) + throw new ObjectDisposedException(nameof(AdaptiveSemaphore)); + + var oldMaxCount = _maxCount; + _maxCount = newMaxCount; + + // If we're increasing the max count, we might be able to release some waiters + if (newMaxCount > oldMaxCount) + { + var additionalCapacity = newMaxCount - oldMaxCount; + _currentCount += additionalCapacity; + + // Release waiters if we have capacity + while (_currentCount > 0 && _waiters.TryDequeue(out var waiter)) + { + waitersToRelease.Add(waiter); + _currentCount--; + } + } + else if (newMaxCount < oldMaxCount) + { + // If decreasing, cap the current count at the new max + // This prevents issues but allows running tests to complete + if (_currentCount > newMaxCount) + { + _currentCount = newMaxCount; + } + } + } + + // Release waiters outside the lock to avoid potential deadlocks + foreach (var waiter in waitersToRelease) + { + waiter.TrySetResult(true); + } + } + + public void Dispose() + { + List> waitersToCancel; + + lock (_lock) + { + if (_disposed) + return; + + _disposed = true; + waitersToCancel = new List>(); + + while (_waiters.TryDequeue(out var waiter)) + { + waitersToCancel.Add(waiter); + } + } + + foreach (var waiter in waitersToCancel) + { + waiter.TrySetCanceled(); + } + } +} \ No newline at end of file diff --git a/TUnit.Engine/Scheduling/ParallelismAdjustmentStrategy.cs b/TUnit.Engine/Scheduling/ParallelismAdjustmentStrategy.cs new file mode 100644 index 0000000000..d57164df05 --- /dev/null +++ b/TUnit.Engine/Scheduling/ParallelismAdjustmentStrategy.cs @@ -0,0 +1,226 @@ +using System.Collections.Concurrent; +using TUnit.Engine.Services; + +namespace TUnit.Engine.Scheduling; + +/// +/// Strategy for adjusting parallelism based on system metrics +/// +internal sealed class ParallelismAdjustmentStrategy +{ + private readonly int _minParallelism; + private readonly int _maxParallelism; + private readonly ConcurrentQueue _completedTests = new(); + private DateTime _lastMeasurementTime = DateTime.UtcNow; + private int _lastCompletedCount; + private const double CpuLowThreshold = 70.0; + private const double CpuHighThreshold = 90.0; + private const double MinIncreaseFactor = 0.25; // 25% increase (more aggressive) + private const double MinDecreaseFactor = 0.15; // 15% decrease (more conservative) + + public ParallelismAdjustmentStrategy(int minParallelism, int maxParallelism) + { + _minParallelism = Math.Max(1, minParallelism); + _maxParallelism = Math.Max(_minParallelism, maxParallelism); + } + + /// + /// Records a test completion for rate calculation + /// + public void RecordTestCompletion(TimeSpan executionTime) + { + _completedTests.Enqueue(new TestCompletionInfo + { + CompletionTime = DateTime.UtcNow, + ExecutionTime = executionTime + }); + + // Clean up old entries (older than 10 seconds) + var cutoff = DateTime.UtcNow.AddSeconds(-10); + while (_completedTests.TryPeek(out var oldest) && oldest.CompletionTime < cutoff) + { + _completedTests.TryDequeue(out _); + } + } + + /// + /// Calculates the recommended parallelism adjustment + /// + public AdjustmentRecommendation CalculateAdjustment(SystemMetrics metrics, int currentParallelism) + { + var decision = MakeDecision(metrics, currentParallelism); + + // No sliding window - make immediate adjustments + if (decision.Direction == AdjustmentDirection.None) + { + return new AdjustmentRecommendation + { + NewParallelism = currentParallelism, + Direction = AdjustmentDirection.None, + Reason = decision.Reason + }; + } + + // Calculate new parallelism based on direction and current metrics + int newParallelism; + if (decision.Direction == AdjustmentDirection.Increase) + { + // Calculate optimal increase based on current CPU usage + // If we're at 2% CPU with 176 tests, and we want to reach ~70% CPU, + // we can estimate: newParallelism = currentParallelism * (targetCPU / currentCPU) + var currentCpu = metrics.SystemCpuUsagePercent; + if (currentCpu > 0 && currentCpu < 10.0) // Very low CPU usage + { + // Aggressive scaling: try to reach 60% CPU utilization + var scaleFactor = Math.Min(60.0 / currentCpu, 3.0); // Cap at 3x to avoid overshooting + var targetParallelism = (int)(currentParallelism * scaleFactor); + newParallelism = Math.Min(_maxParallelism, targetParallelism); + } + else + { + // Normal increase by 25% + var adjustmentSize = Math.Max(1, (int)(currentParallelism * MinIncreaseFactor)); + newParallelism = Math.Min(_maxParallelism, currentParallelism + adjustmentSize); + } + } + else + { + // Decrease more conservatively + var adjustmentSize = Math.Max(1, (int)(currentParallelism * MinDecreaseFactor)); + newParallelism = Math.Max(_minParallelism, currentParallelism - adjustmentSize); + } + + return new AdjustmentRecommendation + { + NewParallelism = newParallelism, + Direction = decision.Direction, + Reason = decision.Reason + }; + } + + private AdjustmentDecision MakeDecision(SystemMetrics metrics, int currentParallelism) + { + // Check for thread pool starvation + var threadUtilization = CalculateThreadUtilization(metrics); + if (threadUtilization > 0.9 || metrics.PendingWorkItems > 100) + { + return new AdjustmentDecision + { + Direction = AdjustmentDirection.Decrease, + Reason = $"Thread pool starvation detected (utilization: {threadUtilization:P1}, pending: {metrics.PendingWorkItems})" + }; + } + + // Check CPU usage + if (metrics.SystemCpuUsagePercent > CpuHighThreshold) + { + return new AdjustmentDecision + { + Direction = AdjustmentDirection.Decrease, + Reason = $"High CPU usage ({metrics.SystemCpuUsagePercent:F1}%)" + }; + } + + // Check memory pressure + if (metrics.TotalMemoryBytes > 1_000_000_000) // Over 1GB + { + return new AdjustmentDecision + { + Direction = AdjustmentDirection.Decrease, + Reason = $"High memory usage ({metrics.TotalMemoryBytes / 1_000_000}MB)" + }; + } + + // Calculate test completion rate + var completionRate = CalculateCompletionRate(); + + // Check if we can increase parallelism + if (metrics.SystemCpuUsagePercent < CpuLowThreshold && + threadUtilization < 0.7 && + currentParallelism < _maxParallelism) + { + // Check if completion rate is stable or improving + if (completionRate >= 0) // Not declining + { + return new AdjustmentDecision + { + Direction = AdjustmentDirection.Increase, + Reason = $"Resources available (CPU: {metrics.SystemCpuUsagePercent:F1}%, threads: {threadUtilization:P1})" + }; + } + } + + // If completion rate is declining significantly, decrease + if (completionRate < -0.2) // More than 20% decline + { + return new AdjustmentDecision + { + Direction = AdjustmentDirection.Decrease, + Reason = $"Test completion rate declining ({completionRate:P1})" + }; + } + + return new AdjustmentDecision + { + Direction = AdjustmentDirection.None, + Reason = "System metrics stable" + }; + } + + private double CalculateThreadUtilization(SystemMetrics metrics) + { + if (metrics.MaxWorkerThreads == 0) return 0; + return 1.0 - (double)metrics.AvailableWorkerThreads / metrics.MaxWorkerThreads; + } + + private double CalculateCompletionRate() + { + var now = DateTime.UtcNow; + var timeDelta = (now - _lastMeasurementTime).TotalSeconds; + if (timeDelta < 1) return 0; // Not enough time passed + + var currentCount = _completedTests.Count; + var completedInPeriod = currentCount - _lastCompletedCount; + + var currentRate = completedInPeriod / timeDelta; + var previousRate = _lastCompletedCount / 10.0; // Over 10 second window + + _lastCompletedCount = currentCount; + _lastMeasurementTime = now; + + if (previousRate == 0) return 0; + return (currentRate - previousRate) / previousRate; // Percentage change + } + + private sealed class AdjustmentDecision + { + public AdjustmentDirection Direction { get; init; } + public string Reason { get; init; } = ""; + } + + private sealed class TestCompletionInfo + { + public DateTime CompletionTime { get; init; } + public TimeSpan ExecutionTime { get; init; } + } +} + +/// +/// Adjustment direction +/// +internal enum AdjustmentDirection +{ + None, + Increase, + Decrease +} + +/// +/// Adjustment recommendation +/// +internal sealed class AdjustmentRecommendation +{ + public int NewParallelism { get; init; } + public AdjustmentDirection Direction { get; init; } + public string Reason { get; init; } = ""; +} \ No newline at end of file diff --git a/TUnit.Engine/Scheduling/SchedulerConfiguration.cs b/TUnit.Engine/Scheduling/SchedulerConfiguration.cs index 862979fe08..1d307a2a0c 100644 --- a/TUnit.Engine/Scheduling/SchedulerConfiguration.cs +++ b/TUnit.Engine/Scheduling/SchedulerConfiguration.cs @@ -31,6 +31,22 @@ public sealed class SchedulerConfiguration /// public ParallelismStrategy Strategy { get; set; } = ParallelismStrategy.Adaptive; + /// + /// Minimum parallelism for adaptive strategy + /// + public int AdaptiveMinParallelism { get; set; } = Environment.ProcessorCount; + + /// + /// Maximum parallelism for adaptive strategy + /// Default: unlimited unless explicitly set via command-line or environment variable + /// + public int AdaptiveMaxParallelism { get; set; } = int.MaxValue; + + /// + /// Enable detailed metrics logging for adaptive strategy + /// + public bool EnableAdaptiveMetrics { get; set; } = false; + /// /// Creates default configuration /// diff --git a/TUnit.Engine/Scheduling/TestScheduler.cs b/TUnit.Engine/Scheduling/TestScheduler.cs index 3e3d2bef21..65d4e5d053 100644 --- a/TUnit.Engine/Scheduling/TestScheduler.cs +++ b/TUnit.Engine/Scheduling/TestScheduler.cs @@ -10,23 +10,25 @@ namespace TUnit.Engine.Scheduling; /// /// A clean, simplified test scheduler that uses an execution plan /// -internal sealed class TestScheduler : ITestScheduler +internal sealed class TestScheduler : ITestScheduler, IDisposable { private readonly TUnitFrameworkLogger _logger; private readonly ITestGroupingService _groupingService; private readonly ITUnitMessageBus _messageBus; - private readonly int _maxParallelism; + private readonly SchedulerConfiguration _configuration; + private AdaptiveParallelismController? _adaptiveController; + private IDisposable? _semaphore; public TestScheduler( TUnitFrameworkLogger logger, ITestGroupingService groupingService, ITUnitMessageBus messageBus, - int maxParallelism) + SchedulerConfiguration configuration) { _logger = logger; _groupingService = groupingService; _messageBus = messageBus; - _maxParallelism = maxParallelism > 0 ? maxParallelism : Environment.ProcessorCount * 4; + _configuration = configuration; } public async Task ScheduleAndExecuteAsync( @@ -62,7 +64,30 @@ private async Task ExecuteGroupedTestsAsync( { var runningTasks = new ConcurrentDictionary(); var completedTests = new ConcurrentDictionary(); - var semaphore = new SemaphoreSlim(_maxParallelism, _maxParallelism); + + // Create appropriate semaphore based on strategy + object semaphore; + if (_configuration.Strategy == ParallelismStrategy.Adaptive) + { + var initialParallelism = Math.Min(Environment.ProcessorCount * 4, _configuration.AdaptiveMaxParallelism); + var adaptiveSemaphore = new AdaptiveSemaphore(initialParallelism, _configuration.AdaptiveMaxParallelism); + _adaptiveController = new AdaptiveParallelismController( + adaptiveSemaphore, + _logger, + _configuration.AdaptiveMinParallelism, + _configuration.AdaptiveMaxParallelism, + initialParallelism, + _configuration.EnableAdaptiveMetrics); + semaphore = adaptiveSemaphore; + _semaphore = adaptiveSemaphore; + } + else + { + var maxParallelism = _configuration.MaxParallelism > 0 ? _configuration.MaxParallelism : Environment.ProcessorCount * 4; + var fixedSemaphore = new SemaphoreSlim(maxParallelism, maxParallelism); + semaphore = fixedSemaphore; + _semaphore = fixedSemaphore; + } // Process all test groups var allTestTasks = new List(); @@ -180,7 +205,7 @@ private async Task ExecuteParallelGroupAsync(SortedDictionary runningTasks, ConcurrentDictionary completedTests, - SemaphoreSlim semaphore, + object semaphore, CancellationToken cancellationToken) { // Execute order groups sequentially @@ -190,7 +215,10 @@ private async Task ExecuteParallelGroupAsync(SortedDictionary ITestExecutor executor, ConcurrentDictionary runningTasks, ConcurrentDictionary completedTests, - SemaphoreSlim semaphore, + object semaphore, CancellationToken cancellationToken) { var tasks = new List(); foreach (var test in tests) { - await semaphore.WaitAsync(cancellationToken); + if (semaphore is AdaptiveSemaphore adaptive) + await adaptive.WaitAsync(cancellationToken); + else + await ((SemaphoreSlim)semaphore).WaitAsync(cancellationToken); // Create a task that releases semaphore on completion without Task.Run overhead async Task ExecuteWithSemaphoreRelease() @@ -235,7 +269,10 @@ async Task ExecuteWithSemaphoreRelease() } finally { - semaphore.Release(); + if (semaphore is AdaptiveSemaphore adaptive) + adaptive.Release(); + else + ((SemaphoreSlim)semaphore).Release(); } } @@ -278,6 +315,7 @@ await _messageBus.Failed(test.Context, private async Task ExecuteTestDirectlyAsync(AbstractExecutableTest test, ITestExecutor executor, ConcurrentDictionary completedTests, CancellationToken cancellationToken) { + var startTime = DateTime.UtcNow; try { await executor.ExecuteTestAsync(test, cancellationToken); @@ -285,6 +323,19 @@ private async Task ExecuteTestDirectlyAsync(AbstractExecutableTest test, ITestEx finally { completedTests[test] = true; + + // Record completion for adaptive metrics + if (_adaptiveController != null) + { + var executionTime = DateTime.UtcNow - startTime; + _adaptiveController.RecordTestCompletion(executionTime); + } } } + + public void Dispose() + { + _adaptiveController?.Dispose(); + _semaphore?.Dispose(); + } } diff --git a/TUnit.Engine/Scheduling/TestSchedulerFactory.cs b/TUnit.Engine/Scheduling/TestSchedulerFactory.cs index 38546ec2da..a6d7376bed 100644 --- a/TUnit.Engine/Scheduling/TestSchedulerFactory.cs +++ b/TUnit.Engine/Scheduling/TestSchedulerFactory.cs @@ -16,11 +16,11 @@ public static ITestScheduler Create(SchedulerConfiguration configuration, TUnitF { var groupingService = new TestGroupingService(); - // Use the new clean scheduler + // Use the new clean scheduler with configuration return new TestScheduler( logger, groupingService, messageBus, - configuration.MaxParallelism); + configuration); } } diff --git a/TUnit.Engine/Services/SystemMetricsCollector.cs b/TUnit.Engine/Services/SystemMetricsCollector.cs new file mode 100644 index 0000000000..214355b2ed --- /dev/null +++ b/TUnit.Engine/Services/SystemMetricsCollector.cs @@ -0,0 +1,143 @@ +using System.Diagnostics; + +namespace TUnit.Engine.Services; + +/// +/// Collects system metrics for adaptive parallelism +/// +internal sealed class SystemMetricsCollector : IDisposable +{ + private readonly Process _currentProcess; + private readonly Timer _gcMemoryTimer; + private long _lastGcMemory; + private DateTime _lastCpuTime; + private TimeSpan _lastTotalProcessorTime; + private double _lastCpuUsage; + + public SystemMetricsCollector() + { + _currentProcess = Process.GetCurrentProcess(); + _lastCpuTime = DateTime.UtcNow; + _lastTotalProcessorTime = _currentProcess.TotalProcessorTime; + + // Update GC memory periodically to avoid blocking + _gcMemoryTimer = new Timer(_ => _lastGcMemory = GC.GetTotalMemory(false), null, TimeSpan.Zero, TimeSpan.FromSeconds(1)); + } + + /// + /// Gets current system metrics snapshot + /// + public SystemMetrics GetMetrics() + { + var now = DateTime.UtcNow; + var currentTotalProcessorTime = _currentProcess.TotalProcessorTime; + var timeDelta = (now - _lastCpuTime).TotalMilliseconds; + + double processCpuUsage = 0; + if (timeDelta > 0) + { + var cpuTimeDelta = (currentTotalProcessorTime - _lastTotalProcessorTime).TotalMilliseconds; + processCpuUsage = (cpuTimeDelta / timeDelta) / Environment.ProcessorCount * 100; + _lastCpuUsage = processCpuUsage; + } + else + { + processCpuUsage = _lastCpuUsage; + } + + _lastCpuTime = now; + _lastTotalProcessorTime = currentTotalProcessorTime; + + // Thread pool statistics + ThreadPool.GetAvailableThreads(out var workerThreads, out var ioThreads); + ThreadPool.GetMaxThreads(out var maxWorkerThreads, out var maxIoThreads); + long pendingWorkItems = 0; +#if NET5_0_OR_GREATER + pendingWorkItems = ThreadPool.PendingWorkItemCount; +#endif + + // Memory metrics + var totalMemory = GC.GetTotalMemory(false); + var gen0Collections = GC.CollectionCount(0); + var gen1Collections = GC.CollectionCount(1); + var gen2Collections = GC.CollectionCount(2); + + return new SystemMetrics + { + ProcessCpuUsagePercent = processCpuUsage, + SystemCpuUsagePercent = processCpuUsage, // Use process CPU as approximation + AvailableWorkerThreads = workerThreads, + AvailableIoThreads = ioThreads, + MaxWorkerThreads = maxWorkerThreads, + MaxIoThreads = maxIoThreads, + PendingWorkItems = pendingWorkItems, + TotalMemoryBytes = totalMemory, + Gen0Collections = gen0Collections, + Gen1Collections = gen1Collections, + Gen2Collections = gen2Collections, + Timestamp = now + }; + } + + /// + /// Detects if the thread pool is experiencing starvation + /// + public bool IsThreadPoolStarved() + { + ThreadPool.GetAvailableThreads(out var workerThreads, out var ioThreads); + ThreadPool.GetMaxThreads(out var maxWorkerThreads, out var maxIoThreads); + + // Consider starved if less than 10% of threads available + var workerUtilization = 1.0 - (double)workerThreads / maxWorkerThreads; + var ioUtilization = 1.0 - (double)ioThreads / maxIoThreads; + + bool hasPendingWork = false; +#if NET5_0_OR_GREATER + hasPendingWork = ThreadPool.PendingWorkItemCount > 100; +#endif + return workerUtilization > 0.9 || ioUtilization > 0.9 || hasPendingWork; + } + + /// + /// Detects memory pressure + /// + public bool IsMemoryPressureHigh() + { + // Simple heuristic: check if we're using more than 90% of max generation size + // or if Gen2 collections are happening frequently + var totalMemory = GC.GetTotalMemory(false); + var gen2Size = GC.GetGeneration(new object()) == 2 ? totalMemory : 0; + + // Check if memory is growing rapidly + var memoryGrowth = totalMemory - _lastGcMemory; + _lastGcMemory = totalMemory; + + return totalMemory > 1_000_000_000 || // Over 1GB + memoryGrowth > 100_000_000; // Growing by more than 100MB/sec + } + + public void Dispose() + { + _gcMemoryTimer?.Dispose(); + _currentProcess?.Dispose(); + } +} + +/// +/// System metrics snapshot +/// +internal sealed class SystemMetrics +{ + public double ProcessCpuUsagePercent { get; init; } + public double SystemCpuUsagePercent { get; init; } + public int AvailableWorkerThreads { get; init; } + public int AvailableIoThreads { get; init; } + public int MaxWorkerThreads { get; init; } + public int MaxIoThreads { get; init; } + public long PendingWorkItems { get; init; } + public long TotalMemoryBytes { get; init; } + public int Gen0Collections { get; init; } + public int Gen1Collections { get; init; } + public int Gen2Collections { get; init; } + public DateTime Timestamp { get; init; } +} \ No newline at end of file diff --git a/TUnit.Engine/TestExecutor.cs b/TUnit.Engine/TestExecutor.cs index b3141e6c43..ac4709d7ed 100644 --- a/TUnit.Engine/TestExecutor.cs +++ b/TUnit.Engine/TestExecutor.cs @@ -161,6 +161,23 @@ private ITestScheduler CreateDefaultScheduler() { var config = SchedulerConfiguration.Default; + // Check environment variables first (can be overridden by command-line) + if (int.TryParse(Environment.GetEnvironmentVariable("TUNIT_ADAPTIVE_MIN_PARALLELISM"), out var envMinParallelism) && envMinParallelism > 0) + { + config.AdaptiveMinParallelism = envMinParallelism; + } + + if (int.TryParse(Environment.GetEnvironmentVariable("TUNIT_ADAPTIVE_MAX_PARALLELISM"), out var envMaxParallelism) && envMaxParallelism > 0) + { + config.AdaptiveMaxParallelism = envMaxParallelism; + } + + if (bool.TryParse(Environment.GetEnvironmentVariable("TUNIT_ADAPTIVE_METRICS"), out var envMetrics)) + { + config.EnableAdaptiveMetrics = envMetrics; + } + + // Handle --maximum-parallel-tests (applies to both fixed and adaptive strategies) if (_commandLineOptions.TryGetOptionArgumentList( MaximumParallelTestsCommandProvider.MaximumParallelTests, out var args) && args.Length > 0) @@ -168,9 +185,26 @@ private ITestScheduler CreateDefaultScheduler() if (int.TryParse(args[0], out var maxParallelTests) && maxParallelTests > 0) { config.MaxParallelism = maxParallelTests; + config.AdaptiveMaxParallelism = maxParallelTests; + // Don't change strategy - let it be controlled by --parallelism-strategy } } + // Handle --parallelism-strategy + if (_commandLineOptions.TryGetOptionArgumentList( + ParallelismStrategyCommandProvider.ParallelismStrategy, + out var strategyArgs) && strategyArgs.Length > 0) + { + var strategy = strategyArgs[0].ToLowerInvariant(); + config.Strategy = strategy == "fixed" ? ParallelismStrategy.Fixed : ParallelismStrategy.Adaptive; + } + + // Handle --adaptive-metrics + if (_commandLineOptions.IsOptionSet(AdaptiveMetricsCommandProvider.AdaptiveMetrics)) + { + config.EnableAdaptiveMetrics = true; + } + var eventReceiverOrchestrator = _serviceProvider.GetService(typeof(EventReceiverOrchestrator)) as EventReceiverOrchestrator; var hookOrchestrator = _serviceProvider.GetService(typeof(HookOrchestrator)) as HookOrchestrator; return TestSchedulerFactory.Create(config, _logger, _serviceProvider.MessageBus, _serviceProvider.CancellationToken, eventReceiverOrchestrator!, hookOrchestrator!); @@ -187,6 +221,9 @@ public void Dispose() } _disposed = true; + + // Dispose the scheduler if it implements IDisposable + (_testScheduler as IDisposable)?.Dispose(); } public ValueTask DisposeAsync() @@ -197,6 +234,9 @@ public ValueTask DisposeAsync() } _disposed = true; + + // Dispose the scheduler if it implements IDisposable + (_testScheduler as IDisposable)?.Dispose(); return default(ValueTask); } diff --git a/TUnit.TestProject/AdaptiveParallelismTests.cs b/TUnit.TestProject/AdaptiveParallelismTests.cs new file mode 100644 index 0000000000..920f750312 --- /dev/null +++ b/TUnit.TestProject/AdaptiveParallelismTests.cs @@ -0,0 +1,84 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace TUnit.TestProject; + +public class AdaptiveParallelismTests +{ + private static int _currentlyRunning; + private static int _maxConcurrent; + private static readonly object _lock = new(); + + [Test] + [Repeat(1000)] + public async Task IoIntensiveTest() + { + // Track concurrency + lock (_lock) + { + _currentlyRunning++; + if (_currentlyRunning > _maxConcurrent) + _maxConcurrent = _currentlyRunning; + } + + try + { + // Simulate I/O work + await Task.Delay(1000); + } + finally + { + lock (_lock) + { + _currentlyRunning--; + } + } + } + + [Test] + [Repeat(500)] + public async Task CpuIntensiveTest() + { + // Track concurrency + lock (_lock) + { + _currentlyRunning++; + if (_currentlyRunning > _maxConcurrent) + _maxConcurrent = _currentlyRunning; + } + + try + { + // Simulate CPU-intensive work + var endTime = DateTime.UtcNow.AddMilliseconds(50); + while (DateTime.UtcNow < endTime) + { + // Busy wait to consume CPU + Thread.SpinWait(1000); + } + await Task.Yield(); + } + finally + { + lock (_lock) + { + _currentlyRunning--; + } + } + } + + [After(Class)] + public static void ReportStats() + { + Console.WriteLine($"Max concurrent tests: {_maxConcurrent}"); + Console.WriteLine($"Processor count: {Environment.ProcessorCount}"); + + // With adaptive parallelism, we expect max concurrent to be higher than processor count + // for I/O-bound tests, as the system should detect low CPU usage and increase parallelism + if (_maxConcurrent > Environment.ProcessorCount) + { + Console.WriteLine("Adaptive parallelism appears to be working - concurrency exceeded processor count!"); + } + } +} diff --git a/TUnit.TestProject/NotInParallelExecutionTests.cs b/TUnit.TestProject/NotInParallelExecutionTests.cs new file mode 100644 index 0000000000..f59d455f89 --- /dev/null +++ b/TUnit.TestProject/NotInParallelExecutionTests.cs @@ -0,0 +1,129 @@ +using System.Collections.Concurrent; +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +[EngineTest(ExpectedResult.Pass)] +[Retry(3)] +public class NotInParallelExecutionTests +{ + private static readonly ConcurrentBag ExecutionRecords = []; + private static readonly object Lock = new(); + private static int CurrentlyRunning = 0; + private static int MaxConcurrentTests = 0; + + [Before(Test)] + public void RecordTestStart() + { + lock (Lock) + { + CurrentlyRunning++; + if (CurrentlyRunning > MaxConcurrentTests) + { + MaxConcurrentTests = CurrentlyRunning; + } + } + + ExecutionRecords.Add(new TestExecutionRecord( + TestContext.Current!.TestDetails.TestName, + TestContext.Current.TestStart.DateTime, + null, + CurrentlyRunning + )); + } + + [After(Test)] + public async Task RecordTestEnd() + { + lock (Lock) + { + CurrentlyRunning--; + } + + var record = ExecutionRecords.FirstOrDefault(r => + r.TestName == TestContext.Current!.TestDetails.TestName && + r.EndTime == null); + + if (record != null) + { + record.EndTime = TestContext.Current.Result!.End!.Value.DateTime; + } + + await AssertNoParallelExecution(); + } + + [Test, NotInParallel, Repeat(3)] + public async Task NotInParallel_ExecutionTest1() + { + await Task.Delay(300); + } + + [Test, NotInParallel, Repeat(3)] + public async Task NotInParallel_ExecutionTest2() + { + await Task.Delay(300); + } + + [Test, NotInParallel, Repeat(3)] + public async Task NotInParallel_ExecutionTest3() + { + await Task.Delay(300); + } + + [Test, NotInParallel, Repeat(3)] + public async Task NotInParallel_ExecutionTest4() + { + await Task.Delay(300); + } + + [Test, NotInParallel, Repeat(3)] + public async Task NotInParallel_ExecutionTest5() + { + await Task.Delay(300); + } + + private async Task AssertNoParallelExecution() + { + await Assert.That(MaxConcurrentTests) + .IsEqualTo(1) + .Because($"Tests with NotInParallel should not run concurrently. Max concurrent: {MaxConcurrentTests}"); + + var completedRecords = ExecutionRecords.Where(r => r.EndTime != null).ToList(); + + foreach (var record in completedRecords) + { + var overlappingTests = completedRecords + .Where(r => r != record && r.OverlapsWith(record)) + .Select(r => r.TestName) + .ToList(); + + await Assert.That(overlappingTests) + .IsEmpty() + .Because($"Test {record.TestName} overlapped with: {string.Join(", ", overlappingTests)}"); + } + } + + private class TestExecutionRecord + { + public string TestName { get; } + public DateTime StartTime { get; } + public DateTime? EndTime { get; set; } + public int ConcurrentCount { get; } + + public TestExecutionRecord(string testName, DateTime startTime, DateTime? endTime, int concurrentCount) + { + TestName = testName; + StartTime = startTime; + EndTime = endTime; + ConcurrentCount = concurrentCount; + } + + public bool OverlapsWith(TestExecutionRecord other) + { + if (EndTime == null || other.EndTime == null) + return false; + + return StartTime < other.EndTime.Value && other.StartTime < EndTime.Value; + } + } +} \ No newline at end of file diff --git a/TUnit.TestProject/NotInParallelMixedTests.cs b/TUnit.TestProject/NotInParallelMixedTests.cs new file mode 100644 index 0000000000..b6c610ac8a --- /dev/null +++ b/TUnit.TestProject/NotInParallelMixedTests.cs @@ -0,0 +1,345 @@ +using System.Collections.Concurrent; +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +[EngineTest(ExpectedResult.Pass)] +[Retry(3)] +public class NotInParallelMixedTests +{ + private static readonly ConcurrentDictionary> ExecutionsByGroup = new(); + private static readonly ConcurrentDictionary MaxConcurrentByGroup = new(); + private static readonly ConcurrentDictionary CurrentlyRunningByGroup = new(); + private static readonly object GlobalLock = new(); + private static int GlobalMaxConcurrent = 0; + private static int GlobalCurrentlyRunning = 0; + + [Before(Test)] + public void RecordTestStart() + { + var testName = TestContext.Current!.TestDetails.TestName; + var groupKey = GetGroupKeyForTest(testName); + var startTime = DateTime.Now; + + lock (GlobalLock) + { + GlobalCurrentlyRunning++; + if (GlobalCurrentlyRunning > GlobalMaxConcurrent) + GlobalMaxConcurrent = GlobalCurrentlyRunning; + } + + if (groupKey != null) + { + var current = CurrentlyRunningByGroup.AddOrUpdate(groupKey, 1, (_, v) => v + 1); + MaxConcurrentByGroup.AddOrUpdate(groupKey, current, (_, v) => Math.Max(v, current)); + } + + var executions = ExecutionsByGroup.GetOrAdd(groupKey ?? "None", _ => new List()); + lock (executions) + { + executions.Add(new ExecutionInfo(testName, startTime, null)); + } + } + + [After(Test)] + public async Task RecordTestEnd() + { + var testName = TestContext.Current!.TestDetails.TestName; + var groupKey = GetGroupKeyForTest(testName); + var endTime = DateTime.Now; + + lock (GlobalLock) + { + GlobalCurrentlyRunning--; + } + + if (groupKey != null) + { + CurrentlyRunningByGroup.AddOrUpdate(groupKey, 0, (_, v) => Math.Max(0, v - 1)); + } + + var executions = ExecutionsByGroup.GetOrAdd(groupKey ?? "None", _ => new List()); + lock (executions) + { + var execution = executions.FirstOrDefault(e => e.TestName == testName && e.EndTime == null); + if (execution != null) + execution.EndTime = endTime; + } + + await ValidateExecutionRules(testName, groupKey); + } + + // Tests with no constraint key - should not run in parallel with each other + [Test, NotInParallel] + public async Task NoKey_Test1() + { + await Task.Delay(200); + } + + [Test, NotInParallel] + public async Task NoKey_Test2() + { + await Task.Delay(200); + } + + [Test, NotInParallel] + public async Task NoKey_Test3() + { + await Task.Delay(200); + } + + // Tests with "GroupA" key and order - should run sequentially in order + [Test, NotInParallel("GroupA", Order = 2)] + public async Task GroupA_Second() + { + await Task.Delay(150); + } + + [Test, NotInParallel("GroupA", Order = 1)] + public async Task GroupA_First() + { + await Task.Delay(150); + } + + [Test, NotInParallel("GroupA", Order = 3)] + public async Task GroupA_Third() + { + await Task.Delay(150); + } + + // Tests with "GroupB" key without order - should not run in parallel with each other + [Test, NotInParallel("GroupB")] + public async Task GroupB_Test1() + { + await Task.Delay(100); + } + + [Test, NotInParallel("GroupB")] + public async Task GroupB_Test2() + { + await Task.Delay(100); + } + + [Test, NotInParallel("GroupB")] + public async Task GroupB_Test3() + { + await Task.Delay(100); + } + + // Tests with "GroupC" key and mixed orders + [Test, NotInParallel("GroupC", Order = 10)] + public async Task GroupC_Last() + { + await Task.Delay(80); + } + + [Test, NotInParallel("GroupC", Order = 1)] + public async Task GroupC_First() + { + await Task.Delay(80); + } + + [Test, NotInParallel("GroupC", Order = 5)] + public async Task GroupC_Middle() + { + await Task.Delay(80); + } + + // Tests without NotInParallel - can run in parallel with everything + [Test] + public async Task Parallel_Test1() + { + await Task.Delay(50); + } + + [Test] + public async Task Parallel_Test2() + { + await Task.Delay(50); + } + + [Test] + public async Task Parallel_Test3() + { + await Task.Delay(50); + } + + // Tests with multiple constraint keys + [Test, NotInParallel(["GroupD", "GroupE"])] + public async Task MultiGroup_Test1() + { + await Task.Delay(120); + } + + [Test, NotInParallel(["GroupD", "GroupF"])] + public async Task MultiGroup_Test2() + { + await Task.Delay(120); + } + + [Test, NotInParallel(["GroupE", "GroupF"])] + public async Task MultiGroup_Test3() + { + await Task.Delay(120); + } + + [After(Class)] + public static async Task ValidateFinalResults() + { + // Validate that tests without keys didn't run in parallel + if (ExecutionsByGroup.TryGetValue("NoKey", out var noKeyExecutions)) + { + await ValidateNoOverlaps(noKeyExecutions, "NoKey"); + + if (MaxConcurrentByGroup.TryGetValue("NoKey", out var maxConcurrent)) + { + await Assert.That(maxConcurrent) + .IsEqualTo(1) + .Because($"Tests with NotInParallel and no key should not run concurrently. Max was: {maxConcurrent}"); + } + } + + // Validate GroupA ran in order + if (ExecutionsByGroup.TryGetValue("GroupA", out var groupAExecutions)) + { + var orderedNames = groupAExecutions.OrderBy(e => e.StartTime).Select(e => e.TestName).ToList(); + await Assert.That(orderedNames) + .IsEquivalentTo(["GroupA_First", "GroupA_Second", "GroupA_Third"]) + .Because("GroupA tests should execute in Order property sequence"); + + await ValidateNoOverlaps(groupAExecutions, "GroupA"); + } + + // Validate GroupB didn't run in parallel + if (ExecutionsByGroup.TryGetValue("GroupB", out var groupBExecutions)) + { + await ValidateNoOverlaps(groupBExecutions, "GroupB"); + } + + // Validate GroupC ran in order + if (ExecutionsByGroup.TryGetValue("GroupC", out var groupCExecutions)) + { + var orderedNames = groupCExecutions.OrderBy(e => e.StartTime).Select(e => e.TestName).ToList(); + await Assert.That(orderedNames) + .IsEquivalentTo(["GroupC_First", "GroupC_Middle", "GroupC_Last"]) + .Because("GroupC tests should execute in Order property sequence"); + + await ValidateNoOverlaps(groupCExecutions, "GroupC"); + } + + // Validate multi-group tests respected all their constraints + await ValidateMultiGroupConstraints(); + + // Log summary + Console.WriteLine($"Global max concurrent: {GlobalMaxConcurrent}"); + foreach (var kvp in MaxConcurrentByGroup) + { + Console.WriteLine($"Max concurrent in {kvp.Key}: {kvp.Value}"); + } + } + + private async Task ValidateExecutionRules(string testName, string? groupKey) + { + if (groupKey == null && testName.StartsWith("NoKey_")) + { + groupKey = "NoKey"; + } + + // Don't validate multi-group tests here - they have special handling + if (groupKey != null && !groupKey.Contains(",")) + { + if (MaxConcurrentByGroup.TryGetValue(groupKey, out var maxConcurrent)) + { + await Assert.That(maxConcurrent) + .IsLessThanOrEqualTo(1) + .Because($"Tests in {groupKey} should not run concurrently. Max was: {maxConcurrent}"); + } + } + } + + private static async Task ValidateNoOverlaps(List executions, string groupName) + { + foreach (var execution in executions.Where(e => e.EndTime != null)) + { + var overlapping = executions + .Where(e => e != execution && e.EndTime != null && e.OverlapsWith(execution)) + .Select(e => e.TestName) + .ToList(); + + await Assert.That(overlapping) + .IsEmpty() + .Because($"In {groupName}, {execution.TestName} should not overlap with other tests"); + } + } + + private static async Task ValidateMultiGroupConstraints() + { + var multiGroupTests = new[] { "MultiGroup_Test1", "MultiGroup_Test2", "MultiGroup_Test3" }; + + // Test1 and Test2 share GroupD - should not overlap + await ValidatePairNoOverlap("MultiGroup_Test1", "MultiGroup_Test2", "GroupD"); + + // Test1 and Test3 share GroupE - should not overlap + await ValidatePairNoOverlap("MultiGroup_Test1", "MultiGroup_Test3", "GroupE"); + + // Test2 and Test3 share GroupF - should not overlap + await ValidatePairNoOverlap("MultiGroup_Test2", "MultiGroup_Test3", "GroupF"); + } + + private static async Task ValidatePairNoOverlap(string test1, string test2, string sharedGroup) + { + var allExecutions = ExecutionsByGroup.Values.SelectMany(v => v).ToList(); + var exec1 = allExecutions.FirstOrDefault(e => e.TestName == test1); + var exec2 = allExecutions.FirstOrDefault(e => e.TestName == test2); + + if (exec1 != null && exec2 != null && exec1.EndTime != null && exec2.EndTime != null) + { + await Assert.That(exec1.OverlapsWith(exec2)) + .IsFalse() + .Because($"{test1} and {test2} share {sharedGroup} constraint and should not overlap"); + } + } + + private static string? GetGroupKeyForTest(string testName) + { + if (testName.StartsWith("NoKey_")) return "NoKey"; + if (testName.StartsWith("GroupA_")) return "GroupA"; + if (testName.StartsWith("GroupB_")) return "GroupB"; + if (testName.StartsWith("GroupC_")) return "GroupC"; + if (testName.StartsWith("MultiGroup_")) + { + // For multi-group tests, we track them separately + return testName switch + { + "MultiGroup_Test1" => "GroupD,GroupE", + "MultiGroup_Test2" => "GroupD,GroupF", + "MultiGroup_Test3" => "GroupE,GroupF", + _ => null + }; + } + if (testName.StartsWith("Parallel_")) return null; // No constraint + return null; + } + + private class ExecutionInfo + { + public string TestName { get; } + public DateTime StartTime { get; } + public DateTime? EndTime { get; set; } + + public ExecutionInfo(string testName, DateTime startTime, DateTime? endTime) + { + TestName = testName; + StartTime = startTime; + EndTime = endTime; + } + + public bool OverlapsWith(ExecutionInfo other) + { + if (EndTime == null || other.EndTime == null) + return false; + + return StartTime < other.EndTime.Value && other.StartTime < EndTime.Value; + } + } +} \ No newline at end of file diff --git a/TUnit.TestProject/NotInParallelOrderExecutionTests.cs b/TUnit.TestProject/NotInParallelOrderExecutionTests.cs new file mode 100644 index 0000000000..715e672230 --- /dev/null +++ b/TUnit.TestProject/NotInParallelOrderExecutionTests.cs @@ -0,0 +1,217 @@ +using System.Collections.Concurrent; +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +[EngineTest(ExpectedResult.Pass)] +[Retry(3)] +public class NotInParallelOrderExecutionTests +{ + private static readonly ConcurrentBag OrderedExecutionRecords = []; + private static readonly ConcurrentDictionary> ExecutionOrderByGroup = new(); + private static readonly ConcurrentDictionary GroupLocks = new(); + private static readonly ConcurrentDictionary CurrentlyExecutingPerGroup = new(); + private static readonly ConcurrentDictionary MaxConcurrentPerGroup = new(); + + [Before(Test)] + public void RecordOrderedTestStart() + { + var testName = TestContext.Current!.TestDetails.TestName; + var groupKey = GetGroupKey(testName); + + var groupLock = GroupLocks.GetOrAdd(groupKey, new object()); + lock (groupLock) + { + var current = CurrentlyExecutingPerGroup.AddOrUpdate(groupKey, 1, (_, v) => v + 1); + MaxConcurrentPerGroup.AddOrUpdate(groupKey, current, (_, v) => Math.Max(v, current)); + + var orderList = ExecutionOrderByGroup.GetOrAdd(groupKey, new List()); + orderList.Add(testName); + } + + OrderedExecutionRecords.Add(new OrderedExecutionRecord( + testName, + groupKey, + TestContext.Current.TestStart.DateTime, + null + )); + } + + [After(Test)] + public async Task RecordOrderedTestEnd() + { + var testName = TestContext.Current!.TestDetails.TestName; + var groupKey = GetGroupKey(testName); + + var groupLock = GroupLocks.GetOrAdd(groupKey, new object()); + lock (groupLock) + { + CurrentlyExecutingPerGroup.AddOrUpdate(groupKey, 0, (_, v) => Math.Max(0, v - 1)); + } + + var record = OrderedExecutionRecords.FirstOrDefault(r => + r.TestName == testName && + r.EndTime == null); + + if (record != null) + { + record.EndTime = TestContext.Current.Result!.End!.Value.DateTime; + } + + await AssertOrderedExecutionWithinGroup(groupKey); + } + + [Test, NotInParallel("OrderGroup1", Order = 3)] + public async Task OrderedTest_Third() + { + await Task.Delay(200); + } + + [Test, NotInParallel("OrderGroup1", Order = 1)] + public async Task OrderedTest_First() + { + await Task.Delay(200); + } + + [Test, NotInParallel("OrderGroup1", Order = 5)] + public async Task OrderedTest_Fifth() + { + await Task.Delay(200); + } + + [Test, NotInParallel("OrderGroup1", Order = 2)] + public async Task OrderedTest_Second() + { + await Task.Delay(200); + } + + [Test, NotInParallel("OrderGroup1", Order = 4)] + public async Task OrderedTest_Fourth() + { + await Task.Delay(200); + } + + [Test, NotInParallel("OrderGroup2", Order = 2)] + public async Task OrderedGroup2_Second() + { + await Task.Delay(150); + } + + [Test, NotInParallel("OrderGroup2", Order = 1)] + public async Task OrderedGroup2_First() + { + await Task.Delay(150); + } + + [Test, NotInParallel("OrderGroup2", Order = 3)] + public async Task OrderedGroup2_Third() + { + await Task.Delay(150); + } + + [After(Class)] + public static async Task VerifyExecutionOrder() + { + if (ExecutionOrderByGroup.TryGetValue("OrderGroup1", out var group1Tests) && group1Tests.Count == 5) + { + await Assert.That(group1Tests) + .IsEquivalentTo([ + "OrderedTest_First", + "OrderedTest_Second", + "OrderedTest_Third", + "OrderedTest_Fourth", + "OrderedTest_Fifth" + ]) + .Because("Tests in OrderGroup1 should execute in order defined by Order property"); + } + + if (ExecutionOrderByGroup.TryGetValue("OrderGroup2", out var group2Tests) && group2Tests.Count == 3) + { + await Assert.That(group2Tests) + .IsEquivalentTo([ + "OrderedGroup2_First", + "OrderedGroup2_Second", + "OrderedGroup2_Third" + ]) + .Because("Tests in OrderGroup2 should execute in order defined by Order property"); + } + + // Verify that tests from different groups could run in parallel + var allRecords = OrderedExecutionRecords.Where(r => r.EndTime != null).ToList(); + var group1Records = allRecords.Where(r => r.GroupKey == "OrderGroup1").ToList(); + var group2Records = allRecords.Where(r => r.GroupKey == "OrderGroup2").ToList(); + + if (group1Records.Any() && group2Records.Any()) + { + var group1Start = group1Records.Min(r => r.StartTime); + var group1End = group1Records.Max(r => r.EndTime!.Value); + var group2Start = group2Records.Min(r => r.StartTime); + var group2End = group2Records.Max(r => r.EndTime!.Value); + + // Check if the groups overlapped in time (which is allowed and expected) + var overlapped = group1Start < group2End && group2Start < group1End; + Console.WriteLine($"Groups ran in parallel: {overlapped}"); + } + } + + private async Task AssertOrderedExecutionWithinGroup(string groupKey) + { + // Check that within this group, tests did not run in parallel + if (MaxConcurrentPerGroup.TryGetValue(groupKey, out var maxConcurrent)) + { + await Assert.That(maxConcurrent) + .IsEqualTo(1) + .Because($"Tests within {groupKey} should not run concurrently. Max concurrent: {maxConcurrent}"); + } + + // Check for overlaps within the same group + var completedRecords = OrderedExecutionRecords + .Where(r => r.EndTime != null && r.GroupKey == groupKey) + .ToList(); + + foreach (var record in completedRecords) + { + var overlappingTests = completedRecords + .Where(r => r != record && r.OverlapsWith(record)) + .Select(r => r.TestName) + .ToList(); + + await Assert.That(overlappingTests) + .IsEmpty() + .Because($"In {groupKey}, test {record.TestName} overlapped with: {string.Join(", ", overlappingTests)}"); + } + } + + private static string GetGroupKey(string testName) + { + if (testName.StartsWith("OrderedTest_")) + return "OrderGroup1"; + if (testName.StartsWith("OrderedGroup2_")) + return "OrderGroup2"; + return "Unknown"; + } + + private class OrderedExecutionRecord + { + public string TestName { get; } + public string GroupKey { get; } + public DateTime StartTime { get; } + public DateTime? EndTime { get; set; } + + public OrderedExecutionRecord(string testName, string groupKey, DateTime startTime, DateTime? endTime) + { + TestName = testName; + GroupKey = groupKey; + StartTime = startTime; + EndTime = endTime; + } + + public bool OverlapsWith(OrderedExecutionRecord other) + { + if (EndTime == null || other.EndTime == null) + return false; + + return StartTime < other.EndTime.Value && other.StartTime < EndTime.Value; + } + } +} \ No newline at end of file