diff --git a/Directory.Packages.props b/Directory.Packages.props index 95fa47b2f0..1052b3a500 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -81,18 +81,18 @@ - - + + - + - - - + + + \ No newline at end of file diff --git a/TUnit.Analyzers.Tests/Verifiers/CSharpCodeFixVerifier.cs b/TUnit.Analyzers.Tests/Verifiers/CSharpCodeFixVerifier.cs index 3683793e41..e28299d735 100644 --- a/TUnit.Analyzers.Tests/Verifiers/CSharpCodeFixVerifier.cs +++ b/TUnit.Analyzers.Tests/Verifiers/CSharpCodeFixVerifier.cs @@ -5,6 +5,7 @@ using Microsoft.CodeAnalysis.CSharp.Testing; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Testing; +using Microsoft.CodeAnalysis.Text; namespace TUnit.Analyzers.Tests.Verifiers; @@ -16,6 +17,12 @@ public class Test : CSharpCodeFixTest { var project = solution.GetProject(projectId); diff --git a/TUnit.Analyzers.Tests/Verifiers/LineEndingNormalizingVerifier.cs b/TUnit.Analyzers.Tests/Verifiers/LineEndingNormalizingVerifier.cs index e12d6722ca..cecbb774c4 100644 --- a/TUnit.Analyzers.Tests/Verifiers/LineEndingNormalizingVerifier.cs +++ b/TUnit.Analyzers.Tests/Verifiers/LineEndingNormalizingVerifier.cs @@ -60,7 +60,17 @@ public void NotEmpty(string collectionName, IEnumerable collection) public void SequenceEqual(IEnumerable expected, IEnumerable actual, IEqualityComparer? equalityComparer = null, string? message = null) { - _defaultVerifier.SequenceEqual(expected, actual, equalityComparer, message); + // Normalize line endings for string sequence comparisons + if (typeof(T) == typeof(string)) + { + var normalizedExpected = expected.Cast().Select(NormalizeLineEndings).Cast(); + var normalizedActual = actual.Cast().Select(NormalizeLineEndings).Cast(); + _defaultVerifier.SequenceEqual(normalizedExpected, normalizedActual, equalityComparer, message); + } + else + { + _defaultVerifier.SequenceEqual(expected, actual, equalityComparer, message); + } } public IVerifier PushContext(string context) diff --git a/TUnit.Assertions/Chaining/AndAssertion.cs b/TUnit.Assertions/Chaining/AndAssertion.cs index b9186e64ee..486c947b73 100644 --- a/TUnit.Assertions/Chaining/AndAssertion.cs +++ b/TUnit.Assertions/Chaining/AndAssertion.cs @@ -139,10 +139,10 @@ private string BuildCombinedExpectation() var becausePrefix = firstBecause.StartsWith("because ", StringComparison.OrdinalIgnoreCase) ? firstBecause : $"because {firstBecause}"; - return $"{firstExpectation}, {becausePrefix}{Environment.NewLine}and {secondExpectation}"; + return $"{firstExpectation}, {becausePrefix}\nand {secondExpectation}"; } - return $"{firstExpectation}{Environment.NewLine}and {secondExpectation}"; + return $"{firstExpectation}\nand {secondExpectation}"; } protected override string GetExpectation() => "both conditions"; diff --git a/TUnit.Assertions/Chaining/OrAssertion.cs b/TUnit.Assertions/Chaining/OrAssertion.cs index 47aa2e87c2..15b042a531 100644 --- a/TUnit.Assertions/Chaining/OrAssertion.cs +++ b/TUnit.Assertions/Chaining/OrAssertion.cs @@ -140,10 +140,10 @@ private string BuildCombinedExpectation() var becausePrefix = firstBecause.StartsWith("because ", StringComparison.OrdinalIgnoreCase) ? firstBecause : $"because {firstBecause}"; - return $"{firstExpectation}, {becausePrefix}{Environment.NewLine}or {secondExpectation}"; + return $"{firstExpectation}, {becausePrefix}\nor {secondExpectation}"; } - return $"{firstExpectation}{Environment.NewLine}or {secondExpectation}"; + return $"{firstExpectation}\nor {secondExpectation}"; } protected override string GetExpectation() => "either condition"; diff --git a/TUnit.Assertions/Extensions/AssertionExtensions.cs b/TUnit.Assertions/Extensions/AssertionExtensions.cs index 95aee50a34..d065942ad1 100644 --- a/TUnit.Assertions/Extensions/AssertionExtensions.cs +++ b/TUnit.Assertions/Extensions/AssertionExtensions.cs @@ -262,7 +262,7 @@ public static MemberAssertionResult Member( /// Example: await Assert.That(myObject).Member(x => x.Attributes, attrs => attrs.ContainsKey("status").And.IsNotEmpty()); /// Note: This overload exists for backward compatibility. For AOT compatibility, use the TTransformed overload instead. /// - [OverloadResolutionPriority(1)] + [OverloadResolutionPriority(2)] [RequiresDynamicCode("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] public static MemberAssertionResult Member( this IAssertionSource source, @@ -421,7 +421,7 @@ public static MemberAssertionResult Member( /// Example: await Assert.That(myObject).Member(x => x.Tags, tags => tags.HasCount(1).And.Contains("value")); /// Note: This overload exists for backward compatibility. For AOT compatibility, use the TTransformed overload instead. /// - [OverloadResolutionPriority(0)] + [OverloadResolutionPriority(1)] [RequiresDynamicCode("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] public static MemberAssertionResult Member( this IAssertionSource source, @@ -527,7 +527,6 @@ public static MemberAssertionResult Member x.PropertyName, value => value.IsEqualTo(expectedValue)); /// - [OverloadResolutionPriority(0)] public static MemberAssertionResult Member( this IAssertionSource source, Expression> memberSelector, @@ -580,7 +579,6 @@ public static MemberAssertionResult Member( /// Example: await Assert.That(myObject).Member(x => x.PropertyName, value => value.IsEqualTo(expectedValue)); /// Note: This overload exists for backward compatibility. For AOT compatibility, use the TTransformed overload instead. /// - [OverloadResolutionPriority(-1)] [RequiresDynamicCode("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] public static MemberAssertionResult Member( this IAssertionSource source, diff --git a/TUnit.Core/EngineCancellationToken.cs b/TUnit.Core/EngineCancellationToken.cs index 3ed56c20b1..395cc7c5dd 100644 --- a/TUnit.Core/EngineCancellationToken.cs +++ b/TUnit.Core/EngineCancellationToken.cs @@ -33,6 +33,7 @@ internal void Initialise(CancellationToken cancellationToken) { #endif Console.CancelKeyPress += OnCancelKeyPress; + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; #if NET5_0_OR_GREATER } #endif @@ -71,6 +72,21 @@ private void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) e.Cancel = true; } + private void OnProcessExit(object? sender, EventArgs e) + { + // Process is exiting (SIGTERM, kill, etc.) - trigger cancellation to execute After hooks + // Note: ProcessExit runs on a background thread with limited time (~3 seconds on Windows) + // The After hooks registered via CancellationToken.Register() will execute when we cancel + if (!CancellationTokenSource.IsCancellationRequested) + { + CancellationTokenSource.Cancel(); + + // Give After hooks a brief moment to execute via registered callbacks + // ProcessExit has limited time, so we can only wait briefly + Task.Delay(TimeSpan.FromMilliseconds(500)).GetAwaiter().GetResult(); + } + } + /// /// Disposes the cancellation token source. /// @@ -82,6 +98,7 @@ public void Dispose() { #endif Console.CancelKeyPress -= OnCancelKeyPress; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; #if NET5_0_OR_GREATER } #endif diff --git a/TUnit.Core/Executors/GenericAbstractExecutor.cs b/TUnit.Core/Executors/GenericAbstractExecutor.cs index 8be73248ba..8631771b9e 100644 --- a/TUnit.Core/Executors/GenericAbstractExecutor.cs +++ b/TUnit.Core/Executors/GenericAbstractExecutor.cs @@ -57,11 +57,6 @@ public ValueTask ExecuteAfterTestHook(MethodMetadata hookMethodInfo, TestContext return ExecuteAsync(action); } - public ValueTask ExecuteDisposal(TestContext context, Func action) - { - return ExecuteAsync(action); - } - public ValueTask ExecuteTest(TestContext context, Func action) { return ExecuteAsync(action); diff --git a/TUnit.Core/Interfaces/IHookExecutor.cs b/TUnit.Core/Interfaces/IHookExecutor.cs index 13ec72d8fd..6d3d1c8e81 100644 --- a/TUnit.Core/Interfaces/IHookExecutor.cs +++ b/TUnit.Core/Interfaces/IHookExecutor.cs @@ -13,11 +13,4 @@ public interface IHookExecutor ValueTask ExecuteAfterAssemblyHook(MethodMetadata hookMethodInfo, AssemblyHookContext context, Func action); ValueTask ExecuteAfterClassHook(MethodMetadata hookMethodInfo, ClassHookContext context, Func action); ValueTask ExecuteAfterTestHook(MethodMetadata hookMethodInfo, TestContext context, Func action); - -#if NETSTANDARD2_0 - ValueTask ExecuteDisposal(TestContext context, Func action); -#else - ValueTask ExecuteDisposal(TestContext context, Func action) - => action(); -#endif } diff --git a/TUnit.Engine.Tests/CancellationAfterHooksTests.cs b/TUnit.Engine.Tests/CancellationAfterHooksTests.cs new file mode 100644 index 0000000000..750bb9ef38 --- /dev/null +++ b/TUnit.Engine.Tests/CancellationAfterHooksTests.cs @@ -0,0 +1,122 @@ +using Shouldly; +using TUnit.Engine.Tests.Enums; +using TUnit.Engine.Tests.Extensions; + +namespace TUnit.Engine.Tests; + +/// +/// Validates that After hooks execute even when tests are cancelled (Issue #3882). +/// These tests run the cancellation test scenarios and verify that After hooks created marker files. +/// +public class CancellationAfterHooksTests(TestMode testMode) : InvokableTestBase(testMode) +{ + private static readonly string TempPath = Path.GetTempPath(); + + [Test] + public async Task TestLevel_AfterHook_Runs_OnCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_Tests", "after_Test_ThatGets_Cancelled.txt"); + + // Clean up any existing marker files + if (File.Exists(afterMarkerFile)) + { + File.Delete(afterMarkerFile); + } + + await RunTestsWithFilter( + "/*/*/CancellationAfterHooksTests/*", + [ + // Test run completes even though the test itself fails (timeout is expected) + result => result.ResultSummary.Counters.Total.ShouldBe(1), + // Test should fail due to timeout + result => result.ResultSummary.Counters.Failed.ShouldBe(1), + // After hook should have created the marker file - this proves After hooks ran on cancellation + _ => File.Exists(afterMarkerFile).ShouldBeTrue($"After hook marker file should exist at {afterMarkerFile}") + ]); + + // Verify marker file content + if (File.Exists(afterMarkerFile)) + { + var content = await File.ReadAllTextAsync(afterMarkerFile); + content.ShouldContain("After hook executed"); + } + } + + [Test] + public async Task SessionLevel_AfterHook_Runs_OnCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_Session_After.txt"); + + // Clean up any existing marker files + if (File.Exists(afterMarkerFile)) + { + File.Delete(afterMarkerFile); + } + + await RunTestsWithFilter( + "/*/*/SessionLevelCancellationTests/*", + [ + // After Session hook should have created the marker file - this proves Session After hooks ran on cancellation + _ => File.Exists(afterMarkerFile).ShouldBeTrue($"Session After hook marker file should exist at {afterMarkerFile}") + ]); + + // Verify marker file content + if (File.Exists(afterMarkerFile)) + { + var content = await File.ReadAllTextAsync(afterMarkerFile); + content.ShouldContain("Session After hook executed"); + } + } + + [Test] + public async Task AssemblyLevel_AfterHook_Runs_OnCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_Assembly_After.txt"); + + // Clean up any existing marker files + if (File.Exists(afterMarkerFile)) + { + File.Delete(afterMarkerFile); + } + + await RunTestsWithFilter( + "/*/*/AssemblyLevelCancellationTests/*", + [ + // After Assembly hook should have created the marker file - this proves Assembly After hooks ran on cancellation + _ => File.Exists(afterMarkerFile).ShouldBeTrue($"Assembly After hook marker file should exist at {afterMarkerFile}") + ]); + + // Verify marker file content + if (File.Exists(afterMarkerFile)) + { + var content = await File.ReadAllTextAsync(afterMarkerFile); + content.ShouldContain("Assembly After hook executed"); + } + } + + [Test] + public async Task ClassLevel_AfterHook_Runs_OnCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_Class_After.txt"); + + // Clean up any existing marker files + if (File.Exists(afterMarkerFile)) + { + File.Delete(afterMarkerFile); + } + + await RunTestsWithFilter( + "/*/*/ClassLevelCancellationTests/*", + [ + // After Class hook should have created the marker file - this proves Class After hooks ran on cancellation + _ => File.Exists(afterMarkerFile).ShouldBeTrue($"Class After hook marker file should exist at {afterMarkerFile}") + ]); + + // Verify marker file content + if (File.Exists(afterMarkerFile)) + { + var content = await File.ReadAllTextAsync(afterMarkerFile); + content.ShouldContain("Class After hook executed"); + } + } +} diff --git a/TUnit.Engine.Tests/ExternalCancellationTests.cs b/TUnit.Engine.Tests/ExternalCancellationTests.cs new file mode 100644 index 0000000000..88dce62572 --- /dev/null +++ b/TUnit.Engine.Tests/ExternalCancellationTests.cs @@ -0,0 +1,147 @@ +using System.Diagnostics; +using CliWrap; +using Shouldly; +using TUnit.Engine.Tests.Enums; + +namespace TUnit.Engine.Tests; + +/// +/// Validates that After hooks execute even when tests are cancelled EXTERNALLY (Issue #3882). +/// These tests start the test process asynchronously, cancel it mid-execution (simulating Ctrl+C or Stop button), +/// and verify that After hooks still execute by checking for marker files. +/// +public class ExternalCancellationTests(TestMode testMode) : InvokableTestBase(testMode) +{ + private static readonly string TempPath = Path.GetTempPath(); + private static readonly string GetEnvironmentVariable = Environment.GetEnvironmentVariable("NET_VERSION") ?? "net10.0"; + + /// + /// Runs a test with external cancellation (simulates Ctrl+C, VS Test Explorer Stop button). + /// + /// Test filter pattern + /// Path to the marker file that proves After hook executed + /// Expected content in the marker file + private async Task RunTestWithExternalCancellation(string filter, string markerFile, string expectedMarkerContent) + { + // Clean up any existing marker files + if (File.Exists(markerFile)) + { + File.Delete(markerFile); + } + + var testProject = Sourcy.DotNet.Projects.TUnit_TestProject; + var guid = Guid.NewGuid().ToString("N"); + var trxFilename = guid + ".trx"; + + using var gracefulCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var forcefulCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(25)); + + // Use cross-platform executable detection (Linux: no extension, Windows: .exe) + var binDir = new DirectoryInfo(Path.Combine(testProject.DirectoryName!, "bin", "Release", GetEnvironmentVariable)); + var file = binDir.GetFiles("TUnit.TestProject").FirstOrDefault()?.FullName + ?? binDir.GetFiles("TUnit.TestProject.exe").First().FullName; + + var command = testMode switch + { + TestMode.SourceGenerated => Cli.Wrap(file) + .WithArguments( + [ + "--treenode-filter", filter, + "--report-trx", "--report-trx-filename", trxFilename, + "--diagnostic-verbosity", "Debug", + "--diagnostic", "--diagnostic-file-prefix", $"log_ExternalCancellation_{GetType().Name}_", + ]) + .WithWorkingDirectory(testProject.DirectoryName!) + .WithValidation(CommandResultValidation.None), + + TestMode.Reflection => Cli.Wrap(file) + .WithArguments( + [ + "--treenode-filter", filter, + "--report-trx", "--report-trx-filename", trxFilename, + "--diagnostic-verbosity", "Debug", + "--diagnostic", "--diagnostic-file-prefix", $"log_ExternalCancellation_{GetType().Name}_", + "--reflection" + ]) + .WithWorkingDirectory(testProject.DirectoryName!) + .WithValidation(CommandResultValidation.None), + + // Skip AOT and SingleFile modes for external cancellation (only test in CI) + TestMode.AOT => null, + TestMode.SingleFileApplication => null, + _ => throw new ArgumentOutOfRangeException(nameof(testMode), testMode, null) + }; + + // Skip AOT and SingleFile modes + if (command == null) + { + return; + } + + try + { + await command.ExecuteAsync(forcefulCancellationTokenSource.Token, gracefulCancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + // Expected: Process was cancelled via CancellationToken + Console.WriteLine("[ExternalCancellation] Process cancelled successfully (expected)"); + } + catch (Exception ex) + { + // Log unexpected exceptions but don't fail - After hooks might still execute + Console.WriteLine($"[ExternalCancellation] Unexpected exception: {ex.Message}"); + } + + // Verify marker file exists - this proves After hook executed even on external cancellation + File.Exists(markerFile).ShouldBeTrue($"After hook marker file should exist at {markerFile}"); + + // Verify marker file content + var content = await File.ReadAllTextAsync(markerFile); + content.ShouldContain(expectedMarkerContent); + } + + [Test] + public async Task TestLevel_AfterHook_Runs_OnExternalCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_External", "after_Test_ThatGets_Cancelled_Externally.txt"); + + await RunTestWithExternalCancellation( + "/*/*/ExternalCancellationTests/*", + afterMarkerFile, + "After hook executed"); + } + + [Test] + public async Task SessionLevel_AfterHook_Runs_OnExternalCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_External_Session_After.txt"); + + await RunTestWithExternalCancellation( + "/*/*/ExternalSessionLevelCancellationTests/*", + afterMarkerFile, + "Session After hook executed"); + } + + [Test] + public async Task AssemblyLevel_AfterHook_Runs_OnExternalCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_External_Assembly_After.txt"); + + await RunTestWithExternalCancellation( + "/*/*/ExternalAssemblyLevelCancellationTests/*", + afterMarkerFile, + "Assembly After hook executed"); + } + + [Test] + public async Task ClassLevel_AfterHook_Runs_OnExternalCancellation() + { + var afterMarkerFile = Path.Combine(TempPath, "TUnit_3882_External_Class_After.txt"); + + await RunTestWithExternalCancellation( + "/*/*/ExternalClassLevelCancellationTests/*", + afterMarkerFile, + "Class After hook executed"); + } +} diff --git a/TUnit.Engine/Discovery/ReflectionHookDiscoveryService.cs b/TUnit.Engine/Discovery/ReflectionHookDiscoveryService.cs index de062bd768..cc3954b121 100644 --- a/TUnit.Engine/Discovery/ReflectionHookDiscoveryService.cs +++ b/TUnit.Engine/Discovery/ReflectionHookDiscoveryService.cs @@ -1029,7 +1029,4 @@ public ValueTask ExecuteBeforeTestSessionHook(MethodMetadata testMethod, TestSes public ValueTask ExecuteAfterTestSessionHook(MethodMetadata testMethod, TestSessionContext context, Func action) => action(); - - public ValueTask ExecuteDisposal(TestContext context, Func action) - => action(); } diff --git a/TUnit.Engine/Framework/TUnitServiceProvider.cs b/TUnit.Engine/Framework/TUnitServiceProvider.cs index 823d643784..dd6896971d 100644 --- a/TUnit.Engine/Framework/TUnitServiceProvider.cs +++ b/TUnit.Engine/Framework/TUnitServiceProvider.cs @@ -145,8 +145,9 @@ public TUnitServiceProvider(IExtension extension, var hookExecutor = Register(new HookExecutor(HookCollectionService, ContextProvider, EventReceiverOrchestrator)); var lifecycleCoordinator = Register(new TestLifecycleCoordinator()); var beforeHookTaskCache = Register(new BeforeHookTaskCache()); + var afterHookPairTracker = Register(new AfterHookPairTracker()); - TestExecutor = Register(new TestExecutor(hookExecutor, lifecycleCoordinator, beforeHookTaskCache, ContextProvider, EventReceiverOrchestrator)); + TestExecutor = Register(new TestExecutor(hookExecutor, lifecycleCoordinator, beforeHookTaskCache, afterHookPairTracker, ContextProvider, EventReceiverOrchestrator)); var testExecutionGuard = Register(new TestExecutionGuard()); var testStateManager = Register(new TestStateManager()); diff --git a/TUnit.Engine/Services/AfterHookPairTracker.cs b/TUnit.Engine/Services/AfterHookPairTracker.cs new file mode 100644 index 0000000000..5e6e12deb5 --- /dev/null +++ b/TUnit.Engine/Services/AfterHookPairTracker.cs @@ -0,0 +1,155 @@ +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using TUnit.Core.Data; + +namespace TUnit.Engine.Services; + +/// +/// Responsible for ensuring After hooks run even when tests are cancelled. +/// When a Before hook completes, this tracker registers the corresponding After hook +/// to run on cancellation, guaranteeing cleanup even if the test is aborted. +/// Follows Single Responsibility Principle - only handles After hook pairing and cancellation registration. +/// +internal sealed class AfterHookPairTracker +{ + // Cached After hook tasks to ensure they run only once (prevent double execution) + private readonly ThreadSafeDictionary>> _afterClassTasks = new(); + private readonly ThreadSafeDictionary>> _afterAssemblyTasks = new(); + private Task>? _afterTestSessionTask; + private readonly object _testSessionLock = new(); + private readonly object _classLock = new(); + + // Track cancellation registrations for cleanup + private readonly ConcurrentBag _registrations = []; + + /// + /// Registers Session After hooks to run on cancellation or normal completion. + /// Ensures After hooks run exactly once even if called both ways. + /// + public void RegisterAfterTestSessionHook( + CancellationToken cancellationToken, + Func>> afterHookExecutor) + { + // Register callback to run After hook on cancellation + var registration = cancellationToken.Register(() => + { + // Use sync-over-async here because CancellationToken.Register requires Action (not Func) + // Fire-and-forget is acceptable here - exceptions will be collected when hooks run normally + _ = GetOrCreateAfterTestSessionTask(afterHookExecutor); + }); + + _registrations.Add(registration); + } + + /// + /// Registers Assembly After hooks to run on cancellation or normal completion. + /// Ensures After hooks run exactly once even if called both ways. + /// + public void RegisterAfterAssemblyHook( + Assembly assembly, + CancellationToken cancellationToken, + Func>> afterHookExecutor) + { + var registration = cancellationToken.Register(() => + { + _ = GetOrCreateAfterAssemblyTask(assembly, afterHookExecutor); + }); + + _registrations.Add(registration); + } + + /// + /// Registers Class After hooks to run on cancellation or normal completion. + /// Ensures After hooks run exactly once even if called both ways. + /// + public void RegisterAfterClassHook( + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicMethods)] + Type testClass, + HookExecutor hookExecutor, + CancellationToken cancellationToken) + { + var registration = cancellationToken.Register(() => + { + _ = GetOrCreateAfterClassTask(testClass, hookExecutor, CancellationToken.None); + }); + + _registrations.Add(registration); + } + + /// + /// Gets or creates the After Test Session task, ensuring it runs only once. + /// Thread-safe using double-checked locking. + /// Returns the exceptions from hook execution. + /// + public ValueTask> GetOrCreateAfterTestSessionTask(Func>> taskFactory) + { + if (_afterTestSessionTask != null) + { + return new ValueTask>(_afterTestSessionTask); + } + + lock (_testSessionLock) + { + if (_afterTestSessionTask == null) + { + _afterTestSessionTask = taskFactory().AsTask(); + } + return new ValueTask>(_afterTestSessionTask); + } + } + + /// + /// Gets or creates the After Assembly task for the specified assembly. + /// Thread-safe using ThreadSafeDictionary. + /// Returns the exceptions from hook execution. + /// + public ValueTask> GetOrCreateAfterAssemblyTask(Assembly assembly, Func>> taskFactory) + { + var task = _afterAssemblyTasks.GetOrAdd(assembly, a => taskFactory(a).AsTask()); + return new ValueTask>(task); + } + + /// + /// Gets or creates the After Class task for the specified test class. + /// Thread-safe using double-checked locking. + /// Returns the exceptions from hook execution. + /// + public ValueTask> GetOrCreateAfterClassTask( + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicMethods)] + Type testClass, + HookExecutor hookExecutor, + CancellationToken cancellationToken) + { + if (_afterClassTasks.TryGetValue(testClass, out var existingTask)) + { + return new ValueTask>(existingTask); + } + + lock (_classLock) + { + if (_afterClassTasks.TryGetValue(testClass, out existingTask)) + { + return new ValueTask>(existingTask); + } + + // Call ExecuteAfterClassHooksAsync directly with the annotated testClass + // The factory ignores the key since we've already created the task with the annotated type + var newTask = hookExecutor.ExecuteAfterClassHooksAsync(testClass, cancellationToken).AsTask(); + _afterClassTasks.GetOrAdd(testClass, _ => newTask); + return new ValueTask>(newTask); + } + } + + /// + /// Disposes all cancellation token registrations. + /// Should be called at the end of test execution to clean up resources. + /// + public void Dispose() + { + foreach (var registration in _registrations) + { + registration.Dispose(); + } + } +} diff --git a/TUnit.Engine/Services/BeforeHookTaskCache.cs b/TUnit.Engine/Services/BeforeHookTaskCache.cs index 3b76326dac..1a23fe93af 100644 --- a/TUnit.Engine/Services/BeforeHookTaskCache.cs +++ b/TUnit.Engine/Services/BeforeHookTaskCache.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; using System.Reflection; using TUnit.Core.Data; @@ -16,8 +15,9 @@ internal sealed class BeforeHookTaskCache private readonly ThreadSafeDictionary _beforeAssemblyTasks = new(); private Task? _beforeTestSessionTask; private readonly object _testSessionLock = new(); + private readonly object _classLock = new(); - public ValueTask GetOrCreateBeforeTestSessionTask(Func taskFactory) + public ValueTask GetOrCreateBeforeTestSessionTask(Func taskFactory, CancellationToken cancellationToken) { if (_beforeTestSessionTask != null) { @@ -28,23 +28,41 @@ public ValueTask GetOrCreateBeforeTestSessionTask(Func taskFactory) { if (_beforeTestSessionTask == null) { - _beforeTestSessionTask = taskFactory().AsTask(); + _beforeTestSessionTask = taskFactory(cancellationToken).AsTask(); } return new ValueTask(_beforeTestSessionTask); } } - public ValueTask GetOrCreateBeforeAssemblyTask(Assembly assembly, Func taskFactory) + public ValueTask GetOrCreateBeforeAssemblyTask(Assembly assembly, Func taskFactory, CancellationToken cancellationToken) { - var task = _beforeAssemblyTasks.GetOrAdd(assembly, a => taskFactory(a).AsTask()); + var task = _beforeAssemblyTasks.GetOrAdd(assembly, a => taskFactory(a, cancellationToken).AsTask()); return new ValueTask(task); } public ValueTask GetOrCreateBeforeClassTask( [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicMethods)] - Type testClass, Func taskFactory) + Type testClass, + HookExecutor hookExecutor, + CancellationToken cancellationToken) { - var task = _beforeClassTasks.GetOrAdd(testClass, t => taskFactory(t).AsTask()); - return new ValueTask(task); + if (_beforeClassTasks.TryGetValue(testClass, out var existingTask)) + { + return new ValueTask(existingTask); + } + + lock (_classLock) + { + if (_beforeClassTasks.TryGetValue(testClass, out existingTask)) + { + return new ValueTask(existingTask); + } + + // Call ExecuteBeforeClassHooksAsync directly with the annotated testClass + // The factory ignores the key since we've already created the task with the annotated type + var newTask = hookExecutor.ExecuteBeforeClassHooksAsync(testClass, cancellationToken).AsTask(); + _beforeClassTasks.GetOrAdd(testClass, _ => newTask); + return new ValueTask(newTask); + } } } diff --git a/TUnit.Engine/Services/TestExecution/TestCoordinator.cs b/TUnit.Engine/Services/TestExecution/TestCoordinator.cs index 928f603df0..b448e459e6 100644 --- a/TUnit.Engine/Services/TestExecution/TestCoordinator.cs +++ b/TUnit.Engine/Services/TestExecution/TestCoordinator.cs @@ -1,7 +1,6 @@ using System.Linq; using TUnit.Core; using TUnit.Core.Exceptions; -using TUnit.Core.Interfaces; using TUnit.Core.Logging; using TUnit.Core.Tracking; using TUnit.Engine.Helpers; @@ -91,7 +90,7 @@ private async ValueTask ExecuteTestInternalAsync(AbstractExecutableTest test, Ca } // Ensure TestSession hooks run before creating test instances - await _testExecutor.EnsureTestSessionHooksExecutedAsync().ConfigureAwait(false); + await _testExecutor.EnsureTestSessionHooksExecutedAsync(cancellationToken).ConfigureAwait(false); // Execute test with retry logic - each retry gets a fresh instance // Timeout is applied per retry attempt, not across all retries @@ -153,8 +152,7 @@ await TimeoutHelper.ExecuteWithTimeoutAsync( try { - var hookExecutor = test.Context.CustomHookExecutor; - await TestExecutor.DisposeTestInstance(test, hookExecutor).ConfigureAwait(false); + await TestExecutor.DisposeTestInstance(test).ConfigureAwait(false); } catch (Exception disposeEx) { diff --git a/TUnit.Engine/TestExecutor.cs b/TUnit.Engine/TestExecutor.cs index 8418bcb322..e7d65b92f0 100644 --- a/TUnit.Engine/TestExecutor.cs +++ b/TUnit.Engine/TestExecutor.cs @@ -19,6 +19,7 @@ internal class TestExecutor private readonly HookExecutor _hookExecutor; private readonly TestLifecycleCoordinator _lifecycleCoordinator; private readonly BeforeHookTaskCache _beforeHookTaskCache; + private readonly AfterHookPairTracker _afterHookPairTracker; private readonly IContextProvider _contextProvider; private readonly EventReceiverOrchestrator _eventReceiverOrchestrator; @@ -26,12 +27,14 @@ public TestExecutor( HookExecutor hookExecutor, TestLifecycleCoordinator lifecycleCoordinator, BeforeHookTaskCache beforeHookTaskCache, + AfterHookPairTracker afterHookPairTracker, IContextProvider contextProvider, EventReceiverOrchestrator eventReceiverOrchestrator) { _hookExecutor = hookExecutor; _lifecycleCoordinator = lifecycleCoordinator; _beforeHookTaskCache = beforeHookTaskCache; + _afterHookPairTracker = afterHookPairTracker; _contextProvider = contextProvider; _eventReceiverOrchestrator = eventReceiverOrchestrator; } @@ -40,12 +43,19 @@ public TestExecutor( /// /// Ensures that Before(TestSession) hooks have been executed. /// This is called before creating test instances to ensure resources are available. + /// Registers the corresponding After(TestSession) hook to run on cancellation. /// - public async Task EnsureTestSessionHooksExecutedAsync() + public async Task EnsureTestSessionHooksExecutedAsync(CancellationToken cancellationToken) { // Get or create and cache Before hooks - these run only once - await _beforeHookTaskCache.GetOrCreateBeforeTestSessionTask(() => - _hookExecutor.ExecuteBeforeTestSessionHooksAsync(CancellationToken.None)).ConfigureAwait(false); + await _beforeHookTaskCache.GetOrCreateBeforeTestSessionTask( + ct => _hookExecutor.ExecuteBeforeTestSessionHooksAsync(ct), + cancellationToken).ConfigureAwait(false); + + // Register After Session hook to run on cancellation (guarantees cleanup) + _afterHookPairTracker.RegisterAfterTestSessionHook( + cancellationToken, + () => new ValueTask>(_hookExecutor.ExecuteAfterTestSessionHooksAsync(CancellationToken.None).AsTask())); } /// @@ -63,7 +73,7 @@ public async ValueTask ExecuteAsync(AbstractExecutableTest executableTest, Cance try { - await EnsureTestSessionHooksExecutedAsync().ConfigureAwait(false); + await EnsureTestSessionHooksExecutedAsync(cancellationToken).ConfigureAwait(false); await _eventReceiverOrchestrator.InvokeFirstTestInSessionEventReceiversAsync( executableTest.Context, @@ -72,8 +82,16 @@ await _eventReceiverOrchestrator.InvokeFirstTestInSessionEventReceiversAsync( executableTest.Context.ClassContext.AssemblyContext.TestSessionContext.RestoreExecutionContext(); - await _beforeHookTaskCache.GetOrCreateBeforeAssemblyTask(testAssembly, assembly => _hookExecutor.ExecuteBeforeAssemblyHooksAsync(assembly, CancellationToken.None)) - .ConfigureAwait(false); + await _beforeHookTaskCache.GetOrCreateBeforeAssemblyTask( + testAssembly, + (assembly, ct) => _hookExecutor.ExecuteBeforeAssemblyHooksAsync(assembly, ct), + cancellationToken).ConfigureAwait(false); + + // Register After Assembly hook to run on cancellation (guarantees cleanup) + _afterHookPairTracker.RegisterAfterAssemblyHook( + testAssembly, + cancellationToken, + (assembly) => new ValueTask>(_hookExecutor.ExecuteAfterAssemblyHooksAsync(assembly, CancellationToken.None).AsTask())); await _eventReceiverOrchestrator.InvokeFirstTestInAssemblyEventReceiversAsync( executableTest.Context, @@ -82,8 +100,10 @@ await _eventReceiverOrchestrator.InvokeFirstTestInAssemblyEventReceiversAsync( executableTest.Context.ClassContext.AssemblyContext.RestoreExecutionContext(); - await _beforeHookTaskCache.GetOrCreateBeforeClassTask(testClass, _ => _hookExecutor.ExecuteBeforeClassHooksAsync(testClass, CancellationToken.None)) - .ConfigureAwait(false); + await _beforeHookTaskCache.GetOrCreateBeforeClassTask(testClass, _hookExecutor, cancellationToken).ConfigureAwait(false); + + // Register After Class hook to run on cancellation (guarantees cleanup) + _afterHookPairTracker.RegisterAfterClassHook(testClass, _hookExecutor, cancellationToken); await _eventReceiverOrchestrator.InvokeFirstTestInClassEventReceiversAsync( executableTest.Context, @@ -121,13 +141,16 @@ await _eventReceiverOrchestrator.InvokeFirstTestInClassEventReceiversAsync( } finally { + // After hooks must use CancellationToken.None to ensure cleanup runs even when cancelled + // This matches the pattern used for After Class/Assembly hooks in TestCoordinator + // Early stage test end receivers run before instance-level hooks - var earlyStageExceptions = await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, cancellationToken, EventReceiverStage.Early).ConfigureAwait(false); + var earlyStageExceptions = await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, CancellationToken.None, EventReceiverStage.Early).ConfigureAwait(false); - var hookExceptions = await _hookExecutor.ExecuteAfterTestHooksAsync(executableTest, cancellationToken).ConfigureAwait(false); + var hookExceptions = await _hookExecutor.ExecuteAfterTestHooksAsync(executableTest, CancellationToken.None).ConfigureAwait(false); // Late stage test end receivers run after instance-level hooks (default behavior) - var lateStageExceptions = await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, cancellationToken, EventReceiverStage.Late).ConfigureAwait(false); + var lateStageExceptions = await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, CancellationToken.None, EventReceiverStage.Late).ConfigureAwait(false); // Combine all exceptions from event receivers var eventReceiverExceptions = new List(earlyStageExceptions.Count + lateStageExceptions.Count); @@ -201,13 +224,17 @@ internal async Task> ExecuteAfterClassAssemblyHooks(AbstractExec if (flags.ShouldExecuteAfterClass) { - var classExceptions = await _hookExecutor.ExecuteAfterClassHooksAsync(testClass, cancellationToken).ConfigureAwait(false); + // Use AfterHookPairTracker to prevent double execution if already triggered by cancellation + var classExceptions = await _afterHookPairTracker.GetOrCreateAfterClassTask(testClass, _hookExecutor, cancellationToken).ConfigureAwait(false); exceptions.AddRange(classExceptions); } if (flags.ShouldExecuteAfterAssembly) { - var assemblyExceptions = await _hookExecutor.ExecuteAfterAssemblyHooksAsync(testAssembly, cancellationToken).ConfigureAwait(false); + // Use AfterHookPairTracker to prevent double execution if already triggered by cancellation + var assemblyExceptions = await _afterHookPairTracker.GetOrCreateAfterAssemblyTask( + testAssembly, + (assembly) => new ValueTask>(_hookExecutor.ExecuteAfterAssemblyHooksAsync(assembly, cancellationToken).AsTask())).ConfigureAwait(false); exceptions.AddRange(assemblyExceptions); } @@ -217,10 +244,15 @@ internal async Task> ExecuteAfterClassAssemblyHooks(AbstractExec /// /// Execute session-level after hooks once at the end of test execution. /// Returns any exceptions that occurred during hook execution. + /// Uses AfterHookPairTracker to prevent double execution if already triggered by cancellation. /// public async Task> ExecuteAfterTestSessionHooksAsync(CancellationToken cancellationToken) { - return await _hookExecutor.ExecuteAfterTestSessionHooksAsync(cancellationToken).ConfigureAwait(false); + // Use AfterHookPairTracker to prevent double execution if already triggered by cancellation + var exceptions = await _afterHookPairTracker.GetOrCreateAfterTestSessionTask( + () => new ValueTask>(_hookExecutor.ExecuteAfterTestSessionHooksAsync(cancellationToken).AsTask())).ConfigureAwait(false); + + return exceptions; } /// @@ -247,12 +279,12 @@ public IContextProvider GetContextProvider() return _contextProvider; } - internal static async Task DisposeTestInstance(AbstractExecutableTest test, IHookExecutor? hookExecutor = null) + internal static async Task DisposeTestInstance(AbstractExecutableTest test) { // Dispose the test instance if it's disposable if (test.Context.Metadata.TestDetails.ClassInstance is not SkippedTestInstance) { - async ValueTask DisposeAsync() + try { var instance = test.Context.Metadata.TestDetails.ClassInstance; @@ -266,18 +298,6 @@ async ValueTask DisposeAsync() break; } } - - try - { - if (hookExecutor != null) - { - await hookExecutor.ExecuteDisposal(test.Context, DisposeAsync).ConfigureAwait(false); - } - else - { - await DisposeAsync().ConfigureAwait(false); - } - } catch { // Swallow disposal errors - they shouldn't fail the test diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt index e67b9f26b0..e1e5f6bbf0 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -1649,13 +1649,11 @@ namespace .Extensions public static . Member(this . source, .<>> memberSelector, <.<., TItem>, .<.>> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(0)] + [.(1)] public static . Member(this . source, .<>> memberSelector, <.<., TItem>, object> assertions) { } - [.(0)] public static . Member(this . source, .<> memberSelector, <., .> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(-1)] public static . Member(this . source, .<> memberSelector, <., object> assertions) { } [.(2)] public static . Member(this . source, .<>> memberSelector, <.<., TItem>, .> assertions) { } @@ -1663,7 +1661,7 @@ namespace .Extensions public static . Member(this . source, .<>> memberSelector, <.<., TKey, TValue>, .<.>> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(1)] + [.(2)] public static . Member(this . source, .<>> memberSelector, <.<., TKey, TValue>, object> assertions) { } [.(1)] public static . Member(this . source, .<> memberSelector, <., .> assertions) { } diff --git a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt index 9a12c7e10b..a6c9c27621 100644 --- a/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Assertions_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -1649,13 +1649,11 @@ namespace .Extensions public static . Member(this . source, .<>> memberSelector, <.<., TItem>, .<.>> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(0)] + [.(1)] public static . Member(this . source, .<>> memberSelector, <.<., TItem>, object> assertions) { } - [.(0)] public static . Member(this . source, .<> memberSelector, <., .> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(-1)] public static . Member(this . source, .<> memberSelector, <., object> assertions) { } [.(2)] public static . Member(this . source, .<>> memberSelector, <.<., TItem>, .> assertions) { } @@ -1663,7 +1661,7 @@ namespace .Extensions public static . Member(this . source, .<>> memberSelector, <.<., TKey, TValue>, .<.>> assertions) { } [.("Uses reflection for legacy compatibility. For AOT compatibility, use the Member overload with strongly-typed assertions.")] - [.(1)] + [.(2)] public static . Member(this . source, .<>> memberSelector, <.<., TKey, TValue>, object> assertions) { } [.(1)] public static . Member(this . source, .<> memberSelector, <., .> assertions) { } diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 8dd30a5468..556486d6e4 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -749,7 +749,6 @@ namespace public . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action) { } public . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action) { } public . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action) { } - public . ExecuteDisposal(.TestContext context, <.> action) { } public . ExecuteTest(.TestContext context, <.> action) { } } public sealed class GenericMethodInfo @@ -2270,7 +2269,6 @@ namespace .Interfaces . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action); . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action); . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action); - . ExecuteDisposal(.TestContext context, <.> action); } public interface IHookRegisteredEventReceiver : . { diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt index 917f3ce1f2..8bbca55ff3 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -749,7 +749,6 @@ namespace public . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action) { } public . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action) { } public . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action) { } - public . ExecuteDisposal(.TestContext context, <.> action) { } public . ExecuteTest(.TestContext context, <.> action) { } } public sealed class GenericMethodInfo @@ -2270,7 +2269,6 @@ namespace .Interfaces . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action); . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action); . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action); - . ExecuteDisposal(.TestContext context, <.> action); } public interface IHookRegisteredEventReceiver : . { diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt index dc18aa6f69..e7945a9846 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -749,7 +749,6 @@ namespace public . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action) { } public . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action) { } public . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action) { } - public . ExecuteDisposal(.TestContext context, <.> action) { } public . ExecuteTest(.TestContext context, <.> action) { } } public sealed class GenericMethodInfo @@ -2270,7 +2269,6 @@ namespace .Interfaces . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action); . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action); . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action); - . ExecuteDisposal(.TestContext context, <.> action); } public interface IHookRegisteredEventReceiver : . { diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt index 7cfcb2a8fc..dd3923472e 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -726,7 +726,6 @@ namespace public . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action) { } public . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action) { } public . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action) { } - public . ExecuteDisposal(.TestContext context, <.> action) { } public . ExecuteTest(.TestContext context, <.> action) { } } public sealed class GenericMethodInfo @@ -2202,7 +2201,6 @@ namespace .Interfaces . ExecuteBeforeTestDiscoveryHook(.MethodMetadata hookMethodInfo, .BeforeTestDiscoveryContext context, <.> action); . ExecuteBeforeTestHook(.MethodMetadata hookMethodInfo, .TestContext context, <.> action); . ExecuteBeforeTestSessionHook(.MethodMetadata hookMethodInfo, .TestSessionContext context, <.> action); - . ExecuteDisposal(.TestContext context, <.> action); } public interface IHookRegisteredEventReceiver : . { diff --git a/TUnit.TestProject/Bugs/_3882/CancellationAfterHooksTests.cs b/TUnit.TestProject/Bugs/_3882/CancellationAfterHooksTests.cs new file mode 100644 index 0000000000..1f1b19030b --- /dev/null +++ b/TUnit.TestProject/Bugs/_3882/CancellationAfterHooksTests.cs @@ -0,0 +1,174 @@ +using System.Diagnostics; + +namespace TUnit.TestProject.Bugs._3882; + +/// +/// Tests for issue #3882: After Test hook is not run when test is cancelled +/// https://github.com/thomhurst/TUnit/issues/3882 +/// +/// This test demonstrates that After hooks now execute even when tests are cancelled. +/// The Before hook starts a process, the test delays, and the After hook cleans up the process. +/// When cancelled via Test Explorer or timeout, the After hook should still execute. +/// +public class CancellationAfterHooksTests +{ + private static readonly string MarkerFileDirectory = Path.Combine(Path.GetTempPath(), "TUnit_3882_Tests"); + + [Before(Test)] + public async Task StartProcess(TestContext context) + { + // Create marker directory + Directory.CreateDirectory(MarkerFileDirectory); + + // Write marker to prove Before hook ran + var beforeMarker = Path.Combine(MarkerFileDirectory, $"before_{context.Metadata.TestName}.txt"); + await File.WriteAllTextAsync(beforeMarker, $"Before hook executed at {DateTime.Now:O}"); + } + + [Test] + [Timeout(2000)] // 2 second timeout to force cancellation + public async Task Test_ThatGets_Cancelled(CancellationToken cancellationToken) + { + // This test delays longer than the timeout, causing cancellation + await Task.Delay(10000, cancellationToken); + } + + [After(Test)] + public async Task StopProcess(TestContext context) + { + try + { + // Write marker to prove After hook ran EVEN ON CANCELLATION + var afterMarker = Path.Combine(MarkerFileDirectory, $"after_{context.Metadata.TestName}.txt"); + await File.WriteAllTextAsync(afterMarker, $"After hook executed at {DateTime.Now:O} - Outcome: {context.Execution.Result?.State}"); + } + catch (Exception ex) + { + // Don't let marker file creation failure prevent process cleanup + Console.WriteLine($"[AfterTest] Failed to write marker file: {ex.Message}"); + } + } +} + +/// +/// Tests for Session-level After hooks with cancellation +/// +public class SessionLevelCancellationTests +{ + private static readonly string SessionMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_Session_After.txt"); + + [Before(TestSession)] + public static async Task SessionSetup(TestSessionContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_Session_Before.txt"), + $"Session Before hook executed at {DateTime.Now:O}"); + } + + [After(TestSession)] + public static async Task SessionCleanup(TestSessionContext context) + { + // This should run even if tests are cancelled + try + { + await File.WriteAllTextAsync( + SessionMarkerFile, + $"Session After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterTestSession] Session After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterTestSession] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + [Timeout(1000)] + public async Task SessionTest_ThatGets_Cancelled(CancellationToken cancellationToken) + { + await Task.Delay(5000, cancellationToken); + } +} + +/// +/// Tests for Assembly-level After hooks with cancellation +/// +public class AssemblyLevelCancellationTests +{ + private static readonly string AssemblyMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_Assembly_After.txt"); + + [Before(Assembly)] + public static async Task AssemblySetup(AssemblyHookContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_Assembly_Before.txt"), + $"Assembly Before hook executed at {DateTime.Now:O}"); + } + + [After(Assembly)] + public static async Task AssemblyCleanup(AssemblyHookContext context) + { + // This should run even if tests are cancelled + try + { + await File.WriteAllTextAsync( + AssemblyMarkerFile, + $"Assembly After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterAssembly] Assembly After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterAssembly] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + [Timeout(1000)] + public async Task AssemblyTest_ThatGets_Cancelled(CancellationToken cancellationToken) + { + await Task.Delay(5000, cancellationToken); + } +} + +/// +/// Tests for Class-level After hooks with cancellation +/// +public class ClassLevelCancellationTests +{ + private static readonly string ClassMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_Class_After.txt"); + + [Before(Class)] + public static async Task ClassSetup(ClassHookContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_Class_Before.txt"), + $"Class Before hook executed at {DateTime.Now:O}"); + } + + [After(Class)] + public static async Task ClassCleanup(ClassHookContext context) + { + // This should run even if tests are cancelled + try + { + await File.WriteAllTextAsync( + ClassMarkerFile, + $"Class After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterClass] Class After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterClass] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + [Timeout(1000)] + public async Task ClassTest_ThatGets_Cancelled(CancellationToken cancellationToken) + { + await Task.Delay(5000, cancellationToken); + } +} diff --git a/TUnit.TestProject/Bugs/_3882/ExternalCancellationTests.cs b/TUnit.TestProject/Bugs/_3882/ExternalCancellationTests.cs new file mode 100644 index 0000000000..509a8326aa --- /dev/null +++ b/TUnit.TestProject/Bugs/_3882/ExternalCancellationTests.cs @@ -0,0 +1,167 @@ +using System.Diagnostics; + +namespace TUnit.TestProject.Bugs._3882; + +[Timeout(300_000)] // Overall timeout for the test class to prevent indefinite hangs +public class ExternalCancellationTests +{ + private static readonly string MarkerFileDirectory = Path.Combine(Path.GetTempPath(), "TUnit_3882_External"); + + [Before(Test)] + public async Task StartProcess(TestContext context, CancellationToken cancellationToken) + { + // Create marker directory + Directory.CreateDirectory(MarkerFileDirectory); + + // Write marker to prove Before hook ran + var beforeMarker = Path.Combine(MarkerFileDirectory, $"before_{context.Metadata.TestName}.txt"); + await File.WriteAllTextAsync(beforeMarker, $"Before hook executed at {DateTime.Now:O}"); + } + + [Test] + // NO [Timeout] attribute - test runs indefinitely until cancelled externally + public async Task Test_ThatGets_Cancelled_Externally(CancellationToken cancellationToken) + { + // This test delays indefinitely, only stops via external cancellation + await Task.Delay(TimeSpan.FromHours(1), cancellationToken); + } + + [After(Test)] + public async Task StopProcess(TestContext context, CancellationToken cancellationToken) + { + try + { + // Write marker to prove After hook ran EVEN ON EXTERNAL CANCELLATION + var afterMarker = Path.Combine(MarkerFileDirectory, $"after_{context.Metadata.TestName}.txt"); + await File.WriteAllTextAsync(afterMarker, $"After hook executed at {DateTime.Now:O} - Outcome: {context.Execution.Result?.State}"); + } + catch (Exception ex) + { + // Don't let marker file creation failure prevent process cleanup + Console.WriteLine($"[AfterTest] Failed to write marker file: {ex.Message}"); + } + } +} + +/// +/// Tests for Session-level After hooks with external cancellation +/// +public class ExternalSessionLevelCancellationTests +{ + private static readonly string SessionMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Session_After.txt"); + + [Before(TestSession)] + public static async Task SessionSetup(TestSessionContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Session_Before.txt"), + $"Session Before hook executed at {DateTime.Now:O}"); + } + + [After(TestSession)] + public static async Task SessionCleanup(TestSessionContext context) + { + // This should run even if tests are cancelled externally + try + { + await File.WriteAllTextAsync( + SessionMarkerFile, + $"Session After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterTestSession] Session After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterTestSession] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + // NO [Timeout] attribute - test runs indefinitely until cancelled externally + public async Task SessionTest_ThatGets_Cancelled_Externally(CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromHours(1), cancellationToken); + } +} + +/// +/// Tests for Assembly-level After hooks with external cancellation +/// +public class ExternalAssemblyLevelCancellationTests +{ + private static readonly string AssemblyMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Assembly_After.txt"); + + [Before(Assembly)] + public static async Task AssemblySetup(AssemblyHookContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Assembly_Before.txt"), + $"Assembly Before hook executed at {DateTime.Now:O}"); + } + + [After(Assembly)] + public static async Task AssemblyCleanup(AssemblyHookContext context) + { + // This should run even if tests are cancelled externally + try + { + await File.WriteAllTextAsync( + AssemblyMarkerFile, + $"Assembly After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterAssembly] Assembly After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterAssembly] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + // NO [Timeout] attribute - test runs indefinitely until cancelled externally + public async Task AssemblyTest_ThatGets_Cancelled_Externally(CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromHours(1), cancellationToken); + } +} + +/// +/// Tests for Class-level After hooks with external cancellation +/// +public class ExternalClassLevelCancellationTests +{ + private static readonly string ClassMarkerFile = Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Class_After.txt"); + + [Before(Class)] + public static async Task ClassSetup(ClassHookContext context) + { + await File.WriteAllTextAsync( + Path.Combine(Path.GetTempPath(), "TUnit_3882_External_Class_Before.txt"), + $"Class Before hook executed at {DateTime.Now:O}"); + } + + [After(Class)] + public static async Task ClassCleanup(ClassHookContext context) + { + // This should run even if tests are cancelled externally + try + { + await File.WriteAllTextAsync( + ClassMarkerFile, + $"Class After hook executed at {DateTime.Now:O}"); + Console.WriteLine($"[AfterClass] Class After hook completed successfully"); + } + catch (Exception ex) + { + Console.WriteLine($"[AfterClass] Failed to write marker file: {ex.Message}"); + throw; // Re-throw to signal failure, but after logging + } + } + + [Test] + // NO [Timeout] attribute - test runs indefinitely until cancelled externally + public async Task ClassTest_ThatGets_Cancelled_Externally(CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromHours(1), cancellationToken); + } +} diff --git a/TUnit.TestProject/SetHookExecutorTests.cs b/TUnit.TestProject/SetHookExecutorTests.cs index dc2427011b..5d11036a32 100644 --- a/TUnit.TestProject/SetHookExecutorTests.cs +++ b/TUnit.TestProject/SetHookExecutorTests.cs @@ -117,36 +117,3 @@ public async Task Test_StaticHooksExecuteInCustomExecutor() await Assert.That(Thread.CurrentThread.Name).IsEqualTo("CrossPlatformTestExecutor"); } } - -/// -/// Tests demonstrating SetHookExecutor affects disposal execution - Issue #3918 -/// -[EngineTest(ExpectedResult.Pass)] -[SetBothExecutors] // This attribute sets both executors -public class DisposalWithHookExecutorTests : IAsyncDisposable -{ - private static bool _disposalExecutedInCustomExecutor; - - [Test] - public async Task Test_ExecutesInCustomExecutor() - { - // Test should execute in custom executor - await Assert.That(Thread.CurrentThread.Name).IsEqualTo("CrossPlatformTestExecutor"); - await Assert.That(CrossPlatformTestExecutor.IsRunningInTestExecutor.Value).IsTrue(); - } - - public ValueTask DisposeAsync() - { - // Verify disposal runs in the custom executor - _disposalExecutedInCustomExecutor = - Thread.CurrentThread.Name == "CrossPlatformTestExecutor" && - CrossPlatformTestExecutor.IsRunningInTestExecutor.Value; - return default; - } - - [After(Class)] - public static async Task VerifyDisposalRanInCustomExecutor(ClassHookContext context) - { - await Assert.That(_disposalExecutedInCustomExecutor).IsTrue(); - } -} diff --git a/docs/docs/benchmarks/AsyncTests.md b/docs/docs/benchmarks/AsyncTests.md index 06a025db9c..98eed244c2 100644 --- a/docs/docs/benchmarks/AsyncTests.md +++ b/docs/docs/benchmarks/AsyncTests.md @@ -7,7 +7,7 @@ sidebar_position: 2 # AsyncTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 573.3 ms | 573.7 ms | 4.43 ms | -| NUnit | 4.4.0 | 658.8 ms | 660.6 ms | 4.99 ms | -| MSTest | 4.0.2 | 631.5 ms | 631.2 ms | 8.74 ms | -| xUnit3 | 3.2.1 | 716.2 ms | 715.4 ms | 6.28 ms | -| **TUnit (AOT)** | 1.2.11 | 124.1 ms | 124.2 ms | 0.20 ms | +| **TUnit** | 1.2.11 | 567.1 ms | 566.4 ms | 2.57 ms | +| NUnit | 4.4.0 | 688.2 ms | 683.1 ms | 13.62 ms | +| MSTest | 4.0.2 | 658.5 ms | 659.6 ms | 10.71 ms | +| xUnit3 | 3.2.0 | 736.1 ms | 733.4 ms | 13.54 ms | +| **TUnit (AOT)** | 1.2.11 | 124.7 ms | 124.8 ms | 0.42 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "AsyncTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 860 - bar [573.3, 658.8, 631.5, 716.2, 124.1] + y-axis "Time (ms)" 0 --> 884 + bar [567.1, 688.2, 658.5, 736.1, 124.7] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.985Z* +*Last generated: 2025-11-27T00:28:35.446Z* diff --git a/docs/docs/benchmarks/BuildTime.md b/docs/docs/benchmarks/BuildTime.md index 54a0ea8d23..0e293d83f3 100644 --- a/docs/docs/benchmarks/BuildTime.md +++ b/docs/docs/benchmarks/BuildTime.md @@ -7,7 +7,7 @@ sidebar_position: 8 # Build Performance Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -18,10 +18,10 @@ Compilation time comparison across frameworks: | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 2.050 s | 2.047 s | 0.0253 s | -| Build_NUnit | 4.4.0 | 1.644 s | 1.646 s | 0.0243 s | -| Build_MSTest | 4.0.2 | 1.714 s | 1.709 s | 0.0303 s | -| Build_xUnit3 | 3.2.1 | 1.639 s | 1.639 s | 0.0300 s | +| **TUnit** | 1.2.11 | 2.043 s | 2.035 s | 0.0196 s | +| Build_NUnit | 4.4.0 | 1.639 s | 1.639 s | 0.0088 s | +| Build_MSTest | 4.0.2 | 1.715 s | 1.712 s | 0.0172 s | +| Build_xUnit3 | 3.2.0 | 1.616 s | 1.621 s | 0.0283 s | ## 📈 Visual Comparison @@ -60,7 +60,7 @@ xychart-beta title "Build Time Comparison" x-axis ["Build_TUnit", "Build_NUnit", "Build_MSTest", "Build_xUnit3"] y-axis "Time (s)" 0 --> 3 - bar [2.05, 1.644, 1.714, 1.639] + bar [2.043, 1.639, 1.715, 1.616] ``` --- @@ -69,4 +69,4 @@ xychart-beta View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.987Z* +*Last generated: 2025-11-27T00:28:35.447Z* diff --git a/docs/docs/benchmarks/DataDrivenTests.md b/docs/docs/benchmarks/DataDrivenTests.md index 55089d1f10..3c58c40c9d 100644 --- a/docs/docs/benchmarks/DataDrivenTests.md +++ b/docs/docs/benchmarks/DataDrivenTests.md @@ -7,7 +7,7 @@ sidebar_position: 3 # DataDrivenTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 501.86 ms | 501.49 ms | 4.693 ms | -| NUnit | 4.4.0 | 535.87 ms | 534.70 ms | 12.790 ms | -| MSTest | 4.0.2 | 502.71 ms | 503.69 ms | 12.277 ms | -| xUnit3 | 3.2.1 | 579.28 ms | 578.02 ms | 8.138 ms | -| **TUnit (AOT)** | 1.2.11 | 24.82 ms | 24.80 ms | 0.166 ms | +| **TUnit** | 1.2.11 | 497.25 ms | 497.97 ms | 3.518 ms | +| NUnit | 4.4.0 | 526.05 ms | 524.98 ms | 6.301 ms | +| MSTest | 4.0.2 | 482.94 ms | 483.13 ms | 10.339 ms | +| xUnit3 | 3.2.0 | 573.56 ms | 576.44 ms | 10.043 ms | +| **TUnit (AOT)** | 1.2.11 | 24.70 ms | 24.72 ms | 0.182 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "DataDrivenTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 696 - bar [501.86, 535.87, 502.71, 579.28, 24.82] + y-axis "Time (ms)" 0 --> 689 + bar [497.25, 526.05, 482.94, 573.56, 24.7] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.985Z* +*Last generated: 2025-11-27T00:28:35.446Z* diff --git a/docs/docs/benchmarks/MassiveParallelTests.md b/docs/docs/benchmarks/MassiveParallelTests.md index 726200db85..9136b82712 100644 --- a/docs/docs/benchmarks/MassiveParallelTests.md +++ b/docs/docs/benchmarks/MassiveParallelTests.md @@ -7,7 +7,7 @@ sidebar_position: 4 # MassiveParallelTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 597.5 ms | 597.8 ms | 3.59 ms | -| NUnit | 4.4.0 | 1,183.2 ms | 1,182.1 ms | 4.90 ms | -| MSTest | 4.0.2 | 2,966.5 ms | 2,968.0 ms | 5.16 ms | -| xUnit3 | 3.2.1 | 3,066.1 ms | 3,065.1 ms | 15.64 ms | -| **TUnit (AOT)** | 1.2.11 | 132.2 ms | 132.3 ms | 0.51 ms | +| **TUnit** | 1.2.11 | 623.3 ms | 624.8 ms | 4.14 ms | +| NUnit | 4.4.0 | 1,212.4 ms | 1,210.2 ms | 9.48 ms | +| MSTest | 4.0.2 | 3,001.4 ms | 2,999.8 ms | 10.05 ms | +| xUnit3 | 3.2.0 | 3,086.1 ms | 3,086.8 ms | 7.57 ms | +| **TUnit (AOT)** | 1.2.11 | 132.7 ms | 132.6 ms | 0.54 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "MassiveParallelTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 3680 - bar [597.5, 1183.2, 2966.5, 3066.1, 132.2] + y-axis "Time (ms)" 0 --> 3704 + bar [623.3, 1212.4, 3001.4, 3086.1, 132.7] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.986Z* +*Last generated: 2025-11-27T00:28:35.446Z* diff --git a/docs/docs/benchmarks/MatrixTests.md b/docs/docs/benchmarks/MatrixTests.md index 8e74222e9c..7b17a6c761 100644 --- a/docs/docs/benchmarks/MatrixTests.md +++ b/docs/docs/benchmarks/MatrixTests.md @@ -7,7 +7,7 @@ sidebar_position: 5 # MatrixTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 574.81 ms | 575.40 ms | 5.417 ms | -| NUnit | 4.4.0 | 1,582.59 ms | 1,581.77 ms | 6.462 ms | -| MSTest | 4.0.2 | 1,540.42 ms | 1,540.75 ms | 5.446 ms | -| xUnit3 | 3.2.1 | 1,630.94 ms | 1,627.40 ms | 11.609 ms | -| **TUnit (AOT)** | 1.2.11 | 79.00 ms | 79.08 ms | 0.235 ms | +| **TUnit** | 1.2.11 | 566.13 ms | 565.43 ms | 6.050 ms | +| NUnit | 4.4.0 | 1,578.62 ms | 1,578.58 ms | 6.773 ms | +| MSTest | 4.0.2 | 1,539.67 ms | 1,540.17 ms | 3.979 ms | +| xUnit3 | 3.2.0 | 1,625.28 ms | 1,624.99 ms | 11.364 ms | +| **TUnit (AOT)** | 1.2.11 | 79.52 ms | 79.55 ms | 0.341 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "MatrixTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1958 - bar [574.81, 1582.59, 1540.42, 1630.94, 79] + y-axis "Time (ms)" 0 --> 1951 + bar [566.13, 1578.62, 1539.67, 1625.28, 79.52] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.986Z* +*Last generated: 2025-11-27T00:28:35.446Z* diff --git a/docs/docs/benchmarks/ScaleTests.md b/docs/docs/benchmarks/ScaleTests.md index 4a084a622a..c459176451 100644 --- a/docs/docs/benchmarks/ScaleTests.md +++ b/docs/docs/benchmarks/ScaleTests.md @@ -7,7 +7,7 @@ sidebar_position: 6 # ScaleTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,11 +16,11 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 535.07 ms | 534.68 ms | 4.870 ms | -| NUnit | 4.4.0 | 585.44 ms | 584.11 ms | 12.531 ms | -| MSTest | 4.0.2 | 509.11 ms | 505.09 ms | 11.032 ms | -| xUnit3 | 3.2.1 | 594.97 ms | 593.95 ms | 6.401 ms | -| **TUnit (AOT)** | 1.2.11 | 46.00 ms | 46.41 ms | 3.968 ms | +| **TUnit** | 1.2.11 | 522.37 ms | 521.16 ms | 5.026 ms | +| NUnit | 4.4.0 | 611.57 ms | 612.00 ms | 8.234 ms | +| MSTest | 4.0.2 | 615.51 ms | 617.04 ms | 9.675 ms | +| xUnit3 | 3.2.0 | 614.63 ms | 610.65 ms | 7.412 ms | +| **TUnit (AOT)** | 1.2.11 | 43.96 ms | 44.10 ms | 3.394 ms | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "ScaleTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 714 - bar [535.07, 585.44, 509.11, 594.97, 46] + y-axis "Time (ms)" 0 --> 739 + bar [522.37, 611.57, 615.51, 614.63, 43.96] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.986Z* +*Last generated: 2025-11-27T00:28:35.446Z* diff --git a/docs/docs/benchmarks/SetupTeardownTests.md b/docs/docs/benchmarks/SetupTeardownTests.md index 9ebe377bbf..348cafc187 100644 --- a/docs/docs/benchmarks/SetupTeardownTests.md +++ b/docs/docs/benchmarks/SetupTeardownTests.md @@ -7,7 +7,7 @@ sidebar_position: 7 # SetupTeardownTests Benchmark :::info Last Updated -This benchmark was automatically generated on **2025-11-29** from the latest CI run. +This benchmark was automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -16,10 +16,10 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI | Framework | Version | Mean | Median | StdDev | |-----------|---------|------|--------|--------| -| **TUnit** | 1.2.11 | 558.0 ms | 557.6 ms | 3.82 ms | -| NUnit | 4.4.0 | 1,148.5 ms | 1,146.8 ms | 8.24 ms | -| MSTest | 4.0.2 | 1,120.1 ms | 1,121.5 ms | 6.42 ms | -| xUnit3 | 3.2.1 | 1,198.8 ms | 1,197.8 ms | 9.10 ms | +| **TUnit** | 1.2.11 | 575.6 ms | 575.7 ms | 4.64 ms | +| NUnit | 4.4.0 | 1,194.3 ms | 1,195.1 ms | 6.50 ms | +| MSTest | 4.0.2 | 1,165.8 ms | 1,164.6 ms | 9.87 ms | +| xUnit3 | 3.2.0 | 1,258.5 ms | 1,258.8 ms | 7.83 ms | | **TUnit (AOT)** | 1.2.11 | NA | NA | NA | ## 📈 Visual Comparison @@ -58,8 +58,8 @@ This benchmark was automatically generated on **2025-11-29** from the latest CI xychart-beta title "SetupTeardownTests Performance Comparison" x-axis ["TUnit", "NUnit", "MSTest", "xUnit3", "TUnit_AOT"] - y-axis "Time (ms)" 0 --> 1439 - bar [558, 1148.5, 1120.1, 1198.8, 0] + y-axis "Time (ms)" 0 --> 1511 + bar [575.6, 1194.3, 1165.8, 1258.5, 0] ``` ## 🎯 Key Insights @@ -72,4 +72,4 @@ This benchmark compares TUnit's performance against NUnit, MSTest, xUnit3 using View the [benchmarks overview](/docs/benchmarks) for methodology details and environment information. ::: -*Last generated: 2025-11-29T00:27:17.987Z* +*Last generated: 2025-11-27T00:28:35.447Z* diff --git a/docs/docs/benchmarks/index.md b/docs/docs/benchmarks/index.md index e5bccf4190..871d642265 100644 --- a/docs/docs/benchmarks/index.md +++ b/docs/docs/benchmarks/index.md @@ -7,7 +7,7 @@ sidebar_position: 1 # Performance Benchmarks :::info Last Updated -These benchmarks were automatically generated on **2025-11-29** from the latest CI run. +These benchmarks were automatically generated on **2025-11-27** from the latest CI run. **Environment:** Ubuntu Latest • .NET SDK 10.0.100 ::: @@ -38,7 +38,7 @@ These benchmarks compare TUnit against the most popular .NET testing frameworks: | Framework | Version Tested | |-----------|----------------| | **TUnit** | 1.2.11 | -| **xUnit v3** | 3.2.1 | +| **xUnit v3** | 3.2.0 | | **NUnit** | 4.4.0 | | **MSTest** | 4.0.2 | @@ -80,4 +80,4 @@ These benchmarks run automatically daily via [GitHub Actions](https://github.com Each benchmark runs multiple iterations with statistical analysis to ensure accuracy. Results may vary based on hardware and test characteristics. ::: -*Last generated: 2025-11-29T00:27:17.987Z* +*Last generated: 2025-11-27T00:28:35.447Z* diff --git a/docs/docs/extensions/extensions.md b/docs/docs/extensions/extensions.md index 1aad0ec492..f8869d3589 100644 --- a/docs/docs/extensions/extensions.md +++ b/docs/docs/extensions/extensions.md @@ -22,7 +22,7 @@ dotnet run --configuration Release --coverage # Specify output location dotnet run --configuration Release --coverage --coverage-output ./coverage/ -# Specify output format (default is binary .coverage file) +# Specify output format (cobertura is default) dotnet run --configuration Release --coverage --coverage-output-format cobertura # Multiple formats diff --git a/docs/docs/migration/mstest.md b/docs/docs/migration/mstest.md index eaa77fb76b..05ef1b9e45 100644 --- a/docs/docs/migration/mstest.md +++ b/docs/docs/migration/mstest.md @@ -1013,7 +1013,7 @@ dotnet run --configuration Release --coverage # Specify output location dotnet run --configuration Release --coverage --coverage-output ./coverage/ -# Specify coverage format (default is binary .coverage file) +# Specify coverage format (default is cobertura) dotnet run --configuration Release --coverage --coverage-output-format cobertura # Multiple formats @@ -1055,7 +1055,7 @@ If you have CI/CD pipelines that reference Coverlet, update them to use the new The Microsoft coverage tool supports multiple output formats: ```bash -# Cobertura (widely supported) +# Cobertura (default, widely supported) dotnet run --configuration Release --coverage --coverage-output-format cobertura # XML (Visual Studio format) diff --git a/docs/docs/migration/nunit.md b/docs/docs/migration/nunit.md index 63933fc045..1e996268a1 100644 --- a/docs/docs/migration/nunit.md +++ b/docs/docs/migration/nunit.md @@ -746,7 +746,7 @@ dotnet run --configuration Release --coverage # Specify output location dotnet run --configuration Release --coverage --coverage-output ./coverage/ -# Specify coverage format (default is binary .coverage file) +# Specify coverage format (default is cobertura) dotnet run --configuration Release --coverage --coverage-output-format cobertura # Multiple formats @@ -788,7 +788,7 @@ If you have CI/CD pipelines that reference Coverlet, update them to use the new The Microsoft coverage tool supports multiple output formats: ```bash -# Cobertura (widely supported) +# Cobertura (default, widely supported) dotnet run --configuration Release --coverage --coverage-output-format cobertura # XML (Visual Studio format) diff --git a/docs/docs/migration/xunit.md b/docs/docs/migration/xunit.md index 32c1ed15ec..f452d62eb7 100644 --- a/docs/docs/migration/xunit.md +++ b/docs/docs/migration/xunit.md @@ -1054,7 +1054,7 @@ dotnet run --configuration Release --coverage # Specify output location dotnet run --configuration Release --coverage --coverage-output ./coverage/ -# Specify coverage format (default is binary .coverage file) +# Specify coverage format (default is cobertura) dotnet run --configuration Release --coverage --coverage-output-format cobertura # Multiple formats @@ -1096,7 +1096,7 @@ If you have CI/CD pipelines that reference Coverlet, update them to use the new The Microsoft coverage tool supports multiple output formats: ```bash -# Cobertura (widely supported) +# Cobertura (default, widely supported) dotnet run --configuration Release --coverage --coverage-output-format cobertura # XML (Visual Studio format) diff --git a/docs/docs/troubleshooting.md b/docs/docs/troubleshooting.md index 300dd182bb..4dfa0bb1ab 100644 --- a/docs/docs/troubleshooting.md +++ b/docs/docs/troubleshooting.md @@ -2240,7 +2240,7 @@ dotnet run --configuration Release --coverage --coverage-settings coverage.runse #### 1. Check Output Format ```bash -# Cobertura (widely supported) +# Default is Cobertura (widely supported) dotnet run --configuration Release --coverage --coverage-output-format cobertura # For Visual Studio diff --git a/docs/static/benchmarks/AsyncTests.json b/docs/static/benchmarks/AsyncTests.json index 6ea3ffc595..d31c3d85de 100644 --- a/docs/static/benchmarks/AsyncTests.json +++ b/docs/static/benchmarks/AsyncTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.985Z", + "timestamp": "2025-11-27T00:28:35.446Z", "category": "AsyncTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "573.3 ms", - "Error": "4.73 ms", - "StdDev": "4.43 ms", - "Median": "573.7 ms" + "Mean": "567.1 ms", + "Error": "3.08 ms", + "StdDev": "2.57 ms", + "Median": "566.4 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "658.8 ms", - "Error": "5.97 ms", - "StdDev": "4.99 ms", - "Median": "660.6 ms" + "Mean": "688.2 ms", + "Error": "13.26 ms", + "StdDev": "13.62 ms", + "Median": "683.1 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "631.5 ms", - "Error": "9.34 ms", - "StdDev": "8.74 ms", - "Median": "631.2 ms" + "Mean": "658.5 ms", + "Error": "12.82 ms", + "StdDev": "10.71 ms", + "Median": "659.6 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "716.2 ms", - "Error": "7.08 ms", - "StdDev": "6.28 ms", - "Median": "715.4 ms" + "Version": "3.2.0", + "Mean": "736.1 ms", + "Error": "14.47 ms", + "StdDev": "13.54 ms", + "Median": "733.4 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "124.1 ms", - "Error": "0.23 ms", - "StdDev": "0.20 ms", - "Median": "124.2 ms" + "Mean": "124.7 ms", + "Error": "0.45 ms", + "StdDev": "0.42 ms", + "Median": "124.8 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/BuildTime.json b/docs/static/benchmarks/BuildTime.json index 747f5ae6e2..c71476e2e7 100644 --- a/docs/static/benchmarks/BuildTime.json +++ b/docs/static/benchmarks/BuildTime.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.987Z", + "timestamp": "2025-11-27T00:28:35.447Z", "category": "BuildTime", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,34 +10,34 @@ { "Method": "Build_TUnit", "Version": "1.2.11", - "Mean": "2.050 s", - "Error": "0.0285 s", - "StdDev": "0.0253 s", - "Median": "2.047 s" + "Mean": "2.043 s", + "Error": "0.0210 s", + "StdDev": "0.0196 s", + "Median": "2.035 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.644 s", - "Error": "0.0260 s", - "StdDev": "0.0243 s", - "Median": "1.646 s" + "Mean": "1.639 s", + "Error": "0.0094 s", + "StdDev": "0.0088 s", + "Median": "1.639 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.714 s", - "Error": "0.0324 s", - "StdDev": "0.0303 s", - "Median": "1.709 s" + "Mean": "1.715 s", + "Error": "0.0184 s", + "StdDev": "0.0172 s", + "Median": "1.712 s" }, { "Method": "Build_xUnit3", - "Version": "3.2.1", - "Mean": "1.639 s", - "Error": "0.0321 s", - "StdDev": "0.0300 s", - "Median": "1.639 s" + "Version": "3.2.0", + "Mean": "1.616 s", + "Error": "0.0302 s", + "StdDev": "0.0283 s", + "Median": "1.621 s" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/DataDrivenTests.json b/docs/static/benchmarks/DataDrivenTests.json index a903f638a7..cf5ab9e10b 100644 --- a/docs/static/benchmarks/DataDrivenTests.json +++ b/docs/static/benchmarks/DataDrivenTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.985Z", + "timestamp": "2025-11-27T00:28:35.446Z", "category": "DataDrivenTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "501.86 ms", - "Error": "5.017 ms", - "StdDev": "4.693 ms", - "Median": "501.49 ms" + "Mean": "497.25 ms", + "Error": "4.213 ms", + "StdDev": "3.518 ms", + "Median": "497.97 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "535.87 ms", - "Error": "10.415 ms", - "StdDev": "12.790 ms", - "Median": "534.70 ms" + "Mean": "526.05 ms", + "Error": "7.107 ms", + "StdDev": "6.301 ms", + "Median": "524.98 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "502.71 ms", - "Error": "9.708 ms", - "StdDev": "12.277 ms", - "Median": "503.69 ms" + "Mean": "482.94 ms", + "Error": "9.302 ms", + "StdDev": "10.339 ms", + "Median": "483.13 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "579.28 ms", - "Error": "9.180 ms", - "StdDev": "8.138 ms", - "Median": "578.02 ms" + "Version": "3.2.0", + "Mean": "573.56 ms", + "Error": "10.737 ms", + "StdDev": "10.043 ms", + "Median": "576.44 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "24.82 ms", - "Error": "0.187 ms", - "StdDev": "0.166 ms", - "Median": "24.80 ms" + "Mean": "24.70 ms", + "Error": "0.194 ms", + "StdDev": "0.182 ms", + "Median": "24.72 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MassiveParallelTests.json b/docs/static/benchmarks/MassiveParallelTests.json index b01106537b..3bbbbc8549 100644 --- a/docs/static/benchmarks/MassiveParallelTests.json +++ b/docs/static/benchmarks/MassiveParallelTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.986Z", + "timestamp": "2025-11-27T00:28:35.446Z", "category": "MassiveParallelTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "597.5 ms", - "Error": "4.05 ms", - "StdDev": "3.59 ms", - "Median": "597.8 ms" + "Mean": "623.3 ms", + "Error": "4.42 ms", + "StdDev": "4.14 ms", + "Median": "624.8 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,183.2 ms", - "Error": "6.27 ms", - "StdDev": "4.90 ms", - "Median": "1,182.1 ms" + "Mean": "1,212.4 ms", + "Error": "10.13 ms", + "StdDev": "9.48 ms", + "Median": "1,210.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "2,966.5 ms", - "Error": "6.60 ms", - "StdDev": "5.16 ms", - "Median": "2,968.0 ms" + "Mean": "3,001.4 ms", + "Error": "12.87 ms", + "StdDev": "10.05 ms", + "Median": "2,999.8 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "3,066.1 ms", - "Error": "17.64 ms", - "StdDev": "15.64 ms", - "Median": "3,065.1 ms" + "Version": "3.2.0", + "Mean": "3,086.1 ms", + "Error": "9.06 ms", + "StdDev": "7.57 ms", + "Median": "3,086.8 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "132.2 ms", - "Error": "0.55 ms", - "StdDev": "0.51 ms", - "Median": "132.3 ms" + "Mean": "132.7 ms", + "Error": "0.58 ms", + "StdDev": "0.54 ms", + "Median": "132.6 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/MatrixTests.json b/docs/static/benchmarks/MatrixTests.json index 578206c245..8d10cf0aa4 100644 --- a/docs/static/benchmarks/MatrixTests.json +++ b/docs/static/benchmarks/MatrixTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.986Z", + "timestamp": "2025-11-27T00:28:35.446Z", "category": "MatrixTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "574.81 ms", - "Error": "5.792 ms", - "StdDev": "5.417 ms", - "Median": "575.40 ms" + "Mean": "566.13 ms", + "Error": "6.468 ms", + "StdDev": "6.050 ms", + "Median": "565.43 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,582.59 ms", - "Error": "7.738 ms", - "StdDev": "6.462 ms", - "Median": "1,581.77 ms" + "Mean": "1,578.62 ms", + "Error": "7.240 ms", + "StdDev": "6.773 ms", + "Median": "1,578.58 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,540.42 ms", - "Error": "5.822 ms", - "StdDev": "5.446 ms", - "Median": "1,540.75 ms" + "Mean": "1,539.67 ms", + "Error": "4.489 ms", + "StdDev": "3.979 ms", + "Median": "1,540.17 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "1,630.94 ms", - "Error": "12.411 ms", - "StdDev": "11.609 ms", - "Median": "1,627.40 ms" + "Version": "3.2.0", + "Mean": "1,625.28 ms", + "Error": "13.608 ms", + "StdDev": "11.364 ms", + "Median": "1,624.99 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "79.00 ms", - "Error": "0.251 ms", - "StdDev": "0.235 ms", - "Median": "79.08 ms" + "Mean": "79.52 ms", + "Error": "0.365 ms", + "StdDev": "0.341 ms", + "Median": "79.55 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/ScaleTests.json b/docs/static/benchmarks/ScaleTests.json index 2786b68df5..7cd77e77f9 100644 --- a/docs/static/benchmarks/ScaleTests.json +++ b/docs/static/benchmarks/ScaleTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.986Z", + "timestamp": "2025-11-27T00:28:35.446Z", "category": "ScaleTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,42 +10,42 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "535.07 ms", - "Error": "5.832 ms", - "StdDev": "4.870 ms", - "Median": "534.68 ms" + "Mean": "522.37 ms", + "Error": "5.373 ms", + "StdDev": "5.026 ms", + "Median": "521.16 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "585.44 ms", - "Error": "11.274 ms", - "StdDev": "12.531 ms", - "Median": "584.11 ms" + "Mean": "611.57 ms", + "Error": "9.860 ms", + "StdDev": "8.234 ms", + "Median": "612.00 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "509.11 ms", - "Error": "9.925 ms", - "StdDev": "11.032 ms", - "Median": "505.09 ms" + "Mean": "615.51 ms", + "Error": "10.914 ms", + "StdDev": "9.675 ms", + "Median": "617.04 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "594.97 ms", - "Error": "7.221 ms", - "StdDev": "6.401 ms", - "Median": "593.95 ms" + "Version": "3.2.0", + "Mean": "614.63 ms", + "Error": "8.361 ms", + "StdDev": "7.412 ms", + "Median": "610.65 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "46.00 ms", - "Error": "1.346 ms", - "StdDev": "3.968 ms", - "Median": "46.41 ms" + "Mean": "43.96 ms", + "Error": "1.151 ms", + "StdDev": "3.394 ms", + "Median": "44.10 ms" } ] } \ No newline at end of file diff --git a/docs/static/benchmarks/SetupTeardownTests.json b/docs/static/benchmarks/SetupTeardownTests.json index be6ddf85ba..d3f8eaa939 100644 --- a/docs/static/benchmarks/SetupTeardownTests.json +++ b/docs/static/benchmarks/SetupTeardownTests.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.987Z", + "timestamp": "2025-11-27T00:28:35.447Z", "category": "SetupTeardownTests", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", @@ -10,34 +10,34 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "558.0 ms", - "Error": "4.08 ms", - "StdDev": "3.82 ms", - "Median": "557.6 ms" + "Mean": "575.6 ms", + "Error": "4.96 ms", + "StdDev": "4.64 ms", + "Median": "575.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,148.5 ms", - "Error": "8.81 ms", - "StdDev": "8.24 ms", - "Median": "1,146.8 ms" + "Mean": "1,194.3 ms", + "Error": "7.33 ms", + "StdDev": "6.50 ms", + "Median": "1,195.1 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,120.1 ms", - "Error": "7.69 ms", - "StdDev": "6.42 ms", - "Median": "1,121.5 ms" + "Mean": "1,165.8 ms", + "Error": "10.55 ms", + "StdDev": "9.87 ms", + "Median": "1,164.6 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "1,198.8 ms", - "Error": "9.73 ms", - "StdDev": "9.10 ms", - "Median": "1,197.8 ms" + "Version": "3.2.0", + "Mean": "1,258.5 ms", + "Error": "8.37 ms", + "StdDev": "7.83 ms", + "Median": "1,258.8 ms" }, { "Method": "TUnit_AOT", diff --git a/docs/static/benchmarks/historical.json b/docs/static/benchmarks/historical.json index 2ca74d50fc..943a897347 100644 --- a/docs/static/benchmarks/historical.json +++ b/docs/static/benchmarks/historical.json @@ -82,13 +82,5 @@ { "date": "2025-11-27", "environment": "Ubuntu" - }, - { - "date": "2025-11-28", - "environment": "Ubuntu" - }, - { - "date": "2025-11-29", - "environment": "Ubuntu" } ] \ No newline at end of file diff --git a/docs/static/benchmarks/latest.json b/docs/static/benchmarks/latest.json index ebc2bb3f26..45e79f39d6 100644 --- a/docs/static/benchmarks/latest.json +++ b/docs/static/benchmarks/latest.json @@ -1,5 +1,5 @@ { - "timestamp": "2025-11-29T00:27:17.988Z", + "timestamp": "2025-11-27T00:28:35.447Z", "environment": { "benchmarkDotNetVersion": "BenchmarkDotNet v0.15.7, Linux Ubuntu 24.04.3 LTS (Noble Numbat)", "sdk": ".NET SDK 10.0.100", @@ -10,244 +10,244 @@ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "573.3 ms", - "Error": "4.73 ms", - "StdDev": "4.43 ms", - "Median": "573.7 ms" + "Mean": "567.1 ms", + "Error": "3.08 ms", + "StdDev": "2.57 ms", + "Median": "566.4 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "658.8 ms", - "Error": "5.97 ms", - "StdDev": "4.99 ms", - "Median": "660.6 ms" + "Mean": "688.2 ms", + "Error": "13.26 ms", + "StdDev": "13.62 ms", + "Median": "683.1 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "631.5 ms", - "Error": "9.34 ms", - "StdDev": "8.74 ms", - "Median": "631.2 ms" + "Mean": "658.5 ms", + "Error": "12.82 ms", + "StdDev": "10.71 ms", + "Median": "659.6 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "716.2 ms", - "Error": "7.08 ms", - "StdDev": "6.28 ms", - "Median": "715.4 ms" + "Version": "3.2.0", + "Mean": "736.1 ms", + "Error": "14.47 ms", + "StdDev": "13.54 ms", + "Median": "733.4 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "124.1 ms", - "Error": "0.23 ms", - "StdDev": "0.20 ms", - "Median": "124.2 ms" + "Mean": "124.7 ms", + "Error": "0.45 ms", + "StdDev": "0.42 ms", + "Median": "124.8 ms" } ], "DataDrivenTests": [ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "501.86 ms", - "Error": "5.017 ms", - "StdDev": "4.693 ms", - "Median": "501.49 ms" + "Mean": "497.25 ms", + "Error": "4.213 ms", + "StdDev": "3.518 ms", + "Median": "497.97 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "535.87 ms", - "Error": "10.415 ms", - "StdDev": "12.790 ms", - "Median": "534.70 ms" + "Mean": "526.05 ms", + "Error": "7.107 ms", + "StdDev": "6.301 ms", + "Median": "524.98 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "502.71 ms", - "Error": "9.708 ms", - "StdDev": "12.277 ms", - "Median": "503.69 ms" + "Mean": "482.94 ms", + "Error": "9.302 ms", + "StdDev": "10.339 ms", + "Median": "483.13 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "579.28 ms", - "Error": "9.180 ms", - "StdDev": "8.138 ms", - "Median": "578.02 ms" + "Version": "3.2.0", + "Mean": "573.56 ms", + "Error": "10.737 ms", + "StdDev": "10.043 ms", + "Median": "576.44 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "24.82 ms", - "Error": "0.187 ms", - "StdDev": "0.166 ms", - "Median": "24.80 ms" + "Mean": "24.70 ms", + "Error": "0.194 ms", + "StdDev": "0.182 ms", + "Median": "24.72 ms" } ], "MassiveParallelTests": [ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "597.5 ms", - "Error": "4.05 ms", - "StdDev": "3.59 ms", - "Median": "597.8 ms" + "Mean": "623.3 ms", + "Error": "4.42 ms", + "StdDev": "4.14 ms", + "Median": "624.8 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,183.2 ms", - "Error": "6.27 ms", - "StdDev": "4.90 ms", - "Median": "1,182.1 ms" + "Mean": "1,212.4 ms", + "Error": "10.13 ms", + "StdDev": "9.48 ms", + "Median": "1,210.2 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "2,966.5 ms", - "Error": "6.60 ms", - "StdDev": "5.16 ms", - "Median": "2,968.0 ms" + "Mean": "3,001.4 ms", + "Error": "12.87 ms", + "StdDev": "10.05 ms", + "Median": "2,999.8 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "3,066.1 ms", - "Error": "17.64 ms", - "StdDev": "15.64 ms", - "Median": "3,065.1 ms" + "Version": "3.2.0", + "Mean": "3,086.1 ms", + "Error": "9.06 ms", + "StdDev": "7.57 ms", + "Median": "3,086.8 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "132.2 ms", - "Error": "0.55 ms", - "StdDev": "0.51 ms", - "Median": "132.3 ms" + "Mean": "132.7 ms", + "Error": "0.58 ms", + "StdDev": "0.54 ms", + "Median": "132.6 ms" } ], "MatrixTests": [ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "574.81 ms", - "Error": "5.792 ms", - "StdDev": "5.417 ms", - "Median": "575.40 ms" + "Mean": "566.13 ms", + "Error": "6.468 ms", + "StdDev": "6.050 ms", + "Median": "565.43 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,582.59 ms", - "Error": "7.738 ms", - "StdDev": "6.462 ms", - "Median": "1,581.77 ms" + "Mean": "1,578.62 ms", + "Error": "7.240 ms", + "StdDev": "6.773 ms", + "Median": "1,578.58 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,540.42 ms", - "Error": "5.822 ms", - "StdDev": "5.446 ms", - "Median": "1,540.75 ms" + "Mean": "1,539.67 ms", + "Error": "4.489 ms", + "StdDev": "3.979 ms", + "Median": "1,540.17 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "1,630.94 ms", - "Error": "12.411 ms", - "StdDev": "11.609 ms", - "Median": "1,627.40 ms" + "Version": "3.2.0", + "Mean": "1,625.28 ms", + "Error": "13.608 ms", + "StdDev": "11.364 ms", + "Median": "1,624.99 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "79.00 ms", - "Error": "0.251 ms", - "StdDev": "0.235 ms", - "Median": "79.08 ms" + "Mean": "79.52 ms", + "Error": "0.365 ms", + "StdDev": "0.341 ms", + "Median": "79.55 ms" } ], "ScaleTests": [ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "535.07 ms", - "Error": "5.832 ms", - "StdDev": "4.870 ms", - "Median": "534.68 ms" + "Mean": "522.37 ms", + "Error": "5.373 ms", + "StdDev": "5.026 ms", + "Median": "521.16 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "585.44 ms", - "Error": "11.274 ms", - "StdDev": "12.531 ms", - "Median": "584.11 ms" + "Mean": "611.57 ms", + "Error": "9.860 ms", + "StdDev": "8.234 ms", + "Median": "612.00 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "509.11 ms", - "Error": "9.925 ms", - "StdDev": "11.032 ms", - "Median": "505.09 ms" + "Mean": "615.51 ms", + "Error": "10.914 ms", + "StdDev": "9.675 ms", + "Median": "617.04 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "594.97 ms", - "Error": "7.221 ms", - "StdDev": "6.401 ms", - "Median": "593.95 ms" + "Version": "3.2.0", + "Mean": "614.63 ms", + "Error": "8.361 ms", + "StdDev": "7.412 ms", + "Median": "610.65 ms" }, { "Method": "TUnit_AOT", "Version": "1.2.11", - "Mean": "46.00 ms", - "Error": "1.346 ms", - "StdDev": "3.968 ms", - "Median": "46.41 ms" + "Mean": "43.96 ms", + "Error": "1.151 ms", + "StdDev": "3.394 ms", + "Median": "44.10 ms" } ], "SetupTeardownTests": [ { "Method": "TUnit", "Version": "1.2.11", - "Mean": "558.0 ms", - "Error": "4.08 ms", - "StdDev": "3.82 ms", - "Median": "557.6 ms" + "Mean": "575.6 ms", + "Error": "4.96 ms", + "StdDev": "4.64 ms", + "Median": "575.7 ms" }, { "Method": "NUnit", "Version": "4.4.0", - "Mean": "1,148.5 ms", - "Error": "8.81 ms", - "StdDev": "8.24 ms", - "Median": "1,146.8 ms" + "Mean": "1,194.3 ms", + "Error": "7.33 ms", + "StdDev": "6.50 ms", + "Median": "1,195.1 ms" }, { "Method": "MSTest", "Version": "4.0.2", - "Mean": "1,120.1 ms", - "Error": "7.69 ms", - "StdDev": "6.42 ms", - "Median": "1,121.5 ms" + "Mean": "1,165.8 ms", + "Error": "10.55 ms", + "StdDev": "9.87 ms", + "Median": "1,164.6 ms" }, { "Method": "xUnit3", - "Version": "3.2.1", - "Mean": "1,198.8 ms", - "Error": "9.73 ms", - "StdDev": "9.10 ms", - "Median": "1,197.8 ms" + "Version": "3.2.0", + "Mean": "1,258.5 ms", + "Error": "8.37 ms", + "StdDev": "7.83 ms", + "Median": "1,258.8 ms" }, { "Method": "TUnit_AOT", @@ -264,34 +264,34 @@ { "Method": "Build_TUnit", "Version": "1.2.11", - "Mean": "2.050 s", - "Error": "0.0285 s", - "StdDev": "0.0253 s", - "Median": "2.047 s" + "Mean": "2.043 s", + "Error": "0.0210 s", + "StdDev": "0.0196 s", + "Median": "2.035 s" }, { "Method": "Build_NUnit", "Version": "4.4.0", - "Mean": "1.644 s", - "Error": "0.0260 s", - "StdDev": "0.0243 s", - "Median": "1.646 s" + "Mean": "1.639 s", + "Error": "0.0094 s", + "StdDev": "0.0088 s", + "Median": "1.639 s" }, { "Method": "Build_MSTest", "Version": "4.0.2", - "Mean": "1.714 s", - "Error": "0.0324 s", - "StdDev": "0.0303 s", - "Median": "1.709 s" + "Mean": "1.715 s", + "Error": "0.0184 s", + "StdDev": "0.0172 s", + "Median": "1.712 s" }, { "Method": "Build_xUnit3", - "Version": "3.2.1", - "Mean": "1.639 s", - "Error": "0.0321 s", - "StdDev": "0.0300 s", - "Median": "1.639 s" + "Version": "3.2.0", + "Mean": "1.616 s", + "Error": "0.0302 s", + "StdDev": "0.0283 s", + "Median": "1.621 s" } ] }, @@ -299,6 +299,6 @@ "runtimeCategories": 6, "buildCategories": 1, "totalBenchmarks": 7, - "lastUpdated": "2025-11-29T00:27:17.984Z" + "lastUpdated": "2025-11-27T00:28:35.445Z" } } \ No newline at end of file diff --git a/docs/static/benchmarks/summary.json b/docs/static/benchmarks/summary.json index 78d36d0b54..ff4e003094 100644 --- a/docs/static/benchmarks/summary.json +++ b/docs/static/benchmarks/summary.json @@ -10,6 +10,6 @@ "build": [ "BuildTime" ], - "timestamp": "2025-11-29", + "timestamp": "2025-11-27", "environment": "Ubuntu Latest • .NET SDK 10.0.100" } \ No newline at end of file