diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 70e0fda6d2..a950d1a14e 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -8,6 +8,13 @@ "dotnet-template-authoring" ], "rollForward": false + }, + "verify.tool": { + "version": "0.6.0", + "commands": [ + "dotnet-verify" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/TUnit.Core/Interfaces/ITestExecution.cs b/TUnit.Core/Interfaces/ITestExecution.cs index 88e12ea337..16a3bd6d50 100644 --- a/TUnit.Core/Interfaces/ITestExecution.cs +++ b/TUnit.Core/Interfaces/ITestExecution.cs @@ -47,18 +47,52 @@ public interface ITestExecution /// Func>? RetryFunc { get; } - /// - /// Overrides the test result with a passed state and custom reason. - /// Useful for marking tests as passed under special conditions. - /// - /// The reason for overriding the result - void OverrideResult(string reason); - /// /// Overrides the test result with a specific state and custom reason. /// - /// The desired test state (Passed, Failed, Skipped, etc.) - /// The reason for overriding the result + /// The desired test state (Passed, Failed, Skipped, Timeout, or Cancelled) + /// The reason for overriding the result (cannot be empty) + /// Thrown when reason is empty, whitespace, or state is invalid (NotStarted, WaitingForDependencies, Queued, Running) + /// Thrown when result has already been overridden + /// + /// This method can only be called once per test. Subsequent calls will throw an exception. + /// Only final states are allowed: Passed, Failed, Skipped, Timeout, or Cancelled. Intermediate states like Running, Queued, NotStarted, or WaitingForDependencies are rejected. + /// The original exception (if any) is preserved in . + /// When overriding to Failed, the original exception is retained in . + /// When overriding to Passed or Skipped, the Exception property is cleared but preserved in OriginalException. + /// Best practice: Call this from or After(Test) hooks. + /// + /// + /// + /// // Override failed test to passed + /// public class RetryOnInfrastructureErrorAttribute : Attribute, ITestEndEventReceiver + /// { + /// public ValueTask OnTestEnd(TestContext context) + /// { + /// if (context.Result?.Exception is HttpRequestException) + /// { + /// context.Execution.OverrideResult(TestState.Passed, "Infrastructure error - not a test failure"); + /// } + /// return default; + /// } + /// public int Order => 0; + /// } + /// + /// // Override failed test to skipped + /// public class IgnoreOnWeekendAttribute : Attribute, ITestEndEventReceiver + /// { + /// public ValueTask OnTestEnd(TestContext context) + /// { + /// if (context.Result?.State == TestState.Failed && DateTime.Now.DayOfWeek is DayOfWeek.Saturday or DayOfWeek.Sunday) + /// { + /// context.Execution.OverrideResult(TestState.Skipped, "Failures ignored on weekends"); + /// } + /// return default; + /// } + /// public int Order => 0; + /// } + /// + /// void OverrideResult(TestState state, string reason); /// diff --git a/TUnit.Core/TestContext.Execution.cs b/TUnit.Core/TestContext.Execution.cs index bf5da16dec..6f75db3580 100644 --- a/TUnit.Core/TestContext.Execution.cs +++ b/TUnit.Core/TestContext.Execution.cs @@ -62,32 +62,62 @@ bool ITestExecution.ReportResult set => ReportResult = value; } - void ITestExecution.OverrideResult(string reason) => OverrideResult(reason); void ITestExecution.OverrideResult(TestState state, string reason) => OverrideResult(state, reason); void ITestExecution.AddLinkedCancellationToken(CancellationToken cancellationToken) => AddLinkedCancellationToken(cancellationToken); // Internal implementation methods - internal void OverrideResult(string reason) - { - OverrideResult(TestState.Passed, reason); - } - internal void OverrideResult(TestState state, string reason) { - Result = new TestResult + lock (Lock) { - State = state, - OverrideReason = reason, - IsOverridden = true, - Start = TestStart ?? DateTimeOffset.UtcNow, - End = DateTimeOffset.UtcNow, - Duration = DateTimeOffset.UtcNow - (TestStart ?? DateTimeOffset.UtcNow), - Exception = null, - ComputerName = Environment.MachineName, - TestContext = this - }; - - InternalExecutableTest.State = state; + if (string.IsNullOrWhiteSpace(reason)) + { + throw new ArgumentException("Override reason cannot be empty or whitespace.", nameof(reason)); + } + + if (Result?.IsOverridden == true) + { + throw new InvalidOperationException( + $"Result has already been overridden to {Result.State} with reason: '{Result.OverrideReason}'. " + + "Cannot override a result multiple times. Check Result.IsOverridden before calling OverrideResult()."); + } + + if (state is TestState.NotStarted or TestState.WaitingForDependencies or TestState.Queued or TestState.Running) + { + throw new ArgumentException( + $"Cannot override to intermediate state '{state}'. " + + "Only final states (Passed, Failed, Skipped, Timeout, Cancelled) are allowed.", + nameof(state)); + } + + var originalException = Result?.Exception; + + Exception? exceptionForResult; + if (state == TestState.Failed) + { + exceptionForResult = originalException ?? new InvalidOperationException($"Test overridden to failed: {reason}"); + } + else + { + exceptionForResult = null; + } + + Result = new TestResult + { + State = state, + OverrideReason = reason, + IsOverridden = true, + OriginalException = originalException, + Start = TestStart ?? DateTimeOffset.UtcNow, + End = DateTimeOffset.UtcNow, + Duration = DateTimeOffset.UtcNow - (TestStart ?? DateTimeOffset.UtcNow), + Exception = exceptionForResult, + ComputerName = Environment.MachineName, + TestContext = this + }; + + InternalExecutableTest.State = state; + } } internal void AddLinkedCancellationToken(CancellationToken cancellationToken) diff --git a/TUnit.Core/TestResult.cs b/TUnit.Core/TestResult.cs index bdf32d85f9..c37fe74e38 100644 --- a/TUnit.Core/TestResult.cs +++ b/TUnit.Core/TestResult.cs @@ -24,6 +24,19 @@ public record TestResult [JsonIgnore] internal TestContext? TestContext { get; init; } - public string? OverrideReason { get; set; } - public bool IsOverridden { get; set; } + /// + /// The reason provided when this result was overridden. + /// + public string? OverrideReason { get; init; } + + /// + /// Indicates whether this result was explicitly overridden via . + /// + public bool IsOverridden { get; init; } + + /// + /// The original exception that occurred before the result was overridden. + /// Useful for debugging and audit trails when a test failure is overridden to pass or skip. + /// + public Exception? OriginalException { get; init; } } diff --git a/TUnit.Engine.Tests/OverrideResultsTests.cs b/TUnit.Engine.Tests/OverrideResultsTests.cs index 4192d1eb1e..ea130230e7 100644 --- a/TUnit.Engine.Tests/OverrideResultsTests.cs +++ b/TUnit.Engine.Tests/OverrideResultsTests.cs @@ -11,11 +11,11 @@ public async Task Test() await RunTestsWithFilter( "/*/*/OverrideResultsTests/*", [ - result => result.ResultSummary.Outcome.ShouldBe("Completed"), - result => result.ResultSummary.Counters.Total.ShouldBe(1), - result => result.ResultSummary.Counters.Passed.ShouldBe(1), - result => result.ResultSummary.Counters.Failed.ShouldBe(0), - result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0) + result => result.ResultSummary.Outcome.ShouldBe("Failed"), + result => result.ResultSummary.Counters.Total.ShouldBe(4), + result => result.ResultSummary.Counters.Passed.ShouldBe(2), + result => result.ResultSummary.Counters.Failed.ShouldBe(1), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(1) ]); } } diff --git a/TUnit.Engine/TestExecutor.cs b/TUnit.Engine/TestExecutor.cs index bde883ffac..5e92baec55 100644 --- a/TUnit.Engine/TestExecutor.cs +++ b/TUnit.Engine/TestExecutor.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Reflection; +using System.Runtime.ExceptionServices; using TUnit.Core; using TUnit.Core.Exceptions; using TUnit.Core.Interfaces; @@ -57,11 +58,13 @@ public async Task ExecuteAsync(AbstractExecutableTest executableTest, Cancellati var testClass = executableTest.Metadata.TestClassType; var testAssembly = testClass.Assembly; + Exception? capturedException = null; + Exception? hookException = null; + try { await EnsureTestSessionHooksExecutedAsync().ConfigureAwait(false); - // Event receivers have their own internal coordination to run once await _eventReceiverOrchestrator.InvokeFirstTestInSessionEventReceiversAsync( executableTest.Context, executableTest.Context.ClassContext.AssemblyContext.TestSessionContext, @@ -72,7 +75,6 @@ await _eventReceiverOrchestrator.InvokeFirstTestInSessionEventReceiversAsync( await _beforeHookTaskCache.GetOrCreateBeforeAssemblyTask(testAssembly, assembly => _hookExecutor.ExecuteBeforeAssemblyHooksAsync(assembly, CancellationToken.None)) .ConfigureAwait(false); - // Event receivers for first test in assembly await _eventReceiverOrchestrator.InvokeFirstTestInAssemblyEventReceiversAsync( executableTest.Context, executableTest.Context.ClassContext.AssemblyContext, @@ -83,7 +85,6 @@ await _eventReceiverOrchestrator.InvokeFirstTestInAssemblyEventReceiversAsync( await _beforeHookTaskCache.GetOrCreateBeforeClassTask(testClass, _ => _hookExecutor.ExecuteBeforeClassHooksAsync(testClass, CancellationToken.None)) .ConfigureAwait(false); - // Event receivers for first test in class await _eventReceiverOrchestrator.InvokeFirstTestInClassEventReceiversAsync( executableTest.Context, executableTest.Context.ClassContext, @@ -97,7 +98,6 @@ await _eventReceiverOrchestrator.InvokeFirstTestInClassEventReceiversAsync( executableTest.Context.RestoreExecutionContext(); - // Only wrap the actual test execution with timeout, not the hooks var testTimeout = executableTest.Context.Metadata.TestDetails.Timeout; var timeoutMessage = testTimeout.HasValue ? $"Test '{executableTest.Context.Metadata.TestDetails.TestName}' execution timed out after {testTimeout.Value}" @@ -111,51 +111,40 @@ await TimeoutHelper.ExecuteWithTimeoutAsync( executableTest.SetResult(TestState.Passed); } - catch (SkipTestException) + catch (SkipTestException ex) { executableTest.SetResult(TestState.Skipped); - throw; + capturedException = ex; } catch (Exception ex) { executableTest.SetResult(TestState.Failed, ex); - - // Run per-retry cleanup hooks before re-throwing + capturedException = ex; + } + finally + { try { - // Run After(Test) hooks await _hookExecutor.ExecuteAfterTestHooksAsync(executableTest, cancellationToken).ConfigureAwait(false); - await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, cancellationToken).ConfigureAwait(false); } - catch + catch (Exception ex) { - // Swallow any exceptions from hooks when we already have a test failure + hookException = ex; } + } - // Check if the result was overridden - if so, don't re-throw - if (executableTest.Context.Execution.Result is { IsOverridden: true, State: TestState.Passed }) - { - // Result was overridden to passed, don't re-throw the exception - executableTest.SetResult(TestState.Passed); - } - else - { - throw; - } + if (capturedException == null && hookException != null) + { + ExceptionDispatchInfo.Capture(hookException).Throw(); } - finally + else if (capturedException is SkipTestException) { - // Per-retry cleanup - runs for success path only - if (executableTest.State != TestState.Failed) - { - // Run After(Test) hooks - await _hookExecutor.ExecuteAfterTestHooksAsync(executableTest, cancellationToken).ConfigureAwait(false); - - await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(executableTest.Context, cancellationToken).ConfigureAwait(false); - } - // Note: Test instance disposal and After(Class/Assembly/Session) hooks - // are now handled in TestCoordinator after the retry loop completes + ExceptionDispatchInfo.Capture(capturedException).Throw(); + } + else if (capturedException != null && executableTest.Context.Execution.Result?.IsOverridden != true) + { + ExceptionDispatchInfo.Capture(capturedException).Throw(); } } diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 89f8aef8ae..1b07fe67bc 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -1460,9 +1460,10 @@ namespace public required ? Duration { get; init; } public required ? End { get; init; } public required ? Exception { get; init; } - public bool IsOverridden { get; set; } + public bool IsOverridden { get; init; } + public ? OriginalException { get; init; } public string? Output { get; } - public string? OverrideReason { get; set; } + public string? OverrideReason { get; init; } public required ? Start { get; init; } public required .TestState State { get; init; } } @@ -2329,7 +2330,6 @@ namespace .Interfaces ? TestEnd { get; } ? TestStart { get; } void AddLinkedCancellationToken(.CancellationToken cancellationToken); - void OverrideResult(string reason); void OverrideResult(.TestState state, string reason); } public interface ITestExecutor diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt index 3b5e11f8a7..697b82f798 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -1460,9 +1460,10 @@ namespace public required ? Duration { get; init; } public required ? End { get; init; } public required ? Exception { get; init; } - public bool IsOverridden { get; set; } + public bool IsOverridden { get; init; } + public ? OriginalException { get; init; } public string? Output { get; } - public string? OverrideReason { get; set; } + public string? OverrideReason { get; init; } public required ? Start { get; init; } public required .TestState State { get; init; } } @@ -2329,7 +2330,6 @@ namespace .Interfaces ? TestEnd { get; } ? TestStart { get; } void AddLinkedCancellationToken(.CancellationToken cancellationToken); - void OverrideResult(string reason); void OverrideResult(.TestState state, string reason); } public interface ITestExecutor diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt index f18ebe2c0f..430287c741 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -1460,9 +1460,10 @@ namespace public required ? Duration { get; init; } public required ? End { get; init; } public required ? Exception { get; init; } - public bool IsOverridden { get; set; } + public bool IsOverridden { get; init; } + public ? OriginalException { get; init; } public string? Output { get; } - public string? OverrideReason { get; set; } + public string? OverrideReason { get; init; } public required ? Start { get; init; } public required .TestState State { get; init; } } @@ -2329,7 +2330,6 @@ namespace .Interfaces ? TestEnd { get; } ? TestStart { get; } void AddLinkedCancellationToken(.CancellationToken cancellationToken); - void OverrideResult(string reason); void OverrideResult(.TestState state, string reason); } public interface ITestExecutor diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt index e7c1406e71..144d2cf634 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -1413,9 +1413,10 @@ namespace public required ? Duration { get; init; } public required ? End { get; init; } public required ? Exception { get; init; } - public bool IsOverridden { get; set; } + public bool IsOverridden { get; init; } + public ? OriginalException { get; init; } public string? Output { get; } - public string? OverrideReason { get; set; } + public string? OverrideReason { get; init; } public required ? Start { get; init; } public required .TestState State { get; init; } } @@ -2260,7 +2261,6 @@ namespace .Interfaces ? TestEnd { get; } ? TestStart { get; } void AddLinkedCancellationToken(.CancellationToken cancellationToken); - void OverrideResult(string reason); void OverrideResult(.TestState state, string reason); } public interface ITestExecutor diff --git a/TUnit.TestProject/OverrideResultsTests.cs b/TUnit.TestProject/OverrideResultsTests.cs index 167a5a758a..f561b5219e 100644 --- a/TUnit.TestProject/OverrideResultsTests.cs +++ b/TUnit.TestProject/OverrideResultsTests.cs @@ -3,27 +3,102 @@ namespace TUnit.TestProject; -[EngineTest(ExpectedResult.Pass)] public class OverrideResultsTests { [Test, OverridePass] - public void OverrideResult_Throws_When_TestResult_Is_Null() + public void OverrideFailedTestToPassed() { - throw new InvalidOperationException(); + throw new InvalidOperationException("This test should fail but will be overridden to passed"); + } + + [Test, OverrideToSkipped] + public void OverrideFailedTestToSkipped() + { + throw new ArgumentException("This test should fail but will be overridden to skipped"); + } + + [Test, OverrideToFailed] + public void OverridePassedTestToFailed() + { + // This test passes but will be overridden to failed + } + + [Test, PreserveOriginalException] + public void VerifyOriginalExceptionIsPreserved() + { + throw new InvalidOperationException("Original exception that should be preserved"); } [After(Class)] public static async Task AfterClass(ClassHookContext classHookContext) { - await Assert.That(classHookContext.Tests).HasSingleItem(); - await Assert.That(classHookContext.Tests).ContainsOnly(t => t.Execution.Result?.State == TestState.Passed); + await Assert.That(classHookContext.Tests.Count).IsEqualTo(4); + + // Test 1: Failed -> Passed + var test1 = classHookContext.Tests.First(t => t.Metadata.TestDetails.TestName == "OverrideFailedTestToPassed"); + await Assert.That(test1.Execution.Result?.State).IsEqualTo(TestState.Passed); + await Assert.That(test1.Execution.Result?.IsOverridden).IsTrue(); + await Assert.That(test1.Execution.Result?.OverrideReason).IsEqualTo("Overridden to passed"); + await Assert.That(test1.Execution.Result?.OriginalException).IsNotNull(); + await Assert.That(test1.Execution.Result?.OriginalException).IsTypeOf(); + + // Test 2: Failed -> Skipped + var test2 = classHookContext.Tests.First(t => t.Metadata.TestDetails.TestName == "OverrideFailedTestToSkipped"); + await Assert.That(test2.Execution.Result?.State).IsEqualTo(TestState.Skipped); + await Assert.That(test2.Execution.Result?.IsOverridden).IsTrue(); + await Assert.That(test2.Execution.Result?.OverrideReason).IsEqualTo("Overridden to skipped"); + await Assert.That(test2.Execution.Result?.OriginalException).IsNotNull(); + await Assert.That(test2.Execution.Result?.OriginalException).IsTypeOf(); + + // Test 3: Passed -> Failed + var test3 = classHookContext.Tests.First(t => t.Metadata.TestDetails.TestName == "OverridePassedTestToFailed"); + await Assert.That(test3.Execution.Result?.State).IsEqualTo(TestState.Failed); + await Assert.That(test3.Execution.Result?.IsOverridden).IsTrue(); + await Assert.That(test3.Execution.Result?.OverrideReason).IsEqualTo("Overridden to failed"); + + // Test 4: Verify original exception preservation + var test4 = classHookContext.Tests.First(t => t.Metadata.TestDetails.TestName == "VerifyOriginalExceptionIsPreserved"); + await Assert.That(test4.Execution.Result?.OriginalException?.Message).IsEqualTo("Original exception that should be preserved"); } public class OverridePassAttribute : Attribute, ITestEndEventReceiver { public ValueTask OnTestEnd(TestContext afterTestContext) { - afterTestContext.Execution.OverrideResult(TestState.Passed, "Because I said so"); + afterTestContext.Execution.OverrideResult(TestState.Passed, "Overridden to passed"); + return default(ValueTask); + } + + public int Order => 0; + } + + public class OverrideToSkippedAttribute : Attribute, ITestEndEventReceiver + { + public ValueTask OnTestEnd(TestContext afterTestContext) + { + afterTestContext.Execution.OverrideResult(TestState.Skipped, "Overridden to skipped"); + return default(ValueTask); + } + + public int Order => 0; + } + + public class OverrideToFailedAttribute : Attribute, ITestEndEventReceiver + { + public ValueTask OnTestEnd(TestContext afterTestContext) + { + afterTestContext.Execution.OverrideResult(TestState.Failed, "Overridden to failed"); + return default(ValueTask); + } + + public int Order => 0; + } + + public class PreserveOriginalExceptionAttribute : Attribute, ITestEndEventReceiver + { + public ValueTask OnTestEnd(TestContext afterTestContext) + { + afterTestContext.Execution.OverrideResult(TestState.Passed, "Test passed after retry"); return default(ValueTask); }