From 3145c1a3e493154467d2dd0357ea39dc6d76564c Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 19 Feb 2026 04:19:45 +0000 Subject: [PATCH 1/2] feat: add intelligent test failure categorization Automatically categorize test failures to help users quickly understand what went wrong. Categories include: Assertion, Timeout, NullReference, Setup (Before hooks), Teardown (After hooks), Infrastructure (IO/network), and Unknown. The category label is prefixed in the failure explanation visible in test reports. --- TUnit.Engine/Enums/FailureCategory.cs | 42 ++++++++++ TUnit.Engine/Services/FailureCategorizer.cs | 92 +++++++++++++++++++++ TUnit.Engine/TUnitMessageBus.cs | 9 +- 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 TUnit.Engine/Enums/FailureCategory.cs create mode 100644 TUnit.Engine/Services/FailureCategorizer.cs diff --git a/TUnit.Engine/Enums/FailureCategory.cs b/TUnit.Engine/Enums/FailureCategory.cs new file mode 100644 index 0000000000..6b84ad7150 --- /dev/null +++ b/TUnit.Engine/Enums/FailureCategory.cs @@ -0,0 +1,42 @@ +namespace TUnit.Engine.Enums; + +/// +/// Categorizes test failures to help users quickly understand what went wrong. +/// +internal enum FailureCategory +{ + /// + /// TUnit assertion failure (AssertionException). + /// + Assertion, + + /// + /// Timeout or cancellation failure (OperationCanceledException, TaskCanceledException, TimeoutException). + /// + Timeout, + + /// + /// NullReferenceException in user code. + /// + NullReference, + + /// + /// Failure in a Before/BeforeEvery hook. + /// + Setup, + + /// + /// Failure in an After/AfterEvery hook. + /// + Teardown, + + /// + /// File, network, or other I/O exception. + /// + Infrastructure, + + /// + /// Unrecognized failure type. + /// + Unknown +} diff --git a/TUnit.Engine/Services/FailureCategorizer.cs b/TUnit.Engine/Services/FailureCategorizer.cs new file mode 100644 index 0000000000..5b088a257e --- /dev/null +++ b/TUnit.Engine/Services/FailureCategorizer.cs @@ -0,0 +1,92 @@ +using TUnit.Core.Exceptions; +using TUnit.Engine.Enums; + +namespace TUnit.Engine.Services; + +/// +/// Examines exceptions from test failures and categorizes them to help users +/// quickly understand what went wrong. +/// +internal static class FailureCategorizer +{ + /// + /// Categorizes the given exception into a . + /// Unwraps to inspect the first inner exception. + /// + public static FailureCategory Categorize(Exception exception) + { + // Unwrap AggregateException to get the real cause + var ex = exception is AggregateException { InnerExceptions.Count: > 0 } agg + ? agg.InnerExceptions[0] + : exception; + + // Setup hooks (Before*) + if (ex is BeforeTestException + or BeforeClassException + or BeforeAssemblyException + or BeforeTestSessionException + or BeforeTestDiscoveryException) + { + return FailureCategory.Setup; + } + + // Teardown hooks (After*) + if (ex is AfterTestException + or AfterClassException + or AfterAssemblyException + or AfterTestSessionException + or AfterTestDiscoveryException) + { + return FailureCategory.Teardown; + } + + // Assertion failures - check by type name to support third-party assertion libraries + if (ex.GetType().Name.Contains("Assertion", StringComparison.Ordinal) + || ex.GetType().Name.Contains("Assert", StringComparison.Ordinal)) + { + return FailureCategory.Assertion; + } + + // Timeout / cancellation + if (ex is OperationCanceledException + or TaskCanceledException + or System.TimeoutException + or TUnit.Core.Exceptions.TimeoutException) + { + return FailureCategory.Timeout; + } + + // NullReference + if (ex is NullReferenceException) + { + return FailureCategory.NullReference; + } + + // Infrastructure (I/O, network, file system) + if (ex is IOException + or System.Net.Sockets.SocketException + or System.Net.Http.HttpRequestException + or UnauthorizedAccessException) + { + return FailureCategory.Infrastructure; + } + + return FailureCategory.Unknown; + } + + /// + /// Returns a short human-readable label for the category, + /// suitable for prefixing failure messages in reports. + /// + public static string GetLabel(FailureCategory category) => category switch + { + FailureCategory.Assertion => "Assertion Failure", + FailureCategory.Timeout => "Timeout", + FailureCategory.NullReference => "Null Reference", + FailureCategory.Setup => "Setup Failure", + FailureCategory.Teardown => "Teardown Failure", + FailureCategory.Infrastructure => "Infrastructure Failure", + FailureCategory.Unknown => "Test Failure", + _ => "Test Failure" + }; +} diff --git a/TUnit.Engine/TUnitMessageBus.cs b/TUnit.Engine/TUnitMessageBus.cs index e85505dd03..b3f4278091 100644 --- a/TUnit.Engine/TUnitMessageBus.cs +++ b/TUnit.Engine/TUnitMessageBus.cs @@ -143,19 +143,22 @@ public ValueTask PublishOutputUpdate(TestNode testNode) private static TestNodeStateProperty GetFailureStateProperty(TestContext testContext, Exception e, TimeSpan duration) { + var category = FailureCategorizer.Categorize(e); + var categoryLabel = FailureCategorizer.GetLabel(category); + if (testContext.Metadata.TestDetails.Timeout != null && e is TaskCanceledException or OperationCanceledException or TimeoutException && duration >= testContext.Metadata.TestDetails.Timeout.Value) { - return new TimeoutTestNodeStateProperty($"Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms"); + return new TimeoutTestNodeStateProperty($"[{categoryLabel}] Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms"); } if (e.GetType().Name.Contains("Assertion", StringComparison.InvariantCulture)) { - return new FailedTestNodeStateProperty(e); + return new FailedTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}"); } - return new ErrorTestNodeStateProperty(e); + return new ErrorTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}"); } public Task IsEnabledAsync() From 0c877fa9c8f38d78e7514468d5fff06edba774bf Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Thu, 19 Feb 2026 07:46:05 +0000 Subject: [PATCH 2/2] fix: use categorizer result consistently and unwrap AggregateException at top of GetFailureStateProperty The method was computing a category via FailureCategorizer but then ignoring it for routing, using separate type-checking logic against the original (potentially wrapped) exception. This caused two bugs: 1. AggregateException wrapping e.g. TaskCanceledException would be correctly categorized as Timeout by the categorizer, but the method's own `e is TaskCanceledException` check would fail since `e` was still the AggregateException. 2. The assertion check used a different string comparison (InvariantCulture) than the categorizer (Ordinal) and only checked for "Assertion" rather than also checking "Assert". Now AggregateException is unwrapped once at the top, the unwrapped exception is passed to the categorizer, and the FailureCategory enum drives all routing and is used for constructing state properties. --- TUnit.Engine/TUnitMessageBus.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/TUnit.Engine/TUnitMessageBus.cs b/TUnit.Engine/TUnitMessageBus.cs index b3f4278091..24b9bb2fa4 100644 --- a/TUnit.Engine/TUnitMessageBus.cs +++ b/TUnit.Engine/TUnitMessageBus.cs @@ -6,6 +6,7 @@ using Microsoft.Testing.Platform.TestHost; using TUnit.Core; using TUnit.Engine.CommandLineProviders; +using TUnit.Engine.Enums; using TUnit.Engine.Exceptions; using TUnit.Engine.Extensions; using TUnit.Engine.Services; @@ -143,22 +144,27 @@ public ValueTask PublishOutputUpdate(TestNode testNode) private static TestNodeStateProperty GetFailureStateProperty(TestContext testContext, Exception e, TimeSpan duration) { - var category = FailureCategorizer.Categorize(e); + // Unwrap AggregateException once so all downstream logic sees the real cause + var unwrapped = e is AggregateException { InnerExceptions.Count: > 0 } agg + ? agg.InnerExceptions[0] + : e; + + var category = FailureCategorizer.Categorize(unwrapped); var categoryLabel = FailureCategorizer.GetLabel(category); - if (testContext.Metadata.TestDetails.Timeout != null - && e is TaskCanceledException or OperationCanceledException or TimeoutException + if (category == FailureCategory.Timeout + && testContext.Metadata.TestDetails.Timeout != null && duration >= testContext.Metadata.TestDetails.Timeout.Value) { return new TimeoutTestNodeStateProperty($"[{categoryLabel}] Test timed out after {testContext.Metadata.TestDetails.Timeout.Value.TotalMilliseconds}ms"); } - if (e.GetType().Name.Contains("Assertion", StringComparison.InvariantCulture)) + if (category == FailureCategory.Assertion) { - return new FailedTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}"); + return new FailedTestNodeStateProperty(unwrapped, $"[{categoryLabel}] {unwrapped.Message}"); } - return new ErrorTestNodeStateProperty(e, $"[{categoryLabel}] {e.Message}"); + return new ErrorTestNodeStateProperty(unwrapped, $"[{categoryLabel}] {unwrapped.Message}"); } public Task IsEnabledAsync()