Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 143 additions & 0 deletions TrxLib.Tests/TrxParserRegressionTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
using AwesomeAssertions;

namespace TrxLib.Tests;

/// <summary>
/// Regression tests verifying TRX parsing correctness against the vstest object model.
/// </summary>
public class TrxParserRegressionTests
{
// -------------------------------------------------------------------------
// Outcome parsing – All vstest outcome strings (Error, Aborted, NotRunnable,
// Disconnected, Warning, Completed, InProgress, PassedButRunAborted) must
// round-trip correctly through the parser. A missing outcome attribute must
// map to TestOutcome.Error, not NotExecuted.
// -------------------------------------------------------------------------

[Fact]
public void Parse_ErrorOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="Error" when the test adapter itself crashes or the
// test process aborts unexpectedly. This is NOT the same as "not executed".
using var trxFile = new TempTrxFile(MinimalTrx(outcome: "Error"));
var results = TrxParser.Parse(trxFile.FileInfo);

results.Single().Outcome.Should().Be(TestOutcome.Error,
because: "outcome=\"Error\" is a distinct vstest state indicating a system error");
}

[Fact]
public void Parse_AbortedOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="Aborted" when the framework (not the user) terminates
// the test mid-execution.
using var trxFile = new TempTrxFile(MinimalTrx(outcome: "Aborted"));
var results = TrxParser.Parse(trxFile.FileInfo);

results.Single().Outcome.Should().Be(TestOutcome.Aborted,
because: "outcome=\"Aborted\" is a distinct vstest state and must not be coerced to NotExecuted");
}

[Fact]
public void Parse_NotRunnableOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="NotRunnable" when ITestElement.IsRunnable == false.
using var trxFile = new TempTrxFile(MinimalTrx(outcome: "NotRunnable"));
var results = TrxParser.Parse(trxFile.FileInfo);

results.Single().Outcome.Should().Be(TestOutcome.NotRunnable,
because: "outcome=\"NotRunnable\" is a distinct vstest state and must not be coerced to NotExecuted");
}

[Fact]
public void Parse_MissingOutcomeAttribute_MapsToError()
{
// In vstest's serialization, TestOutcome.Error == 0 is the enum default.
// XmlPersistence.SaveSimpleField() skips writing the attribute when value == default,
// so a real TRX file with an attachment error has NO outcome= attribute on
// <UnitTestResult>. A missing attribute is the live form of the Error outcome.
using var trxFile = new TempTrxFile(MinimalTrx());
var results = TrxParser.Parse(trxFile.FileInfo);

results.Single().Outcome.Should().Be(TestOutcome.Error,
because: "a missing outcome attribute means TestOutcome.Error in vstest's serialization model");
}

// -------------------------------------------------------------------------
// Directory resolution – TestProjectDirectory must be resolved correctly for
// all standard .NET SDK output layouts, including RID-qualified and publish
// subdirectories nested under bin/.
// -------------------------------------------------------------------------

[Fact]
public void Parse_TestProjectDirectory_IsCorrectForRidQualifiedOutputPath() =>
AssertProjectRootResolves("fake_project_rid", "bin", "Debug", "net8.0", "win-x64", "MyProject.dll");

[Fact]
public void Parse_TestProjectDirectory_IsCorrectForPublishOutputPath() =>
AssertProjectRootResolves("fake_project_publish", "bin", "Release", "net8.0", "publish", "MyProject.dll");

[Fact]
public void Parse_TestProjectDirectory_IsCorrectForRidPlusPublishOutputPath() =>
AssertProjectRootResolves("fake_project_rid_publish", "bin", "Release", "net8.0", "linux-x64", "publish", "MyProject.dll");

private static void AssertProjectRootResolves(string subfolder, params string[] relativeSegments)
{
var projectRoot = Path.GetFullPath(Path.Combine(Path.GetTempPath(), subfolder));
var codebase = Path.Combine(new[] { projectRoot }.Concat(relativeSegments).ToArray());
using var trxFile = new TempTrxFile(MinimalTrx(codeBase: codebase));
var results = TrxParser.Parse(trxFile.FileInfo);
results.Single().TestProjectDirectory!.FullName.TrimEnd(Path.DirectorySeparatorChar)
.Should().Be(projectRoot.TrimEnd(Path.DirectorySeparatorChar));
}

// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------

private static string MinimalTrx(string? outcome = null, string codeBase = "test.dll")
{
const string testId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const string execId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
var codebaseXml = codeBase.Replace('\\', '/');
return $"""
<?xml version="1.0" encoding="UTF-8"?>
<TestRun id="cccccccc-cccc-cccc-cccc-cccccccccccc" name="test run"
xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Results>
<UnitTestResult executionId="{execId}" testId="{testId}"
testName="SomeNamespace.SomeClass.SomeTest" computerName="host"
duration="00:00:00.0010000"
startTime="2024-01-01T00:00:00.0000000Z"
endTime="2024-01-01T00:00:00.0000000Z"
testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
{(outcome is null ? "" : $"outcome=\"{outcome}\"")}
testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
relativeResultsDirectory="{execId}" />
</Results>
<TestDefinitions>
<UnitTest name="SomeTest" id="{testId}">
<TestMethod className="SomeNamespace.SomeClass" name="SomeTest"
adapterTypeName="executor://mstestadapter/v2"
codeBase="{codebaseXml}" />
</UnitTest>
</TestDefinitions>
</TestRun>
""";
}

/// <summary>Helper that writes a TRX string to a temp file and deletes it on dispose.</summary>
private sealed class TempTrxFile : IDisposable
{
public FileInfo FileInfo { get; }

public TempTrxFile(string content)
{
var path = Path.Combine(Path.GetTempPath(), $"trxtest_{Guid.NewGuid():N}.trx");
File.WriteAllText(path, content);
FileInfo = new FileInfo(path);
}

public void Dispose() => FileInfo.Delete();
}
}
2 changes: 1 addition & 1 deletion TrxLib/TestOutcome.cs
Original file line number Diff line number Diff line change
Expand Up @@ -81,5 +81,5 @@ public enum TestOutcome
/// <summary>
/// Test is in the execution queue, was not started yet.
/// </summary>
Pending
Pending,
}
44 changes: 32 additions & 12 deletions TrxLib/TrxParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,25 @@ public static TestResultSet Parse(FileInfo trxFile)

var outcome = result.Outcome?.ToLowerInvariant() switch
{
"passed" => TestOutcome.Passed,
"failed" => TestOutcome.Failed,
"notexecuted" => TestOutcome.NotExecuted,
"inconclusive" => TestOutcome.Inconclusive,
"timeout" => TestOutcome.Timeout,
"pending" => TestOutcome.Pending,
_ => TestOutcome.NotExecuted
"passed" => TestOutcome.Passed,
"failed" => TestOutcome.Failed,
"notexecuted" => TestOutcome.NotExecuted,
"inconclusive" => TestOutcome.Inconclusive,
"timeout" => TestOutcome.Timeout,
"pending" => TestOutcome.Pending,
"error" => TestOutcome.Error,
"aborted" => TestOutcome.Aborted,
"notrunnable" => TestOutcome.NotRunnable,
"disconnected" => TestOutcome.Disconnected,
"warning" => TestOutcome.Warning,
"completed" => TestOutcome.Completed,
"inprogress" => TestOutcome.InProgress,
"passedbutrunaborted" => TestOutcome.PassedButRunAborted,
// A null/absent outcome attribute means Error in vstest's serialization:
// TestOutcome.Error is ordinal 0 (the enum default), so XmlPersistence
// omits the attribute when outcome == Error.
null => TestOutcome.Error,
_ => TestOutcome.NotExecuted
};

DateTimeOffset? resultStartTime = null, resultEndTime = null;
Expand Down Expand Up @@ -108,17 +120,25 @@ public static TestResultSet Parse(FileInfo trxFile)
codebaseFile = new FileInfo(testMethodDomain.CodeBase);
}

// Calculate test project directory from codebase if available
// Calculate test project directory from codebase if available.
// Walk up to the 'bin' folder, then take its parent — this handles
// all standard .NET SDK output layouts:
// bin/{config}/{tfm}/ (depth 3 — classic)
// bin/{config}/{tfm}/{rid}/ (depth 4 — self-contained)
// bin/{config}/{tfm}/publish/ (depth 4 — publish output)
// bin/{config}/{tfm}/{rid}/publish/ (depth 5 — self-contained publish)
DirectoryInfo? testProjectDirectory = null;
if (codebaseFile != null && codebaseFile.Directory != null)
if (codebaseFile?.Directory != null)
{
// Go up 3 levels to get the project directory (bin/Debug/netcoreappX.X/)
var dir = codebaseFile.Directory;
for (int i = 0; i < 3 && dir?.Parent != null; i++)
while (dir?.Parent != null && !string.Equals(dir.Name, "bin", StringComparison.OrdinalIgnoreCase))
{
dir = dir.Parent;
}
testProjectDirectory = dir;
// dir is now the 'bin' folder (or root if no 'bin' found); its parent is the project root.
testProjectDirectory = string.Equals(dir?.Name, "bin", StringComparison.OrdinalIgnoreCase)
? dir!.Parent
: dir;
}

// Extract StdOut if available
Expand Down
Loading