Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ jobs:
use-github-auto-merge: true
build-and-test:
runs-on: ${{ matrix.os }}
permissions:
contents: read

strategy:
matrix:
Expand All @@ -54,4 +56,11 @@ jobs:
run: dotnet build -p:ContinuousIntegrationBuild=True --no-restore --configuration Release

- name: Test
run: dotnet test --no-build --configuration Release --verbosity normal
run: dotnet test --no-build --configuration Release --verbosity normal

- name: AOT Publish Validation
run: |
$rid = if ($env:RUNNER_OS -eq 'Windows') { 'win-x64' } else { 'linux-x64' }
dotnet publish TrxLib.AotSample/TrxLib.AotSample.csproj -r $rid -c Release --self-contained
$ext = if ($env:RUNNER_OS -eq 'Windows') { '.exe' } else { '' }
& "TrxLib.AotSample/bin/Release/net10.0/$rid/publish/TrxLib.AotSample$ext"
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:

- name: Upload Artifacts
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v7
with:
name: NuGet
path: ${{ github.workspace }}/PackedNuget
Expand Down
20 changes: 20 additions & 0 deletions TrxLib.AotSample/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using TrxLib;

var sampleRoot = Path.Combine(AppContext.BaseDirectory, "SampleTrxFiles");
if (!Directory.Exists(sampleRoot))
throw new DirectoryNotFoundException($"Sample TRX directory not found: {sampleRoot}");

var sampleFiles = Directory
.EnumerateFiles(sampleRoot, "*.trx", SearchOption.AllDirectories)
.OrderBy(path => path, StringComparer.Ordinal)
.ToArray();

if (sampleFiles.Length == 0)
throw new InvalidOperationException($"No TRX sample files found under: {sampleRoot}");

foreach (var file in sampleFiles)
{
_ = TrxParser.Parse(new FileInfo(file));
}

Console.WriteLine($"TrxLib AOT validation passed. Parsed {sampleFiles.Length} sample files.");
22 changes: 22 additions & 0 deletions TrxLib.AotSample/TrxLib.AotSample.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<PublishAot>true</PublishAot>
<IsPublishable>true</IsPublishable>
<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\TrxLib\TrxLib.csproj" />
</ItemGroup>

<ItemGroup>
<Content Include="..\TrxLib.Tests\SampleTrxFiles\**\*.trx"
Link="SampleTrxFiles\%(RecursiveDir)%(Filename)%(Extension)"
CopyToOutputDirectory="PreserveNewest"
CopyToPublishDirectory="PreserveNewest" />
</ItemGroup>

</Project>
236 changes: 236 additions & 0 deletions TrxLib.Tests/KnownBugsTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
using AwesomeAssertions;

namespace TrxLib.Tests;

/// <summary>
/// Tests that document currently known bugs found by comparing our object model to the
/// upstream vstest TRX ObjectModel (commit ba0077af).
/// Every test in this file is expected to FAIL until the corresponding bug is fixed.
/// </summary>
public class KnownBugsTests
{
// -------------------------------------------------------------------------
// Bug 3 – Outcomes defined by vstest (Error, Aborted, NotRunnable, Disconnected,
// Warning, Completed, InProgress, PassedButRunAborted) are absent from
// local TestOutcome enum. The parser's catch-all arm (TrxParser.cs:52)
// maps every unrecognised string to NotExecuted, causing silent
// misclassification of distinct failure modes.
// -------------------------------------------------------------------------

[Fact]
public void Bug_Parse_ErrorOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="Error" when the test adapter itself crashes or the
// test process aborts unexpectedly. This is NOT the same as "not executed".
using var trxFile = new TempTrxFile(MinimalTrxWithOutcome("Error"));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: TrxParser catch-all maps "error" → TestOutcome.NotExecuted (line 52).
results.Single().Outcome.Should().Be(TestOutcome.Error,
because: "outcome=\"Error\" is a distinct vstest state indicating a system error");
}

[Fact]
public void Bug_Parse_AbortedOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="Aborted" when the framework (not the user) terminates
// the test mid-execution.
using var trxFile = new TempTrxFile(MinimalTrxWithOutcome("Aborted"));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: TrxParser catch-all maps "aborted" → TestOutcome.NotExecuted (line 52).
results.Single().Outcome.Should().Be(TestOutcome.Aborted,
because: "outcome=\"Aborted\" is a distinct vstest state and must not be coerced to NotExecuted");
}

[Fact]
public void Bug_Parse_NotRunnableOutcome_IsNotSilentlyMappedToNotExecuted()
{
// vstest writes outcome="NotRunnable" when ITestElement.IsRunnable == false.
using var trxFile = new TempTrxFile(MinimalTrxWithOutcome("NotRunnable"));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: TrxParser catch-all maps "notrunnable" → TestOutcome.NotExecuted (line 52).
results.Single().Outcome.Should().Be(TestOutcome.NotRunnable,
because: "outcome=\"NotRunnable\" is a distinct vstest state and must not be coerced to NotExecuted");
}

[Fact]
public void Bug_Parse_MissingOutcomeAttribute_MapsToError()
{
// In vstest's serialization, TestOutcome.Error == 0 is the enum default.
// XmlPersistence.SaveSimpleField() skips writing the attribute when value == default,
// so a real TRX file with an attachment error has NO outcome= attribute on
// <UnitTestResult>. A missing attribute is the live form of the Error outcome.
using var trxFile = new TempTrxFile(MinimalTrxWithoutOutcome());
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: null result.Outcome hits catch-all → TestOutcome.NotExecuted instead of TestOutcome.Error.
results.Single().Outcome.Should().Be(TestOutcome.Error,
because: "a missing outcome attribute means TestOutcome.Error in vstest's serialization model");
}

// -------------------------------------------------------------------------
// Bug 4 – Test project directory is derived by a hardcoded 3-level upward
// traversal from the codeBase DLL path (TrxParser.cs:96-101).
// This breaks for .NET 8+ RID-qualified output layouts where the DLL
// is 4 levels below the project root:
// <project>/bin/Debug/net8.0/<rid>/Foo.dll
// -------------------------------------------------------------------------

[Fact]
public void Bug_Parse_TestProjectDirectory_IsCorrectForRidQualifiedOutputPath()
{
// Represents: <project>/bin/Debug/net8.0/win-x64/MyProject.dll
// Going up 3 levels lands on bin/ — one level too shallow.
var projectRoot = Path.GetFullPath(Path.Combine(Path.GetTempPath(), "fake_project_rid"));
var codebase = Path.Combine(projectRoot, "bin", "Debug", "net8.0", "win-x64", "MyProject.dll");

using var trxFile = new TempTrxFile(MinimalTrxWithCodebase(codebase));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: hardcoded 3-level traversal returns bin/ instead of the project root.
results.Single().TestProjectDirectory!.FullName.TrimEnd(Path.DirectorySeparatorChar)
.Should().Be(projectRoot.TrimEnd(Path.DirectorySeparatorChar));
}

[Fact]
public void Bug_Parse_TestProjectDirectory_IsCorrectForPublishOutputPath()
{
// Represents: <project>/bin/Release/net8.0/publish/MyProject.dll
// Going up 3 levels lands on bin/ — one level too shallow.
var projectRoot = Path.GetFullPath(Path.Combine(Path.GetTempPath(), "fake_project_publish"));
var codebase = Path.Combine(projectRoot, "bin", "Release", "net8.0", "publish", "MyProject.dll");

using var trxFile = new TempTrxFile(MinimalTrxWithCodebase(codebase));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: hardcoded 3-level traversal returns bin/ instead of the project root.
results.Single().TestProjectDirectory!.FullName.TrimEnd(Path.DirectorySeparatorChar)
.Should().Be(projectRoot.TrimEnd(Path.DirectorySeparatorChar));
}

[Fact]
public void Bug_Parse_TestProjectDirectory_IsCorrectForRidPlusPublishOutputPath()
{
// Represents: <project>/bin/Release/net8.0/linux-x64/publish/MyProject.dll
// Going up 3 levels lands on Release/ — two levels too shallow.
var projectRoot = Path.GetFullPath(Path.Combine(Path.GetTempPath(), "fake_project_rid_publish"));
var codebase = Path.Combine(projectRoot, "bin", "Release", "net8.0", "linux-x64", "publish", "MyProject.dll");

using var trxFile = new TempTrxFile(MinimalTrxWithCodebase(codebase));
var results = TrxParser.Parse(trxFile.FileInfo);

// FAILS: hardcoded 3-level traversal returns Release/ instead of the project root.
results.Single().TestProjectDirectory!.FullName.TrimEnd(Path.DirectorySeparatorChar)
.Should().Be(projectRoot.TrimEnd(Path.DirectorySeparatorChar));
}

// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------

private static string MinimalTrxWithoutOutcome()
{
const string testId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const string execId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
return $"""
<?xml version="1.0" encoding="UTF-8"?>
<TestRun id="cccccccc-cccc-cccc-cccc-cccccccccccc" name="test run"
xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Results>
<UnitTestResult executionId="{execId}" testId="{testId}"
testName="SomeNamespace.SomeClass.SomeTest" computerName="host"
duration="00:00:00.0010000"
startTime="2024-01-01T00:00:00.0000000Z"
endTime="2024-01-01T00:00:00.0000000Z"
testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
relativeResultsDirectory="{execId}" />
</Results>
<TestDefinitions>
<UnitTest name="SomeTest" id="{testId}">
<TestMethod className="SomeNamespace.SomeClass" name="SomeTest"
adapterTypeName="executor://mstestadapter/v2" codeBase="test.dll" />
</UnitTest>
</TestDefinitions>
</TestRun>
""";
}

private static string MinimalTrxWithOutcome(string outcome)
{
const string testId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const string execId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
return $"""
<?xml version="1.0" encoding="UTF-8"?>
<TestRun id="cccccccc-cccc-cccc-cccc-cccccccccccc" name="test run"
xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Results>
<UnitTestResult executionId="{execId}" testId="{testId}"
testName="SomeNamespace.SomeClass.SomeTest" computerName="host"
duration="00:00:00.0010000"
startTime="2024-01-01T00:00:00.0000000Z"
endTime="2024-01-01T00:00:00.0000000Z"
testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
outcome="{outcome}"
testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
relativeResultsDirectory="{execId}" />
</Results>
<TestDefinitions>
<UnitTest name="SomeTest" id="{testId}">
<TestMethod className="SomeNamespace.SomeClass" name="SomeTest"
adapterTypeName="executor://mstestadapter/v2" codeBase="test.dll" />
</UnitTest>
</TestDefinitions>
</TestRun>
""";
}

private static string MinimalTrxWithCodebase(string codebasePath)
{
const string testId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const string execId = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
// XML attribute value must use forward slashes to avoid escaping issues
var codebaseXml = codebasePath.Replace('\\', '/');
return $"""
<?xml version="1.0" encoding="UTF-8"?>
<TestRun id="cccccccc-cccc-cccc-cccc-cccccccccccc" name="test run"
xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Results>
<UnitTestResult executionId="{execId}" testId="{testId}"
testName="SomeNamespace.SomeClass.SomeTest" computerName="host"
duration="00:00:00.0010000"
startTime="2024-01-01T00:00:00.0000000Z"
endTime="2024-01-01T00:00:00.0000000Z"
testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
outcome="Passed"
testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d"
relativeResultsDirectory="{execId}" />
</Results>
<TestDefinitions>
<UnitTest name="SomeTest" id="{testId}">
<TestMethod className="SomeNamespace.SomeClass" name="SomeTest"
adapterTypeName="executor://mstestadapter/v2"
codeBase="{codebaseXml}" />
</UnitTest>
</TestDefinitions>
</TestRun>
""";
}

/// <summary>Helper that writes a TRX string to a temp file and deletes it on dispose.</summary>
private sealed class TempTrxFile : IDisposable
{
public FileInfo FileInfo { get; }

public TempTrxFile(string content)
{
var path = Path.Combine(Path.GetTempPath(), $"trxtest_{Guid.NewGuid():N}.trx");
File.WriteAllText(path, content);
FileInfo = new FileInfo(path);
}

public void Dispose() => FileInfo.Delete();
}
}
46 changes: 46 additions & 0 deletions TrxLib.Tests/SampleTrxFiles/theory-tests.trx
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
<?xml version="1.0" encoding="utf-8"?>
<TestRun id="e70a511e-0bfb-420b-8259-d0a72fdba1a0" name="benja@BAM-Z690 2026-05-16 21:39:55" runUser="BAM-Z690\benja" xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
<Times creation="2026-05-16T21:39:55.3918046-07:00" queuing="2026-05-16T21:39:55.3918050-07:00" start="2026-05-16T21:39:54.5463544-07:00" finish="2026-05-16T21:39:55.3962459-07:00" />
<TestSettings name="default" id="e738a127-b106-49cf-8a3d-f73e3c609f43">
<Deployment runDeploymentRoot="benja_BAM-Z690_2026-05-16_21_39_55" />
</TestSettings>
<Results>
<UnitTestResult executionId="d29a42b5-85e5-4de5-8485-488b36990291" testId="69bf1a74-6bd3-1b8d-520d-c9801d194c51" testName="Acme.Tests.MathTests.AddNumbers(left: 1, right: 2)" computerName="BAM-Z690" duration="00:00:00.0002264" startTime="2026-05-16T21:39:55.2985338-07:00" endTime="2026-05-16T21:39:55.2989500-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="d29a42b5-85e5-4de5-8485-488b36990291" />
<UnitTestResult executionId="a1fa9069-44cf-41cb-96c9-11f88e879548" testId="718be9ec-e1ee-2f03-4dd6-5b464b52a593" testName="Acme.Tests.MathTests.AddNumbers(left: 0, right: 0)" computerName="BAM-Z690" duration="00:00:00.0175485" startTime="2026-05-16T21:39:55.2666361-07:00" endTime="2026-05-16T21:39:55.2903128-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="a1fa9069-44cf-41cb-96c9-11f88e879548" />
<UnitTestResult executionId="003fc356-5ae3-4120-a0e8-8ca8ff6d9caa" testId="2f0a2807-568e-bacb-3945-4a25878eaf48" testName="Acme.Tests.MathTests.PlainTest" computerName="BAM-Z690" duration="00:00:00.0010569" startTime="2026-05-16T21:39:55.2999872-07:00" endTime="2026-05-16T21:39:55.3000209-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="003fc356-5ae3-4120-a0e8-8ca8ff6d9caa" />
</Results>
<TestDefinitions>
<UnitTest name="Acme.Tests.MathTests.AddNumbers(left: 0, right: 0)" storage="d:\trxlib\trxlib.tests\bin\debug\net10.0\trxlib.tests.dll" id="718be9ec-e1ee-2f03-4dd6-5b464b52a593">
<Execution id="a1fa9069-44cf-41cb-96c9-11f88e879548" />
<TestMethod codeBase="D:\TrxLib\TrxLib.Tests\bin\Debug\net10.0\TrxLib.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner3/netcore/" className="Acme.Tests.MathTests" name="AddNumbers" />
</UnitTest>
<UnitTest name="Acme.Tests.MathTests.PlainTest" storage="d:\trxlib\trxlib.tests\bin\debug\net10.0\trxlib.tests.dll" id="2f0a2807-568e-bacb-3945-4a25878eaf48">
<Execution id="003fc356-5ae3-4120-a0e8-8ca8ff6d9caa" />
<TestMethod codeBase="D:\TrxLib\TrxLib.Tests\bin\Debug\net10.0\TrxLib.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner3/netcore/" className="Acme.Tests.MathTests" name="PlainTest" />
</UnitTest>
<UnitTest name="Acme.Tests.MathTests.AddNumbers(left: 1, right: 2)" storage="d:\trxlib\trxlib.tests\bin\debug\net10.0\trxlib.tests.dll" id="69bf1a74-6bd3-1b8d-520d-c9801d194c51">
<Execution id="d29a42b5-85e5-4de5-8485-488b36990291" />
<TestMethod codeBase="D:\TrxLib\TrxLib.Tests\bin\Debug\net10.0\TrxLib.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner3/netcore/" className="Acme.Tests.MathTests" name="AddNumbers" />
</UnitTest>
</TestDefinitions>
<TestEntries>
<TestEntry testId="69bf1a74-6bd3-1b8d-520d-c9801d194c51" executionId="d29a42b5-85e5-4de5-8485-488b36990291" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestEntry testId="718be9ec-e1ee-2f03-4dd6-5b464b52a593" executionId="a1fa9069-44cf-41cb-96c9-11f88e879548" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestEntry testId="2f0a2807-568e-bacb-3945-4a25878eaf48" executionId="003fc356-5ae3-4120-a0e8-8ca8ff6d9caa" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
</TestEntries>
<TestLists>
<TestList name="Results Not in a List" id="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
<TestList name="All Loaded Results" id="19431567-8539-422a-85d7-44ee4e166bda" />
</TestLists>
<ResultSummary outcome="Completed">
<Counters total="3" executed="3" passed="3" failed="0" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="0" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />
<Output>
<StdOut>[xUnit.net 00:00:00.00] xUnit.net VSTest Adapter v3.1.5+1b188a7b0a (64-bit .NET 10.0.8)&#xD;
[xUnit.net 00:00:00.08] Discovering: TrxLib.Tests&#xD;
[xUnit.net 00:00:00.13] Discovered: TrxLib.Tests&#xD;
[xUnit.net 00:00:00.15] Starting: TrxLib.Tests&#xD;
[xUnit.net 00:00:00.20] Finished: TrxLib.Tests&#xD;
</StdOut>
</Output>
</ResultSummary>
</TestRun>
Loading
Loading