feat!: [junit] #153 Add properties support on test case level for test logger (#154)

* feat: [testlogger] Parse traits for all supported testing frameworks

* feat: [junit] Add properties support for testcases

* ci: Enable color

* test: [junit] Add xunit acceptance tests

Author: timo-reymann

.github/workflows/dotnet.yml

@@ -45,6 +45,9 @@ jobs:
     needs: [version]
     env:
       APP_BUILD_VERSION: ${{ needs.version.outputs.build_version }}
+      # force color output for dotnet
+      DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: "1"
+      TERM: "xterm"
     steps:
       - uses: actions/checkout@v4
       - name: Setup .NET 8.0
@@ -55,6 +58,13 @@ jobs:
         uses: actions/setup-dotnet@v1
         with:
           dotnet-version: "3.1.x"
+      - name: Configure NuGet cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.nuget/packages
+          key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-nuget-
       - name: Package (debug)
         run: dotnet pack -p:PackageVersion=${{ env.APP_BUILD_VERSION }}
       - name: Package (release)

src/JUnit.Xml.TestLogger/JunitXmlSerializer.cs

@@ -176,14 +176,67 @@ private static string Indent(IReadOnlyCollection<TestResultMessage> messages)
                     $"{Environment.NewLine}{indent}",
                     messages.SelectMany(m =>
                         m.Text.Split(new string[] { "\r", "\n" }, StringSplitOptions.None)
-                              .Where(x => !string.IsNullOrWhiteSpace(x))
-                              .Select(x => x.Trim())));
+                            .Where(x => !string.IsNullOrWhiteSpace(x))
+                            .Select(x => x.Trim())));
         }
 
-        private XElement CreateTestSuitesElement(
-            List<TestResultInfo> results,
-            TestRunConfiguration runConfiguration,
-            List<TestMessageInfo> messages)
+        private static IEnumerable<XElement> CreatePropertyElement(string name, params string[] values)
+        {
+            if (string.IsNullOrWhiteSpace(name))
+            {
+                throw new ArgumentException("message", nameof(name));
+            }
+
+            foreach (var value in values)
+            {
+                yield return new XElement(
+                    "property",
+                    new XAttribute("name", name),
+                    new XAttribute("value", value));
+            }
+        }
+
+        private static XElement CreatePropertyElement(Trait trait)
+        {
+            return CreatePropertyElement(trait.Name, trait.Value).Single();
+        }
+
+        private static XElement CreatePropertiesElement(TestResultInfo result)
+        {
+            var propertyElements = new HashSet<XNode>(result.Traits.Select(CreatePropertyElement));
+
+#pragma warning disable CS0618 // Type or member is obsolete
+
+            // Required since TestCase.Properties is a superset of TestCase.Traits
+            // Unfortunately not all NUnit properties are available as traits
+            var traitProperties = result.Properties;
+
+#pragma warning restore CS0618 // Type or member is obsolete
+
+            foreach (var traitProperty in traitProperties)
+            {
+                if (traitProperty.Key != "CustomProperty")
+                {
+                    continue;
+                }
+
+                var propertyDef = traitProperty.Value as string[];
+                if (propertyDef == null || propertyDef.Length != 2)
+                {
+                    continue;
+                }
+
+                var propertyElement = CreatePropertyElement(propertyDef[0], propertyDef[1]);
+                foreach (var element in propertyElement)
+                {
+                    propertyElements.Add(element);
+                }
+            }
+
+            return propertyElements.Any() ? new XElement("properties", propertyElements.Distinct()) : null;
+        }
+
+        private XElement CreateTestSuitesElement(List<TestResultInfo> results, TestRunConfiguration runConfiguration, List<TestMessageInfo> messages)
         {
             var assemblies = results.Select(x => x.AssemblyPath).Distinct().ToList();
             var testsuiteElements = assemblies
@@ -266,9 +319,7 @@ private XElement CreateTestCaseElement(TestResultInfo result)
 
             // Ensure time value is never zero because gitlab treats 0 like its null. 0.1 micro
             // seconds should be low enough it won't interfere with anyone monitoring test duration.
-            testcaseElement.SetAttributeValue(
-                "time",
-                Math.Max(0.0000001f, result.Duration.TotalSeconds).ToString("0.0000000", CultureInfo.InvariantCulture));
+            testcaseElement.SetAttributeValue("time", Math.Max(0.0000001f, result.Duration.TotalSeconds).ToString("0.0000000", CultureInfo.InvariantCulture));
 
             if (result.Outcome == TestOutcome.Failed)
             {
@@ -343,6 +394,8 @@ private XElement CreateTestCaseElement(TestResultInfo result)
                 testcaseElement.Add(new XElement("system-err", new XCData(stdErr.ToString())));
             }
 
+            testcaseElement.Add(CreatePropertiesElement(result));
+
             return testcaseElement;
         }
 

src/TestLogger/Extensions/MSTestAdapter.cs

@@ -26,9 +26,24 @@ public List<TestResultInfo> TransformResults(List<TestResultInfo> results, List<
                     // Preserving method because result display name was empty
                 }
                 else if (method != displayName)
-                {
+                {
                     result.Method = displayName;
                 }
+
+                // Parse property traits
+                var properties = new List<KeyValuePair<string, object>>();
+                foreach (var property in result.TestCase.Properties)
+                {
+                    switch (property.Id)
+                    {
+                        case "Microsoft.VisualStudio.TestTools.UnitTesting.TestContext.TestProperty":
+                            var propertyValue = result.TestCase.GetPropertyValue(property) as string[];
+                            properties.Add(new KeyValuePair<string, object>("CustomProperty", propertyValue));
+                            break;
+                    }
+                }
+
+                result.Properties = properties;
             }
 
             return results;

src/TestLogger/Extensions/NUnitTestAdapter.cs

@@ -38,6 +38,9 @@ public List<TestResultInfo> TransformResults(List<TestResultInfo> results, List<
                         case "NUnit.TestCategory":
                             properties.Add(new KeyValuePair<string, object>(property.Id, result.TestCase.GetPropertyValue(property)));
                             break;
+                        case "NUnit.Category":
+                            properties.Add(new KeyValuePair<string, object>("CustomProperty", result.TestCase.GetPropertyValue(property)));
+                            break;
                     }
                 }
 

src/TestLogger/Extensions/XunitTestAdapter.cs

@@ -56,6 +56,23 @@ public List<TestResultInfo> TransformResults(
                     result.Method += displayName.Substring(i);
                 }
 
+                var properties = new List<KeyValuePair<string, object>>();
+
+                // Parse test traits via Trait decorator
+                foreach (var property in result.TestCase.Properties)
+                {
+                    switch (property.Id)
+                    {
+                        case "Xunit.Trait":
+                            var propertyValue = result.TestCase.GetPropertyValue(property) as string[];
+
+                            properties.Add(new KeyValuePair<string, object>("CustomProperty", propertyValue));
+                            break;
+                    }
+                }
+
+                result.Properties = properties;
+
                 transformedResults.Add(result);
             }
 

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnit.Xml.TestLogger.AcceptanceTests.csproj

@@ -27,6 +27,7 @@
 
     <TestAssets Include="$(MSBuildThisFileDirectory)../assets/JUnit.Xml.TestLogger.NetCore.Tests/JUnit.Xml.TestLogger.NetCore.Tests.csproj" />
     <TestAssets Include="$(MSBuildThisFileDirectory)../assets/JUnit.Xml.TestLogger.NetMulti.Tests/JUnit.Xml.TestLogger.NetMulti.Tests.csproj" />
+    <TestAssets Include="$(MSBuildThisFileDirectory)../assets/JUnit.Xml.TestLogger.XUnit.NetCore.Tests/JUnit.Xml.TestLogger.XUnit.NetCore.Tests.csproj" />
   </ItemGroup>
   <Target Name="TestTarget" BeforeTargets="Build">
     <Message Importance="High" Text="... Building test assets" />

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerAcceptanceTests.cs

@@ -37,9 +37,9 @@ public static void SuiteInitialize(TestContext context)
 
             // Enable reporting of internal properties in the adapter using runsettings
             _ = DotnetTestFixture
-                    .Create()
-                    .WithBuild()
-                    .Execute(AssetName, loggerArgs, collectCoverage: false, "test-results.xml");
+                .Create()
+                .WithBuild()
+                .Execute(AssetName, loggerArgs, collectCoverage: false, "test-results.xml");
         }
 
         [TestMethod]
@@ -123,6 +123,31 @@ public void TestResultFileShouldContainErrordOut()
             Assert.IsTrue(node.Value.Contains("{33F5FD22-6F40-499D-98E4-481D87FAEAA1}"));
         }
 
+        [TestMethod]
+        public void TestResultFileShouldContainNUnitCategoryAsProperty()
+        {
+            var tesuites = this.resultsXml.XPathSelectElement("/testsuites/testsuite");
+            var testcase = tesuites
+                .Nodes()
+                .FirstOrDefault(n =>
+                {
+                    var element = n as XElement;
+                    return element.Attribute("classname")?.Value == "JUnit.Xml.TestLogger.NetFull.Tests.UnitTest1" &&
+                           element.Attribute("name")?.Value == "WithProperties";
+                });
+            Assert.IsNotNull(testcase);
+
+            var properties = (testcase as XElement)
+                .Nodes()
+                .FirstOrDefault(n => (n as XElement)?.Name == "properties") as XElement;
+            Assert.IsNotNull(properties);
+            Assert.AreEqual(2, properties.Nodes().Count());
+            var propertyElements = properties.Nodes().ToList();
+
+            Assert.AreEqual("Property name", (propertyElements[0] as XElement).Attribute("name").Value);
+            Assert.AreEqual("Property value 1", (propertyElements[0] as XElement).Attribute("value").Value);
+        }
+
         [TestMethod]
         public void LoggedXmlValidatesAgainstXsdSchema()
         {

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerFormatOptionsAcceptanceTests.cs

@@ -98,7 +98,8 @@ public void FailureBodyFormat_Verbose_ShouldStartWithMessage()
                 Assert.IsTrue(body.Trim().StartsWith(message.Trim()));
             }
 
-            Assert.IsTrue(new JunitXmlValidator().IsValid(resultsXml));
+            var validator = new JunitXmlValidator();
+            Assert.IsTrue(validator.IsValid(resultsXml));
         }
 
         [TestMethod]

test/JUnit.Xml.TestLogger.AcceptanceTests/JUnitTestLoggerXUnitAcceptanceTests.cs

@@ -0,0 +1,78 @@
+// Copyright (c) Spekt Contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace JUnit.Xml.TestLogger.AcceptanceTests
+{
+    using System.IO;
+    using System.Linq;
+    using System.Xml.Linq;
+    using System.Xml.XPath;
+    using global::TestLogger.Fixtures;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    /// <summary>
+    /// Acceptance tests evaluate the most recent output of the build.ps1 script, NOT the most
+    /// recent build performed by visual studio or dotnet.build
+    ///
+    /// These acceptance tests look at the specific structure and contents of the produced Xml,
+    /// when running using the xUnit vstest runner.
+    /// </summary>
+    [TestClass]
+    public class JUnitTestLoggerXunitAcceptanceTests
+    {
+        private const string AssetName = "JUnit.Xml.TestLogger.XUnit.NetCore.Tests";
+        private readonly string resultsFile;
+        private readonly XDocument resultsXml;
+
+        public JUnitTestLoggerXunitAcceptanceTests()
+        {
+            this.resultsFile = Path.Combine(AssetName.ToAssetDirectoryPath(), "test-results.xml");
+            this.resultsXml = XDocument.Load(this.resultsFile);
+        }
+
+        [ClassInitialize]
+        public static void SuiteInitialize(TestContext context)
+        {
+            var loggerArgs = "junit;LogFilePath=test-results.xml";
+
+            // Enable reporting of internal properties in the adapter using runsettings
+            _ = DotnetTestFixture
+                .Create()
+                .WithBuild()
+                .Execute(AssetName, loggerArgs, collectCoverage: false, "test-results.xml");
+        }
+
+        [TestMethod]
+        public void LoggedXmlValidatesAgainstXsdSchema()
+        {
+            var validator = new JunitXmlValidator();
+            var result = validator.IsValid(File.ReadAllText(this.resultsFile));
+            Assert.IsTrue(result);
+        }
+
+        [TestMethod]
+        public void TestResultFileShouldContainXUnitTraitAsProperty()
+        {
+            var properties = this.resultsXml.XPathSelectElement(
+                "/testsuites/testsuite//testcase[@classname=\"NUnit.Xml.TestLogger.Tests2.ApiTest\"]/properties");
+            Assert.IsNotNull(properties);
+            Assert.AreEqual(2, properties.Nodes().Count());
+            foreach (XElement node in properties.Nodes())
+            {
+                Assert.IsNotNull(node);
+                if (node.Attribute("name").Value == "SomeProp")
+                {
+                    Assert.AreEqual("SomeVal", node.Attribute("value").Value);
+                }
+                else if (node.Attribute("name").Value == "Category")
+                {
+                    Assert.AreEqual("ApiTest", node.Attribute("value").Value);
+                }
+                else
+                {
+                    Assert.Fail($"Unexpted property found");
+                }
+            }
+        }
+    }
+}

test/TestLogger.UnitTests/Extensions/MSTestAdapterTests.cs

@@ -55,5 +55,25 @@ public void TransformShouldNoitOverwriteMethodEmptyValues(string displayName)
             Assert.AreEqual(1, transformedResults.Count);
             Assert.AreEqual(Method, transformedResults.Single().Method);
         }
+
+        [TestMethod]
+        public void TransformResultShouldAddProperties()
+        {
+            var testResults = new List<TestResultInfo>
+            {
+                new TestResultInfoBuilder("namespace", "type", Method)
+                    .WithProperty("Microsoft.VisualStudio.TestTools.UnitTesting.TestContext.TestProperty", new[] { "c1", "c2" })
+                    .Build(),
+            };
+
+            var sut = new MSTestAdapter();
+
+            var transformedResults = sut.TransformResults(testResults, new List<TestMessageInfo>());
+
+            Assert.AreEqual(1, transformedResults.Count);
+            Assert.AreEqual(1, transformedResults[0].Properties.Count);
+            Assert.AreEqual(Method, transformedResults.Single().Method);
+            CollectionAssert.AreEquivalent(new[] { "c1", "c2" }, (string[])transformedResults[0].Properties.Single(p => p.Key == "CustomProperty").Value);
+        }
     }
 }