Fix multiple results not returned with custom TestMethod and TestClass (#358)

src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs

@@ -247,22 +247,30 @@ internal UnitTestResult[] RunTestMethod()
                                     watch.Start();
 
                                     this.testContext.SetDataRow(dataRow);
-                                    UTF.TestResult currentResult;
+                                    UTF.TestResult[] testResults;
 
                                     try
                                     {
-                                        currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+                                        testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
                                     }
                                     catch (Exception ex)
                                     {
-                                        currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+                                        testResults = new[]
+                                        {
+                                            new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+                                        };
                                     }
 
-                                    currentResult.DatarowIndex = rowIndex++;
                                     watch.Stop();
-                                    currentResult.Duration = watch.Elapsed;
+                                    foreach (var testResult in testResults)
+                                    {
+                                        testResult.DatarowIndex = rowIndex;
+                                        testResult.Duration = watch.Elapsed;
+                                    }
 
-                                    results.Add(currentResult);
+                                    rowIndex++;
+
+                                    results.AddRange(testResults);
                                 }
                             }
                             finally
@@ -293,18 +301,25 @@ internal UnitTestResult[] RunTestMethod()
                             foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
                             {
                                 this.testMethodInfo.SetArguments(data);
-                                UTF.TestResult currentResult;
+                                UTF.TestResult[] testResults;
                                 try
                                 {
-                                    currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+                                    testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
                                 }
                                 catch (Exception ex)
                                 {
-                                    currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+                                    testResults = new[]
+                                    {
+                                        new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+                                    };
+                                }
+
+                                foreach (var testResult in testResults)
+                                {
+                                    testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
                                 }
 
-                                currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
-                                results.Add(currentResult);
+                                results.AddRange(testResults);
                                 this.testMethodInfo.SetArguments(null);
                             }
                         }
@@ -313,7 +328,7 @@ internal UnitTestResult[] RunTestMethod()
                     {
                         try
                         {
-                            results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]);
+                            results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
                         }
                         catch (Exception ex)
                         {

test/E2ETests/Automation.CLI/CLITestBase.cs

@@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation
     using System.Linq;
     using System.Xml;
     using Microsoft.TestPlatform.VsTestConsole.TranslationLayer;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
     public class CLITestBase
@@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "")
         /// </summary>
         /// <param name="sources">List of test assemblies.</param>
         /// <param name="runSettings">Run settings for execution.</param>
-        public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
+        /// <param name="testCaseFilter">Test Case filter for execution.</param>
+        public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null)
         {
             for (var iterator = 0; iterator < sources.Length; iterator++)
             {
@@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
 
             // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
             vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
-            vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler);
+            vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
         }
 
         /// <summary>

test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace MSTestAdapter.Smoke.E2ETests
+{
+    using Microsoft.MSTestV2.CLIAutomation;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class CustomTestExecutionExtensibilityTests : CLITestBase
+    {
+        private const string TestAssembly = "FxExtensibilityTestProject.dll";
+
+        [TestMethod]
+        public void ExecuteCustomTestExtensibilityTests()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly });
+            this.ValidatePassedTestsContain(
+                "CustomTestMethod1 - Execution number 1",
+                "CustomTestMethod1 - Execution number 2",
+                "CustomTestMethod1 - Execution number 4",
+                "CustomTestMethod1 - Execution number 5",
+                "CustomTestClass1 - Execution number 1",
+                "CustomTestClass1 - Execution number 2",
+                "CustomTestClass1 - Execution number 4",
+                "CustomTestClass1 - Execution number 5");
+            this.ValidateFailedTestsContain(
+                TestAssembly,
+                "CustomTestMethod1 - Execution number 3",
+                "CustomTestClass1 - Execution number 3");
+        }
+
+        [TestMethod]
+        public void ExecuteCustomTestExtensibilityWithTestDataTests()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2");
+            this.ValidatePassedTests(
+                "CustomTestMethod2 (B)",
+                "CustomTestMethod2 (B)",
+                "CustomTestMethod2 (B)");
+            this.ValidateFailedTests(
+                TestAssembly,
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (C)",
+                "CustomTestMethod2 (C)",
+                "CustomTestMethod2 (C)");
+        }
+    }
+}

test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj

@@ -39,6 +39,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AssertExtensibilityTests.cs" />
+    <Compile Include="CustomTestExecutionExtensibilityTests.cs" />
     <Compile Include="DataSourceTests.cs" />
     <Compile Include="DesktopCSharpCLITests.cs" />
     <Compile Include="ParallelExecutionTests.cs" />

test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs

@@ -0,0 +1,77 @@
+namespace FxExtensibilityTestProject
+{
+    using System.Collections.Generic;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [IterativeTestClass(5)]
+    public class CustomTestExTests
+    {
+        private static int customTestMethod1ExecutionCount;
+        [IterativeTestMethod(5)]
+        public void CustomTestMethod1()
+        {
+            customTestMethod1ExecutionCount++;
+            Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
+        }
+
+        [IterativeTestMethod(3)]
+        [DataRow("A")]
+        [DataRow("B")]
+        [DataRow("C")]
+        public void CustomTestMethod2(string value)
+        {
+            Assert.AreEqual("B", value);
+        }
+
+        private static int customTestClass1ExecutionCount;
+        [TestMethod]
+        public void CustomTestClass1()
+        {
+            customTestClass1ExecutionCount++;
+            Assert.AreNotEqual(3, customTestClass1ExecutionCount);
+        }
+    }
+
+    public class IterativeTestMethodAttribute : TestMethodAttribute
+    {
+        private readonly int stabilityThreshold;
+
+        public IterativeTestMethodAttribute(int stabilityThreshold)
+        {
+            this.stabilityThreshold = stabilityThreshold;
+        }
+
+        public override TestResult[] Execute(ITestMethod testMethod)
+        {
+            var results = new List<TestResult>();
+            for (int count = 0; count < this.stabilityThreshold; count++)
+            {
+                var testResults = base.Execute(testMethod);
+                foreach (var testResult in testResults)
+                {
+                    testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}";
+                }
+                results.AddRange(testResults);
+            }
+
+            return results.ToArray();
+        }
+    }
+
+    public class IterativeTestClassAttribute : TestClassAttribute
+    {
+        private readonly int stabilityThreshold;
+
+        public IterativeTestClassAttribute(int stabilityThreshold)
+        {
+            this.stabilityThreshold = stabilityThreshold;
+        }
+
+        public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute)
+        {
+            if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute;
+
+            return new IterativeTestMethodAttribute(this.stabilityThreshold);
+        }
+    }
+}

test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj

@@ -44,6 +44,7 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="DynamicDataExTests.cs" />
     <Compile Include="DynamicDataExMoreTests.cs" />
+    <Compile Include="CustomTestExTests.cs" />
     <Compile Include="TestDataSourceExTests.cs" />
   </ItemGroup>
   <ItemGroup>

test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs

@@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
             StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
         }
 
+        [TestMethodV1]
+        public void RunTestMethodForMultipleResultsReturnMultipleResults()
+        {
+            var testMethodAttributeMock = new Mock<UTF.TestMethodAttribute>();
+            testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny<UTF.ITestMethod>())).Returns(new[]
+            {
+                new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed },
+                new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed }
+            });
+
+            var localTestMethodOptions = new TestMethodOptions
+            {
+                Timeout = 200,
+                Executor = testMethodAttributeMock.Object,
+                TestContext = this.testContextImplementation,
+                ExpectedException = null
+            };
+
+            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null);
+            var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
+
+            var results = testMethodRunner.Execute();
+            Assert.AreEqual(2, results.Length);
+            Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome);
+            Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome);
+        }
+
         [TestMethodV1]
         public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome()
         {
-            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo,  this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
+            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
             var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
 
             var results = testMethodRunner.Execute();