diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
index 5559f91338..2ece924fcc 100644
--- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
+++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
@@ -247,22 +247,30 @@ internal UnitTestResult[] RunTestMethod()
watch.Start();
this.testContext.SetDataRow(dataRow);
- UTF.TestResult currentResult;
+ UTF.TestResult[] testResults;
try
{
- currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+ testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
- currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+ testResults = new[]
+ {
+ new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+ };
}
- currentResult.DatarowIndex = rowIndex++;
watch.Stop();
- currentResult.Duration = watch.Elapsed;
+ foreach (var testResult in testResults)
+ {
+ testResult.DatarowIndex = rowIndex;
+ testResult.Duration = watch.Elapsed;
+ }
- results.Add(currentResult);
+ rowIndex++;
+
+ results.AddRange(testResults);
}
}
finally
@@ -293,18 +301,25 @@ internal UnitTestResult[] RunTestMethod()
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
this.testMethodInfo.SetArguments(data);
- UTF.TestResult currentResult;
+ UTF.TestResult[] testResults;
try
{
- currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+ testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
- currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+ testResults = new[]
+ {
+ new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+ };
+ }
+
+ foreach (var testResult in testResults)
+ {
+ testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
}
- currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
- results.Add(currentResult);
+ results.AddRange(testResults);
this.testMethodInfo.SetArguments(null);
}
}
@@ -313,7 +328,7 @@ internal UnitTestResult[] RunTestMethod()
{
try
{
- results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]);
+ results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
}
catch (Exception ex)
{
diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs
index 88cd82f668..f846b3e69a 100644
--- a/test/E2ETests/Automation.CLI/CLITestBase.cs
+++ b/test/E2ETests/Automation.CLI/CLITestBase.cs
@@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation
using System.Linq;
using System.Xml;
using Microsoft.TestPlatform.VsTestConsole.TranslationLayer;
+ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestTools.UnitTesting;
public class CLITestBase
@@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "")
///
/// List of test assemblies.
/// Run settings for execution.
- public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
+ /// Test Case filter for execution.
+ public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null)
{
for (var iterator = 0; iterator < sources.Length; iterator++)
{
@@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
// this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
- vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler);
+ vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
}
///
diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
new file mode 100644
index 0000000000..23f7cd81c9
--- /dev/null
+++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace MSTestAdapter.Smoke.E2ETests
+{
+ using Microsoft.MSTestV2.CLIAutomation;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [TestClass]
+ public class CustomTestExecutionExtensibilityTests : CLITestBase
+ {
+ private const string TestAssembly = "FxExtensibilityTestProject.dll";
+
+ [TestMethod]
+ public void ExecuteCustomTestExtensibilityTests()
+ {
+ this.InvokeVsTestForExecution(new string[] { TestAssembly });
+ this.ValidatePassedTestsContain(
+ "CustomTestMethod1 - Execution number 1",
+ "CustomTestMethod1 - Execution number 2",
+ "CustomTestMethod1 - Execution number 4",
+ "CustomTestMethod1 - Execution number 5",
+ "CustomTestClass1 - Execution number 1",
+ "CustomTestClass1 - Execution number 2",
+ "CustomTestClass1 - Execution number 4",
+ "CustomTestClass1 - Execution number 5");
+ this.ValidateFailedTestsContain(
+ TestAssembly,
+ "CustomTestMethod1 - Execution number 3",
+ "CustomTestClass1 - Execution number 3");
+ }
+
+ [TestMethod]
+ public void ExecuteCustomTestExtensibilityWithTestDataTests()
+ {
+ this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2");
+ this.ValidatePassedTests(
+ "CustomTestMethod2 (B)",
+ "CustomTestMethod2 (B)",
+ "CustomTestMethod2 (B)");
+ this.ValidateFailedTests(
+ TestAssembly,
+ "CustomTestMethod2 (A)",
+ "CustomTestMethod2 (A)",
+ "CustomTestMethod2 (A)",
+ "CustomTestMethod2 (C)",
+ "CustomTestMethod2 (C)",
+ "CustomTestMethod2 (C)");
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
index 6005ad1c50..8b694c0e14 100644
--- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
+++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
@@ -39,6 +39,7 @@
+
diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
new file mode 100644
index 0000000000..c42be4a431
--- /dev/null
+++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
@@ -0,0 +1,77 @@
+namespace FxExtensibilityTestProject
+{
+ using System.Collections.Generic;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [IterativeTestClass(5)]
+ public class CustomTestExTests
+ {
+ private static int customTestMethod1ExecutionCount;
+ [IterativeTestMethod(5)]
+ public void CustomTestMethod1()
+ {
+ customTestMethod1ExecutionCount++;
+ Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
+ }
+
+ [IterativeTestMethod(3)]
+ [DataRow("A")]
+ [DataRow("B")]
+ [DataRow("C")]
+ public void CustomTestMethod2(string value)
+ {
+ Assert.AreEqual("B", value);
+ }
+
+ private static int customTestClass1ExecutionCount;
+ [TestMethod]
+ public void CustomTestClass1()
+ {
+ customTestClass1ExecutionCount++;
+ Assert.AreNotEqual(3, customTestClass1ExecutionCount);
+ }
+ }
+
+ public class IterativeTestMethodAttribute : TestMethodAttribute
+ {
+ private readonly int stabilityThreshold;
+
+ public IterativeTestMethodAttribute(int stabilityThreshold)
+ {
+ this.stabilityThreshold = stabilityThreshold;
+ }
+
+ public override TestResult[] Execute(ITestMethod testMethod)
+ {
+ var results = new List();
+ for (int count = 0; count < this.stabilityThreshold; count++)
+ {
+ var testResults = base.Execute(testMethod);
+ foreach (var testResult in testResults)
+ {
+ testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}";
+ }
+ results.AddRange(testResults);
+ }
+
+ return results.ToArray();
+ }
+ }
+
+ public class IterativeTestClassAttribute : TestClassAttribute
+ {
+ private readonly int stabilityThreshold;
+
+ public IterativeTestClassAttribute(int stabilityThreshold)
+ {
+ this.stabilityThreshold = stabilityThreshold;
+ }
+
+ public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute)
+ {
+ if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute;
+
+ return new IterativeTestMethodAttribute(this.stabilityThreshold);
+ }
+ }
+}
diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
index db9dc23cba..9e587e801d 100644
--- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
+++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
@@ -44,6 +44,7 @@
+
diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
index 9f66bad592..3dbd30d23e 100644
--- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
+++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
@@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
}
+ [TestMethodV1]
+ public void RunTestMethodForMultipleResultsReturnMultipleResults()
+ {
+ var testMethodAttributeMock = new Mock();
+ testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[]
+ {
+ new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed },
+ new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed }
+ });
+
+ var localTestMethodOptions = new TestMethodOptions
+ {
+ Timeout = 200,
+ Executor = testMethodAttributeMock.Object,
+ TestContext = this.testContextImplementation,
+ ExpectedException = null
+ };
+
+ var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null);
+ var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
+
+ var results = testMethodRunner.Execute();
+ Assert.AreEqual(2, results.Length);
+ Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome);
+ Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome);
+ }
+
[TestMethodV1]
public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome()
{
- var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
+ var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
var results = testMethodRunner.Execute();