diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 5559f91338..2ece924fcc 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -247,22 +247,30 @@ internal UnitTestResult[] RunTestMethod() watch.Start(); this.testContext.SetDataRow(dataRow); - UTF.TestResult currentResult; + UTF.TestResult[] testResults; try { - currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]; + testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { - currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }; + testResults = new[] + { + new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } + }; } - currentResult.DatarowIndex = rowIndex++; watch.Stop(); - currentResult.Duration = watch.Elapsed; + foreach (var testResult in testResults) + { + testResult.DatarowIndex = rowIndex; + testResult.Duration = watch.Elapsed; + } - results.Add(currentResult); + rowIndex++; + + results.AddRange(testResults); } } finally @@ -293,18 +301,25 @@ internal UnitTestResult[] RunTestMethod() foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) { this.testMethodInfo.SetArguments(data); - UTF.TestResult currentResult; + UTF.TestResult[] testResults; try { - currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]; + testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); } catch (Exception ex) { - currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }; + testResults = new[] + { + new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } + }; + } + + foreach (var testResult in testResults) + { + testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); } - currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); - results.Add(currentResult); + results.AddRange(testResults); this.testMethodInfo.SetArguments(null); } } @@ -313,7 +328,7 @@ internal UnitTestResult[] RunTestMethod() { try { - results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]); + results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); } catch (Exception ex) { diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs index 88cd82f668..f846b3e69a 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.cs @@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation using System.Linq; using System.Xml; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; public class CLITestBase @@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "") /// /// List of test assemblies. /// Run settings for execution. - public void InvokeVsTestForExecution(string[] sources, string runSettings = "") + /// Test Case filter for execution. + public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null) { for (var iterator = 0; iterator < sources.Length; iterator++) { @@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "") // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll")); - vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler); + vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler); } /// diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs new file mode 100644 index 0000000000..23f7cd81c9 --- /dev/null +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace MSTestAdapter.Smoke.E2ETests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CustomTestExecutionExtensibilityTests : CLITestBase + { + private const string TestAssembly = "FxExtensibilityTestProject.dll"; + + [TestMethod] + public void ExecuteCustomTestExtensibilityTests() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }); + this.ValidatePassedTestsContain( + "CustomTestMethod1 - Execution number 1", + "CustomTestMethod1 - Execution number 2", + "CustomTestMethod1 - Execution number 4", + "CustomTestMethod1 - Execution number 5", + "CustomTestClass1 - Execution number 1", + "CustomTestClass1 - Execution number 2", + "CustomTestClass1 - Execution number 4", + "CustomTestClass1 - Execution number 5"); + this.ValidateFailedTestsContain( + TestAssembly, + "CustomTestMethod1 - Execution number 3", + "CustomTestClass1 - Execution number 3"); + } + + [TestMethod] + public void ExecuteCustomTestExtensibilityWithTestDataTests() + { + this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2"); + this.ValidatePassedTests( + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)"); + this.ValidateFailedTests( + TestAssembly, + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)"); + } + } +} \ No newline at end of file diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj index 6005ad1c50..8b694c0e14 100644 --- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj +++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj @@ -39,6 +39,7 @@ + diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs new file mode 100644 index 0000000000..c42be4a431 --- /dev/null +++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs @@ -0,0 +1,77 @@ +namespace FxExtensibilityTestProject +{ + using System.Collections.Generic; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [IterativeTestClass(5)] + public class CustomTestExTests + { + private static int customTestMethod1ExecutionCount; + [IterativeTestMethod(5)] + public void CustomTestMethod1() + { + customTestMethod1ExecutionCount++; + Assert.AreNotEqual(3, customTestMethod1ExecutionCount); + } + + [IterativeTestMethod(3)] + [DataRow("A")] + [DataRow("B")] + [DataRow("C")] + public void CustomTestMethod2(string value) + { + Assert.AreEqual("B", value); + } + + private static int customTestClass1ExecutionCount; + [TestMethod] + public void CustomTestClass1() + { + customTestClass1ExecutionCount++; + Assert.AreNotEqual(3, customTestClass1ExecutionCount); + } + } + + public class IterativeTestMethodAttribute : TestMethodAttribute + { + private readonly int stabilityThreshold; + + public IterativeTestMethodAttribute(int stabilityThreshold) + { + this.stabilityThreshold = stabilityThreshold; + } + + public override TestResult[] Execute(ITestMethod testMethod) + { + var results = new List(); + for (int count = 0; count < this.stabilityThreshold; count++) + { + var testResults = base.Execute(testMethod); + foreach (var testResult in testResults) + { + testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}"; + } + results.AddRange(testResults); + } + + return results.ToArray(); + } + } + + public class IterativeTestClassAttribute : TestClassAttribute + { + private readonly int stabilityThreshold; + + public IterativeTestClassAttribute(int stabilityThreshold) + { + this.stabilityThreshold = stabilityThreshold; + } + + public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute) + { + if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute; + + return new IterativeTestMethodAttribute(this.stabilityThreshold); + } + } +} diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj index db9dc23cba..9e587e801d 100644 --- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj +++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj @@ -44,6 +44,7 @@ + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index 9f66bad592..3dbd30d23e 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test"); } + [TestMethodV1] + public void RunTestMethodForMultipleResultsReturnMultipleResults() + { + var testMethodAttributeMock = new Mock(); + testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] + { + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, + new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } + }); + + var localTestMethodOptions = new TestMethodOptions + { + Timeout = 200, + Executor = testMethodAttributeMock.Object, + TestContext = this.testContextImplementation, + ExpectedException = null + }; + + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); + var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); + + var results = testMethodRunner.Execute(); + Assert.AreEqual(2, results.Length); + Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); + Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); + } + [TestMethodV1] public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome() { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed }); + var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed }); var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); var results = testMethodRunner.Execute();