Skip to content

Fix multiple results not returned with custom TestMethod and TestClass (#358) #363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 28, 2018
Merged

Fix multiple results not returned with custom TestMethod and TestClass (#358) #363

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f51024e
Fix Multiple results in VS2017 test runner (#336)
Feb 6, 2018
0b48abb
Fixed PR comments
Feb 7, 2018
d80a026
Added tests with data-driven tests
Feb 7, 2018
25bea17
Fixed build
Feb 7, 2018
a9f76bc
Merge branch 'master' into multipletestresultsbug
jayaranigarg Feb 12, 2018
fbee0f1
Merge branch 'master' into multipletestresultsbug
jayaranigarg Feb 12, 2018
b2af498
Merge remote-tracking branch 'upstream/master' into multipletestresul…
jayaranigarg Mar 27, 2018
97bca2b
Merge branch 'multipletestresultsbug' of https://github.com/bignonced…
jayaranigarg Mar 27, 2018
5374612
Merge branch 'master' into multipletestresultsbug
jayaranigarg Mar 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,22 +247,28 @@ internal UnitTestResult[] RunTestMethod()
watch.Start();

this.testContext.SetDataRow(dataRow);
UTF.TestResult currentResult;
UTF.TestResult[] testResults;

try
{
currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
testResults = new[]
{
new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
};
}

currentResult.DatarowIndex = rowIndex++;
watch.Stop();
currentResult.Duration = watch.Elapsed;
foreach (var testResult in testResults)
{
testResult.DatarowIndex = rowIndex++;
Copy link
Member

@jayaranigarg jayaranigarg Feb 6, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DatarowIndex is used for each row of data in dataSource.
By framework extensibility you have now started running each dataRow multiple times(say 5), but we should not increase dataRowIndex for each of the 5 runs of a DataRow. It should be incremented only once per dataRow and not once per testResult.
This statement should be testResult.DataRowIndex = rowIndex;
Do rowIndex++ separately outside of the loop.

testResult.Duration = watch.Elapsed;
}

results.Add(currentResult);
results.AddRange(testResults);
}
}
finally
Expand Down Expand Up @@ -293,18 +299,25 @@ internal UnitTestResult[] RunTestMethod()
foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
{
this.testMethodInfo.SetArguments(data);
UTF.TestResult currentResult;
UTF.TestResult[] testResults;
try
{
currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
}
catch (Exception ex)
{
currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
testResults = new[]
{
new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
};
}

foreach (var testResult in testResults)
{
testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
}

currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
results.Add(currentResult);
results.AddRange(testResults);
this.testMethodInfo.SetArguments(null);
}
}
Expand All @@ -313,7 +326,7 @@ internal UnitTestResult[] RunTestMethod()
{
try
{
results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]);
results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
}
catch (Exception ex)
{
Expand Down
1 change: 1 addition & 0 deletions test/E2ETests/Automation.CLI/CLITestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ public void ValidatePassedTestsContain(params string[] passedTests)
{
foreach (var test in passedTests)
{
var x = string.Join(", ", this.runEventsHandler.PassedTests.Select(_ => _.DisplayName));
Copy link
Member

@jayaranigarg jayaranigarg Feb 6, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this.

var testFound = this.runEventsHandler.PassedTests.Any(
p => test.Equals(p.TestCase?.FullyQualifiedName)
|| test.Equals(p.DisplayName));
Expand Down
32 changes: 32 additions & 0 deletions test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace MSTestAdapter.Smoke.E2ETests
{
using Microsoft.MSTestV2.CLIAutomation;
using Microsoft.VisualStudio.TestTools.UnitTesting;

[TestClass]
public class CustomTestExecutionExtensibilityTests : CLITestBase
{
private const string TestAssembly = "FxExtensibilityTestProject.dll";

[TestMethod]
public void ExecuteCustomTestExtensibilityTests()
{
this.InvokeVsTestForExecution(new string[] { TestAssembly });
this.ValidatePassedTestsContain(
"CustomTestMethod1 - Execution number 1",
"CustomTestMethod1 - Execution number 2",
"CustomTestMethod1 - Execution number 4",
"CustomTestMethod1 - Execution number 5",
"CustomTestClass1 - Execution number 1",
"CustomTestClass1 - Execution number 2",
"CustomTestClass1 - Execution number 4",
"CustomTestClass1 - Execution number 5");
this.ValidateFailedTestsContain(
"CustomTestMethod1 - Execution number 3",
"CustomTestClass1 - Execution number 3");
}
}
}
1 change: 1 addition & 0 deletions test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="AssertExtensibilityTests.cs" />
<Compile Include="CustomTestExecutionExtensibilityTests.cs" />
<Compile Include="DataSourceTests.cs" />
<Compile Include="DesktopCSharpCLITests.cs" />
<Compile Include="ParallelExecutionTests.cs" />
Expand Down
69 changes: 69 additions & 0 deletions test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
namespace FxExtensibilityTestProject
{
using System;
using System.Collections.Generic;
using Microsoft.VisualStudio.TestTools.UnitTesting;

[IterativeTestClass(5)]
public class CustomTestExTests
{
private static int customTestMethod1ExecutionCount;
[IterativeTestMethod(5)]
public void CustomTestMethod1()
{
customTestMethod1ExecutionCount++;
Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
}

private static int customTestClass1ExecutionCount;
[TestMethod]
public void CustomTestClass1()
{
customTestClass1ExecutionCount++;
Assert.AreNotEqual(3, customTestClass1ExecutionCount);
}
}

Copy link
Member

@jayaranigarg jayaranigarg Feb 6, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider adding a data-driven test with [IterativeTestMethod] attribute

public class IterativeTestMethodAttribute : TestMethodAttribute
{
private readonly int stabilityThreshold;

public IterativeTestMethodAttribute(int stabilityThreshold)
{
this.stabilityThreshold = stabilityThreshold;
}

public override TestResult[] Execute(ITestMethod testMethod)
{
var results = new List<TestResult>();
for (int count = 0; count < this.stabilityThreshold; count++)
{
var testResults = base.Execute(testMethod);
foreach (var testResult in testResults)
{
testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}";
}
results.AddRange(testResults);
}

return results.ToArray();
}
}

public class IterativeTestClassAttribute : TestClassAttribute
{
private readonly int stabilityThreshold;

public IterativeTestClassAttribute(int stabilityThreshold)
{
this.stabilityThreshold = stabilityThreshold;
}

public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute)
{
if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute;

return new IterativeTestMethodAttribute(this.stabilityThreshold);
}
}
}
1 change: 1 addition & 0 deletions test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="DynamicDataExTests.cs" />
<Compile Include="DynamicDataExMoreTests.cs" />
<Compile Include="CustomTestExTests.cs" />
<Compile Include="TestDataSourceExTests.cs" />
</ItemGroup>
<ItemGroup>
Expand Down
29 changes: 28 additions & 1 deletion test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
}

[TestMethodV1]
public void RunTestMethodForMultipleResultsReturnMultipleResults()
{
var testMethodAttributeMock = new Mock<UTF.TestMethodAttribute>();
testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny<UTF.ITestMethod>())).Returns(new[]
{
new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed },
new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed }
});

var localTestMethodOptions = new TestMethodOptions
{
Timeout = 200,
Executor = testMethodAttributeMock.Object,
TestContext = this.testContextImplementation,
ExpectedException = null
};

var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null);
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);

var results = testMethodRunner.Execute();
Assert.AreEqual(2, results.Length);
Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome);
Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome);
}

[TestMethodV1]
public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome()
{
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);

var results = testMethodRunner.Execute();
Expand Down