Skip to content

Support for CancellationTokenSource in TestContext to help in timeout… #585

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 28, 2019
Merged

Support for CancellationTokenSource in TestContext to help in timeout… #585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a3c2978
Support for CancellationTokenSource in TestContext to help in timeout…
cltshivash Mar 16, 2019
fedd47e
Fixed build failures in the tests
cltshivash Mar 19, 2019
20753dd
Added couple of unit tests
cltshivash Mar 26, 2019
6833766
Added E2E tests and took care of minor review comments
cltshivash Mar 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion TestFx.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CompatTestProject", "test\E
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProjectForAssemblyResolution", "test\ComponentTests\TestAssets\TestProjectForAssemblyResolution\TestProjectForAssemblyResolution.csproj", "{0B057B99-DCDD-417A-BC19-3E63DDD86F24}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataRowTestProject", "test\E2ETests\TestAssets\DataRowTestProject\DataRowTestProject.csproj", "{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProject", "test\E2ETests\TestAssets\TimeoutTestProject\TimeoutTestProject.csproj", "{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}"
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
Expand Down Expand Up @@ -1065,6 +1067,30 @@ Global
{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x64.Build.0 = Release|Any CPU
{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.ActiveCfg = Release|Any CPU
{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1}.Release|x86.Build.0 = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|ARM.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|ARM.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x64.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x64.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x86.ActiveCfg = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Debug|x86.Build.0 = Debug|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|Any CPU.Build.0 = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|ARM.ActiveCfg = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|ARM.Build.0 = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x64.ActiveCfg = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x64.Build.0 = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x86.ActiveCfg = Release|Any CPU
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -1125,6 +1151,7 @@ Global
{2D2C5B73-F1F1-47C8-BC5C-A172E9BB3D16} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
{0B057B99-DCDD-417A-BC19-3E63DDD86F24} = {1899187D-8B9C-40C2-9F04-9E9A76C9A919}
{7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}
Expand Down
49 changes: 26 additions & 23 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
using System.Globalization;
using System.Reflection;
using System.Text;
using System.Threading;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
Expand Down Expand Up @@ -453,8 +454,8 @@ private void RunTestCleanupMethod(object classInstance, TestResult result)

if (cleanupStackTrace.Length > 0)
{
cleanupStackTrace.Append(Resource.UTA_CleanupStackTrace);
cleanupStackTrace.Append(Environment.NewLine);
cleanupStackTrace.Append(Resource.UTA_CleanupStackTrace);
cleanupStackTrace.Append(Environment.NewLine);
}

Exception realException = ex.GetInnerExceptionOrDefault();
Expand Down Expand Up @@ -668,40 +669,42 @@ private TestResult ExecuteInternalWithTimeout(object[] arguments)
TestResult result = null;
Exception failure = null;

Action executeAsyncAction = () =>
void executeAsyncAction()
{
try
{
try
{
result = this.ExecuteInternal(arguments);
}
catch (Exception ex)
{
failure = ex;
}
};
result = this.ExecuteInternal(arguments);
}
catch (Exception ex)
{
failure = ex;
}
}

if (PlatformServiceProvider.Instance.ThreadOperations.Execute(executeAsyncAction, this.TestMethodOptions.Timeout))
CancellationToken cancelToken = this.TestMethodOptions.TestContext.Context.CancellationTokenSource.Token;
if (PlatformServiceProvider.Instance.ThreadOperations.Execute(executeAsyncAction, this.TestMethodOptions.Timeout, cancelToken))
{
if (failure != null)
{
throw failure;
}

Debug.Assert(result != null, "no timeout, no failure result should not be null");
return result;
}
else
{
// Timed out

// If the method times out, then
//
// 1. If the test is stuck, then we can get CannotUnloadAppDomain exception.
//
// Which are handled as follows: -
//
// For #1, we are now restarting the execution process if adapter fails to unload app-domain.
// Timed out or canceled
string errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Timeout, this.TestMethodName);
if (this.TestMethodOptions.TestContext.Context.CancellationTokenSource.IsCancellationRequested)
{
errorMessage = string.Format(CultureInfo.CurrentCulture, Resource.Execution_Test_Cancelled, this.TestMethodName);
}
else
{
// Cancel the token source as test has timedout
this.TestMethodOptions.TestContext.Context.CancellationTokenSource.Cancel();
}

TestResult timeoutResult = new TestResult() { Outcome = TestTools.UnitTesting.UnitTestOutcome.Timeout, TestFailureException = new TestFailedException(UnitTestOutcome.Timeout, errorMessage) };
return timeoutResult;
}
Expand Down
15 changes: 8 additions & 7 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,13 +183,14 @@ internal UnitTestResult[] Execute()
result = new[] { new UnitTestResult() };
}

var newResult =
new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()));
newResult.StandardOut = result[result.Length - 1].StandardOut;
newResult.StandardError = result[result.Length - 1].StandardError;
newResult.DebugTrace = result[result.Length - 1].DebugTrace;
newResult.TestContextMessages = result[result.Length - 1].TestContextMessages;
newResult.Duration = result[result.Length - 1].Duration;
var newResult = new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()))
{
StandardOut = result[result.Length - 1].StandardOut,
StandardError = result[result.Length - 1].StandardError,
DebugTrace = result[result.Length - 1].DebugTrace,
TestContextMessages = result[result.Length - 1].TestContextMessages,
Duration = result[result.Length - 1].Duration
};
result[result.Length - 1] = newResult;
}
finally
Expand Down
9 changes: 9 additions & 0 deletions src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,4 +317,7 @@ Error: {1}</value>
<data name="UTA_TestMethodExpectedParameters" xml:space="preserve">
<value>Only data driven test methods can have parameters. Did you intend to use [DataRow] or [DynamicData]?</value>
</data>
<data name="Execution_Test_Cancelled" xml:space="preserve">
Copy link
Contributor

@singhsarab singhsarab Mar 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please xlf file changes associated with this change

Copy link
Member

@jayaranigarg jayaranigarg Mar 28, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Run command "build.cmd -uxlf". You would have to commit the xlf files changes post running the script.

<value>Test '{0}' execution has been aborted.</value>
</data>
</root>
3 changes: 2 additions & 1 deletion src/Adapter/PlatformServices.Desktop/Services/DesktopTestContextImplementation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Deployment;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ObjectModel;
Expand Down Expand Up @@ -82,7 +83,7 @@ public TestContextImplementation(ITestMethod testMethod, StringWriter stringWrit
this.testMethod = testMethod;
this.stringWriter = stringWriter;
this.properties = new Dictionary<string, object>(properties);

this.CancellationTokenSource = new CancellationTokenSource();
this.InitializeProperties();

this.testResultFiles = new List<string>();
Expand Down
76 changes: 57 additions & 19 deletions src/Adapter/PlatformServices.Desktop/Services/DesktopThreadOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices
using System.Threading;

using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;

#pragma warning disable SA1649 // SA1649FileNameMustMatchTypeName

Expand All @@ -21,36 +22,44 @@ public class ThreadOperations : IThreadOperations
/// </summary>
/// <param name="action">The action to execute.</param>
/// <param name="timeout">Timeout for the specified action in milliseconds.</param>
/// <param name="cancelToken">Token to cancel the execution</param>
/// <returns>Returns true if the action executed before the timeout. returns false otherwise.</returns>
public bool Execute(Action action, int timeout)
public bool Execute(Action action, int timeout, CancellationToken cancelToken)
{
Thread executionThread = new Thread(new ThreadStart(action));
executionThread.IsBackground = true;
executionThread.Name = "MSTestAdapter Thread";
bool executionAborted = false;
Thread executionThread = new Thread(new ThreadStart(action))
{
IsBackground = true,
Name = "MSTestAdapter Thread"
};

executionThread.SetApartmentState(Thread.CurrentThread.GetApartmentState());
executionThread.Start();
cancelToken.Register(() =>
{
executionAborted = true;
AbortThread(executionThread);
});

if (executionThread.Join(timeout))
if (JoinThread(timeout, executionThread))
{
if (executionAborted)
{
return false;
}

// Successfully completed
return true;
}
else if (executionAborted)
{
// Execution aborted due to user choice
return false;
}
else
{
// Timed out
try
{
// Abort test thread after timeout.
executionThread.Abort();
}
catch (ThreadStateException)
{
// Catch and discard ThreadStateException. If Abort is called on a thread that has been suspended,
// a ThreadStateException is thrown in the thread that called Abort,
// and AbortRequested is added to the ThreadState property of the thread being aborted.
// A ThreadAbortException is not thrown in the suspended thread until Resume is called.
}

AbortThread(executionThread);
return false;
}
}
Expand All @@ -75,7 +84,36 @@ public void ExecuteWithAbortSafety(Action action)
throw new TargetInvocationException(exception);
}
}
}

private static bool JoinThread(int timeout, Thread executionThread)
{
try
{
return executionThread.Join(timeout);
}
catch (ThreadStateException)
{
// Join was called on a thread not started
}

return false;
}

private static void AbortThread(Thread executionThread)
{
try
{
// Abort test thread after timeout.
executionThread.Abort();
}
catch (ThreadStateException)
{
// Catch and discard ThreadStateException. If Abort is called on a thread that has been suspended,
// a ThreadStateException is thrown in the thread that called Abort,
// and AbortRequested is added to the ThreadState property of the thread being aborted.
// A ThreadAbortException is not thrown in the suspended thread until Resume is called.
}
}
}
#pragma warning restore SA1649 // SA1649FileNameMustMatchTypeName
}
4 changes: 3 additions & 1 deletion src/Adapter/PlatformServices.Interface/IThreadOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface
{
using System;
using System.Threading;

/// <summary>
/// This service is responsible for any thread operations specific to a platform.
Expand All @@ -15,8 +16,9 @@ public interface IThreadOperations
/// </summary>
/// <param name="action">The action to execute.</param>
/// <param name="timeout">Timeout for the specified action.</param>
/// <param name="cancelToken">Token to cancel the execution</param>
/// <returns>Returns true if the action executed before the timeout. returns false otherwise.</returns>
bool Execute(Action action, int timeout);
bool Execute(Action action, int timeout, CancellationToken cancelToken);

/// <summary>
/// Execute an action with handling for Thread Aborts (if possible) so the main thread of the adapter does not die.
Expand Down
3 changes: 2 additions & 1 deletion src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10TestContextImplementation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices
using System.Diagnostics;
using System.Globalization;
using System.IO;

using System.Threading;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ObjectModel;

Expand Down Expand Up @@ -61,6 +61,7 @@ public TestContextImplementation(ITestMethod testMethod, StringWriter writer, ID
this.testMethod = testMethod;
this.properties = new Dictionary<string, object>(properties);
this.stringWriter = writer;
this.CancellationTokenSource = new CancellationTokenSource();
this.InitializeProperties();
}

Expand Down
7 changes: 4 additions & 3 deletions src/Adapter/PlatformServices.Shared/netstandard1.0/Services/ns10ThreadOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices
{
using System;
using System.Threading;
using System.Threading.Tasks;

using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;
Expand All @@ -20,12 +21,12 @@ public class ThreadOperations : IThreadOperations
/// </summary>
/// <param name="action">The action to execute.</param>
/// <param name="timeout">Timeout for the specified action.</param>
/// <param name="cancelToken">Token to cancel the execution</param>
/// <returns>Returns true if the action executed before the timeout. returns false otherwise.</returns>
public bool Execute(Action action, int timeout)
public bool Execute(Action action, int timeout, CancellationToken cancelToken)
{
var executionTask = Task.Factory.StartNew(action);

if (executionTask.Wait(timeout))
if (executionTask.Wait(timeout, cancelToken))
{
return true;
}
Expand Down
6 changes: 6 additions & 0 deletions src/TestFramework/Extension.Desktop/TestContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace Microsoft.VisualStudio.TestTools.UnitTesting
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Threading;

/// <summary>
/// Used to store information that is provided to unit tests.
Expand All @@ -22,6 +23,11 @@ public abstract class TestContext
/// </summary>
public abstract IDictionary Properties { get; }

/// <summary>
/// Gets or sets the cancellation token source. This token source is cancelled when test timesout. Also when explicitly cancelled the test will be aborted
/// </summary>
public virtual CancellationTokenSource CancellationTokenSource { get; protected set; }

/// <summary>
/// Gets the current data row when test is used for data driven testing.
/// </summary>
Expand Down
Loading