Skip to content

Refined runsettings matching criteria for test sessions #3610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cvpoienaru merged 3 commits into from
Apr 29, 2022
Merged

Refined runsettings matching criteria for test sessions #3610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1b0f8cb
Refined runsettings matching criteria for test sessions
Apr 27, 2022
94a1a9e
Refined runsettings matching criteria for test sessions
Apr 27, 2022
6c1b8da
Refactored caching of test session env variables
Apr 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;

using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
using Microsoft.VisualStudio.TestPlatform.Utilities;

using CrossPlatResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources;

Expand Down Expand Up @@ -42,6 +45,22 @@ private enum TestSessionState
private readonly IList<ProxyOperationManagerContainer> _proxyContainerList;
private readonly IDictionary<string, int> _proxyMap;
private readonly Stopwatch _testSessionStopwatch;
private IDictionary<string, string> _testSessionEnvironmentVariables = new Dictionary<string, string>();

private IDictionary<string, string> TestSessionEnvironmentVariables
{
get
{
if (_testSessionEnvironmentVariables.Count == 0)
{
_testSessionEnvironmentVariables = InferRunSettingsHelper.GetEnvironmentVariables(
_testSessionCriteria.RunSettings)
?? _testSessionEnvironmentVariables;
}

return _testSessionEnvironmentVariables;
}
}

/// <summary>
/// Initializes a new instance of the <see cref="ProxyTestSessionManager"/> class.
Expand Down Expand Up @@ -227,10 +246,7 @@ public virtual ProxyOperationManager DequeueProxy(string source, string runSetti
// We must ensure the current run settings match the run settings from when the
// testhost was started. If not, throw an exception to force the caller to create
// its own proxy instead.
//
// TODO (copoiena): This run settings match is rudimentary. We should refine the
// match criteria in the future.
if (!_testSessionCriteria.RunSettings.Equals(runSettings))
if (!CheckRunSettingsAreCompatible(runSettings))
{
throw new InvalidOperationException(
string.Format(
Expand Down Expand Up @@ -382,6 +398,23 @@ private void DisposeProxies()
_proxyMap.Clear();
}
}

private bool CheckRunSettingsAreCompatible(string requestRunSettings)
{
// Environment variable sets should be identical, otherwise it's not safe to reuse the
// already running testhosts.
var requestEnvironmentVariables = InferRunSettingsHelper.GetEnvironmentVariables(requestRunSettings);
if (requestEnvironmentVariables != null
&& TestSessionEnvironmentVariables != null
&& (requestEnvironmentVariables.Count != TestSessionEnvironmentVariables.Count
|| requestEnvironmentVariables.Except(TestSessionEnvironmentVariables).Any()))
{
return false;
}

// Data collection is not supported for test sessions yet.
return !XmlRunSettingsUtilities.IsDataCollectionEnabled(requestRunSettings);
}
}

/// <summary>
Expand Down
155 changes: 152 additions & 3 deletions test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,42 @@ public class ProxyTestSessionManagerTests
@"C:\temp\FakeTestAsset7.dll",
@"C:\temp\FakeTestAsset8.dll",
};
private readonly string _runSettingsNoEnvVars = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsOneEnvVar = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsTwoEnvVars = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
<BBB>Test2</BBB>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsTwoEnvVarsAndDataCollectors = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
<BBB>Test2</BBB>
</EnvironmentVariables>
</RunConfiguration>
<DataCollectionRunSettings>
<DataCollectors>
<DataCollector friendlyName=""blame"" enabled=""True""></DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>";
private readonly string _fakeRunSettings = "FakeRunSettings";
private readonly ProtocolConfig _protocolConfig = new() { Version = 1 };
private Mock<ITestSessionEventsHandler> _mockEventsHandler;
Expand Down Expand Up @@ -281,7 +317,7 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _fakeRunSettings);
var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsNoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Expand All @@ -302,7 +338,7 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
// Second call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
"DummyRunSettings"));
_runSettingsOneEnvVar));

// Third call to DequeueProxy succeeds.
Assert.AreEqual(proxyManager.DequeueProxy(
Expand All @@ -316,14 +352,127 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
testSessionCriteria.RunSettings));
}

[TestMethod]
public void DequeueProxyTwoConsecutiveTimesWithEnqueueShouldBeSuccessful()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// Call to DequeueProxy succeeds.
Assert.AreEqual(proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
testSessionCriteria.RunSettings),
mockProxyOperationManager.Object);

Assert.AreEqual(proxyManager.EnqueueProxy(mockProxyOperationManager.Object.Id), true);

// Call to DequeueProxy succeeds when called with the same runsettings as before.
Assert.AreEqual(proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
testSessionCriteria.RunSettings),
mockProxyOperationManager.Object);
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFails()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsOneEnvVar);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsTwoEnvVars));
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFailsFor2EnvVariables()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsOneEnvVar));
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFailsForDataCollectors()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsTwoEnvVarsAndDataCollectors));
}

[TestMethod]
public void EnqueueProxyShouldSucceedIfIdentificationCriteriaAreMet()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _fakeRunSettings);
var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsNoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// Validate sanity checks.
Expand Down