diff --git a/TestPlatform.sln b/TestPlatform.sln index 5824a83e66..05b0609b3c 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -179,6 +179,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTest1", "playground\MSTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataCollector", "test\TestAssets\AttachmentProcessorDataCollector\AttachmentProcessorDataCollector.csproj", "{B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testhost.arm64", "src\testhost.arm64\testhost.arm64.csproj", "{186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}" EndProject Global @@ -883,6 +889,42 @@ Global {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x64.Build.0 = Release|Any CPU {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.ActiveCfg = Release|Any CPU {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B}.Release|x86.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x64.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x64.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x86.ActiveCfg = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Debug|x86.Build.0 = Debug|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|Any CPU.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x64.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x64.Build.0 = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x86.ActiveCfg = Release|Any CPU + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}.Release|x86.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x64.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x64.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x86.ActiveCfg = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Debug|x86.Build.0 = Debug|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|Any CPU.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x64.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x64.Build.0 = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x86.ActiveCfg = Release|Any CPU + {BFBB35C9-6437-480A-8DCC-AE3700110E7D}.Release|x86.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x64.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x64.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x86.ActiveCfg = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Debug|x86.Build.0 = Debug|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|Any CPU.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x64.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x64.Build.0 = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x86.ActiveCfg = Release|Any CPU + {29270853-90DC-4C39-9621-F47AE40A79B6}.Release|x86.Build.0 = Release|Any CPU {186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}.Debug|Any CPU.Build.0 = Debug|Any CPU {186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -970,6 +1012,9 @@ Global {545A88D3-1AE2-4D39-9B7C-C691768AD17F} = {6CE2F530-582B-4695-A209-41065E103426} {57A61A09-10AD-44BE-8DF4-A6FD108F7DF7} = {6CE2F530-582B-4695-A209-41065E103426} {B6AF6BCD-64C6-4F4E-ABCA-C8AA2AA66B7B} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} + {B1F84FD8-6150-4ECA-9AD7-C316E04E17D8} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} + {BFBB35C9-6437-480A-8DCC-AE3700110E7D} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} + {29270853-90DC-4C39-9621-F47AE40A79B6} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} {186069FE-E1E8-4DE1-BEA4-0FF1484D22D1} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/src/Microsoft.TestPlatform.Client/Friends.cs b/src/Microsoft.TestPlatform.Client/Friends.cs index 90a5d9db6f..81da967ae4 100644 --- a/src/Microsoft.TestPlatform.Client/Friends.cs +++ b/src/Microsoft.TestPlatform.Client/Friends.cs @@ -10,5 +10,6 @@ #region Test Assemblies [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Client.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] #endregion diff --git a/src/Microsoft.TestPlatform.Client/TestPlatform.cs b/src/Microsoft.TestPlatform.Client/TestPlatform.cs index 2676096fdd..f46702a5fc 100644 --- a/src/Microsoft.TestPlatform.Client/TestPlatform.cs +++ b/src/Microsoft.TestPlatform.Client/TestPlatform.cs @@ -36,7 +36,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Client; /// internal class TestPlatform : ITestPlatform { - private readonly TestRuntimeProviderManager _testHostProviderManager; + private readonly ITestRuntimeProviderManager _testHostProviderManager; private readonly IFileHelper _fileHelper; @@ -66,10 +66,10 @@ public TestPlatform() /// The test engine. /// The file helper. /// The data. - protected TestPlatform( + protected internal TestPlatform( ITestEngine testEngine, IFileHelper filehelper, - TestRuntimeProviderManager testHostProviderManager) + ITestRuntimeProviderManager testHostProviderManager) { TestEngine = testEngine; _fileHelper = filehelper; @@ -117,6 +117,11 @@ public ITestRunRequest CreateTestRunRequest( ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData); loggerManager.Initialize(testRunCriteria.TestRunSettings); + // TODO: PERF: this will create a testhost manager, and then it will pass that to GetExecutionManager, where it will + // be used only when we will run in-process. If we don't run in process, we will throw away the manager we just + // created and let the proxy parallel callbacks to create a new one. This seems to be very easy to move to the GetExecutionManager, + // and safe as well, so we create the manager only once. + // TODO: Of course TestEngine.GetExecutionManager is public api... ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings); TestPlatform.ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings); diff --git a/src/Microsoft.TestPlatform.Common/Friends.cs b/src/Microsoft.TestPlatform.Common/Friends.cs index ea3f0a478d..23875a52a0 100644 --- a/src/Microsoft.TestPlatform.Common/Friends.cs +++ b/src/Microsoft.TestPlatform.Common/Friends.cs @@ -25,4 +25,6 @@ [assembly: InternalsVisibleTo("Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("Microsoft.TestPlatform.AcceptanceTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] + #endregion diff --git a/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs new file mode 100644 index 0000000000..57366c4669 --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; + +internal interface ITestRuntimeProviderManager +{ + ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration); + ITestRuntimeProvider GetTestHostManagerByUri(string hostUri); +} diff --git a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs index dfbd3db82d..9752ec6f57 100644 --- a/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs +++ b/src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs @@ -14,7 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting; /// /// Responsible for managing TestRuntimeProviderManager extensions /// -public class TestRuntimeProviderManager +public class TestRuntimeProviderManager : ITestRuntimeProviderManager { private static TestRuntimeProviderManager s_testHostManager; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs index 777230a217..63bae93f18 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/DataCollectionRequestHandler.cs @@ -152,6 +152,14 @@ public static DataCollectionRequestHandler Create( { ValidateArg.NotNull(communicationManager, nameof(communicationManager)); ValidateArg.NotNull(messageSink, nameof(messageSink)); + // TODO: The MessageSink and DataCollectionRequestHandler have circular dependency. + // Message sink is injected into this Create method and then into constructor + // and into the constructor of DataCollectionRequestHandler. Data collection manager + // is then assigned to .Instace (which unlike many other .Instance is not populated + // directly in that property, but is created here). And then MessageSink depends on + // the .Instance. This is a very complicated way of solving the circular dependency, + // and should be replaced by adding a property to Message and assigning it. + // .Instance can then be removed. if (Instance == null) { lock (SyncObject) diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs index 6ea4bbcd12..bf583737b6 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/Communication/ConnectedEventArgs.cs @@ -15,6 +15,7 @@ public class ConnectedEventArgs : EventArgs /// /// Initializes a new instance of the class. /// + // TODO: Do we need this constructor? public ConnectedEventArgs() { } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs index 269ae7d8b0..0abaa266a8 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ICommunicationEndpoint.cs @@ -15,7 +15,7 @@ public interface ICommunicationEndPoint event EventHandler Connected; /// - /// Event raised when an endPoint is disconnected. + /// Event raised when an endPoint is disconnected on failure. It should not be notified when we are just closing the connection after success. /// event EventHandler Disconnected; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs index d8602e29a0..f339111a90 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/Message.cs @@ -21,6 +21,9 @@ public class Message /// /// Gets or sets the payload. /// + // TODO: Our public contract says that we should be able to communicate over JSON, but we should not be stopping ourselves from + // negotiating a different protocol. Or using a different serialization library than NewtonsoftJson. Check why this is published as JToken + // and not as a string. public JToken Payload { get; set; } /// @@ -29,6 +32,8 @@ public class Message /// The . public override string ToString() { + // TODO: Review where this is used, we should avoid extensive serialization and deserialization, + // and this might be happening in multiple places that are not the edge of our process. return "(" + MessageType + ") -> " + (Payload == null ? "null" : Payload.ToString(Formatting.Indented)); } } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs index eb6bc7feff..a2a1120f57 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketClient.cs @@ -96,17 +96,17 @@ private void OnServerConnected(Task connectAsyncTask) // Start the message loop Task.Run(() => _tcpClient.MessageLoopAsync( _channel, - Stop, + StopOnError, _cancellation.Token)) .ConfigureAwait(false); } } } - private void Stop(Exception error) + private void StopOnError(Exception error) { EqtTrace.Info("SocketClient.PrivateStop: Stop communication from server endpoint: {0}, error:{1}", _endPoint, error); - + // This is here to prevent stack overflow. if (!_stopped) { // Do not allow stop to be called multiple times. diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs index c990b6ffcd..afdc029774 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/SocketServer.cs @@ -102,11 +102,16 @@ private void OnClientConnected(TcpClient client) EqtTrace.Verbose("SocketServer.OnClientConnected: Client connected for endPoint: {0}, starting MessageLoopAsync:", _endPoint); // Start the message loop - Task.Run(() => _tcpClient.MessageLoopAsync(_channel, error => Stop(error), _cancellation.Token)).ConfigureAwait(false); + Task.Run(() => _tcpClient.MessageLoopAsync(_channel, error => StopOnError(error), _cancellation.Token)).ConfigureAwait(false); } } - private void Stop(Exception error) + /// + /// Stop the connection when error was encountered. Dispose all communication, and notify subscribers of Disconnected event + /// that we aborted. + /// + /// + private void StopOnError(Exception error) { EqtTrace.Info("SocketServer.PrivateStop: Stopping server endPoint: {0} error: {1}", _endPoint, error); diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index ff818b46f5..e5c78bb33f 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -39,7 +39,7 @@ public class TestRequestSender : ITestRequestSender private readonly int _clientExitedWaitTime; - private ICommunicationEndPoint _communicationEndpoint; + private readonly ICommunicationEndPoint _communicationEndpoint; private ICommunicationChannel _channel; @@ -80,7 +80,6 @@ public TestRequestSender(ProtocolConfig protocolConfig, ITestRuntimeProvider run protocolConfig, ClientProcessExitWaitTimeout) { - SetCommunicationEndPoint(); } internal TestRequestSender( @@ -101,7 +100,14 @@ internal TestRequestSender( // The connectionInfo here is that of RuntimeProvider, so reverse the role of runner. _runtimeProvider = runtimeProvider; - _communicationEndpoint = communicationEndPoint; + + // TODO: In various places TestRequest sender is instantiated, and we can't easily inject the factory, so this is last + // resort of getting the dependency into the execution flow. + _communicationEndpoint = communicationEndPoint +#if DEBUG + ?? TestServiceLocator.Get(connectionInfo.Endpoint) +#endif + ?? SetCommunicationEndPoint(); _connectionInfo.Endpoint = connectionInfo.Endpoint; _connectionInfo.Role = connectionInfo.Role == ConnectionRole.Host ? ConnectionRole.Client @@ -145,6 +151,12 @@ public int InitializeCommunication() _communicationEndpoint.Connected += (sender, args) => { _channel = args.Channel; + // TODO: I suspect that Channel can be null only because of some unit tests, + // and being connected and actually not setting any channel should be error + // rather than silently waiting for timeout + // TODO: also this event is called back on connected, why are the event args holding + // the Connected boolean and why do we check it here. If we did not connect we should + // have not fired this event. if (args.Connected && _channel != null) { _connected.Set(); @@ -158,6 +170,7 @@ public int InitializeCommunication() // Server start returns the listener port // return int.Parse(this.communicationServer.Start()); var endpoint = _communicationEndpoint.Start(_connectionInfo.Endpoint); + // TODO: This is forcing us to use IP address and port for communication return endpoint.GetIpEndPoint().Port; } @@ -183,7 +196,8 @@ public void CheckVersionWithTestHost() { // Negotiation follows these steps: // Runner sends highest supported version to Test host - // Test host sends the version it can support (must be less than highest) to runner + // Test host compares the version with the highest version it can support. + // Test host sends back the lower number of the two. So the highest protocol version, that both sides support is used. // Error case: test host can send a protocol error if it cannot find a supported version var protocolNegotiated = new ManualResetEvent(false); _onMessageReceived = (sender, args) => @@ -530,7 +544,9 @@ private void OnExecutionMessageReceived(object sender, MessageReceivedEventArgs } catch (Exception exception) { - OnTestRunAbort(testRunEventsHandler, exception, false); + // If we failed to process the incoming message, initiate client (testhost) abort, because we can't recover, and don't wait + // for it to exit and write into error stream, because it did not do anything wrong, so no error is coming there + OnTestRunAbort(testRunEventsHandler, exception, getClientError: false); } } @@ -640,20 +656,25 @@ private void OnDiscoveryAbort(ITestDiscoveryEventsHandler2 eventHandler, Excepti private string GetAbortErrorMessage(Exception exception, bool getClientError) { + EqtTrace.Verbose("TestRequestSender: GetAbortErrorMessage: Exception: " + exception); // It is also possible for an operation to abort even if client has not - // disconnected, e.g. if there's an error parsing the response from test host. We - // want the exception to be available in those scenarios. + // disconnected, because we initiate client abort when there is error in processing incoming messages. + // in this case, we will use the exception as the failure result, if it is present. Otherwise we will + // try to wait for the client process to exit, and capture it's error output (we are listening to it's standard and + // error output in the ClientExited callback). var reason = exception?.Message; if (getClientError) { EqtTrace.Verbose("TestRequestSender: GetAbortErrorMessage: Client has disconnected. Wait for standard error."); // Wait for test host to exit for a moment + // TODO: this timeout is 10 seconds, make it also configurable like the other famous timeout that is 100ms if (_clientExited.Wait(_clientExitedWaitTime)) { - // Set a default message of test host process exited and additionally specify the error if present + // Set a default message of test host process exited and additionally specify the error if we were able to get it + // from error output of the process EqtTrace.Info("TestRequestSender: GetAbortErrorMessage: Received test host error message."); reason = CommonResources.TestHostProcessCrashed; if (!string.IsNullOrWhiteSpace(_clientExitErrorMessage)) @@ -712,18 +733,18 @@ private void SetOperationComplete() Interlocked.CompareExchange(ref _operationCompleted, 1, 0); } - private void SetCommunicationEndPoint() + private ICommunicationEndPoint SetCommunicationEndPoint() { // TODO: Use factory to get the communication endpoint. It will abstract out the type of communication endpoint like socket, shared memory or named pipe etc., if (_connectionInfo.Role == ConnectionRole.Client) { - _communicationEndpoint = new SocketClient(); EqtTrace.Verbose("TestRequestSender is acting as client."); + return new SocketClient(); } else { - _communicationEndpoint = new SocketServer(); EqtTrace.Verbose("TestRequestSender is acting as server."); + return new SocketServer(); } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 960afeb051..6a7e86d2c3 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -72,6 +72,7 @@ public ITestRunStatistics GetAggregatedRunStats() { foreach (var runStats in _testRunStatsList) { + // TODO: we get nullref here if the stats are empty. foreach (var outcome in runStats.Stats.Keys) { if (!testOutcomeMap.ContainsKey(outcome)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs index 772b079a07..47886ec36b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyExecutionManager.cs @@ -359,6 +359,9 @@ public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) /// public void HandleRawMessage(string rawMessage) { + // TODO: PERF: - why do we have to deserialize the messages here only to read that this is + // execution complete? Why can't we act on it somewhere else where the result of deserialization is not + // thrown away? var message = _dataSerializer.DeserializeMessage(rawMessage); if (string.Equals(message.MessageType, MessageType.ExecutionComplete)) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs index 530aca8b9c..88f0c020aa 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyOperationManager.cs @@ -324,8 +324,16 @@ public virtual void Close() { _initialized = false; - // Please clean up test host. - TestHostManager.CleanTestHostAsync(CancellationToken.None).Wait(); + // This is calling external code, make sure we don't fail when it throws + try + { + // Please clean up test host. + TestHostManager.CleanTestHostAsync(CancellationToken.None).Wait(); + } + catch (Exception ex) + { + EqtTrace.Error($"ProxyOperationManager: Cleaning testhost failed: {ex}"); + } TestHostManager.HostExited -= TestHostManagerHostExited; TestHostManager.HostLaunched -= TestHostManagerHostLaunched; @@ -407,6 +415,9 @@ private void CompatIssueWithVersionCheckAndRunsettings() { var properties = TestHostManager.GetType().GetRuntimeProperties(); + // The field is actually defaulting to true, so this is just a complicated way to set or not set + // this to true (modern testhosts should have it set to true). Bad thing about this is that we are checking + // internal "undocumented" property. Good thing is that if you don't implement it you get the modern behavior. var versionCheckProperty = properties.FirstOrDefault(p => string.Equals(p.Name, _versionCheckPropertyName, StringComparison.OrdinalIgnoreCase)); if (versionCheckProperty != null) { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs index b52f4adf02..25f5726567 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Friends.cs @@ -11,5 +11,6 @@ [assembly: InternalsVisibleTo("testhost, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("testhost.x86, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("vstest.console, PublicKey = 002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs index e8ea9e69c3..d8d5373ef8 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs @@ -34,7 +34,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; /// public class TestEngine : ITestEngine { - private readonly TestRuntimeProviderManager _testHostProviderManager; + private readonly ITestRuntimeProviderManager _testHostProviderManager; private ITestExtensionManager _testExtensionManager; private readonly IProcessHelper _processHelper; @@ -42,8 +42,14 @@ public class TestEngine : ITestEngine { } - protected TestEngine( + protected internal TestEngine( TestRuntimeProviderManager testHostProviderManager, + IProcessHelper processHelper) : this((ITestRuntimeProviderManager) testHostProviderManager, processHelper) + { + } + + internal TestEngine( + ITestRuntimeProviderManager testHostProviderManager, IProcessHelper processHelper) { _testHostProviderManager = testHostProviderManager; diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs index 76fce94c04..4db76fbc4c 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/TestRunCriteria.cs @@ -482,7 +482,7 @@ public TestRunCriteria( runStatsChangeEventTimeout, testHostLauncher) { - var testCases = tests as IList ?? tests.ToList(); + var testCases = tests as IList ?? tests?.ToList(); ValidateArg.NotNullOrEmpty(testCases, nameof(tests)); Tests = testCases; diff --git a/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs new file mode 100644 index 0000000000..6264588d08 --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/TestServiceLocator.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; + +// We don't want this in our shipped code. Build only for debug until I am able to remove it. +#if DEBUG + +#if !NETSTANDARD1_0 +using System; +#endif +using System.Collections.Generic; + +#pragma warning disable RS0016 // Add public types and members to the declared API +#pragma warning disable RS0037 // Enable tracking of nullability of reference types in the declared API + +public static class TestServiceLocator +{ + public static Dictionary Instances { get; } = new Dictionary(); + public static List Resolves { get; } = new(); + + public static void Register(string name, TRegistration instance) where TRegistration : notnull + { + Instances.Add(name, instance); + } + + public static TRegistration? Get(string name) + { + if (!Instances.TryGetValue(name, out var instance)) + { + return default; + // TODO: Add enable flag for the whole provider to activate so I can leverage throwing in programmer tests, but not run into it in Playground, or other debug builds. + // throw new InvalidOperationException($"Cannot find an instance for name {name}."); + } + +#if !NETSTANDARD1_0 + Resolves.Add(new Resolve(name, typeof(TRegistration).FullName, Environment.StackTrace)); +#endif + return (TRegistration)instance; + } + + public static void Clear() + { + Instances.Clear(); + Resolves.Clear(); + } +} + +// TODO: Make this internal, I am just trying to have easier time trying this out. +public class Resolve +{ + public Resolve(string name, string type, string stackTrace) + { + Name = name; + Type = type; + StackTrace = stackTrace; + } + + public string Name { get; } + public string Type { get; } + public string StackTrace { get; } +} + +#pragma warning restore RS0037 // Enable tracking of nullability of reference types in the declared API +#pragma warning restore RS0016 // Add public types and members to the declared API +#endif diff --git a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs index e65aef5e0a..4e7e37a311 100644 --- a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs +++ b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs @@ -499,6 +499,7 @@ private static void AddNodeIfNotPresent(XmlDocument xmlDocument, string nodeP if (root.SelectSingleNode(RunConfigurationNodePath) == null) { + // TODO: When runsettings are incomplete this will silently return, when we run just TestRequestManager we don't get full settings. EqtTrace.Error("InferRunSettingsHelper.UpdateNodeIfNotPresent: Unable to navigate to RunConfiguration. Current node: " + xmlDocument.LocalName); return; } diff --git a/src/vstest.console/CommandLine/TestRunResultAggregator.cs b/src/vstest.console/CommandLine/TestRunResultAggregator.cs index 47d6784a29..05b0505558 100644 --- a/src/vstest.console/CommandLine/TestRunResultAggregator.cs +++ b/src/vstest.console/CommandLine/TestRunResultAggregator.cs @@ -20,7 +20,7 @@ internal class TestRunResultAggregator /// Initializes the TestRunResultAggregator /// /// Constructor is private since the factory method should be used to get the instance. - protected TestRunResultAggregator() + protected internal TestRunResultAggregator() { // Outcome is passed until we see a failure. Outcome = TestOutcome.Passed; diff --git a/src/vstest.console/Friends.cs b/src/vstest.console/Friends.cs index 9b2a48fabe..9dd880e408 100644 --- a/src/vstest.console/Friends.cs +++ b/src/vstest.console/Friends.cs @@ -10,6 +10,7 @@ #region Test Assemblies [assembly: InternalsVisibleTo("vstest.console.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("vstest.ProgrammerTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("vstest.console.PlatformTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs b/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs index b085ba3637..f651207cb5 100644 --- a/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs +++ b/src/vstest.console/Processors/ArtifactProcessingCollectModeProcessor.cs @@ -91,9 +91,9 @@ internal class ArtifactProcessingCollectModeProcessorExecutor : IArgumentExecuto { private readonly CommandLineOptions _commandLineOptions; - public ArtifactProcessingCollectModeProcessorExecutor(CommandLineOptions options) + public ArtifactProcessingCollectModeProcessorExecutor(CommandLineOptions options!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); + _commandLineOptions = options; } public void Initialize(string argument) diff --git a/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs b/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs index 1693e0d058..65eee5d2b4 100644 --- a/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs +++ b/src/vstest.console/Processors/ArtifactProcessingPostProcessModeProcessor.cs @@ -95,10 +95,10 @@ internal class ArtifactProcessingPostProcessModeProcessorExecutor : IArgumentExe private readonly CommandLineOptions _commandLineOptions; private readonly IArtifactProcessingManager _artifactProcessingManage; - public ArtifactProcessingPostProcessModeProcessorExecutor(CommandLineOptions options, IArtifactProcessingManager artifactProcessingManager) + public ArtifactProcessingPostProcessModeProcessorExecutor(CommandLineOptions options!!, IArtifactProcessingManager artifactProcessingManager!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); - _artifactProcessingManage = artifactProcessingManager ?? throw new ArgumentNullException(nameof(artifactProcessingManager)); ; + _commandLineOptions = options; + _artifactProcessingManage = artifactProcessingManager; ; } public void Initialize(string argument) diff --git a/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs b/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs index 053ce3f9d5..52b00e8273 100644 --- a/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs +++ b/src/vstest.console/Processors/TestSessionCorrelationIdProcessor.cs @@ -86,9 +86,9 @@ internal class TestSessionCorrelationIdProcessorModeProcessorExecutor : IArgumen { private readonly CommandLineOptions _commandLineOptions; - public TestSessionCorrelationIdProcessorModeProcessorExecutor(CommandLineOptions options) + public TestSessionCorrelationIdProcessorModeProcessorExecutor(CommandLineOptions options!!) { - _commandLineOptions = options ?? throw new ArgumentNullException(nameof(options)); + _commandLineOptions = options; } public void Initialize(string argument) diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index dc0f43cde7..0888ad1e9b 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -52,6 +52,8 @@ internal class TestRequestManager : ITestRequestManager private readonly ITestPlatform _testPlatform; private readonly ITestPlatformEventSource _testPlatformEventSource; + // TODO: No idea what is Task supposed to buy us, Tasks start immediately on instantiation + // and the work done to produce the metrics publisher is minimal. private readonly Task _metricsPublisher; private readonly object _syncObject = new(); diff --git a/test/Intent.Primitives/ExcludeAttribute.cs b/test/Intent.Primitives/ExcludeAttribute.cs new file mode 100644 index 0000000000..4fe517e902 --- /dev/null +++ b/test/Intent.Primitives/ExcludeAttribute.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +public class ExcludeAttribute : Attribute +{ +} diff --git a/test/Intent.Primitives/IRunLogger.cs b/test/Intent.Primitives/IRunLogger.cs new file mode 100644 index 0000000000..474369e78c --- /dev/null +++ b/test/Intent.Primitives/IRunLogger.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +using System.Reflection; + +public interface IRunLogger +{ + void WriteTestPassed(MethodInfo m); + void WriteTestInconclusive(MethodInfo m); + void WriteTestFailure(MethodInfo m, Exception ex); + void WriteFrameworkError(Exception ex); +} diff --git a/test/Intent.Primitives/Intent.Primitives.csproj b/test/Intent.Primitives/Intent.Primitives.csproj new file mode 100644 index 0000000000..93ec3f6e75 --- /dev/null +++ b/test/Intent.Primitives/Intent.Primitives.csproj @@ -0,0 +1,13 @@ + + + + $(MSBuildThisFileDirectory)..\..\ + net6.0 + enable + enable + True + $(TestPlatformRoot)\scripts\key.snk + True + + + diff --git a/test/Intent.Primitives/OnlyAttribute.cs b/test/Intent.Primitives/OnlyAttribute.cs new file mode 100644 index 0000000000..3d327e1f55 --- /dev/null +++ b/test/Intent.Primitives/OnlyAttribute.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Constructor | AttributeTargets.Method)] +public class OnlyAttribute : Attribute +{ +} diff --git a/test/Intent.Primitives/TestResult.cs b/test/Intent.Primitives/TestResult.cs new file mode 100644 index 0000000000..053167718f --- /dev/null +++ b/test/Intent.Primitives/TestResult.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +public enum TestResult +{ + None, + Passed, + Failed, + Error, +} diff --git a/test/Intent/ConsoleLogger.cs b/test/Intent/ConsoleLogger.cs new file mode 100644 index 0000000000..b41c484532 --- /dev/null +++ b/test/Intent/ConsoleLogger.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent.Console; +using System.Reflection; +using System.Text.RegularExpressions; + +using static System.Console; +using static System.ConsoleColor; + +internal class ConsoleLogger : IRunLogger +{ + public void WriteTestInconclusive(MethodInfo m) + { + var currentColor = ForegroundColor; + ForegroundColor = Yellow; + WriteLine($"[?] {FormatMethodName(m.Name)}"); + ForegroundColor = currentColor; + } + + public void WriteTestPassed(MethodInfo m) + { + var currentColor = ForegroundColor; + ForegroundColor = Green; + WriteLine($"[+] {FormatMethodName(m.Name)}"); + ForegroundColor = currentColor; + } + + public void WriteTestFailure(MethodInfo m, Exception ex) + { + var currentColor = ForegroundColor; + ForegroundColor = Red; + WriteLine($"[-] {FormatMethodName(m.Name)}{Environment.NewLine}{ex}"); + ForegroundColor = currentColor; + } + + public void WriteFrameworkError(Exception ex) + { + var currentColor = ForegroundColor; + ForegroundColor = DarkRed; + WriteLine($"[-] framework failed{Environment.NewLine}{ex}{Environment.NewLine}{Environment.NewLine}"); + ForegroundColor = currentColor; + } + + private static string FormatMethodName(string methodName) + { + var noUnderscores = methodName.Replace('_', ' '); + // insert space before every capital letter or number that is after a non-capital letter + var spaced = Regex.Replace(noUnderscores, "(?<=[a-z])([A-Z0-9])", " $1"); + // insert space before every capital leter that is after a number + var spaced2 = Regex.Replace(spaced, "(?<=[0-9]|^)([A-Z])", " $1"); + var newLines = spaced2.Replace("When", $"{Environment.NewLine} When") + .Replace("Then", $"{Environment.NewLine} Then"); + + return newLines.ToLowerInvariant(); + } +} diff --git a/test/Intent/Extensions.cs b/test/Intent/Extensions.cs new file mode 100644 index 0000000000..78a9bef318 --- /dev/null +++ b/test/Intent/Extensions.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +using System.Reflection; + +public static class Extensions +{ + public static bool IsExcluded(this Assembly asm) + { + return asm.CustomAttributes.Any(a => a.AttributeType == typeof(ExcludeAttribute)); + } + + public static List SkipExcluded(this IEnumerable e) + { + return e.Where(i => i.GetCustomAttribute() == null).ToList(); + } + + public static List SkipNonPublic(this IEnumerable e) + { + return e.Where(i => i.IsPublic).ToList(); + } + + public static List SkipExcluded(this IEnumerable e) + { + return e.Where(i => + i.Name != nameof(object.ToString) + && i.Name != nameof(object.GetType) + && i.Name != nameof(object.GetHashCode) + && i.Name != nameof(object.Equals) + && i.GetCustomAttribute() == null).ToList(); + } +} diff --git a/test/Intent/Intent.csproj b/test/Intent/Intent.csproj new file mode 100644 index 0000000000..97b464df79 --- /dev/null +++ b/test/Intent/Intent.csproj @@ -0,0 +1,18 @@ + + + + $(MSBuildThisFileDirectory)..\..\ + Exe + net6.0 + enable + enable + True + True + $(TestPlatformRoot)\scripts\key.snk + + + + + + + diff --git a/test/Intent/Program.cs b/test/Intent/Program.cs new file mode 100644 index 0000000000..75d50702b1 --- /dev/null +++ b/test/Intent/Program.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent.Console; + +public class Program +{ + public static void Main(string[] path) + { + Runner.Run(path, new ConsoleLogger()); + } +} diff --git a/test/Intent/Runner.cs b/test/Intent/Runner.cs new file mode 100644 index 0000000000..3bab3db2c7 --- /dev/null +++ b/test/Intent/Runner.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Intent; + +using System.Reflection; + +public class Runner +{ + public static void Run(IEnumerable paths, IRunLogger logger) + { + foreach (var path in paths) + { + try + { + var assembly = Assembly.LoadFrom(path); + if (assembly.IsExcluded()) + continue; + + var types = assembly.GetTypes().SkipNonPublic().SkipExcluded(); + foreach (var type in types) + { + var methods = type.GetMethods().SkipExcluded(); + + // TODO: This chooses the Only tests only for single assembly and single class, + // to support this full we would have to enumerate all classes and methods first, + // it is easy, I just don't need it right now. + var methodsWithOnly = methods.Where(m => m.GetCustomAttribute() != null).ToList(); + if (methodsWithOnly.Count > 0) + methods = methodsWithOnly; + + foreach (var method in methods) + { + try + { + var instance = Activator.CreateInstance(type); + var testResult = method.Invoke(instance, Array.Empty()); + if (testResult is Task task) + { + // When the result is a task we need to await it. + // TODO: this can be improved with await, imho + task.GetAwaiter().GetResult(); + }; + + logger.WriteTestPassed(method); + } + catch (Exception ex) + { + if (ex is TargetInvocationException tex && tex.InnerException != null) + { + logger.WriteTestFailure(method, tex.InnerException); + } + else + { + logger.WriteTestFailure(method, ex); + } + } + } + } + } + catch (Exception ex) + { + logger.WriteFrameworkError(ex); + } + } + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs new file mode 100644 index 0000000000..9e461e0093 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/DebugOptions.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#pragma warning disable IDE1006 // Naming Styles +// For some reason only this occurence of vstest is flagged in build, and no other. +namespace vstest.ProgrammerTests.Fakes; +#pragma warning restore IDE1006 // Naming Styles + +internal class DebugOptions +{ + public const int DefaultTimeout = 5; + // TODO: This setting is actually quite pointless, because I cannot come up with + // a useful way to abort quickly enough when debugger is attached and I am just running my tests (pressing F5) + // but at the same time not abort when I am in the middle of debugging some behavior. Maybe looking at debugger, + // and asking it if any breakpoints were hit / are set. But that is difficult. + // + // So normally I press F5 to investigate, but Ctrl+F5 (run without debugger), to run tests. + public const int DefaultDebugTimeout = 30 * 60; + public const bool DefaultBreakOnAbort = true; + public int Timeout { get; init; } = DefaultTimeout; + public int DebugTimeout { get; init; } = DefaultDebugTimeout; + public bool BreakOnAbort { get; init; } = DefaultBreakOnAbort; +} diff --git a/test/vstest.ProgrammerTests/Fakes/EventRecord.cs b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs new file mode 100644 index 0000000000..0bfcd03776 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/EventRecord.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class EventRecord +{ + public object? Sender { get; } + + public T Data { get; } + + public EventRecord(object? sender, T data) + { + Sender = sender; + Data = data; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs new file mode 100644 index 0000000000..fef8088fd0 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeAssemblyMetadataProvider.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeAssemblyMetadataProvider : IAssemblyMetadataProvider +{ + public FakeFileHelper FakeFileHelper { get; } + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public FakeAssemblyMetadataProvider(FakeFileHelper fakeFileHelper, FakeErrorAggregator fakeErrorAggregator) + { + FakeFileHelper = fakeFileHelper; + FakeErrorAggregator = fakeErrorAggregator; + } + + public Architecture GetArchitecture(string filePath) + { + var file = FakeFileHelper.GetFakeFile(filePath); + return file.Architecture; + } + + public FrameworkName GetFrameWork(string filePath) + { + var file = FakeFileHelper.GetFakeFile(filePath); + return file.FrameworkName; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs new file mode 100644 index 0000000000..3d26712557 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationChannel.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Collections.Concurrent; +using System.Diagnostics; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal abstract class FakeCommunicationChannel : ICommunicationChannel +{ + public FakeCommunicationChannel(int id) + { + Id = id; + } + + public int Id { get; } + public CancellationTokenSource CancellationTokenSource { get; } = new(); + + public BlockingCollection InQueue { get; } = new(); + public BlockingCollection OutQueue { get; } = new(); + + /// + /// True if we encountered unexpected message (e.g. unknown message, message out of order) or when we sent all our prepared responses, and there were still more requests coming. + /// + public bool Faulted { get; protected set; } + + #region ICommunicationChannel + // The naming for ICommunicationChannel is a bit confusing when this is implemented in-process. + // Normally one side in one process would have one end of the communication channel, + // and would use Send to pass message to another process. The other side would get notified + // about new data by NotifyDataAvailable, read the data there, and send them to other consumers + // using MessageReceived event. These consumers would then call Send, the channel and post the data + // to the other process. The other process would recieve the data and be notified by MessageReceived. + // + // But when we are in the same process, one side sends data using Send. We recieve them here by reading + // them from the queue (NotifyDataAvailable is not used, because we monitor the queue directly here, instead of + // in communication manager). And then we reply back to the sender, by invoking MessageReceived. + + public event EventHandler? MessageReceived; + + public void Dispose() + { + CancellationTokenSource.Cancel(); + InQueue.CompleteAdding(); + } + + public Task NotifyDataAvailable() + { + // This is used only by communication manager. vstest.console does not use communication manager. + throw new NotImplementedException(); + } + + public Task Send(string data) + { + InQueue.Add(data); + return Task.CompletedTask; + } + #endregion + + public void OnMessageReceived(object sender, MessageReceivedEventArgs eventArgs) + { + // This is still a race condition. In real code we solve this via SafeInvoke that does null check + // and catches the exception. In this code I prefer doing it this way, to see if it is fragile. + MessageReceived?.Invoke(this, eventArgs); + } +} + +internal class FakeCommunicationChannel : FakeCommunicationChannel, ICommunicationChannel +{ + + /// + /// Queue of MessageType of the incoming request, and the response that will be sent back. + /// + public Queue> NextResponses { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeMessage? OutgoingMessage { get; private set; } + public TContext? Context { get; private set; } + public List> ProcessedMessages { get; } = new(); + public Task? ProcessIncomingMessagesTask { get; private set; } + public Task? ProcessOutgoingMessagesTask { get; private set; } + + public FakeCommunicationChannel(List> responses, FakeErrorAggregator fakeErrorAggregator, int id) : base (id) + { + FakeErrorAggregator = fakeErrorAggregator; + responses.ForEach(NextResponses.Enqueue); + } + + public void Start(TContext context) + { + Context = context; + ProcessIncomingMessagesTask = Task.Run(() => ProcessIncomingMessages(context), CancellationTokenSource.Token); + ProcessOutgoingMessagesTask = Task.Run(ProcessOutgoingMessages, CancellationTokenSource.Token); + } + + private void ProcessOutgoingMessages() + { + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + // TODO: better name? this is message that we are currently trying to send + OutgoingMessage = OutQueue.Take(); + OnMessageReceived(this, new MessageReceivedEventArgs { Data = OutgoingMessage.SerializedMessage }); + OutgoingMessage = null; + } + catch (Exception ex) + { + FakeErrorAggregator.Add(ex); + } + } + } + + private void ProcessIncomingMessages(TContext context) + { + var token = CancellationTokenSource.Token; + while (!token.IsCancellationRequested) + { + try + { + var rawMessage = InQueue.Take(token); + var requestMessage = JsonDataSerializer.Instance.DeserializeMessage(rawMessage); + + if (Faulted) + { + // We already failed, when there are more requests coming, just save them and respond with error. We want to avoid + // a situation where server ignores our error message and responds with another request, for which we accidentally + // have the right answer in queue. + // + // E.g. We have VersionCheck, TestRunStart prepared, and server sends: VersionCheck, TestInitialize, TestRunStart. + // The first request has a valid response. The next TestInitialize does not have a valid response and errors out, + // but the server ignores it, and sends TestRunStart, which would normally have a prepared response, and lead to + // possibly overlooking the error response to TestInitialize. + // + // With this check in place we will not respond to TestRunStart with success, but with error. + // TODO: Better way to map MessageType and the payload type. + // TODO: simpler way to report error, and add it to the error aggregator + var errorMessage = $"FakeCommunicationChannel: FakeCommunicationChannel: Got message {requestMessage.MessageType}. But a message that was unexptected was received previously and the channel is now faulted. Review {nameof(ProcessedMessages)}, and {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + + // Just peek at it so we can keep the message on the the queue in case of error. + if (!NextResponses.TryPeek(out var nextResponsePair)) + { + // If there are no more prepared responses then return protocol error. + var errorMessage = $"FakeCommunicationChannel: Got message {requestMessage.MessageType}, but no more requests were expected, because there are no more responses in {nameof(NextResponses)}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else if (nextResponsePair.Request != requestMessage.MessageType) + { + // If the incoming message does not match what we expected return protocol error. The lsat message will remain in the + // NextResponses queue. + var errorMessage = $"FakeCommunicationChannel: Excpected message {nextResponsePair.Request} but got {requestMessage.MessageType}."; + var exception = new Exception(errorMessage); + FakeErrorAggregator.Add(exception); + var errorResponse = new FakeMessage(MessageType.ProtocolError, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = errorMessage }); + ProcessedMessages.Add(new RequestResponsePair(requestMessage, errorResponse)); + Faulted = true; + OutQueue.Add(errorResponse); + } + else + { + var responsePair = NextResponses.Dequeue(); + if (responsePair.Debug && Debugger.IsAttached) + { + // We are about to send an interesting message + Debugger.Break(); + } + + // TODO: passing the raw message in, is strange + responsePair.BeforeAction?.Invoke(context); + var responses = responsePair.Responses; + ProcessedMessages.Add(new RequestResponsePair(requestMessage, responses, false)); + + foreach (var response in responses) + { + // If we created a pair with NoResponse message, we won't send that back to the server. + if (response != FakeMessage.NoResponse) + { + OutQueue.Add(response); + } + } + + responsePair.AfterAction?.Invoke(context); + } + } + catch (Exception ex) + { + FakeErrorAggregator.Add(ex); + } + } + } +} + diff --git a/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs new file mode 100644 index 0000000000..07c53e9ae4 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeCommunicationEndpoint.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.Utilities; + +internal class FakeCommunicationEndpoint : ICommunicationEndPoint +{ + private bool _stopped; + + public FakeCommunicationEndpoint(FakeCommunicationChannel fakeCommunicationChannel, FakeErrorAggregator fakeErrorAggregator) + { + Channel = fakeCommunicationChannel; + FakeErrorAggregator = fakeErrorAggregator; + TestHostConnectionInfo = new TestHostConnectionInfo + { + Endpoint = $"127.0.0.1:{fakeCommunicationChannel.Id}", + Role = ConnectionRole.Client, + Transport = Transport.Sockets, + }; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeCommunicationChannel Channel { get; } + public TestHostConnectionInfo TestHostConnectionInfo { get; } + + /// + /// Notify the caller that we disconnected, this happens if process exits unexpectedly and leads to abort flow. + /// In success case use Stop instead, to just "close" the channel, because the other side already disconnected from us + /// and told us to tear down. + /// + public void Disconnect() + { + Disconnected?.Invoke(this, new DisconnectedEventArgs()); + _stopped = true; + } + + #region ICommunicationEndPoint + + public event EventHandler? Connected; + public event EventHandler? Disconnected; + + public string Start(string endPoint) + { + // In normal run this endpoint can be a client or a server. When we are a client we will get an address and a port and + // we will try to connect to it. + // If we are a server, we will get an address and port 0, which means we should figure out a port that is free + // and return the address and port back to the caller. + // + // In our fake scenario we know the "port" from the get go, we set it to an id that was given to the testhost + // because that is currently the only way for us to check if we are connecting to the expected fake testhost + // that has a list of canned responses, which must correlate with the requests. So e.g. if we get request for mstest1.dll + // we should return the responses we have prepared for mstest1.dll, and not for mstest2.dll. + // + // We use the port number because the rest of the IP address is validated. We force sockets and IP usage in multiple places, + // so we cannot just past the dll path (or something similar) as the endpoint name, because the other side will check if that is + // a valid IP address and port. + if (endPoint != TestHostConnectionInfo.Endpoint) + { + throw new InvalidOperationException($"Expected to connect to {endPoint} but instead got channel with {TestHostConnectionInfo.Endpoint}."); + } + Connected?.SafeInvoke(this, new ConnectedEventArgs(Channel), "FakeCommunicationEndpoint.Start"); + return endPoint; + } + + public void Stop() + { + if (!_stopped) + { + // Do not allow stop to be called multiple times, because it will end up calling us back and stack overflows. + _stopped = true; + } + } + + #endregion +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs new file mode 100644 index 0000000000..664dd699a1 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeDataCollectorAttachmentsProcessorsFactory.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class FakeDataCollectorAttachmentsProcessorsFactory : IDataCollectorAttachmentsProcessorsFactory +{ + public FakeDataCollectorAttachmentsProcessorsFactory(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public DataCollectorAttachmentProcessor[] Create(InvokedDataCollector[] invokedDataCollectors, IMessageLogger logger) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs b/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs new file mode 100644 index 0000000000..25d04fc094 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeErrorAggregator.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class FakeErrorAggregator +{ + public List Errors { get; } = new(); + + public void Add(object error) + { + Errors.Add(error); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs new file mode 100644 index 0000000000..e9764eab4b --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeFile.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class FakeFile +{ + public string Path { get; } + + public FakeFile(string path) + { + Path = path; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs new file mode 100644 index 0000000000..30ebab8f53 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeFileHelper.cs @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +internal class FakeFileHelper : IFileHelper +{ + public FakeFileHelper(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public List Files { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } + + public void CopyFile(string sourcePath, string destinationPath) + { + throw new NotImplementedException(); + } + + public DirectoryInfo CreateDirectory(string path) + { + throw new NotImplementedException(); + } + + public void Delete(string path) + { + throw new NotImplementedException(); + } + + public void DeleteDirectory(string directoryPath, bool recursive) + { + throw new NotImplementedException(); + } + + public void DeleteEmptyDirectroy(string directoryPath) + { + throw new NotImplementedException(); + } + + public bool DirectoryExists(string path) + { + // TODO: Check if any file has the directory in name. This will improve. + var directoryExists = Files.Select(f => Path.GetDirectoryName(f.Path)).Any(p => p != null && p.StartsWith(path)); + return directoryExists; + } + + public IEnumerable EnumerateFiles(string directory, SearchOption searchOption, params string[] endsWithSearchPatterns) + { + Func predicate = searchOption == SearchOption.TopDirectoryOnly + ? (f, dir) => Path.GetDirectoryName(f.Path) == dir + : (f, dir) => Path.GetDirectoryName(f.Path)!.Contains(dir); + + var files = Files.Where(f => predicate(f, directory)).Select(f => f.Path).ToList(); + return files; + } + + public bool Exists(string path) + { + throw new NotImplementedException(); + } + + public string GetCurrentDirectory() + { + throw new NotImplementedException(); + } + + public FileAttributes GetFileAttributes(string path) + { + throw new NotImplementedException(); + } + + public long GetFileLength(string path) + { + throw new NotImplementedException(); + } + + public string[] GetFiles(string path, string searchPattern, SearchOption searchOption) + { + throw new NotImplementedException(); + } + + public Version GetFileVersion(string path) + { + throw new NotImplementedException(); + } + + public string GetFullPath(string path) + { + throw new NotImplementedException(); + } + + public Stream GetStream(string filePath, FileMode mode, FileAccess access = FileAccess.ReadWrite) + { + throw new NotImplementedException(); + } + + public Stream GetStream(string filePath, FileMode mode, FileAccess access, FileShare share) + { + throw new NotImplementedException(); + } + + public string GetTempPath() + { + throw new NotImplementedException(); + } + + public void MoveFile(string sourcePath, string destinationPath) + { + throw new NotImplementedException(); + } + + public void WriteAllTextToFile(string filePath, string content) + { + throw new NotImplementedException(); + } + + internal void AddFakeFile(T file) where T : FakeFile + { + if (Files.Any(f => f.Path.Equals(file.Path, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Fake file '{file.Path}' already exists."); + } + + Files.Add(file); + } + + internal T GetFakeFile(string path) where T : FakeFile + { + var matchingFiles = Files.Where(f => f.Path == path).ToList(); + if (matchingFiles.Count == 0) + throw new FileNotFoundException($"Fake file {path}, was not found. Check if file was previously added to FakeFileHelper."); + + // TODO: The public collection of files should probably be made readonly / immutable, and internally be made a concurrent dictionary, because it does not make + // sense to have more than 1 file object with the same name, and we check for that in AddFakeFile anyway. + if (matchingFiles.Count > 1) + throw new InvalidOperationException($"Fake file {path}, exists more than once. Are you modifying the Files collection in FakeFileHelper manually?"); + + var file = matchingFiles.Single(); + if (file is not T result) + throw new InvalidOperationException($"Fake file {path}, was supposed to be a {typeof(T)}, but was {file.GetType()}."); + + return result; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs new file mode 100644 index 0000000000..ffbeb8666d --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeMessage.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; + +/// +/// Marker for Fake message so we can put put all FakeMessages into one collection, without making it too wide. +/// +internal abstract class FakeMessage +{ + /// + /// The message serialized using the default JsonDataSerializer. + /// + // TODO: Is there a better way to ensure that is is not null, we will always set it in the inherited types, but it would be nice to have warning if we did not. + // And adding constructor makes it difficult to use the serializer, especially if we wanted to the serializer dynamic and not a static instance. + public string SerializedMessage { get; init; } = string.Empty; + + /// + /// + /// + public static FakeMessage NoResponse { get; } = new FakeMessage("NoResponse", 0); +} + +/// +/// A class like Message / VersionedMessage that is easier to create and review during debugging. +/// +internal sealed class FakeMessage : FakeMessage +{ + public FakeMessage(string messageType, T payload, int version = 0) + { + MessageType = messageType; + Payload = payload; + Version = version; + SerializedMessage = JsonDataSerializer.Instance.SerializePayload(MessageType, payload, version); + } + + /// + /// Message identifier, usually coming from the MessageType class. + /// + public string MessageType { get; } + + /// + /// The payload that this message is holding. + /// + public T Payload { get; } + + /// + /// Version of the message to allow the internal serializer to choose the correct serialization strategy. + /// + public int Version { get; } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs new file mode 100644 index 0000000000..39d06384f4 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeMetricsPublisher.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; + +internal class FakeMetricsPublisher : IMetricsPublisher +{ + public FakeMetricsPublisher(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public void Dispose() + { + // do nothing + } + + public void PublishMetrics(string eventName, IDictionary metrics) + { + // TODO: does nothing but probably should + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs new file mode 100644 index 0000000000..ce1d7fbe6e --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeOutput.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Text; + +using Microsoft.VisualStudio.TestPlatform.Utilities; + +internal class FakeOutput : IOutput +{ + public List Messages { get; } = new(); + public StringBuilder CurrentLine { get; } = new(); + public List Lines { get; } = new(); + + public void Write(string message, OutputLevel level) + { + Messages.Add(new OutputMessage(message, level, isNewLine: false)); + CurrentLine.Append(message); + } + + public void WriteLine(string message, OutputLevel level) + { + Lines.Add(CurrentLine + message); + CurrentLine.Clear(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs new file mode 100644 index 0000000000..3f1eb2ea96 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcess.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; + +internal class FakeProcess +{ + public int Id { get; internal set; } + public string Name { get; init; } + public string Path { get; } + public string? Arguments { get; set; } + public string WorkingDirectory { get; } + public IDictionary EnvironmentVariables { get; } + // TODO: Throw if already set + public Action? ErrorCallback { get; set; } + // TODO: Throw if already set + public Action? ExitCallback { get; set; } + // TODO: Throw if already set + public Action? OutputCallback { get; set; } + public PlatformArchitecture Architecture { get; init; } = PlatformArchitecture.X64; + public FakeErrorAggregator FakeErrorAggregator { get; } + public string? ErrorOutput { get; init; } + public int ExitCode { get; init; } = -1; + public bool Started { get; private set; } + public bool Exited { get; private set; } + public TestProcessStartInfo TestProcessStartInfo { get; internal set; } + + public FakeProcess(FakeErrorAggregator fakeErrorAggregator, string path, string? arguments = null, string? workingDirectory = null, IDictionary? environmentVariables = null, Action? errorCallback = null, Action? exitCallBack = null, Action? outputCallback = null) + { + FakeErrorAggregator = fakeErrorAggregator; + Path = path; + Name = System.IO.Path.GetFileName(path); + Arguments = arguments; + WorkingDirectory = workingDirectory ?? System.IO.Path.GetDirectoryName(path) ?? throw new InvalidOperationException($"Path {path} does not have a parent directory."); + EnvironmentVariables = environmentVariables ?? new Dictionary(); + ErrorCallback = errorCallback; + ExitCallback = exitCallBack; + OutputCallback = outputCallback; + + TestProcessStartInfo = new TestProcessStartInfo() + { + FileName = Path, + Arguments = Arguments, + WorkingDirectory = WorkingDirectory, + EnvironmentVariables = EnvironmentVariables, + // TODO: is this even used anywhere + CustomProperties = new Dictionary(), + }; + } + + internal static FakeProcess EnsureFakeProcess(object process) + { + return (FakeProcess)process; + } + + internal void SetId(int id) + { + if (Id != 0) + throw new InvalidOperationException($"Cannot set Id to {id} for fake process {Name}, {Id}, because it was already set."); + + Id = id; + } + + internal void Start() + { + if (Started) + throw new InvalidOperationException($"Cannot start process {Name} - {Id} because it was already started before."); + + Started = true; + } + + internal void Exit() + { + if (!Started) + throw new InvalidOperationException($"Cannot exit process {Name} - {Id} because it was not started before."); + + // We want to call the exit callback just once. This is behavior inherent to being a real process, + // that also exits only once. + var exited = Exited; + Exited = true; + if (!exited && ExitCallback != null) + { + ExitCallback(this); + } + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs new file mode 100644 index 0000000000..2085202009 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeProcessHelper.cs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; + +internal class FakeProcessHelper : IProcessHelper +{ + // starting from 100 for no particular reason + // I want to avoid processId 0 and 1 as they are + // "reserved" on Windows (0) and Linux (both 0 and 1) + private static readonly SequentialId IdSource = new(100); + + public FakeProcess CurrentProcess { get; } + public List Processes { get; } = new(); + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public FakeProcessHelper(FakeErrorAggregator fakeErrorAggregator, FakeProcess currentProcess) + { + FakeErrorAggregator = fakeErrorAggregator; + CurrentProcess = currentProcess; + AddFakeProcess(currentProcess); + } + + public void AddFakeProcess(FakeProcess process) + { + var id = IdSource.Next(); + process.SetId(id); + Processes.Add(process); + } + + public PlatformArchitecture GetCurrentProcessArchitecture() + { + return CurrentProcess.Architecture; + } + + public string GetCurrentProcessFileName() + { + return CurrentProcess.Path; + } + + public int GetCurrentProcessId() + { + return CurrentProcess.Id; + } + + public string GetCurrentProcessLocation() + { + // TODO: how is this different from Path + throw new NotImplementedException(); + } + + public string GetNativeDllDirectory() + { + throw new NotImplementedException(); + } + + public IntPtr GetProcessHandle(int processId) + { + throw new NotImplementedException(); + } + + public int GetProcessId(object process) + { + var fakeProcess = FakeProcess.EnsureFakeProcess(process); + return fakeProcess.Id; + } + + public string GetProcessName(int processId) + { + var process = Processes.Single(p => p.Id == processId); + return process.Name; + } + + public string GetTestEngineDirectory() + { + throw new NotImplementedException(); + } + + public object LaunchProcess(string processPath, string arguments, string workingDirectory, IDictionary environmentVariables, Action errorCallback, Action exitCallBack, Action outputCallback) + { + // TODO: Throw if setting says we can't start new processes; + var process = new FakeProcess(FakeErrorAggregator, processPath, arguments, workingDirectory, environmentVariables, errorCallback, exitCallBack, outputCallback); + Processes.Add(process); + process.Start(); + + return process; + } + + public void SetExitCallback(int processId, Action callbackAction) + { + // TODO: implement? + } + + public void TerminateProcess(object process) + { + var fakeProcess = (FakeProcess)process; + fakeProcess.Exit(); + } + + public bool TryGetExitCode(object process, out int exitCode) + { + exitCode = ((FakeProcess)process).ExitCode; + return true; + } + + public void WaitForProcessExit(object process) + { + // todo: implement for timeouts? + } + + internal void StartFakeProcess(FakeProcess process) + { + // TODO: mark the process as started. Do not add a new process if it did not exist. + if (!Processes.Contains(process)) + throw new InvalidOperationException($"Cannot start process {process.Name} - {process.Id} because it was not found in the list of known fake processes."); + + process.Start(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs new file mode 100644 index 0000000000..f8f7e21127 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestBatchBuilder.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeTestBatchBuilder +{ + public int TotalCount { get; private set; } + public TimeSpan Duration { get; private set; } + public int BatchSize { get; private set; } + public static List> Empty => new(); + + public FakeTestBatchBuilder() + { + } + + /// + /// Total test count in all batches. + /// + internal FakeTestBatchBuilder WithTotalCount(int count) + { + TotalCount = count; + return this; + } + + internal FakeTestBatchBuilder WithDuration(TimeSpan duration) + { + + // TODO: add min duration and max duration, and distribution, if timing becomes relevant + // TODO: and replay rate, if we actually want to simulate stuff like really executing the tests + Duration = duration; + return this; + } + + /// + /// Splits the tests to batches of this size when reporting them back. + /// + /// + /// + internal FakeTestBatchBuilder WithBatchSize(int batchSize) + { + BatchSize = batchSize; + return this; + } + + internal List> Build() + { + if (BatchSize == 0 && TotalCount != 0) + throw new InvalidOperationException("Batch size cannot be 0, unless TotalCount is also 0. Splitting non-zero amount of tests into 0 sized batches does not make sense."); + + if (TotalCount == 0) + return Empty; + + var numberOfBatches = Math.DivRem(TotalCount, BatchSize, out int remainder); + + // TODO: Add adapter uri, and dll name + // TODO: set duration + var batches = Enumerable.Range(0, numberOfBatches) + .Select(batchNumber => Enumerable.Range(0, BatchSize) + .Select((index) => new TestResult(new TestCase($"Test{batchNumber}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList()).ToList(); + + if (remainder > 0) + { + var reminderBatch = Enumerable.Range(0, remainder) + .Select((index) => new TestResult(new TestCase($"Test{numberOfBatches + 1}-{index}", new Uri("some://uri"), "DummySourceFileName"))).ToList(); + + batches.Add(reminderBatch); + } + + return batches; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs new file mode 100644 index 0000000000..036869a762 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllBuilder.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System; +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeTestDllBuilder +{ + private string _path = @$"X:\fake\mstest_{Guid.NewGuid()}.dll"; + private FrameworkName _framework = KnownFrameworkNames.Net5; + private Architecture _architecture = Architecture.X64; + private List>? _testBatches; + + internal FakeTestDllBuilder WithFramework(FrameworkName framework) + { + _framework = framework; + return this; + } + + internal FakeTestDllBuilder WithPath(string path) + { + _path = path; + return this; + } + + internal FakeTestDllBuilder WithArchitecture(Architecture architecture) + { + _architecture = architecture; + return this; + } + + /// + /// Use this together with TestBatchBuilder, or use WithTestCount to get basic test batch. + /// + /// + /// + internal FakeTestDllBuilder WithTestBatches(List> testBatches) + { + _testBatches = testBatches; + return this; + } + + /// + /// Use this to get basic test batch, or use WithTestBatches together with TestBatchBuilder, to get a custom batch. + /// + /// + /// + internal FakeTestDllBuilder WithTestCount(int totalCount, int? batchSize = null) + { + _testBatches = new FakeTestBatchBuilder() + .WithTotalCount(totalCount) + .WithBatchSize(batchSize ?? totalCount) + .Build(); + + return this; + } + + internal FakeTestDllFile Build() + { + if (_testBatches == null) + { + _testBatches = new FakeTestBatchBuilder() + .WithTotalCount(10) + .WithBatchSize(5) + .Build(); + } + return new FakeTestDllFile(_path, _framework, _architecture, _testBatches); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs new file mode 100644 index 0000000000..1239292cd7 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestDllFile.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Runtime.Versioning; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeTestDllFile : FakeFile +{ + public FrameworkName FrameworkName { get; } + public Architecture Architecture { get; } + public List> TestResultBatches { get; } + public int TestCount { get; } + public int BatchCount { get; } + + public FakeTestDllFile(string path, FrameworkName frameworkName, Architecture architecture, List> testResultBatches) : base(path) + { + FrameworkName = frameworkName; + Architecture = architecture; + TestResultBatches = testResultBatches; + + TestCount = testResultBatches.SelectMany(tr => tr).Count(); + BatchCount = testResultBatches.Count; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs new file mode 100644 index 0000000000..1f5adbbbf7 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestExtensionManager.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + +internal class FakeTestExtensionManager : ITestExtensionManager +{ + public void ClearExtensions() + { + throw new NotImplementedException(); + } + + public void UseAdditionalExtensions(IEnumerable pathToAdditionalExtensions, bool skipExtensionFilters) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs new file mode 100644 index 0000000000..ea425831b4 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHost.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; +internal class FakeTestHostFixture +{ + public int Id { get; } + public List Dlls { get; } + public FakeTestRuntimeProvider FakeTestRuntimeProvider { get; } + public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } + public FakeCommunicationChannel FakeCommunicationChannel { get; } + public List> Responses { get; } + public FakeProcess Process { get; internal set; } + + public FakeTestHostFixture( + int id, List dlls, + FakeTestRuntimeProvider fakeTestRuntimeProvider, + FakeCommunicationEndpoint fakeCommunicationEndpoint, + FakeCommunicationChannel fakeCommunicationChannel, + FakeProcess process, + List> responses) + { + Id = id; + Dlls = dlls; + FakeTestRuntimeProvider = fakeTestRuntimeProvider; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeCommunicationChannel = fakeCommunicationChannel; + Process = process; + Responses = responses; + + // The channel will pass back this whole fixture as context for every processed request so we can + // refer back to any part of testhost in message responses. E.g. to abort the channel, or exit + // testhost before or after answering. + fakeCommunicationChannel.Start(this); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs new file mode 100644 index 0000000000..7be196b79c --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostBuilder.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +internal class FakeTestHostFixtureBuilder +{ + // This will be also used as a port number, don't start from 0 + // it skips some paths in the real code, because port 0 has special meaning. + private static readonly SequentialId Id = new(1000); + + private readonly Fixture _fixture; + + // TODO: this would correctly be any test holding container, but let's not get ahead of myself. + private readonly List _dlls = new(); + private FakeProcess? _process; + private List>? _responses; + + public FakeTestHostFixtureBuilder(Fixture fixture) + { + _fixture = fixture; + } + + internal FakeTestHostFixtureBuilder WithTestDll(FakeTestDllFile dll) + { + _dlls.Add(dll); + return this; + } + + internal FakeTestHostFixture Build() + { + + if (_responses == null) + throw new InvalidOperationException("Add some reponses to the testhost by using WithResponses."); + + if (_process == null) + throw new InvalidOperationException("Add some process to the testhost by using WithProcess."); + + var id = Id.Next(); + var fakeCommunicationChannel = new FakeCommunicationChannel(_responses, _fixture.ErrorAggregator, id); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, _fixture.ErrorAggregator); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(_fixture.ProcessHelper, _process, _fixture.FileHelper, _dlls, fakeCommunicationEndpoint, _fixture.ErrorAggregator); + +#if DEBUG + // This registers the endpoint so we can look it up later using the address, the Id from here is propagated to + // testhost connection info, and is used as port in 127.0.0.1:, address so we can lookup the correct channel. + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +# endif + + return new FakeTestHostFixture(id, _dlls, fakeTestRuntimeProvider, fakeCommunicationEndpoint, fakeCommunicationChannel, _process, _responses); + } + + internal FakeTestHostFixtureBuilder WithProcess(FakeProcess process) + { + _process = process; + return this; + } + + internal FakeTestHostFixtureBuilder WithResponses(List> responses) + { + _responses = responses; + return this; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs new file mode 100644 index 0000000000..8492a7bb4a --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostLauncher.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; + +internal class FakeTestHostLauncher : ITestHostLauncher +{ + private readonly FakeProcessHelper _fakeProcessHelper; + + public FakeTestHostLauncher(FakeProcessHelper fakeProcessHelper, bool isDebug = false) + { + IsDebug = isDebug; + _fakeProcessHelper = fakeProcessHelper; + } + + public bool IsDebug { get; } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + throw new NotImplementedException(); + } + + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs new file mode 100644 index 0000000000..2f3302e565 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestHostResponsesBuilder.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +/// +/// Builds a list of RequestResponse pairs, with the provided values. Each method is a name of incoming message type. +/// The order in which the builder methods are called determines the order or responses. +/// +internal class FakeTestHostResponsesBuilder +{ + private readonly List> _responses = new(); + + /// + /// For VersionCheck message it responds with VersionCheck response that has the given version. + /// + /// + /// + internal FakeTestHostResponsesBuilder VersionCheck(int version) + { + AddPairWithValue(MessageType.VersionCheck, version); + return this; + } + + /// + /// For VersionCheck message it responds with the given FakeMessage. + /// + /// Message to respond with, or FakeMessage.NoResponse to not respond. + /// + internal FakeTestHostResponsesBuilder VersionCheck(FakeMessage message) + { + AddPairWithFakeMessage(MessageType.VersionCheck, message); + return this; + } + + /// + /// For VersionCheck message it does the given before action and responds with the given FakeMessage and then does the given after action. + /// Use FakeMessage.NoResponse to not respond. + /// + /// + /// + internal FakeTestHostResponsesBuilder VersionCheck(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.VersionCheck, message, beforeAction, afterAction); + return this; + } + + internal FakeTestHostResponsesBuilder ExecutionInitialize(FakeMessage message) + { + AddPairWithFakeMessage(MessageType.ExecutionInitialize, message); + return this; + } + + internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.StartTestExecutionWithSources, message, beforeAction, afterAction); + return this; + } + + internal FakeTestHostResponsesBuilder StartTestExecutionWithSources(List> testResultBatches!!) + { + List messages; + if (testResultBatches.Count != 0) + { + // this will create as many test stats changes messages, as there are batches -1 + // the last batch will be sent as test run complete event + + // TODO: make the stats agree with the tests below + List changeMessages = testResultBatches.Take(testResultBatches.Count - 1).Select(batch => + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + + // TODO: This is finicky because the statistics processor expects the dictionary to not be null + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), testResultBatches.Last(), new List()), + }); + messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + } + else + { + var completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 0 }), new List(), new List()), + }); + + messages = completedMessage.AsList(); + } + + + AddPairWithMultipleFakeMessages(MessageType.StartTestExecutionWithSources, messages); + return this; + } + + + internal FakeTestHostResponsesBuilder SessionEnd(FakeMessage fakeMessage) + { + AddPairWithFakeMessage(MessageType.SessionEnd, fakeMessage); + return this; + } + + internal FakeTestHostResponsesBuilder SessionEnd(FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(MessageType.SessionEnd, message, beforeAction, afterAction); + return this; + } + + private void AddPairWithValue(string messageType, T value, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithFakeMessage(messageType, new FakeMessage(messageType, value), beforeAction, afterAction); + } + + private void AddPairWithFakeMessage(string messageType, FakeMessage message, Action? beforeAction = null, Action? afterAction = null) + { + AddPairWithMultipleFakeMessages(messageType, new[] { message }, beforeAction, afterAction); + } + + private void AddPairWithMultipleFakeMessages(string messageType, IEnumerable messages, Action? beforeAction = null, Action? afterAction = null) + { + _responses.Add(new RequestResponsePair(messageType, messages, beforeAction, afterAction)); + } + + internal List> Build() + { + return _responses; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs new file mode 100644 index 0000000000..cde02328ba --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestPlatformEventSource.cs @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; + +internal class FakeTestPlatformEventSource : ITestPlatformEventSource +{ + public FakeTestPlatformEventSource(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public void AdapterDiscoveryStart(string executorUri) + { + // do nothing + } + + public void AdapterDiscoveryStop(long numberOfTests) + { + // do nothing + } + + public void AdapterExecutionStart(string executorUri) + { + // do nothing + } + + public void AdapterExecutionStop(long numberOfTests) + { + // do nothing + } + + public void AdapterSearchStart() + { + // do nothing + } + + public void AdapterSearchStop() + { + // do nothing + } + + public void DataCollectionStart(string dataCollectorUri) + { + // do nothing + } + + public void DataCollectionStop() + { + // do nothing + } + + public void DiscoveryRequestStart() + { + // do nothing + } + + public void DiscoveryRequestStop() + { + // do nothing + } + + public void DiscoveryStart() + { + // do nothing + } + + public void DiscoveryStop(long numberOfTests) + { + // do nothing + } + + public void ExecutionRequestStart() + { + // do nothing + } + + public void ExecutionRequestStop() + { + // do nothing + } + + public void ExecutionStart() + { + // do nothing + } + + public void ExecutionStop(long numberOfTests) + { + // do nothing + } + + public void MetricsDisposeStart() + { + // do nothing + } + + public void MetricsDisposeStop() + { + // do nothing + } + + public void StartTestSessionStart() + { + // do nothing + } + + public void StartTestSessionStop() + { + // do nothing + } + + public void StopTestSessionStart() + { + // do nothing + } + + public void StopTestSessionStop() + { + // do nothing + } + + public void TestHostAppDomainCreationStart() + { + // do nothing + } + + public void TestHostAppDomainCreationStop() + { + // do nothing + } + + public void TestHostStart() + { + // do nothing + } + + public void TestHostStop() + { + // do nothing + } + + public void TestRunAttachmentsProcessingRequestStart() + { + // do nothing + } + + public void TestRunAttachmentsProcessingRequestStop() + { + // do nothing + } + + public void TestRunAttachmentsProcessingStart(long numberOfAttachments) + { + // do nothing + } + + public void TestRunAttachmentsProcessingStop(long numberOfAttachments) + { + // do nothing + } + + public void TranslationLayerDiscoveryStart() + { + // do nothing + } + + public void TranslationLayerDiscoveryStop() + { + // do nothing + } + + public void TranslationLayerExecutionStart(long customTestHost, long sourcesCount, long testCasesCount, string runSettings) + { + // do nothing + } + + public void TranslationLayerExecutionStop() + { + // do nothing + } + + public void TranslationLayerInitializeStart() + { + // do nothing + } + + public void TranslationLayerInitializeStop() + { + // do nothing + } + + public void TranslationLayerStartTestSessionStart() + { + // do nothing + } + + public void TranslationLayerStartTestSessionStop() + { + // do nothing + } + + public void TranslationLayerStopTestSessionStart() + { + // do nothing + } + + public void TranslationLayerStopTestSessionStop() + { + // do nothing + } + + public void TranslationLayerTestRunAttachmentsProcessingStart() + { + // do nothing + } + + public void TranslationLayerTestRunAttachmentsProcessingStop() + { + // do nothing + } + + public void VsTestConsoleStart() + { + // do nothing + } + + public void VsTestConsoleStop() + { + // do nothing + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs new file mode 100644 index 0000000000..5610c9a9fd --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRunEventsRegistrar.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class FakeTestRunEventsRegistrar : ITestRunEventsRegistrar +{ + public FakeTestRunEventsRegistrar(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public List AllEvents { get; } = new(); + public List LoggedWarnings { get; } = new(); + public List> RunCompleteEvents { get; } = new(); + public List> RunStartEvents { get; } = new(); + public List> RunChangedEvents { get; } = new(); + public List> RawMessageEvents { get; } = new(); + public List> RunMessageEvents { get; } = new(); + public FakeErrorAggregator FakeErrorAggregator { get; } + + public void LogWarning(string message) + { + AllEvents.Add(message); + LoggedWarnings.Add(message); + } + + public void RegisterTestRunEvents(ITestRunRequest testRunRequest) + { + testRunRequest.TestRunMessage += OnTestRunMessage; + testRunRequest.OnRawMessageReceived += OnRawMessage; + testRunRequest.OnRunStart += OnRunStart; + testRunRequest.OnRunStatsChange += OnRunStatsChange; + testRunRequest.OnRunCompletion += OnRunCompletion; + } + + public void UnregisterTestRunEvents(ITestRunRequest testRunRequest) + { + testRunRequest.TestRunMessage -= OnTestRunMessage; + testRunRequest.OnRawMessageReceived -= OnRawMessage; + testRunRequest.OnRunStart -= OnRunStart; + testRunRequest.OnRunStatsChange -= OnRunStatsChange; + testRunRequest.OnRunCompletion -= OnRunCompletion; + } + + private void OnRunCompletion(object? sender, TestRunCompleteEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunCompleteEvents.Add(eventRecord); + } + + private void OnRunStart(object? sender, TestRunStartEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunStartEvents.Add(eventRecord); + } + + private void OnRunStatsChange(object? sender, TestRunChangedEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RunChangedEvents.Add(eventRecord); + } + + private void OnRawMessage(object? sender, string e) + { + var eventRecord = new EventRecord(sender, e); + AllEvents.Add(eventRecord); + RawMessageEvents.Add(eventRecord); + } + + private void OnTestRunMessage(object? sender, TestRunMessageEventArgs e) + { + var eventRecord = new EventRecord(sender, e); + if (e.Level == TestMessageLevel.Error) + { + FakeErrorAggregator.Errors.Add(eventRecord); + } + AllEvents.Add(eventRecord); + RunMessageEvents.Add(eventRecord); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs new file mode 100644 index 0000000000..c68d50fdab --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProvider.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +internal class FakeTestRuntimeProvider : ITestRuntimeProvider +{ + public FakeProcessHelper FakeProcessHelper { get; } + public FakeCommunicationEndpoint FakeCommunicationEndpoint { get; } + public FakeErrorAggregator FakeErrorAggregator { get; } + public FakeProcess TestHostProcess { get; private set; } + public FakeFileHelper FileHelper { get; } + public List TestDlls { get; } + + // TODO: make this configurable? + public bool Shared => false; + + public event EventHandler? HostLaunched; + public event EventHandler? HostExited; + + public FakeTestRuntimeProvider(FakeProcessHelper fakeProcessHelper, FakeProcess fakeTestHostProcess, FakeFileHelper fakeFileHelper, List fakeTestDlls, FakeCommunicationEndpoint fakeCommunicationEndpoint, FakeErrorAggregator fakeErrorAggregator) + { + FakeProcessHelper = fakeProcessHelper; + TestHostProcess = fakeTestHostProcess; + FileHelper = fakeFileHelper; + TestDlls = fakeTestDlls; + FakeCommunicationEndpoint = fakeCommunicationEndpoint; + FakeErrorAggregator = fakeErrorAggregator; + + var architectures = fakeTestDlls.Select(dll => dll.Architecture).Distinct().ToList(); + var frameworks = fakeTestDlls.Select(dll => dll.FrameworkName).Distinct().ToList(); + + if (architectures.Count > 1) + throw new InvalidOperationException($"The provided dlls have more than 1 architecture {architectures.JoinByComma()}. Fake TestRuntimeProvider cannot have dlls with mulitple architectures, because real testhost process can also run just with a single architecture."); + + if (frameworks.Count > 1) + throw new InvalidOperationException($"The provided dlls have more than 1 target framework {frameworks.JoinByComma()}. Fake TestRuntimeProvider cannot have dlls with mulitple target framework, because real testhost process can also run just a single target framework."); + + fakeTestDlls.ForEach(FileHelper.AddFakeFile); + + fakeProcessHelper.AddFakeProcess(fakeTestHostProcess); + TestHostProcess.ExitCallback = p => + { + // TODO: Validate the process we are passed is actually the same as TestHostProcess + // TODO: Validate we already started the process. + var process = (FakeProcess)p; + if (HostExited != null) + { + // TODO: When we exit, eventually there are no subscribers, maybe we should review if we don't lose the error output sometimes, in unnecessary way + HostExited(this, new HostProviderEventArgs(process.ErrorOutput, process.ExitCode, process.Id)); + } + }; + } + + public bool CanExecuteCurrentRunConfiguration(string runsettingsXml) + { + // x86 + // Framework40 + + return true; + } + + public Task CleanTestHostAsync(CancellationToken cancellationToken) + { + if (TestHostProcess == null) + throw new InvalidOperationException("Cannot clean testhost, no testhost process was started"); + FakeProcessHelper.TerminateProcess(TestHostProcess); + return Task.CompletedTask; + } + + public TestHostConnectionInfo GetTestHostConnectionInfo() + { + return FakeCommunicationEndpoint.TestHostConnectionInfo; + } + + public TestProcessStartInfo GetTestHostProcessStartInfo(IEnumerable sources, IDictionary environmentVariables, TestRunnerConnectionInfo connectionInfo) + { + // TODO: do we need to do more here? How to link testhost to the fake one we "start"? + return TestHostProcess.TestProcessStartInfo; + } + + public IEnumerable GetTestPlatformExtensions(IEnumerable sources, IEnumerable extensions) + { + // send extensions so we send InitializeExecutionMessage + return new[] { @"c:\temp\extension.dll" }; + } + + public IEnumerable GetTestSources(IEnumerable sources) + { + // gives testhost opportunity to translate sources to something else, + // e.g. in uwp the main exe is returned, rather than the dlls that dlls that are tested + return sources; + } + + public void Initialize(IMessageLogger logger, string runsettingsXml) + { + // TODO: this is called twice, is that okay? + // TODO: and also by HandlePartialRunComplete after the test run has completed and we aborted because the client disconnected + + // do nothing + } + + public Task LaunchTestHostAsync(TestProcessStartInfo testHostStartInfo, CancellationToken cancellationToken) + { + if (TestHostProcess.TestProcessStartInfo.FileName != testHostStartInfo.FileName) + throw new InvalidOperationException($"Tried to start a different process than the one associated with this provider: File name is {testHostStartInfo.FileName} is not the same as the fake process associated with this provider {TestHostProcess.TestProcessStartInfo.FileName}."); + + FakeProcessHelper.StartFakeProcess(TestHostProcess); + + if (HostLaunched != null) + { + HostLaunched(this, new HostProviderEventArgs("Fake testhost launched", 0, TestHostProcess.Id)); + } + return Task.FromResult(true); + } + + public void SetCustomLauncher(ITestHostLauncher customLauncher) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs new file mode 100644 index 0000000000..9668bd3f99 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/FakeTestRuntimeProviderManager.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Collections.Concurrent; + +using Microsoft.VisualStudio.TestPlatform.Common.Hosting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; + +internal class FakeTestRuntimeProviderManager : ITestRuntimeProviderManager +{ + public FakeTestRuntimeProviderManager(FakeErrorAggregator fakeErrorAggregator) + { + FakeErrorAggregator = fakeErrorAggregator; + } + + public ConcurrentQueue TestRuntimeProviders { get; } = new(); + public List UsedTestRuntimeProviders { get; } = new(); + + public FakeErrorAggregator FakeErrorAggregator { get; } + + public void AddTestRuntimeProviders(params FakeTestRuntimeProvider[] runtimeProviders) + { + // This is not a bug, I am registering each provider twice because TestPlatform resolves + // them twice for every request that does not run in-process. + foreach (var runtimeProvider in runtimeProviders) + { + TestRuntimeProviders.Enqueue(runtimeProvider); + TestRuntimeProviders.Enqueue(runtimeProvider); + } + } + + public ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration) + { + + + if (!TestRuntimeProviders.TryDequeue(out var next)) + { + throw new InvalidOperationException("There are no more TestRuntimeProviders to be provided"); + } + + UsedTestRuntimeProviders.Add(next); + return next; + } + + public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri) + { + throw new NotImplementedException(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/Fixture.cs b/test/vstest.ProgrammerTests/Fakes/Fixture.cs new file mode 100644 index 0000000000..e8b66c1c4f --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/Fixture.cs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using FluentAssertions; + +using Microsoft.VisualStudio.TestPlatform.Client; +using Microsoft.VisualStudio.TestPlatform.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; +using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +internal class Fixture : IDisposable +{ + public FakeErrorAggregator ErrorAggregator { get; } = new(); + public FakeProcessHelper ProcessHelper { get; } + public FakeProcess CurrentProcess { get; } + public FakeFileHelper FileHelper { get; } + public FakeTestRuntimeProviderManager TestRuntimeProviderManager { get; } + public FakeTestRunEventsRegistrar TestRunEventsRegistrar { get; } + public TestEngine? TestEngine { get; private set; } + public TestPlatform? TestPlatform { get; private set; } + public TestRunResultAggregator? TestRunResultAggregator { get; private set; } + public FakeTestPlatformEventSource? TestPlatformEventSource { get; private set; } + public FakeAssemblyMetadataProvider? AssemblyMetadataProvider { get; private set; } + public InferHelper? InferHelper { get; private set; } + public FakeDataCollectorAttachmentsProcessorsFactory? DataCollectorAttachmentsProcessorsFactory { get; private set; } + public TestRunAttachmentsProcessingManager? TestRunAttachmentsProcessingManager { get; private set; } + public TestRequestManager? TestRequestManager { get; private set; } + public List ExecutedTests => TestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).ToList(); + + public ProtocolConfig ProtocolConfig { get; internal set; } + + public Fixture() + { + // This type is compiled only in DEBUG, and won't exist otherwise. +#if DEBUG + // We need to use static class to find the communication endpoint, this clears all the registrations of previous tests. + TestServiceLocator.Clear(); +#else + // This fools compiler into not being able to tell that the the rest of the code is unreachable. + var a = true; + if (a) + { + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + } +#endif + + CurrentProcess = new FakeProcess(ErrorAggregator, @"X:\fake\vstest.console.exe", string.Empty, null, null, null, null, null); + ProcessHelper = new FakeProcessHelper(ErrorAggregator, CurrentProcess); + FileHelper = new FakeFileHelper(ErrorAggregator); + TestRuntimeProviderManager = new FakeTestRuntimeProviderManager(ErrorAggregator); + TestRunEventsRegistrar = new FakeTestRunEventsRegistrar(ErrorAggregator); + ProtocolConfig = new ProtocolConfig(); + } + public void Dispose() + { + + } + + internal void AddTestHostFixtures(params FakeTestHostFixture[] testhosts) + { + var providers = testhosts.Select(t => t.FakeTestRuntimeProvider).ToArray(); + TestRuntimeProviderManager.AddTestRuntimeProviders(providers); + } + + internal TestRequestManagerTestHelper BuildTestRequestManager( + int? timeout = DebugOptions.DefaultTimeout, + int? debugTimeout = DebugOptions.DefaultDebugTimeout, + bool? breakOnAbort = DebugOptions.DefaultBreakOnAbort) + { + if (!TestRuntimeProviderManager.TestRuntimeProviders.Any()) + throw new InvalidOperationException("There are runtime providers registered for FakeTestRuntimeProviderManager."); + + + TestEngine = new TestEngine(TestRuntimeProviderManager, ProcessHelper); + TestPlatform = new TestPlatform(TestEngine, FileHelper, TestRuntimeProviderManager); + + TestRunResultAggregator = new TestRunResultAggregator(); + TestPlatformEventSource = new FakeTestPlatformEventSource(ErrorAggregator); + + AssemblyMetadataProvider = new FakeAssemblyMetadataProvider(FileHelper, ErrorAggregator); + InferHelper = new InferHelper(AssemblyMetadataProvider); + + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + DataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(ErrorAggregator); + TestRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(TestPlatformEventSource, DataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(ErrorAggregator)); + + var commandLineOptions = CommandLineOptions.Instance; + TestRequestManager testRequestManager = new( + commandLineOptions, + TestPlatform, + TestRunResultAggregator, + TestPlatformEventSource, + InferHelper, + fakeMetricsPublisherTask, + ProcessHelper, + TestRunAttachmentsProcessingManager + ); + + TestRequestManager = testRequestManager; + + return new TestRequestManagerTestHelper(ErrorAggregator, testRequestManager, new DebugOptions + { + Timeout = timeout ?? DebugOptions.DefaultTimeout, + DebugTimeout = debugTimeout ?? DebugOptions.DefaultDebugTimeout, + BreakOnAbort = breakOnAbort ?? DebugOptions.DefaultBreakOnAbort, + }); + } + + internal void AssertNoErrors() + { + ErrorAggregator.Errors.Should().BeEmpty(); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs new file mode 100644 index 0000000000..bdb800d888 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkNames.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Runtime.Versioning; + +internal static class KnownFrameworkNames +{ + public static FrameworkName Netcoreapp1 = new(KnownFrameworkStrings.Netcoreapp1); + public static FrameworkName Netcoreapp2 = new(KnownFrameworkStrings.Netcoreapp2); + public static FrameworkName Netcoreapp21 = new(KnownFrameworkStrings.Netcoreapp21); + public static FrameworkName Netcoreapp3 = new(KnownFrameworkStrings.Netcoreapp3); + public static FrameworkName Netcoreapp31 = new(KnownFrameworkStrings.Netcoreapp31); + public static FrameworkName Net5 = new(KnownFrameworkStrings.Net5); + public static FrameworkName Net6 = new(KnownFrameworkStrings.Net6); + public static FrameworkName Net7 = new(KnownFrameworkStrings.Net7); + + public static FrameworkName Net4 = new(KnownFrameworkStrings.Net4); + public static FrameworkName Net45 = new(KnownFrameworkStrings.Net45); + public static FrameworkName Net451 = new(KnownFrameworkStrings.Net451); + public static FrameworkName Net452 = new(KnownFrameworkStrings.Net452); + public static FrameworkName Net46 = new(KnownFrameworkStrings.Net46); + public static FrameworkName Net461 = new(KnownFrameworkStrings.Net461); + public static FrameworkName Net462 = new(KnownFrameworkStrings.Net462); + public static FrameworkName Net47 = new(KnownFrameworkStrings.Net47); + public static FrameworkName Net471 = new(KnownFrameworkStrings.Net471); + public static FrameworkName Net472 = new(KnownFrameworkStrings.Net472); + public static FrameworkName Net48 = new(KnownFrameworkStrings.Net48); +} diff --git a/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs new file mode 100644 index 0000000000..9605a8ff59 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/KnownFrameworkStrings.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal static class KnownFrameworkStrings +{ + public static string NetCore(int major, int minor = 0) => $".NETCoreApp,Version=v{major}.{minor}"; + private static string NetFramework(int major, int minor, int patch = 0) => $".NETFramework,Version=v{major}.{minor}.{patch}"; + + public static string Netcoreapp1 = NetCore(1); + public static string Netcoreapp2 = NetCore(2); + public static string Netcoreapp21 = NetCore(2, 1); + public static string Netcoreapp3 = NetCore(3); + public static string Netcoreapp31 = NetCore(3, 1); + public static string Net5 = NetCore(5); + public static string Net6 = NetCore(6); + public static string Net7 = NetCore(7); + + public static string Net4 = NetFramework(4,0); + public static string Net45 = NetFramework(4, 5); + public static string Net451 = NetFramework(4, 5, 1); + public static string Net452 = NetFramework(4, 5, 2); + public static string Net46 = NetFramework(4, 6); + public static string Net461 = NetFramework(4, 6, 1); + public static string Net462 = NetFramework(4, 6, 2); + public static string Net47 = NetFramework(4, 7); + public static string Net471 = NetFramework(4, 7, 1); + public static string Net472 = NetFramework(4, 7, 2); + public static string Net48 = NetFramework(4, 8); +} diff --git a/test/vstest.ProgrammerTests/Fakes/OutputMessage.cs b/test/vstest.ProgrammerTests/Fakes/OutputMessage.cs new file mode 100644 index 0000000000..498f7fcd51 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/OutputMessage.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using Microsoft.VisualStudio.TestPlatform.Utilities; + +internal class OutputMessage +{ + public OutputMessage(string message, OutputLevel level, bool isNewLine) + { + Message = message; + Level = level; + IsNewLine = isNewLine; + } + + public string Message { get; } + public OutputLevel Level { get; } + public bool IsNewLine { get; } +} diff --git a/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs new file mode 100644 index 0000000000..5384f9cace --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/RequestResponsePair.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +internal class RequestResponsePair where TRequest : class +{ + public RequestResponsePair(TRequest request, TResponse response, bool debug = false) + { + Request = request; + Responses = new List { response }; + Debug = debug; + } + + public RequestResponsePair(TRequest request, IEnumerable responses, bool debug = false) + { + Request = request; + Responses = responses.ToList(); + Debug = debug; + } + + public RequestResponsePair(TRequest request, IEnumerable responses, Action? beforeAction = null, Action? afterAction = null, bool debug = false) + { + Request = request; + Responses = responses.ToList(); + BeforeAction = beforeAction; + AfterAction = afterAction; + Debug = debug; + } + + public TRequest Request { get; } + + // TODO: make this Expression< so we can get some info about what this is doing when looking directly at this instance + public Action? BeforeAction { get; } + public Action? AfterAction { get; } + public List Responses { get; } + public bool Debug { get; } +} diff --git a/test/vstest.ProgrammerTests/Fakes/SequentialId.cs b/test/vstest.ProgrammerTests/Fakes/SequentialId.cs new file mode 100644 index 0000000000..815245110b --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/SequentialId.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +/// +/// A sequential Id that starts from 0 or a given number. Put this in a static field in your class, and call Next to get the next Id. +/// +internal class SequentialId +{ + private int _id; + + public SequentialId() : this(0) + { + } + + public SequentialId(int firstId) + { + _id = firstId; + } + + public int Next() + { + return Interlocked.Increment(ref _id); + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/StringExtensions.cs b/test/vstest.ProgrammerTests/Fakes/StringExtensions.cs new file mode 100644 index 0000000000..fcca5fbef0 --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/StringExtensions.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Linq; + +internal static class EnumerableExtensions +{ + public static string JoinBySpace(this IEnumerable value) + { + return value.JoinBy(" "); + } + + public static string JoinByComma(this IEnumerable value) + { + return value.JoinBy(", "); + } + + public static string JoinBy(this IEnumerable value, string delimiter) + { + return string.Join(delimiter, value.Select(v => v?.ToString())); + } + + public static List AsList(this T value) + { + return new List { value }; + } +} diff --git a/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs new file mode 100644 index 0000000000..bf83c8599e --- /dev/null +++ b/test/vstest.ProgrammerTests/Fakes/TestRequestManagerHelper.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests.Fakes; + +using System.Diagnostics; + +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; + +internal class TestRequestManagerTestHelper +{ + private readonly FakeErrorAggregator _errorAggregator; + private readonly TestRequestManager _testRequestManager; + private readonly DebugOptions _debugOptions; + + public TestRequestManagerTestHelper(FakeErrorAggregator errorAggregator, TestRequestManager testRequestManager, DebugOptions debugOptions) + { + _errorAggregator = errorAggregator; + _testRequestManager = testRequestManager; + _debugOptions = debugOptions; + } + + public async Task ExecuteWithAbort(Action testRequsestManagerAction) + { + // We make sure the test is running for the timeout time at max and then we try to abort + // if we aborted we write the error to aggregator + + // Start tasks that waits until it is the right time to call abort + // and continue to starting the method. If that method finishes running on time we cancel this + // wait and don't abort. Otherwise we call abort to start our abort flow. + // + // This abort does not guarantee that we won't hang. If our abort flow is broken then we will + // remain hanging. To have that guarantee it needs to be handled by failfast, or something else that will hang dump us. + // Or a simple timer that kill the process after a given timeout, like a simplified blame hang dumper. + var cancelAbort = new CancellationTokenSource(); + var abortOnTimeout = Task.Run(async () => + { + // Wait until timeout or until we are cancelled. + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? _debugOptions.DebugTimeout : _debugOptions.Timeout), cancelAbort.Token); + if (Debugger.IsAttached && _debugOptions.BreakOnAbort) + { + var errors = _errorAggregator.Errors; + // we will abort because we are hanging, look at errors and at concurrent stacks to see where we are hanging. + Debugger.Break(); + } + _errorAggregator.Add(new Exception("errr we aborted")); + _testRequestManager.AbortTestRun(); + }); + + testRequsestManagerAction(_testRequestManager); + + cancelAbort.Cancel(); + if (!abortOnTimeout.IsCanceled) + { + await abortOnTimeout; + } + } +} diff --git a/test/vstest.ProgrammerTests/Program.cs b/test/vstest.ProgrammerTests/Program.cs new file mode 100644 index 0000000000..33f8f4d538 --- /dev/null +++ b/test/vstest.ProgrammerTests/Program.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests; + +using System.Reflection; + +internal class Program +{ + static void Main() + { + Intent.Console.Program.Main(new[] { Assembly.GetExecutingAssembly().Location }); + } +} diff --git a/test/vstest.ProgrammerTests/Properties/launchSettings.json b/test/vstest.ProgrammerTests/Properties/launchSettings.json new file mode 100644 index 0000000000..7425c62a46 --- /dev/null +++ b/test/vstest.ProgrammerTests/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "WSL": { + "commandName": "WSL2", + "distributionName": "" + }, + "vstest.ProgrammerTests": { + "commandName": "Project" + } + } +} \ No newline at end of file diff --git a/test/vstest.ProgrammerTests/UnitTest1.cs b/test/vstest.ProgrammerTests/UnitTest1.cs new file mode 100644 index 0000000000..61ca59cd84 --- /dev/null +++ b/test/vstest.ProgrammerTests/UnitTest1.cs @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.ProgrammerTests; + +using System.Diagnostics; +using System.Runtime.Versioning; + +using FluentAssertions; +using FluentAssertions.Extensions; + +using Microsoft.VisualStudio.TestPlatform.Client; +using Microsoft.VisualStudio.TestPlatform.CommandLine; +using Microsoft.VisualStudio.TestPlatform.CommandLine.Publisher; +using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; +using Microsoft.VisualStudio.TestPlatform.CommandLineUtilities; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestRunAttachmentsProcessing; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + +# if DEBUG +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; +#endif + +using vstest.ProgrammerTests.Fakes; + +// Tests are run by Intent library that is executed from our Program.Main. To debug press F5 in VS, and maybe mark just a single test with [Only]. +// To just run, press Ctrl+F5 to run without debugging. It will use short timeout for abort in case something is wrong with your test. + +public class TestDiscoveryTests +{ + public async Task GivenAnMSTestAssemblyWith108Tests_WhenTestsAreRun_Then108TestsAreExecuted() + { + // -- arrange + var fakeErrorAggregator = new FakeErrorAggregator(); + var commandLineOptions = CommandLineOptions.Instance; + + var fakeCurrentProcess = new FakeProcess(fakeErrorAggregator, @"X:\fake\vstest.console.exe"); + var fakeProcessHelper = new FakeProcessHelper(fakeErrorAggregator, fakeCurrentProcess); + + var fakeFileHelper = new FakeFileHelper(fakeErrorAggregator); + // TODO: Get framework name from constants + // TODO: have mstest1dll canned + var tests = new FakeTestBatchBuilder() + .WithTotalCount(108) + .WithDuration(100.Milliseconds()) + .WithBatchSize(10) + .Build(); + var mstest1Dll = new FakeTestDllFile(@"X:\fake\mstest1.dll", new FrameworkName(".NETCoreApp,Version=v5.0"), Architecture.X64, tests); + + List changeMessages = tests.Take(tests.Count - 1).Select(batch => // TODO: make the stats agree with the tests below + new FakeMessage(MessageType.TestRunStatsChange, + new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = batch.Count }), batch, new List()) + )).ToList(); + FakeMessage completedMessage = new FakeMessage(MessageType.ExecutionComplete, new TestRunCompletePayload + { + // TODO: make the stats agree with the tests below + TestRunCompleteArgs = new TestRunCompleteEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), false, false, null, new System.Collections.ObjectModel.Collection(), TimeSpan.Zero), + LastRunTests = new TestRunChangedEventArgs(new TestRunStatistics(new Dictionary { [TestOutcome.Passed] = 1 }), tests.Last(), new List()), + }); + List messages = changeMessages.Concat(new[] { completedMessage }).ToList(); + var responses = new List> { + new RequestResponsePair(MessageType.VersionCheck, new FakeMessage(MessageType.VersionCheck, 5)), + new RequestResponsePair(MessageType.ExecutionInitialize, FakeMessage.NoResponse), + new RequestResponsePair(MessageType.StartTestExecutionWithSources, messages, false), + new RequestResponsePair(MessageType.SessionEnd, new [] { FakeMessage.NoResponse }, message => + { + // TODO: how do we associate this to the correct process? + var fp = fakeProcessHelper.Processes.Last(); + fakeProcessHelper.TerminateProcess(fp); + }), + }; + + var fakeCommunicationChannel = new FakeCommunicationChannel(responses, fakeErrorAggregator, 1); + fakeCommunicationChannel.Start(new object()); + var fakeCommunicationEndpoint = new FakeCommunicationEndpoint(fakeCommunicationChannel, fakeErrorAggregator); +#if DEBUG + TestServiceLocator.Clear(); + TestServiceLocator.Register(fakeCommunicationEndpoint.TestHostConnectionInfo.Endpoint, fakeCommunicationEndpoint); +#else + // This fools compiler into not being able to tell that the the rest of the code is unreachable. + var a = true; + if (a) + { + throw new InvalidOperationException("Tests cannot run in Release mode, because TestServiceLocator is compiled only for Debug, and so the tests will fail to setup channel and will hang."); + } +#endif + var fakeTestHostProcess = new FakeProcess(fakeErrorAggregator, @"C:\temp\testhost.exe"); + var fakeTestRuntimeProvider = new FakeTestRuntimeProvider(fakeProcessHelper, fakeTestHostProcess, fakeFileHelper, mstest1Dll.AsList(), fakeCommunicationEndpoint, fakeErrorAggregator); + var fakeTestRuntimeProviderManager = new FakeTestRuntimeProviderManager(fakeErrorAggregator); + fakeTestRuntimeProviderManager.AddTestRuntimeProviders(fakeTestRuntimeProvider); + var testEngine = new TestEngine(fakeTestRuntimeProviderManager, fakeProcessHelper); + + var testPlatform = new TestPlatform(testEngine, fakeFileHelper, fakeTestRuntimeProviderManager); + + var testRunResultAggregator = new TestRunResultAggregator(); + var fakeTestPlatformEventSource = new FakeTestPlatformEventSource(fakeErrorAggregator); + + var fakeAssemblyMetadataProvider = new FakeAssemblyMetadataProvider(fakeFileHelper, fakeErrorAggregator); + var inferHelper = new InferHelper(fakeAssemblyMetadataProvider); + + // This is most likely not the correctl place where to cut this off, plugin cache is probably the better place, + // but it is not injected, and I don't want to investigate this now. + var fakeDataCollectorAttachmentsProcessorsFactory = new FakeDataCollectorAttachmentsProcessorsFactory(fakeErrorAggregator); + var testRunAttachmentsProcessingManager = new TestRunAttachmentsProcessingManager(fakeTestPlatformEventSource, fakeDataCollectorAttachmentsProcessorsFactory); + + Task fakeMetricsPublisherTask = Task.FromResult(new FakeMetricsPublisher(fakeErrorAggregator)); + TestRequestManager testRequestManager = new( + commandLineOptions, + testPlatform, + testRunResultAggregator, + fakeTestPlatformEventSource, + inferHelper, + fakeMetricsPublisherTask, + fakeProcessHelper, + testRunAttachmentsProcessingManager + ); + + // -- act + + // TODO: this gives me run configuration that is way too complete, do we a way to generate "bare" runsettings? if not we should add them. Would be also useful to get + // runsettings from parameter set so people can use it + // TODO: TestSessionTimeout gives me way to abort the run without having to cancel it externally, but could probably still lead to hangs if that funtionality is broken + // TODO: few tries later, that is exactly the case when we abort, it still hangs on waiting to complete request, because test run complete was not sent + // var runConfiguration = new Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration { TestSessionTimeout = 40_000 }.ToXml().OuterXml; + var runConfiguration = string.Empty; + var testRunRequestPayload = new TestRunRequestPayload + { + // TODO: passing null sources and null testcases does not fail fast + Sources = mstest1Dll.Path.AsList(), + // TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code + // TODO: passing empty string fails in the xml parser code + RunSettings = $"{runConfiguration}" + }; + + // var fakeTestHostLauncher = new FakeTestHostLauncher(); + var fakeTestRunEventsRegistrar = new FakeTestRunEventsRegistrar(fakeErrorAggregator); + var protocolConfig = new ProtocolConfig(); + + // TODO: we make sure the test is running 10 minutes at max and then we try to abort + // if we aborted we write the error to aggregator, this needs to be made into a pattern + // so we can avoid hanging if the run does not complete correctly + var cancelAbort = new CancellationTokenSource(); + var task = Task.Run(async () => + { + await Task.Delay(TimeSpan.FromSeconds(Debugger.IsAttached ? 100 : 10), cancelAbort.Token); + if (Debugger.IsAttached) + { + // we will abort because we are hanging, look at stacks to see what the problem is + Debugger.Break(); + } + fakeErrorAggregator.Add(new Exception("errr we aborted")); + testRequestManager.AbortTestRun(); + + }); + testRequestManager.RunTests(testRunRequestPayload, testHostLauncher: null, fakeTestRunEventsRegistrar, protocolConfig); + cancelAbort.Cancel(); + if (!task.IsCanceled) + { + await task; + } + // pattern end + + // -- assert + fakeErrorAggregator.Errors.Should().BeEmpty(); + fakeTestRunEventsRegistrar.RunChangedEvents.SelectMany(er => er.Data.NewTestResults).Should().HaveCount(108); + } + + public async Task GivenMultipleMsTestAssembliesThatUseTheSameTargetFrameworkAndArchitecture_WhenTestsAreRun_ThenAllTestsFromAllAssembliesAreRun() + { + // -- arrange + using var fixture = new Fixture(); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) + .WithArchitecture(Architecture.X64) + .WithTestCount(108, 10) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net5) + .WithArchitecture(Architecture.X64) + .WithTestCount(50, 8) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, f => f.Process.Exit()) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + // -- act + var runConfiguration = string.Empty; + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount + mstest2Dll.TestCount); + } + + public async Task GivenMultipleMsTestAssembliesThatUseDifferentTargetFrameworkAndTheSameArchitecture_WhenTestsAreRun_ThenTwoTesthostsAreStartedBothForTheSameTFM() + { + // TODO: make vstest.console not start testhosts for incompatible sources. + + // -- arrange + using var fixture = new Fixture(); + + var mstest1Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest1.dll") + .WithFramework(KnownFrameworkNames.Net5) // <--- + .WithArchitecture(Architecture.X64) + .WithTestCount(2) + .Build(); + + var testhost1Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost1.exe"); + + var runTests1 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest1Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, afterAction: _ => testhost1Process.Exit()) + .Build(); + + var testhost1 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest1Dll) + .WithProcess(testhost1Process) + .WithResponses(runTests1) + .Build(); + + // -- + + var mstest2Dll = new FakeTestDllBuilder() + .WithPath(@"X:\fake\mstest2.dll") + .WithFramework(KnownFrameworkNames.Net48) // <--- + .WithArchitecture(Architecture.X64) + // In reality, the dll would fail to load, and no tests would run from this dll, + // we simulate that by making it have 0 tests. + .WithTestCount(0) + .Build(); + + var testhost2Process = new FakeProcess(fixture.ErrorAggregator, @"X:\fake\testhost2.exe"); + + var runTests2 = new FakeTestHostResponsesBuilder() + .VersionCheck(5) + .ExecutionInitialize(FakeMessage.NoResponse) + .StartTestExecutionWithSources(mstest2Dll.TestResultBatches) + .SessionEnd(FakeMessage.NoResponse, _ => testhost2Process.Exit()) + .Build(); + + var testhost2 = new FakeTestHostFixtureBuilder(fixture) + .WithTestDll(mstest2Dll) + .WithProcess(testhost2Process) + .WithResponses(runTests2) + .Build(); + + fixture.AddTestHostFixtures(testhost1, testhost2); + + var testRequestManager = fixture.BuildTestRequestManager(); + + mstest1Dll.FrameworkName.Should().NotBe(mstest2Dll.FrameworkName); + + // -- act + // TODO: Building whole default runconfiguration is needed here, because TestRequestManager does not ensure the basic settings are populated, + // and all methods that populate them just silently fail, so TestHostProvider does not get any useful settings. + var runConfiguration = new RunConfiguration().ToXml().OuterXml; + var testRunRequestPayload = new TestRunRequestPayload + { + Sources = new List { mstest1Dll.Path, mstest2Dll.Path }, + + RunSettings = $"{runConfiguration}" + }; + + await testRequestManager.ExecuteWithAbort(tm => tm.RunTests(testRunRequestPayload, testHostLauncher: null, fixture.TestRunEventsRegistrar, fixture.ProtocolConfig)); + + // -- assert + fixture.AssertNoErrors(); + // We unify the frameworks to netcoreapp1.0 (because the vstest.console dll we are loading is built for netcoreapp and prefers netcoreapp), and because the + // behavior is to choose the common oldest framework. We then log warning about incompatible sources. + fixture.TestRunEventsRegistrar.LoggedWarnings.Should().ContainMatch($"Test run detected DLL(s) which were built for different framework and platform versions*{KnownFrameworkNames.Netcoreapp1}*"); + + // We started both testhosts, even thought we know one of them is incompatible. + fixture.ProcessHelper.Processes.Where(p => p.Started).Should().HaveCount(2); + var startWithSources1 = testhost1.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources1Text = startWithSources1.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest1.dll + startWithSources1Text.Should().Contain("mstest1.dll"); + // And we sent netcoreapp1.0 as the target framework + startWithSources1Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); + + var startWithSources2 = testhost2.FakeCommunicationChannel.ProcessedMessages.Single(m => m.Request.MessageType == MessageType.StartTestExecutionWithSources); + var startWithSources2Text = startWithSources2.Request.Payload.Select(t => t.ToString()).JoinBySpace(); + // We sent mstest2.dll + startWithSources2Text.Should().Contain("mstest2.dll"); + // And we sent netcoreapp1.0 as the target framework, even though it is incompatible + startWithSources2Text.Should().Contain(KnownFrameworkStrings.Netcoreapp1); + + fixture.ExecutedTests.Should().HaveCount(mstest1Dll.TestCount); + } +} + +// Test and improvmement ideas: +// TODO: passing null runsettings does not fail fast, instead it fails in Fakes settings code +// TODO: passing empty string fails in the xml parser code +// TODO: passing null sources and null testcases does not fail fast +// TODO: Just calling Exit, Close won't stop the run, we will keep waiting for test run to complete, I think in real life when we exit then Disconnected will be called on the vstest.console side, leading to abort flow. +//.StartTestExecutionWithSources(new FakeMessage(MessageType.TestMessage, new TestMessagePayload { MessageLevel = TestMessageLevel.Error, Message = "Loading type failed." }), afterAction: f => { /*f.Process.Exit();*/ f.FakeCommunicationEndpoint.Disconnect(); }) diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj new file mode 100644 index 0000000000..21a59d0ab8 --- /dev/null +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -0,0 +1,40 @@ + + + + ..\..\ + false + false + + + + enable + true + preview + net6.0 + netcoreapp3.1 + Exe + Exe + + + + + + + + + true + + + true + + + + + + + + + + + +