diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 9fa1efee0d..88545cf6ea 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -1,4 +1,4 @@ - + 15.8.3247 diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 31812dacfa..72b2c6569d 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -226,7 +226,16 @@ private void ProcessRequests(ITestRequestManager testRequestManager) case MessageType.CancelDiscovery: { - testRequestManager.CancelDiscovery(); + // If testhost has old version, we should use old cancel logic + // to be consistent and not create regression issues + if (this.protocolConfig.Version < ObjectModel.Constants.MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport) + { + testRequestManager.CancelDiscovery(); + } + else + { + testRequestManager.CancelDiscoveryWithEventHandler(); + } break; } diff --git a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs index d63a2508e2..9d58199b36 100644 --- a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs @@ -143,6 +143,42 @@ public void Abort() } } + /// + public void AbortWithEventHandler() + { + if (EqtTrace.IsVerboseEnabled) + { + EqtTrace.Verbose("DiscoveryRequest.AbortWithEventHandler: Aborting."); + } + + lock (this.syncObject) + { + if (this.disposed) + { + throw new ObjectDisposedException("DiscoveryRequest"); + } + + if (this.discoveryInProgress) + { + this.DiscoveryManager.Abort(this); + } + else + { + if (EqtTrace.IsInfoEnabled) + { + EqtTrace.Info("DiscoveryRequest.AbortWithEventHandler: No operation to abort."); + } + + return; + } + } + + if (EqtTrace.IsInfoEnabled) + { + EqtTrace.Info("DiscoveryRequest.AbortWithEventHandler: Aborted."); + } + } + /// /// Wait for discovery completion /// diff --git a/src/Microsoft.TestPlatform.Client/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.Client/PublicAPI/PublicAPI.Unshipped.txt index e69de29bb2..408b735f89 100644 --- a/src/Microsoft.TestPlatform.Client/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.Client/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,2 @@ +Microsoft.VisualStudio.TestPlatform.Client.Discovery.DiscoveryRequest.AbortWithEventHandler() -> void +Microsoft.VisualStudio.TestPlatform.Client.RequestHelper.ITestRequestManager.CancelDiscoveryWithEventHandler() -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs b/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs index 714e1df9cb..11bad6a82d 100644 --- a/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs +++ b/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs @@ -102,6 +102,11 @@ void StartTestSession( /// void CancelDiscovery(); + /// + /// Cancels the current discovery request with discovery complete event handler + /// + void CancelDiscoveryWithEventHandler(); + /// /// Cancels the current test run attachments processing request. /// diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs index 908b744543..3157d8216a 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs @@ -10,6 +10,11 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine /// public interface IParallelProxyDiscoveryManager : IParallelOperationManager, IProxyDiscoveryManager { + /// + /// Indicates if user requested an abortion + /// + bool IsAbortRequested { get; set; } + /// /// Handles Partial Discovery Complete event coming from a specific concurrent proxy discovery manager /// Each concurrent proxy discovery manager will signal the parallel discovery manager when its complete @@ -25,4 +30,14 @@ bool HandlePartialDiscoveryComplete( IEnumerable lastChunk, bool isAborted); } + + /// + /// Enums for indicating discovery status of source + /// + public enum DiscoveryStatus + { + FullyDiscovered, // Indicates that source was fully discovered + PartiallyDiscovered, // Indicates that we started discovery of the source but something happened (cancel/abort) and we stopped processing it + NotDiscovered // Indicates the sources which were not touched during discovery + } } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs index c77c525236..e03bdf3616 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs @@ -28,6 +28,12 @@ public interface IProxyDiscoveryManager /// void Abort(); + /// + /// Aborts discovery operation with EventHandler. + /// + /// EventHandler for handling discovery events from Engine + void Abort(ITestDiscoveryEventsHandler2 eventHandler); + /// /// Closes the current test operation. /// Send a EndSession message to close the test host and channel gracefully. diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs index e60787a968..1a7c01bcda 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs @@ -29,5 +29,11 @@ public interface IDiscoveryManager /// Aborts the test discovery. /// void Abort(); + + /// + /// Aborts the test discovery with eventHandler + /// + /// EventHandler for handling discovery events from Engine + void Abort(ITestDiscoveryEventsHandler2 eventHandler); } } diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt index e69de29bb2..0f1022956e 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,8 @@ +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 0 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.NotDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.PartiallyDiscovered = 1 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager.IsAbortRequested.get -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager.IsAbortRequested.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs index d65aa7bb0f..950c0ff6f6 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs @@ -87,6 +87,11 @@ public interface ITestRequestSender : IDisposable /// void SendTestRunAbort(); + /// + /// Send the request to abort the discovery + /// + void SendDiscoveryAbort(); + /// /// Handle client process exit /// diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs index 72c44d1e6c..969f5b69f9 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs @@ -230,6 +230,7 @@ private JsonSerializer GetPayloadSerializer(int? version) case 2: case 4: case 5: + case 6: return payloadSerializer2; default: throw new NotSupportedException($"Protocol version {version} is not supported. " + diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs index 1abacd4f4e..4586d0773a 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs @@ -30,5 +30,20 @@ public class DiscoveryCompletePayload /// Gets or sets the Metrics /// public IDictionary Metrics { get; set; } + + /// + /// Gets or sets list of sources which were fully discovered + /// + public IReadOnlyCollection FullyDiscoveredSources { get; set; } = new List(); + + /// + /// Gets or sets list of sources which were partially discovered (started discover tests, but then discovery aborted) + /// + public IReadOnlyCollection PartiallyDiscoveredSources { get; set; } = new List(); + + /// + /// Gets or sets list of sources which were not discovered at all + /// + public IReadOnlyCollection NotDiscoveredSources { get; set; } = new List(); } } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt index e69de29bb2..1c90267bfa 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,8 @@ +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.SendDiscoveryAbort() -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.SendDiscoveryAbort() -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 16ec2d6c03..013d39ffeb 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -14,8 +14,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - using CommonResources = Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources; - using ObjectModelConstants = Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants; + using CommonResources = Resources.Resources; + using ObjectModelConstants = TestPlatform.ObjectModel.Constants; /// /// Test request sender implementation. @@ -54,7 +54,7 @@ public class TestRequestSender : ITestRequestSender // Must be in sync with the highest supported version in // src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs file. - private int highestSupportedVersion = 5; + private int highestSupportedVersion = 6; private TestHostConnectionInfo connectionInfo; @@ -290,6 +290,24 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve this.channel.Send(message); } + + /// + public void SendDiscoveryAbort() + { + if (this.IsOperationComplete()) + { + EqtTrace.Verbose("TestRequestSender.SendDiscoveryAbort: Operation is already complete. Skip error message."); + return; + } + + if (EqtTrace.IsVerboseEnabled) + { + EqtTrace.Verbose("TestRequestSender.SendDiscoveryAbort: Sending discovery abort."); + } + + this.channel?.Send(this.dataSerializer.SerializeMessage(MessageType.CancelDiscovery)); + } + #endregion #region Execution Protocol @@ -595,7 +613,12 @@ private void OnDiscoveryMessageReceived(ITestDiscoveryEventsHandler2 discoveryEv case MessageType.DiscoveryComplete: var discoveryCompletePayload = this.dataSerializer.DeserializePayload(data); - var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(discoveryCompletePayload.TotalTests, discoveryCompletePayload.IsAborted); + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( + discoveryCompletePayload.TotalTests, + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; discoveryEventsHandler.HandleDiscoveryComplete( discoveryCompleteEventArgs, diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs index db1baf2d1a..ae18f5d0a1 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs @@ -95,6 +95,12 @@ public void Abort() Task.Run(() => this.testHostManagerFactory.GetDiscoveryManager().Abort()); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + Task.Run(() => this.testHostManagerFactory.GetDiscoveryManager().Abort(eventHandler)); + } + private void InitializeExtensions(IEnumerable sources) { var extensionsFromSource = this.testHostManager.GetTestPlatformExtensions(sources, Enumerable.Empty()); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs index f41394d598..2d2d9c9987 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs @@ -4,6 +4,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel { using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -42,6 +44,20 @@ public ParallelDiscoveryDataAggregator() #endregion + #region Internal Properties + + /// + /// Dictionary which stores source with corresponding discoveryStatus + /// + internal ConcurrentDictionary SourcesWithDiscoveryStatus = new ConcurrentDictionary(); + + /// + /// Indicates if discovery complete payload already sent back to IDE + /// + internal bool IsMessageSent { get; set; } + + #endregion + #region Public Methods /// @@ -128,6 +144,58 @@ public void AggregateDiscoveryDataMetrics(IDictionary metrics) } } + /// + /// Aggregate the source as fully discovered + /// + /// Fully discovered source + internal void AggregateTheSourcesWithDiscoveryStatus(IEnumerable sources, DiscoveryStatus status) + { + if (sources == null || sources.Count() == 0) return; + + foreach (var source in sources) + { + if (status == DiscoveryStatus.NotDiscovered) SourcesWithDiscoveryStatus[source] = status; + + if (!SourcesWithDiscoveryStatus.ContainsKey(source)) + { + EqtTrace.Warning($"ParallelDiscoveryDataAggregator.AggregateTheSourcesWithDiscoveryStatus : " + + $"{source} is not present in SourcesWithDiscoveryStatus dictionary"); + } + else + { + SourcesWithDiscoveryStatus[source] = status; + + if (EqtTrace.IsInfoEnabled) + { + EqtTrace.Info($"ParallelDiscoveryDataAggregator.AggregateTheSourcesWithDiscoveryStatus : " + + $"{source} is marked with {status} status"); + } + } + } + } + + /// + /// Aggregate the value indicating if we already sent message to IDE + /// + /// Boolean value if we already sent message to IDE + internal void AggregateIsMessageSent(bool isMessageSent) + { + this.IsMessageSent = this.IsMessageSent || isMessageSent; + } + + /// + /// Returning sources with particular discovery status + /// + /// Status to filter + /// + internal ICollection GetSourcesWithStatus(DiscoveryStatus status) + { + if (SourcesWithDiscoveryStatus == null || SourcesWithDiscoveryStatus.IsEmpty) return new List(); + + return SourcesWithDiscoveryStatus.Where(source => source.Value == status) + .Select(source => source.Key).ToList(); + } + #endregion } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs index 1552ca924c..accfbfa4e2 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel { using System.Collections.Generic; + using System.Linq; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; @@ -33,6 +34,8 @@ internal class ParallelDiscoveryEventsHandler : ITestDiscoveryEventsHandler2 private IRequestData requestData; + private readonly object sendMessageLock = new object(); + public ParallelDiscoveryEventsHandler(IRequestData requestData, IProxyDiscoveryManager proxyDiscoveryManager, ITestDiscoveryEventsHandler2 actualDiscoveryEventsHandler, @@ -63,6 +66,12 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete var totalTests = discoveryCompleteEventArgs.TotalCount; var isAborted = discoveryCompleteEventArgs.IsAborted; + // Aggregate for final discovery complete + discoveryDataAggregator.Aggregate(totalTests, isAborted); + + // Aggregate Discovery Data Metrics + discoveryDataAggregator.AggregateDiscoveryDataMetrics(discoveryCompleteEventArgs.Metrics); + // we get discovery complete events from each host process // so we cannot "complete" the actual operation until all sources are consumed // We should not block last chunk results while we aggregate overall discovery data @@ -70,14 +79,13 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete { ConvertToRawMessageAndSend(MessageType.TestCasesFound, lastChunk); this.HandleDiscoveredTests(lastChunk); + // If we come here it means that some source was already fully discovered so we can mark it + if (!discoveryDataAggregator.IsAborted) + { + AggregateComingSourcesAsFullyDiscovered(lastChunk, discoveryCompleteEventArgs); + } } - // Aggregate for final discovery complete - discoveryDataAggregator.Aggregate(totalTests, isAborted); - - // Aggregate Discovery Data Metrics - discoveryDataAggregator.AggregateDiscoveryDataMetrics(discoveryCompleteEventArgs.Metrics); - // Do not send TestDiscoveryComplete to actual test discovery handler // We need to see if there are still sources left - let the parallel manager decide var parallelDiscoveryComplete = this.parallelProxyDiscoveryManager.HandlePartialDiscoveryComplete( @@ -88,13 +96,27 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete if (parallelDiscoveryComplete) { + var fullyDiscovered = discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered) as IReadOnlyCollection; + var partiallyDiscovered = discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered) as IReadOnlyCollection; + var notDiscovered = discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered) as IReadOnlyCollection; + + // As we immediatelly return results to IDE in case of aborting + // we need to set isAborted = true and totalTests = -1 + if (this.parallelProxyDiscoveryManager.IsAbortRequested) + { + discoveryDataAggregator.Aggregate(-1, true); + } + // In case of sequential discovery - RawMessage would have contained a 'DiscoveryCompletePayload' object // To send a raw message - we need to create raw message from an aggregated payload object var testDiscoveryCompletePayload = new DiscoveryCompletePayload() { TotalTests = discoveryDataAggregator.TotalTests, - IsAborted = discoveryDataAggregator.IsAborted, - LastDiscoveredTests = null + IsAborted = discoveryDataAggregator.IsAborted, + LastDiscoveredTests = null, + FullyDiscoveredSources = fullyDiscovered, + PartiallyDiscoveredSources = partiallyDiscovered, + NotDiscoveredSources = notDiscovered }; // Collecting Final Discovery State @@ -104,11 +126,15 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete var aggregatedDiscoveryDataMetrics = discoveryDataAggregator.GetAggregatedDiscoveryDataMetrics(); testDiscoveryCompletePayload.Metrics = aggregatedDiscoveryDataMetrics; - // we have to send raw messages as we block the discovery complete actual raw messages - this.ConvertToRawMessageAndSend(MessageType.DiscoveryComplete, testDiscoveryCompletePayload); + // Sending discovery complete message to IDE + this.SendMessage(testDiscoveryCompletePayload); var finalDiscoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(this.discoveryDataAggregator.TotalTests, - this.discoveryDataAggregator.IsAborted); + this.discoveryDataAggregator.IsAborted, + fullyDiscovered, + partiallyDiscovered, + notDiscovered); + finalDiscoveryCompleteEventArgs.Metrics = aggregatedDiscoveryDataMetrics; // send actual test discovery complete to clients @@ -160,5 +186,53 @@ private void ConvertToRawMessageAndSend(string messageType, object payload) var rawMessage = this.dataSerializer.SerializePayload(messageType, payload); this.actualDiscoveryEventsHandler.HandleRawMessage(rawMessage); } + + /// + /// Sending discovery complete message to IDE + /// + /// Discovery aggregator to know if we already sent this message + /// Discovery complete payload to send + private void SendMessage(DiscoveryCompletePayload testDiscoveryCompletePayload) + { + // In case of abort scenario, we need to send raw message to IDE only once after abortion. + // All other testhosts which will finish after shouldn't send raw message + if (!discoveryDataAggregator.IsMessageSent) + { + lock (sendMessageLock) + { + if (!discoveryDataAggregator.IsMessageSent) + { + // we have to send raw messages as we block the discovery complete actual raw messages + this.ConvertToRawMessageAndSend(MessageType.DiscoveryComplete, testDiscoveryCompletePayload); + discoveryDataAggregator.AggregateIsMessageSent(true); + } + } + } + } + + /// + /// Aggregate source as fully discovered + /// + /// Aggregator to aggregate results + /// Last chunk of discovered test cases + private void AggregateComingSourcesAsFullyDiscovered(IEnumerable lastChunk, DiscoveryCompleteEventArgs discoveryCompleteEventArgs) + { + if (lastChunk == null) return; + + IEnumerable lastChunkSources; + + // Sometimes we get lastChunk as empty list (when number of tests in project dividable by 10) + // Then we will take sources from discoveryCompleteEventArgs coming from testhost + if (lastChunk.Count() == 0) + { + lastChunkSources = discoveryCompleteEventArgs.FullyDiscoveredSources; + } + else + { + lastChunkSources = lastChunk.Select(testcase => testcase.Source); + } + + discoveryDataAggregator.AggregateTheSourcesWithDiscoveryStatus(lastChunkSources, DiscoveryStatus.FullyDiscovered); + } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index c7ea14d07a..c755746895 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -37,8 +37,7 @@ internal class ParallelProxyDiscoveryManager : ParallelOperationManager private object discoveryStatusLockObject = new object(); + private object enumeratorLockObject = new object(); + #endregion public ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, bool sharedHosts) @@ -85,16 +86,30 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve { EqtTrace.Verbose("ParallelProxyDiscoveryManager: Start discovery. Total sources: " + this.availableTestSources); } + + // One data aggregator per parallel discovery + this.currentDiscoveryDataAggregator = new ParallelDiscoveryDataAggregator(); + + // Marking all sources as not discovered before starting actual discovery + this.MarkAllSourcesAsNotDiscovered(discoveryCriteria.Sources); + this.DiscoverTestsPrivate(eventHandler); } /// public void Abort() { - this.discoveryAbortRequested = true; + IsAbortRequested = true; this.DoActionOnAllManagers((proxyManager) => proxyManager.Abort(), doActionsInParallel: true); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + IsAbortRequested = true; + this.DoActionOnAllManagers((proxyManager) => proxyManager.Abort(eventHandler), doActionsInParallel: true); + } + /// public void Close() { @@ -130,7 +145,7 @@ If discovery is complete or discovery aborting was requsted by testPlatfrom(user when testhost crashed by itself and when user requested it (f.e. through TW) Schedule the clean up for managers and handlers. */ - if (allDiscoverersCompleted || discoveryAbortRequested) + if (allDiscoverersCompleted || IsAbortRequested) { // Reset enumerators this.sourceEnumerator = null; @@ -181,9 +196,6 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa // Reset the discovery complete data this.discoveryCompletedClients = 0; - // One data aggregator per parallel discovery - this.currentDiscoveryDataAggregator = new ParallelDiscoveryDataAggregator(); - foreach (var concurrentManager in this.GetConcurrentManagerInstances()) { var parallelEventsHandler = new ParallelDiscoveryEventsHandler( @@ -256,5 +268,22 @@ private void DiscoverTestsOnConcurrentManager(IProxyDiscoveryManager proxyDiscov EqtTrace.Verbose("ProxyParallelDiscoveryManager: No sources available for discovery."); } } + + /// + /// Mark all sources as not discovered before starting actual discovery + /// + /// Sources which will be discovered + private void MarkAllSourcesAsNotDiscovered(IEnumerable sources) + { + if (sources == null || sources.Count() == 0) return; + + lock (enumeratorLockObject) + { + foreach (string source in sources) + { + this.currentDiscoveryDataAggregator.SourcesWithDiscoveryStatus[source] = DiscoveryStatus.NotDiscovered; + } + } + } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs index c8d85fa1b8..06b8bc8e0f 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs @@ -6,7 +6,6 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client using System; using System.Collections.Generic; using System.Linq; - using System.Threading; using Microsoft.VisualStudio.TestPlatform.Common; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; @@ -196,6 +195,26 @@ public void Abort() this.Close(); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + // Do nothing if the proxy is not initialized yet. + if (this.proxyOperationManager == null) + { + return; + } + + if (this.baseTestDiscoveryEventsHandler == null) + { + this.baseTestDiscoveryEventsHandler = eventHandler; + } + + if (this.isCommunicationEstablished) + { + this.proxyOperationManager.RequestSender.SendDiscoveryAbort(); + } + } + /// public void Close() { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs index f85f70fa65..663a5ec0d5 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs @@ -26,7 +26,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Utilities; - using CrossPlatEngineResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources; + using CrossPlatEngineResources = Resources.Resources; /// /// Enumerates through all the discoverers. @@ -200,7 +200,7 @@ private void DiscoverTestsFromSingleDiscoverer( ref double totalAdaptersUsed, ref double totalTimeTakenByAdapters) { - if (!DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType)) + if (!TryToLoadDiscoverer(discoverer, logger, out var discovererType)) { // Fail to instantiate the discoverer type. return; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs index f2751c18e5..addef943c6 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -18,10 +19,11 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - using CrossPlatEngineResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources; + using CrossPlatEngineResources = Resources.Resources; /// /// Orchestrates discovery operations for the engine communicating with the test host process. @@ -34,6 +36,8 @@ public class DiscoveryManager : IDiscoveryManager private ITestDiscoveryEventsHandler2 testDiscoveryEventsHandler; private DiscoveryCriteria discoveryCriteria; private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + private string previousSource = null; + private ConcurrentDictionary SourcesWithDiscoveryStatus { get; set; } = new ConcurrentDictionary(); /// /// Initializes a new instance of the class. @@ -104,6 +108,8 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve if (verifiedSources.Any()) { verifiedExtensionSourceMap.Add(kvp.Key, kvp.Value); + // Mark all sources as NotDiscovered before actual discovery starts + MarkSourcesWithStatus(verifiedSources, DiscoveryStatus.NotDiscovered); } } @@ -133,6 +139,9 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve if (lastChunk != null) { UpdateTestCases(lastChunk, this.discoveryCriteria.Package); + /* When discovery is complete we will have case that the last discovered source is still marked as partiallyDiscovered. + * So we need to mark it as fullyDiscovered.*/ + MarkTheLastChunkSourcesAsFullyDiscovered(lastChunk); } // Collecting Discovery State @@ -140,10 +149,18 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve // Collecting Total Tests Discovered this.requestData.MetricsCollection.Add(TelemetryDataConstants.TotalTestsDiscovered, totalDiscoveredTestCount); - var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(totalDiscoveredTestCount, false) + + if (cancellationTokenSource.IsCancellationRequested) { - Metrics = this.requestData.MetricsCollection.Metrics - }; + totalDiscoveredTestCount = -1; + } + + var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(totalDiscoveredTestCount, cancellationTokenSource.IsCancellationRequested, + GetFilteredSources(DiscoveryStatus.FullyDiscovered), + GetFilteredSources(DiscoveryStatus.PartiallyDiscovered), + GetFilteredSources(DiscoveryStatus.NotDiscovered)); + + discoveryCompleteEventsArgs.Metrics = this.requestData.MetricsCollection.Metrics; eventHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, lastChunk); } @@ -167,6 +184,22 @@ public void Abort() this.cancellationTokenSource.Cancel(); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + if (!cancellationTokenSource.IsCancellationRequested) + { + this.Abort(); + } + + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true, + GetFilteredSources(DiscoveryStatus.FullyDiscovered), + GetFilteredSources(DiscoveryStatus.PartiallyDiscovered), + GetFilteredSources(DiscoveryStatus.NotDiscovered)); + + eventHandler.HandleDiscoveryComplete(discoveryCompleteEventArgs, null); + } + private void OnReportTestCases(IEnumerable testCases) { UpdateTestCases(testCases, this.discoveryCriteria.Package); @@ -174,6 +207,8 @@ private void OnReportTestCases(IEnumerable testCases) if (this.testDiscoveryEventsHandler != null) { this.testDiscoveryEventsHandler.HandleDiscoveredTests(testCases); + // We need to mark sources based on already discovered testcases + MarkSourcesBasedOnDiscoveredTestCases(testCases); } else { @@ -293,5 +328,120 @@ private static void UpdateTestCases(IEnumerable testCases, string pack } } } + + /// + /// Mark sources based on already discovered testCases + /// + /// List of testCases which were already discovered + private void MarkSourcesBasedOnDiscoveredTestCases(IEnumerable testCases) + { + if (testCases == null || testCases.Count() == 0) return; + + foreach (var testCase in testCases) + { + string currentSource = testCase.Source; + + // If it is the first list of testCases which was discovered, we mark sources as partiallyDiscovered. + // Or if current source is the same as previous we again mark them as partiallyDiscovered for consistency + if (previousSource is null || previousSource == currentSource) + { + MarkSourceWithStatus(currentSource, DiscoveryStatus.PartiallyDiscovered); + } + // If source is changed, we need to mark previous source as already fullyDiscovered + // and currentSource should be partiallyDiscovered + else if (currentSource != previousSource) + { + MarkSourceWithStatus(previousSource, DiscoveryStatus.FullyDiscovered); + MarkSourceWithStatus(currentSource, DiscoveryStatus.PartiallyDiscovered); + } + + this.previousSource = currentSource; + } + } + + /// + /// Mark the last sources as fullyDiscovered + /// + /// Last chunk of testCases which were discovered + private void MarkTheLastChunkSourcesAsFullyDiscovered(IList lastChunk) + { + if (lastChunk == null) return; + + var lastChunkSources = lastChunk.Select(testcase => testcase.Source); + + // When all testcases in project is dividable by 10 then lastChunk is coming as empty + // So we need to take the lastSource and mark it as FullyDiscovered + if (lastChunk.Count == 0) lastChunkSources = new List() { previousSource }; + + MarkSourcesWithStatus(lastChunkSources, DiscoveryStatus.FullyDiscovered); + } + + /// + /// Mark the source with particular DiscoveryStatus + /// + /// Sources to mark + /// DiscoveryStatus to mark for source + private void MarkSourceWithStatus(string source, DiscoveryStatus status) + { + if (source == null) return; + + if (!SourcesWithDiscoveryStatus.ContainsKey(source)) + { + EqtTrace.Warning($"DiscoveryManager.MarkSourceWithStatus : SourcesWithDiscoveryStatus does not contain {source}"); + } + else + { + SourcesWithDiscoveryStatus[source] = status; + EqtTrace.Warning($"DiscoveryManager.MarkSourceWithStatus : Marking {source} with {status} status"); + } + } + + /// + /// Mark sources with particular DiscoveryStatus + /// + /// List of sources to mark + /// DiscoveryStatus to mark for list of sources + private void MarkSourcesWithStatus(IEnumerable sources, DiscoveryStatus status) + { + if (sources == null || sources.Count() == 0) return; + + foreach (var source in sources) + { + if (source == null) continue; + + // It is the first time when we fill our map with sources + if (status == DiscoveryStatus.NotDiscovered) SourcesWithDiscoveryStatus[source] = status; + + else + { + if (!SourcesWithDiscoveryStatus.ContainsKey(source)) + { + EqtTrace.Warning($"DiscoveryManager.MarkSourcesWithStatus : SourcesWithDiscoveryStatus does not contain {source}"); + } + else + { + SourcesWithDiscoveryStatus[source] = status; + EqtTrace.Warning($"DiscoveryManager.MarkSourcesWithStatus : Marking {source} with {status} status"); + } + } + } + } + + /// + /// Filter discovery sources based on discovery status condition + /// + /// discoveryStatus indicates if source was fully/partially/not discovered + /// + private IReadOnlyCollection GetFilteredSources(DiscoveryStatus discoveryStatus) + { + // If by some accident SourcesWithDiscoveryStatus map is empty we will return empty list + if (SourcesWithDiscoveryStatus == null || SourcesWithDiscoveryStatus.Count == 0) + { + return new List(); + } + + return SourcesWithDiscoveryStatus.Where(source => source.Value == discoveryStatus) + .Select(source => source.Key).ToList(); + } } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs index bbfe0ee0ca..dff7494fdc 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs @@ -18,8 +18,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommunicationUtilities using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities; - using CrossPlatResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources; - using ObjectModelConstants = Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants; + using CrossPlatResources = CrossPlatEngine.Resources.Resources; + using ObjectModelConstants = TestPlatform.ObjectModel.Constants; public class TestRequestHandler : ITestRequestHandler { @@ -27,7 +27,7 @@ public class TestRequestHandler : ITestRequestHandler // Must be in sync with the highest supported version in // src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs file. - private int highestSupportedVersion = 5; + private int highestSupportedVersion = 6; private readonly IDataSerializer dataSerializer; private ITestHostManagerFactory testHostManagerFactory; @@ -204,7 +204,10 @@ public void DiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventA TotalTests = discoveryCompleteEventArgs.TotalCount, LastDiscoveredTests = discoveryCompleteEventArgs.IsAborted ? null : lastChunk, IsAborted = discoveryCompleteEventArgs.IsAborted, - Metrics = discoveryCompleteEventArgs.Metrics + Metrics = discoveryCompleteEventArgs.Metrics, + FullyDiscoveredSources = discoveryCompleteEventArgs.FullyDiscoveredSources, + PartiallyDiscoveredSources = discoveryCompleteEventArgs.PartiallyDiscoveredSources, + NotDiscoveredSources = discoveryCompleteEventArgs.NotDiscoveredSources }, this.protocolVersion); this.SendData(data); @@ -485,6 +488,12 @@ public void OnMessageReceived(object sender, MessageReceivedEventArgs messageRec this.onAttachDebuggerAckRecieved?.Invoke(message); break; + case MessageType.CancelDiscovery: + jobQueue.Pause(); + this.testHostManagerFactoryReady.Wait(); + testHostManagerFactory.GetDiscoveryManager().Abort(new TestDiscoveryEventHandler(this)); + break; + case MessageType.AbortTestRun: try { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt index e69de29bb2..8a84288ee8 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,2 @@ +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery.DiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs index a17323a347..3bf6f9f6ae 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs @@ -12,6 +12,24 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client /// public class DiscoveryCompleteEventArgs : EventArgs { + /// + /// Constructor for creating event args object + /// + /// Total tests which got discovered + /// Specifies if discovery has been aborted. + /// List of fully discovered sources + /// List of partially discovered sources + /// List of not discovered sources + public DiscoveryCompleteEventArgs(long totalTests, bool isAborted, + IReadOnlyCollection fullyDiscoveredSources, + IReadOnlyCollection partiallyDiscoveredSources, + IReadOnlyCollection notDiscoveredSources) : this(totalTests, isAborted) + { + FullyDiscoveredSources = fullyDiscoveredSources; + PartiallyDiscoveredSources = partiallyDiscoveredSources; + NotDiscoveredSources = notDiscoveredSources; + } + /// /// Constructor for creating event args object /// @@ -42,5 +60,20 @@ public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) /// Metrics /// public IDictionary Metrics { get; set; } + + /// + /// Gets the list of sources which were fully discovered. + /// + public IReadOnlyCollection FullyDiscoveredSources { get; } = new List(); + + /// + /// Gets the list of sources which were partially discovered (started discover tests, but then discovery aborted). + /// + public IReadOnlyCollection PartiallyDiscoveredSources { get; } = new List(); + + /// + /// Gets the list of sources which were not discovered at all. + /// + public IReadOnlyCollection NotDiscoveredSources { get; } = new List(); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IDiscoveryRequest.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IDiscoveryRequest.cs index ce3ab28efc..9b01a3bc16 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IDiscoveryRequest.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Interfaces/IDiscoveryRequest.cs @@ -49,5 +49,10 @@ DiscoveryCriteria DiscoveryCriteria /// Aborts the discovery request /// void Abort(); + + /// + /// Aborts the discovery request with event handler + /// + void AbortWithEventHandler(); } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs index 8b1d57bbb8..c34a572bb6 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs @@ -173,13 +173,18 @@ public static class Constants /// /// The default protocol version /// - public static readonly ProtocolConfig DefaultProtocolConfig = new ProtocolConfig { Version = 5 }; + public static readonly ProtocolConfig DefaultProtocolConfig = new ProtocolConfig { Version = 6 }; /// /// The minimum protocol version that has debug support /// public const int MinimumProtocolVersionWithDebugSupport = 3; + /// + /// The minimum protocol version that has debug support + /// + public const int MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport = 6; + /// /// Name of the results directory /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt index e3d7f85f98..f7c6ee6bdd 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt @@ -4,3 +4,9 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSes Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler) -> bool +const Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants.MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport = 6 -> int +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.DiscoveryCompleteEventArgs(long totalTests, bool isAborted, System.Collections.Generic.IReadOnlyCollection fullyDiscoveredSources, System.Collections.Generic.IReadOnlyCollection partiallyDiscoveredSources, System.Collections.Generic.IReadOnlyCollection notDiscoveredSources) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.FullyDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.NotDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.PartiallyDiscoveredSources.get -> System.Collections.Generic.IReadOnlyCollection +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IDiscoveryRequest.AbortWithEventHandler() -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs index be26f63f1f..f621885063 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs @@ -22,7 +22,7 @@ namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - using TranslationLayerResources = Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources; + using TranslationLayerResources = VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources; /// /// Vstest console request sender for sending requests to vstest.console.exe @@ -43,7 +43,7 @@ internal class VsTestConsoleRequestSender : ITranslationLayerRequestSender private bool handShakeSuccessful = false; - private int protocolVersion = 5; + private int protocolVersion = 6; /// /// Used to cancel blocking tasks associated with the vstest.console process. @@ -1025,7 +1025,10 @@ private void SendMessageAndListenAndReportTestCases( var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( discoveryCompletePayload.TotalTests, - discoveryCompletePayload.IsAborted); + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); // Adding metrics from vstest.console. discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; @@ -1051,7 +1054,7 @@ private void SendMessageAndListenAndReportTestCases( eventHandler.HandleLogMessage( TestMessageLevel.Error, TranslationLayerResources.AbortedTestsDiscovery); - var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true); + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true, new List(), new List(), new List()); eventHandler.HandleDiscoveryComplete(discoveryCompleteEventArgs, null); // Earlier we were closing the connection with vstest.console in case of exceptions. @@ -1115,7 +1118,10 @@ private async Task SendMessageAndListenAndReportTestCasesAsync( var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( discoveryCompletePayload.TotalTests, - discoveryCompletePayload.IsAborted); + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); // Adding Metrics from VsTestConsole discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; @@ -1143,7 +1149,7 @@ private async Task SendMessageAndListenAndReportTestCasesAsync( TestMessageLevel.Error, TranslationLayerResources.AbortedTestsDiscovery); - var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true); + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true, new List(), new List(), new List()); eventHandler.HandleDiscoveryComplete(discoveryCompleteEventArgs, null); // Earlier we were closing the connection with vstest.console in case of exceptions. diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index c450e968c9..42a3231d30 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -525,10 +525,17 @@ public void CancelTestRun() /// public void CancelDiscovery() { - EqtTrace.Info("TestRequestManager.CancelTestDiscovery: Sending cancel request."); + EqtTrace.Info("TestRequestManager.CancelDiscovery: Sending cancel request."); this.currentDiscoveryRequest?.Abort(); } + /// + public void CancelDiscoveryWithEventHandler() + { + EqtTrace.Info("TestRequestManager.CancelDiscoveryWithEventHandler: Sending cancel request."); + this.currentDiscoveryRequest?.AbortWithEventHandler(); + } + /// public void AbortTestRun() { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 35dfc4181d..638fd1af71 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -1,4 +1,5 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. + +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests @@ -143,6 +144,25 @@ public void DiscoverTestsUsingEventHandler1AndBatchSize(RunnerInfo runnerInfo) Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize); } + [TestMethod] + [NetCoreTargetFrameworkDataSource] + [NetFullTargetFrameworkDataSource] + public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovered(RunnerInfo runnerInfo) + { + SetTestEnvironment(this.testEnvironment, runnerInfo); + this.Setup(); + + var eventHandler2 = new DiscoveryEventHandler2(); + + this.vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), + this.GetDefaultRunSettings(), + null, + eventHandler2); + + // Assert. + Assert.AreEqual(2, eventHandler2.FullyDiscoveredSources.Count); + } + [TestMethod] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs index e6e72e7a24..9afb7e5f64 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs @@ -70,6 +70,10 @@ public class DiscoveryEventHandler2 : ITestDiscoveryEventsHandler2 /// public List DiscoveredTestCases { get; } + public IReadOnlyCollection FullyDiscoveredSources { get; private set; } + public IReadOnlyCollection PartiallyDiscoveredSources { get; private set; } + public IReadOnlyCollection NotDiscoveredSources { get; private set; } + public List testMessages; /// @@ -101,6 +105,9 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete } this.Metrics = discoveryCompleteEventArgs.Metrics; + this.FullyDiscoveredSources = discoveryCompleteEventArgs.FullyDiscoveredSources; + this.PartiallyDiscoveredSources = discoveryCompleteEventArgs.PartiallyDiscoveredSources; + this.NotDiscoveredSources = discoveryCompleteEventArgs.NotDiscoveredSources; } public void HandleDiscoveredTests(IEnumerable discoveredTestCases) diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs index 9cf49fb843..acdc7a0a99 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs @@ -41,7 +41,7 @@ public class DesignModeClientTests private readonly DesignModeClient designModeClient; - private readonly int protocolVersion = 5; + private readonly int protocolVersion = 6; private readonly AutoResetEvent completeEvent; @@ -134,7 +134,7 @@ public void DesignModeClientConnectShouldNotSendConnectedIfServerConnectionTimes [TestMethod] public void DesignModeClientDuringConnectShouldHighestCommonVersionWhenReceivedVersionIsGreaterThanSupportedVersion() { - var verCheck = new Message { MessageType = MessageType.VersionCheck, Payload = 5 }; + var verCheck = new Message { MessageType = MessageType.VersionCheck, Payload = 6 }; var sessionEnd = new Message { MessageType = MessageType.SessionEnd }; this.mockCommunicationManager.Setup(cm => cm.WaitForServerConnection(It.IsAny())).Returns(true); this.mockCommunicationManager.SetupSequence(cm => cm.ReceiveMessage()).Returns(verCheck).Returns(sessionEnd); diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs index 3b80962f7b..36a9d4aada 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs @@ -99,6 +99,17 @@ public void AbortIfDiscoveryIsinProgressShouldCallDiscoveryManagerAbort() this.discoveryManager.Verify(dm => dm.Abort(), Times.Once); } + [TestMethod] + public void AbortWithEventHandlerIfDiscoveryIsinProgressShouldCallDiscoveryManagerAbortWithEventHandler() + { + // Just to set the IsDiscoveryInProgress flag + this.discoveryRequest.DiscoverAsync(); + var eventsHandler = this.discoveryRequest as ITestDiscoveryEventsHandler2; + + this.discoveryRequest.AbortWithEventHandler(); + this.discoveryManager.Verify(dm => dm.Abort(eventsHandler), Times.Once); + } + [TestMethod] public void AbortIfDiscoveryIsNotInProgressShouldNotCallDiscoveryManagerAbort() { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs index c8d1e1ec8a..014ae33c99 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs @@ -8,6 +8,7 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client.Parallel using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -224,5 +225,42 @@ public void GetDiscoveryDataMetricsShouldNotAddNumberOfAdapterDiscoveredIfMetric Assert.IsFalse(runMetrics.TryGetValue(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringDiscovery, out var value)); } + + [TestMethod] + public void AggregateShouldAggregateMessageSentCorrectly() + { + var aggregator = new ParallelDiscoveryDataAggregator(); + + aggregator.AggregateIsMessageSent(isMessageSent: false); + Assert.IsFalse(aggregator.IsMessageSent, "Aborted must be false"); + + aggregator.AggregateIsMessageSent(isMessageSent: true); + Assert.IsTrue(aggregator.IsMessageSent, "Aborted must be true"); + + aggregator.AggregateIsMessageSent(isMessageSent: false); + Assert.IsTrue(aggregator.IsMessageSent, "Aborted must be true"); + } + + [TestMethod] + public void AggregateShouldAggregateSourcesCorrectly() + { + // Arrange + var aggregator = new ParallelDiscoveryDataAggregator(); + var sources = new List() { "sample.dll" }; + + // Act + aggregator.AggregateTheSourcesWithDiscoveryStatus(sources, DiscoveryStatus.NotDiscovered); + var sourcesWithNotDiscoveredStatus = aggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered); + + // Assert + Assert.AreEqual(1, sourcesWithNotDiscoveredStatus.Count); + + // Act + aggregator.AggregateTheSourcesWithDiscoveryStatus(sources, DiscoveryStatus.FullyDiscovered); + var sourcesWithFullyDiscoveryStatus = aggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered); + + // Assert + Assert.AreEqual(1, sourcesWithFullyDiscoveryStatus.Count); + } } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs index 4ba1536d27..14ae4b6225 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs @@ -148,6 +148,27 @@ public void HandleDiscoveryCompleteShouldCallTestDiscoveryCompleteOnActualHandle this.mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleDiscoveryComplete(It.IsAny(), null), Times.Once); } + [TestMethod] + public void HandleDiscoveryCompleteShouldCallConvertToRawMessageAndSendOnceIfDiscoveryIsComplete() + { + string payload = "DiscoveryComplete"; + int totalTests = 10; + bool aborted = false; + + this.mockParallelProxyDiscoveryManager.Setup(mp => mp.HandlePartialDiscoveryComplete( + this.mockProxyDiscoveryManager.Object, totalTests, null, aborted)).Returns(true); + + this.mockDataSerializer.Setup(mds => mds.SerializeMessage(MessageType.DiscoveryComplete)).Returns(payload); + + // Act + var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(totalTests, aborted, It.IsAny>(), It.IsAny>(), It.IsAny>()); + this.parallelDiscoveryEventsHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); + + // Verify + this.mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleRawMessage(It.IsAny()), Times.Once); + this.mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleDiscoveryComplete(It.IsAny(), null), Times.Once); + } + [TestMethod] public void HandleDiscoveryTestsShouldJustPassOnTheEventToDiscoveryEventsHandler() { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index d9f85b420b..0dec321e79 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -3,11 +3,6 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client { - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; @@ -19,8 +14,11 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Moq; + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; [TestClass] public class ParallelProxyDiscoveryManagerTests @@ -101,12 +99,32 @@ public void DiscoverTestsShouldProcessAllSources() AssertMissingAndDuplicateSources(processedSources); } - /// - /// Create ParallelProxyDiscoveryManager with parallel level 1 and two source, - /// Abort in any source should not stop discovery for other sources. - /// [TestMethod] - public void DiscoveryTestsShouldProcessAllSourcesOnDiscoveryAbortsForAnySource() + public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedWithEventHandler() + { + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(this.mockRequestData.Object, this.proxyManagerFunc, 1, false); + var proxyDiscovermanager = new ProxyDiscoveryManager(this.mockRequestData.Object, new Mock().Object, new Mock().Object); + + parallelDiscoveryManager.Abort(this.mockHandler.Object); + bool isPartialDiscoveryComplete = parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: false); + + Assert.IsTrue(isPartialDiscoveryComplete); + } + + [TestMethod] + public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAborted() + { + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(this.mockRequestData.Object, this.proxyManagerFunc, 1, false); + var proxyDiscovermanager = new ProxyDiscoveryManager(this.mockRequestData.Object, new Mock().Object, new Mock().Object); + + parallelDiscoveryManager.Abort(); + bool isPartialDiscoveryComplete = parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: false); + + Assert.IsTrue(isPartialDiscoveryComplete); + } + + [TestMethod] + public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); @@ -116,18 +134,15 @@ public void DiscoveryTestsShouldProcessAllSourcesOnDiscoveryAbortsForAnySource() Task.Run(() => { parallelDiscoveryManager.DiscoverTests(this.testDiscoveryCriteria, this.mockHandler.Object); + parallelDiscoveryManager.Abort(); }); - Assert.IsTrue(this.discoveryCompleted.Wait(ParallelProxyDiscoveryManagerTests.taskTimeout), "Test discovery not completed."); - Assert.AreEqual(2, processedSources.Count, "All Sources must be processed."); + Assert.IsTrue(this.discoveryCompleted.Wait(taskTimeout), "Test discovery not completed."); + Assert.AreEqual(1, processedSources.Count, "One source should be processed."); } - /// - /// Create ParallelProxyDiscoveryManager with parallel level 1 and two sources, - /// Overall discovery should stop, if aborting was requested - /// [TestMethod] - public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() + public void DiscoveryTestsShouldStopDiscoveryIfAbortionWithEventHandlerWasRequested() { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); @@ -137,7 +152,7 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() Task.Run(() => { parallelDiscoveryManager.DiscoverTests(this.testDiscoveryCriteria, this.mockHandler.Object); - parallelDiscoveryManager.Abort(); + parallelDiscoveryManager.Abort(mockHandler.Object); }); Assert.IsTrue(this.discoveryCompleted.Wait(taskTimeout), "Test discovery not completed."); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs index d7d956b4be..0f3ed56821 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs @@ -82,10 +82,7 @@ public void DiscoverTestsShouldLogIfTheSourceDoesNotExist() var errorMessage = string.Format(CultureInfo.CurrentCulture, "Could not find file {0}.", Path.Combine(Directory.GetCurrentDirectory(), "imaginary.dll")); mockLogger.Verify( l => - l.HandleLogMessage( - Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.TestMessageLevel.Warning, - errorMessage), - Times.Once); + l.HandleLogMessage(TestMessageLevel.Warning,errorMessage),Times.Once); } [TestMethod] @@ -105,10 +102,7 @@ public void DiscoverTestsShouldLogIfTheSourceDoesNotExistIfItHasAPackage() var errorMessage = string.Format(CultureInfo.CurrentCulture, "Could not find file {0}.", Path.Combine(fakeDirectory, "imaginary.exe")); mockLogger.Verify( l => - l.HandleLogMessage( - Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.TestMessageLevel.Warning, - errorMessage), - Times.Once); + l.HandleLogMessage(TestMessageLevel.Warning,errorMessage),Times.Once); } [TestMethod] @@ -124,10 +118,7 @@ public void DiscoverTestsShouldLogIfThereAreNoValidSources() var errorMessage = string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.NoValidSourceFoundForDiscovery, sourcesString); mockLogger.Verify( l => - l.HandleLogMessage( - Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.TestMessageLevel.Warning, - errorMessage), - Times.Once); + l.HandleLogMessage(TestMessageLevel.Warning,errorMessage),Times.Once); } [TestMethod] @@ -151,10 +142,7 @@ public void DiscoverTestsShouldLogIfTheSameSourceIsSpecifiedTwice() var errorMessage = string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DuplicateSource, sources[0]); mockLogger.Verify( l => - l.HandleLogMessage( - Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.TestMessageLevel.Warning, - errorMessage), - Times.Once); + l.HandleLogMessage(TestMessageLevel.Warning,errorMessage),Times.Once); } [TestMethod] @@ -266,6 +254,34 @@ public void DiscoveryInitializeShouldVerifyWarningMessageIfAdapterFailedToLoad() mockLogger.Verify(rd => rd.HandleLogMessage(TestMessageLevel.Warning, "verify that the HandleLogMessage method getting invoked at least once"), Times.Once); } + [TestMethod] + public void DiscoveryTestsShouldSendAbortValuesCorrectlyIfAbortionHappened() + { + // Arrange + var sources = new List { typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location }; + + var criteria = new DiscoveryCriteria(sources, 100, null); + var mockHandler = new Mock(); + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + mockHandler.Setup(ml => ml.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback( + (DiscoveryCompleteEventArgs complete, + IEnumerable tests) => + { + receivedDiscoveryCompleteEventArgs = complete; + }); + + // Act + this.discoveryManager.DiscoverTests(criteria, mockHandler.Object); + this.discoveryManager.Abort(mockHandler.Object); + + // Assert + Assert.AreEqual(true, receivedDiscoveryCompleteEventArgs.IsAborted); + Assert.AreEqual(-1, receivedDiscoveryCompleteEventArgs.TotalCount); + } + #endregion } } diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs index c07b2698b8..e2ea55a0cc 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs @@ -38,7 +38,7 @@ public class VsTestConsoleRequestSenderTests private readonly int WaitTimeout = 2000; - private int protocolVersion = 5; + private int protocolVersion = 6; private IDataSerializer serializer = JsonDataSerializer.Instance; public VsTestConsoleRequestSenderTests() @@ -429,6 +429,92 @@ public async Task DiscoverTestsAsyncShouldCompleteWithSingleTest() mockHandler.Verify(mh => mh.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never, "TestMessage event must not be called"); } + [TestMethod] + public void DiscoverTestsShouldCompleteWithSingleFullyDiscoveredSource() + { + this.InitializeCommunication(); + + var mockHandler = new Mock(); + + List sources = new List() { "1.dll" }; + + var testCase = new TestCase("hello", new Uri("world://how"), source: sources[0]); + var testsFound = new Message() + { + MessageType = MessageType.TestCasesFound, + Payload = JToken.FromObject(new List() { testCase }) + }; + + var payload = new DiscoveryCompletePayload() { TotalTests = 1, LastDiscoveredTests = null, IsAborted = false, FullyDiscoveredSources = sources }; + var discoveryComplete = new Message() + { + MessageType = MessageType.DiscoveryComplete, + Payload = JToken.FromObject(payload) + }; + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(testsFound)); + mockHandler.Setup(mh => mh.HandleDiscoveredTests(It.IsAny>())).Callback( + () => this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(discoveryComplete))); + + mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback((DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable tests) => + { receivedDiscoveryCompleteEventArgs = discoveryCompleteEventArgs;}); + + this.requestSender.DiscoverTests(sources, null, new TestPlatformOptions(), null, mockHandler.Object); + + mockHandler.Verify(mh => mh.HandleDiscoveryComplete(It.IsAny(), null), Times.Once, "Discovery Complete must be called"); + Assert.IsNotNull(receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources); + Assert.AreEqual(1, receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources.Count); + } + + [TestMethod] + public void DiscoverTestsShouldCompleteWithCorrectAbortedValuesIfAbortingWasRequsted() + { + // Arrange + this.InitializeCommunication(); + + var mockHandler = new Mock(); + + List sources = new List() { "1.dll" }; + + var testCase = new TestCase("hello", new Uri("world://how"), source: sources[0]); + var testsFound = new Message() + { + MessageType = MessageType.TestCasesFound, + Payload = JToken.FromObject(new List() { testCase }) + }; + + var payload = new DiscoveryCompletePayload() { TotalTests = -1, LastDiscoveredTests = null, IsAborted = true, FullyDiscoveredSources = sources }; + var discoveryComplete = new Message() + { + MessageType = MessageType.DiscoveryComplete, + Payload = JToken.FromObject(payload) + }; + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(testsFound)); + mockHandler.Setup(mh => mh.HandleDiscoveredTests(It.IsAny>())).Callback( + () => this.mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(discoveryComplete))); + + mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback((DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable tests) => + { receivedDiscoveryCompleteEventArgs = discoveryCompleteEventArgs; }); + + // Act + this.requestSender.DiscoverTests(sources, null, new TestPlatformOptions(), null, mockHandler.Object); + this.requestSender.CancelDiscovery(); + + // Assert + mockHandler.Verify(mh => mh.HandleDiscoveryComplete(It.IsAny(), null), Times.Once, "Discovery Complete must be called"); + Assert.IsNotNull(receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources); + Assert.AreEqual(1, receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources.Count); + Assert.AreEqual(-1, receivedDiscoveryCompleteEventArgs.TotalCount); + Assert.AreEqual(true, receivedDiscoveryCompleteEventArgs.IsAborted); + } + [TestMethod] public void DiscoverTestsShouldReportBackTestsWithTraitsInTestsFoundMessage() { diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index d8440633b5..a3449ae2af 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -236,7 +236,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); @@ -260,7 +260,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() this.testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + Assert.AreEqual(6, actualRequestData.ProtocolConfig.Version); } @@ -286,7 +286,7 @@ public void DiscoverTestsShouldCollectMetrics() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -335,7 +335,7 @@ public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStri " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -378,7 +378,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -421,7 +421,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -464,7 +464,7 @@ public void DiscoverTestsShouldCollectCommands() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -519,7 +519,7 @@ public void DiscoverTestsShouldCollectTestSettings() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -566,7 +566,7 @@ public void DiscoverTestsShouldCollectVsmdiFile() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -613,7 +613,7 @@ public void DiscoverTestsShouldCollectTestRunConfigFile() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -907,7 +907,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() Sources = new List() { "a" }, RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -920,7 +920,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() this.testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + Assert.AreEqual(6, actualRequestData.ProtocolConfig.Version); } [TestMethod] @@ -934,7 +934,7 @@ public void RunTestsShouldCollectCommands() Sources = new List() { "a" }, RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -1001,7 +1001,7 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -1051,7 +1051,7 @@ public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettin " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -1099,7 +1099,7 @@ public void RunTestsShouldCollectMetrics() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6}; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); this.mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback(