Skip sources when runtime provider is not found

playground/TestPlatform.Playground/Program.cs

@@ -143,6 +143,8 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete
             Console.WriteLine(WriteSources(discoveryCompleteEventArgs.FullyDiscoveredSources));
             Console.WriteLine("Partially discovered:");
             Console.WriteLine(WriteSources(discoveryCompleteEventArgs.PartiallyDiscoveredSources));
+            Console.WriteLine("Skipped discovery:");
+            Console.WriteLine(WriteSources(discoveryCompleteEventArgs.SkippedDiscoveredSources));
             Console.WriteLine("Not discovered:");
             Console.WriteLine(WriteSources(discoveryCompleteEventArgs.NotDiscoveredSources));
             if (lastChunk != null) { TestCases.AddRange(lastChunk); }

src/Microsoft.TestPlatform.Client/DesignMode/DesignModeTestEventsRegistrar.cs

@@ -63,3 +63,4 @@ public void LogWarning(string? message)
         _designModeClient.SendTestMessage(TestMessageLevel.Warning, message);
     }
 }
+

src/Microsoft.TestPlatform.Client/TestPlatform.cs

@@ -80,7 +80,8 @@ public IDiscoveryRequest CreateDiscoveryRequest(
         IRequestData requestData,
         DiscoveryCriteria discoveryCriteria,
         TestPlatformOptions? options,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(discoveryCriteria, nameof(discoveryCriteria));
 
@@ -90,7 +91,7 @@ public IDiscoveryRequest CreateDiscoveryRequest(
         ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(discoveryCriteria.RunSettings);
 
-        IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap);
+        IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap, warningLogger);
         discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager);
@@ -101,7 +102,8 @@ public ITestRunRequest CreateTestRunRequest(
         IRequestData requestData,
         TestRunCriteria testRunCriteria,
         TestPlatformOptions? options,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(testRunCriteria, nameof(testRunCriteria));
 
@@ -112,7 +114,7 @@ public ITestRunRequest CreateTestRunRequest(
         ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(testRunCriteria.TestRunSettings);
 
-        IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap);
+        IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap, warningLogger);
         executionManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager);
@@ -123,7 +125,8 @@ public bool StartTestSession(
         IRequestData requestData,
         StartTestSessionCriteria testSessionCriteria,
         ITestSessionEventsHandler eventsHandler,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(testSessionCriteria, nameof(testSessionCriteria));
 
@@ -137,7 +140,7 @@ public bool StartTestSession(
             return false;
         }
 
-        IProxyTestSessionManager? testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap);
+        IProxyTestSessionManager? testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap, warningLogger);
         if (testSessionManager == null)
         {
             // The test session manager is null because the combination of runsettings and

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs

@@ -22,4 +22,9 @@ public enum DiscoveryStatus
     /// Indicates that source was fully discovered.
     /// </summary>
     FullyDiscovered,
+
+    /// <summary>
+    /// Indicates that source was skipped in discovery.
+    /// </summary>
+    SkippedDiscovery
 }

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs

@@ -29,7 +29,8 @@ public interface ITestEngine
     IProxyDiscoveryManager GetDiscoveryManager(
         IRequestData requestData,
         DiscoveryCriteria discoveryCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the ExecutionManager for this engine. This manager would provide all
@@ -46,7 +47,8 @@ IProxyDiscoveryManager GetDiscoveryManager(
     IProxyExecutionManager GetExecutionManager(
         IRequestData requestData,
         TestRunCriteria testRunCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the TestSessionManager for this engine. This manager would provide all
@@ -64,7 +66,8 @@ IProxyExecutionManager GetExecutionManager(
     IProxyTestSessionManager GetTestSessionManager(
         IRequestData requestData,
         StartTestSessionCriteria testSessionCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the extension manager for this engine. This manager would provide extensibility

src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt

@@ -251,13 +251,14 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.Partially
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager.UpdateParallelLevel(int parallelLevel) -> void
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.Cancel(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.StartTestRun(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> int
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler testRunEventsHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.Cancel(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler testRunEventsHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.StartTestRun(System.Collections.Generic.Dictionary<string, System.Collections.Generic.IEnumerable<string>> adapterSourceMap, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEvents, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.StartTestRun(System.Collections.Generic.IEnumerable<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEvents, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.SkippedDiscovery = 3 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager

src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs

@@ -49,6 +49,11 @@ public class DiscoveryCompletePayload
     /// </summary>
     public IList<string> NotDiscoveredSources { get; set; } = new List<string>();
 
+    /// <summary>
+    /// Gets or sets list of sources which skipped in discovery on purpose, e.g. because they are known dlls that have no tests, or there is no runtime provider to run them.
+    /// </summary>
+    public IList<string> SkippedDiscoverySources { get; set; } = new List<string>();
+
     /// <summary>
     /// Gets or sets the collection of discovered extensions.
     /// </summary>

src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Shipped.txt

@@ -372,11 +372,8 @@ static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Reso
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources.VersionCheckFailed.get -> string
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources.VersionCheckTimedout.get -> string
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.SendDiscoveryAbort() -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.set -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.set -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.set -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.SendDiscoveryAbort() -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.DiscoveredExtensions.get -> System.Collections.Generic.Dictionary<string, System.Collections.Generic.HashSet<string>>
@@ -389,3 +386,8 @@ Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestReque
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithTests runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithSources runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithTests runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IList<string>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IList<string>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList<string>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.SkippedDiscoverySources.get -> System.Collections.Generic.IList<string>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.SkippedDiscoverySources.set -> void

src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs

@@ -586,6 +586,7 @@ private void OnDiscoveryMessageReceived(ITestDiscoveryEventsHandler2 discoveryEv
                         PartiallyDiscoveredSources = payload.PartiallyDiscoveredSources,
                         NotDiscoveredSources = payload.NotDiscoveredSources,
                         DiscoveredExtensions = payload.DiscoveredExtensions,
+                        SkippedDiscoveredSources = payload.SkippedDiscoverySources,
                     };
 
                     discoveryCompleteEventArgs.Metrics = payload.Metrics;

src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/DiscoveryDataAggregator.cs

@@ -212,7 +212,7 @@ public void MarkSourcesWithStatus(IEnumerable<string?>? sources, DiscoveryStatus
                 (_, previousStatus) =>
                 {
                     if (previousStatus == DiscoveryStatus.FullyDiscovered && status != DiscoveryStatus.FullyDiscovered
-                        || previousStatus == DiscoveryStatus.PartiallyDiscovered && status == DiscoveryStatus.NotDiscovered)
+                        || previousStatus == DiscoveryStatus.PartiallyDiscovered && (status == DiscoveryStatus.NotDiscovered || status == DiscoveryStatus.SkippedDiscovery))
                     {
                         EqtTrace.Warning($"DiscoveryDataAggregator.MarkSourcesWithStatus: Downgrading source {source} status from '{previousStatus}' to '{status}'.");
                     }

src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs

@@ -7,6 +7,7 @@
 using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
 using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces;
 using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Utilities;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
@@ -89,6 +90,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete
         var fullyDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered);
         var partiallyDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered);
         var notDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered);
+        var skippedDiscovery = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.SkippedDiscovery);
 
         // As we immediately return results to IDE in case of aborting we need to set
         // isAborted = true and totalTests = -1
@@ -131,6 +133,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete
             FullyDiscoveredSources = fullyDiscovered,
             PartiallyDiscoveredSources = partiallyDiscovered,
             NotDiscoveredSources = notDiscovered,
+            SkippedDiscoverySources = skippedDiscovery,
             DiscoveredExtensions = _discoveryDataAggregator.DiscoveredExtensions,
             Metrics = aggregatedDiscoveryDataMetrics,
         };
@@ -146,6 +149,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete
             FullyDiscoveredSources = fullyDiscovered,
             PartiallyDiscoveredSources = partiallyDiscovered,
             NotDiscoveredSources = notDiscovered,
+            SkippedDiscoveredSources = skippedDiscovery,
             DiscoveredExtensions = _discoveryDataAggregator.DiscoveredExtensions,
             Metrics = aggregatedDiscoveryDataMetrics,
         };