Run Microsoft.Testing.Platform test apps under vstest.console and datacollector

eng/expected-dll-frameworks.json

@@ -182,7 +182,7 @@
       "contentFiles/any/net10.0/Microsoft.Extensions.FileSystemGlobbing.dll": "netstandard",
       "contentFiles/any/net10.0/Microsoft.TestPlatform.CommunicationUtilities.dll": "net",
       "contentFiles/any/net10.0/Microsoft.TestPlatform.CoreUtilities.dll": "net",
-      "contentFiles/any/net10.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "netstandard",
+      "contentFiles/any/net10.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "net",
       "contentFiles/any/net10.0/Microsoft.TestPlatform.PlatformAbstractions.dll": "net",
       "contentFiles/any/net10.0/Microsoft.TestPlatform.Utilities.dll": "netstandard",
       "contentFiles/any/net10.0/Microsoft.TestPlatform.VsTestConsole.TranslationLayer.dll": "net",
@@ -379,7 +379,7 @@
       "tools/net8.0/Microsoft.Extensions.FileSystemGlobbing.dll": "netstandard",
       "tools/net8.0/Microsoft.TestPlatform.CommunicationUtilities.dll": "net",
       "tools/net8.0/Microsoft.TestPlatform.CoreUtilities.dll": "net",
-      "tools/net8.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "netstandard",
+      "tools/net8.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "net",
       "tools/net8.0/Microsoft.TestPlatform.PlatformAbstractions.dll": "net",
       "tools/net8.0/Microsoft.TestPlatform.Utilities.dll": "netstandard",
       "tools/net8.0/Microsoft.TestPlatform.VsTestConsole.TranslationLayer.dll": "net",
@@ -391,7 +391,7 @@
       "tools/net8.0/ru/Microsoft.CodeCoverage.IO.dll": "none",
       "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.CommunicationUtilities.dll": "netframework",
       "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.CoreUtilities.dll": "net",
-      "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.CrossPlatEngine.dll": "netstandard",
+      "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.CrossPlatEngine.dll": "net",
       "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.PlatformAbstractions.dll": "net",
       "tools/net8.0/TestHostNetFramework/Microsoft.TestPlatform.Utilities.dll": "netstandard",
       "tools/net8.0/TestHostNetFramework/Microsoft.VisualStudio.TestPlatform.Common.dll": "netstandard",
@@ -408,7 +408,7 @@
       "build/net8.0/x86/testhost.x86.dll": "net",
       "lib/net8.0/Microsoft.TestPlatform.CommunicationUtilities.dll": "net",
       "lib/net8.0/Microsoft.TestPlatform.CoreUtilities.dll": "net",
-      "lib/net8.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "netstandard",
+      "lib/net8.0/Microsoft.TestPlatform.CrossPlatEngine.dll": "net",
       "lib/net8.0/Microsoft.TestPlatform.PlatformAbstractions.dll": "net",
       "lib/net8.0/Microsoft.TestPlatform.Utilities.dll": "netstandard",
       "lib/net8.0/Microsoft.VisualStudio.TestPlatform.Common.dll": "netstandard",

src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System;
 using System.Collections.Generic;
 
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host;
@@ -11,4 +12,12 @@ internal interface ITestRuntimeProviderManager
 {
     ITestRuntimeProvider? GetTestHostManagerByRunConfiguration(string? runConfiguration, List<string> sources);
     ITestRuntimeProvider? GetTestHostManagerByUri(string hostUri);
+
+    /// <summary>
+    /// Runs only the source-aware first-refusal pass for a single source and returns the <see cref="Type"/> of
+    /// the runtime provider that would claim it, without instantiating the provider. Returns <see langword="null"/>
+    /// when no source-aware provider claims the source (i.e. it will be resolved by a generic, source-blind
+    /// provider). This lets callers group sources by which source-aware provider owns them.
+    /// </summary>
+    Type? GetSourceAwareRuntimeProviderType(string? runConfiguration, string source);
 }

src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs

@@ -43,10 +43,58 @@ public static TestRuntimeProviderManager Instance
         return host?.Value;
     }
 
-    public virtual ITestRuntimeProvider? GetTestHostManagerByRunConfiguration(string? runConfiguration, List<string>? _)
+    Type? ITestRuntimeProviderManager.GetSourceAwareRuntimeProviderType(string? runConfiguration, string source)
     {
+        // Consult only the source-aware providers (the first-refusal pass), mirroring the first loop of
+        // GetTestHostManagerByRunConfiguration, but for a single source and without instantiating anything.
+        // Callers use the returned type purely as a grouping discriminator, so a source claimed by a
+        // source-aware provider is scheduled separately from a generic (source-blind) source.
+        var sources = new List<string> { source };
         foreach (var testExtension in _testHostExtensionManager.TestExtensions)
         {
+            if (testExtension.Value is ISourceAwareTestRuntimeProvider sourceAware
+                && sourceAware.CanExecuteCurrentRunConfiguration(runConfiguration, sources))
+            {
+                return testExtension.Value.GetType();
+            }
+        }
+
+        return null;
+    }
+
+    public virtual ITestRuntimeProvider? GetTestHostManagerByRunConfiguration(string? runConfiguration, List<string>? sources)
+    {
+        // First pass: give source-aware providers first refusal. These providers (e.g. the
+        // Microsoft.Testing.Platform provider) can inspect the actual sources to decide whether they own the
+        // run, so they must be consulted before the generic, source-blind providers that match only by target
+        // framework. This gives the more specific provider priority without any global ordering scheme, and
+        // without relying on the generic providers to decline.
+        if (sources is not null && sources.Count > 0)
+        {
+            foreach (var testExtension in _testHostExtensionManager.TestExtensions)
+            {
+                if (testExtension.Value is ISourceAwareTestRuntimeProvider sourceAware
+                    && sourceAware.CanExecuteCurrentRunConfiguration(runConfiguration, sources))
+                {
+                    // We are creating a new instance of ITestRuntimeProvider so that each POM gets its own object of ITestRuntimeProvider.
+                    return (ITestRuntimeProvider?)Activator.CreateInstance(testExtension.Value.GetType());
+                }
+            }
+        }
+
+        // Second pass: the legacy, source-blind resolution based purely on the run configuration.
+        // Source-aware providers already had their (source-based) first refusal above, so exclude them here.
+        // Re-consulting them would be redundant, and — more importantly — a provider that declined by source
+        // must not be re-admitted by matching only the target framework. Enforcing the exclusion in the manager
+        // keeps the "first refusal" contract here, instead of relying on every source-aware provider to remember
+        // to return false when asked the source-blind question.
+        foreach (var testExtension in _testHostExtensionManager.TestExtensions)
+        {
+            if (testExtension.Value is ISourceAwareTestRuntimeProvider)
+            {
+                continue;
+            }
+
             if (testExtension.Value.CanExecuteCurrentRunConfiguration(runConfiguration))
             {
                 // we are creating a new Instance of ITestRuntimeProvider so that each POM gets it's own object of ITestRuntimeProvider

src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt

@@ -262,6 +262,6 @@ static Microsoft.VisualStudio.TestPlatform.Common.Utilities.RunSettingsUtilities
 virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.TestPluginCache.GetFilteredExtensions(System.Collections.Generic.List<string!>! extensions, string! endsWithPattern) -> System.Collections.Generic.IEnumerable<string!>!
 virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.IdentifierData.get -> string?
 virtual Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities.TestPluginInformation.Metadata.get -> System.Collections.Generic.ICollection<object?>!
-virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string? runConfiguration, System.Collections.Generic.List<string!>? _) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider?
+virtual Microsoft.VisualStudio.TestPlatform.Common.Hosting.TestRuntimeProviderManager.GetTestHostManagerByRunConfiguration(string? runConfiguration, System.Collections.Generic.List<string!>? sources) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Host.ITestRuntimeProvider?
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager.InitializeDiscovery(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria! discoveryCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2! eventHandler, bool skipDefaultAdapters) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.InitializeTestRun(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria! testRunCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler! eventHandler) -> void

src/Microsoft.TestPlatform.CoreUtilities/Helpers/MicrosoftTestingPlatformDetector.cs

@@ -0,0 +1,153 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+namespace Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
+
+/// <summary>
+/// Detects whether an assembly is a Microsoft.Testing.Platform application by reading the
+/// <c>[assembly: AssemblyMetadata("Microsoft.Testing.Platform.Application", "true")]</c> attribute
+/// that the Microsoft.Testing.Platform MSBuild targets stamp onto the entry assembly at build time.
+/// </summary>
+/// <remarks>
+/// This lives in CoreUtilities so both the up-front detection in vstest.console and the routing
+/// decision in the CrossPlatEngine TestEngine can share the exact same logic instead of duplicating
+/// the (subtle) custom-attribute blob parsing.
+/// </remarks>
+internal static class MicrosoftTestingPlatformDetector
+{
+    private const string MicrosoftTestingPlatformApplicationMetadataKey = "Microsoft.Testing.Platform.Application";
+
+    // Detection reads the assembly's PE metadata from disk, and a single run can ask about the same source
+    // several times (grouping, provider resolution, and per-source proxy creation). Memoize per source path so
+    // we read each assembly at most once. Concurrent because resolution can happen on parallel proxy threads.
+    private static readonly ConcurrentDictionary<string, bool> Cache = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// Returns <see langword="true"/> if the assembly at <paramref name="filePath"/> is a
+    /// Microsoft.Testing.Platform application. Never throws; returns <see langword="false"/> on any error.
+    /// </summary>
+    public static bool IsMicrosoftTestingPlatformApp(string filePath)
+    {
+        if (filePath is null)
+        {
+            return false;
+        }
+
+        return Cache.GetOrAdd(filePath, static path =>
+        {
+            try
+            {
+                using var assemblyStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+                var result = IsMicrosoftTestingPlatformApp(assemblyStream);
+                EqtTrace.Info("MicrosoftTestingPlatformDetector.IsMicrosoftTestingPlatformApp: '{0}' for source: '{1}'", result, path);
+                return result;
+            }
+            catch (Exception ex)
+            {
+                EqtTrace.Warning("MicrosoftTestingPlatformDetector.IsMicrosoftTestingPlatformApp: failed to read assembly metadata, exception: {0} for assembly: {1}", ex, path);
+                return false;
+            }
+        });
+    }
+
+    /// <summary>
+    /// Returns <see langword="true"/> if <paramref name="assemblyStream"/> is a Microsoft.Testing.Platform
+    /// application. The caller owns the stream lifetime.
+    /// </summary>
+    public static bool IsMicrosoftTestingPlatformApp(Stream assemblyStream)
+    {
+        using var peReader = new PEReader(assemblyStream);
+        if (!peReader.HasMetadata)
+        {
+            return false;
+        }
+
+        var metadataReader = peReader.GetMetadataReader();
+
+        // Microsoft.Testing.Platform applications are marked at build time with
+        // [assembly: AssemblyMetadata("Microsoft.Testing.Platform.Application", "true")] by the
+        // Microsoft.Testing.Platform MSBuild targets. We only look at assembly-level attributes.
+        foreach (var handle in metadataReader.GetAssemblyDefinition().GetCustomAttributes())
+        {
+            var attribute = metadataReader.GetCustomAttribute(handle);
+            if (!IsAssemblyMetadataAttribute(metadataReader, attribute))
+            {
+                continue;
+            }
+
+            try
+            {
+                // AssemblyMetadataAttribute has a (string key, string value) constructor. The custom attribute
+                // blob is: 2-byte prolog (0x0001), then the two serialized strings, then the named-argument count.
+                var blob = metadataReader.GetBlobReader(attribute.Value);
+                if (blob.ReadUInt16() != 1)
+                {
+                    continue;
+                }
+
+                var key = blob.ReadSerializedString();
+                var value = blob.ReadSerializedString();
+                if (string.Equals(key, MicrosoftTestingPlatformApplicationMetadataKey, StringComparison.Ordinal)
+                    && string.Equals(value, "true", StringComparison.OrdinalIgnoreCase))
+                {
+                    return true;
+                }
+            }
+            catch (Exception ex)
+            {
+                EqtTrace.Verbose("MicrosoftTestingPlatformDetector.IsMicrosoftTestingPlatformApp: could not decode AssemblyMetadata attribute: {0}", ex);
+            }
+        }
+
+        return false;
+    }
+
+    private static bool IsAssemblyMetadataAttribute(MetadataReader metadataReader, CustomAttribute attribute)
+    {
+        StringHandle typeNameHandle;
+        StringHandle typeNamespaceHandle;
+        switch (attribute.Constructor.Kind)
+        {
+            case HandleKind.MemberReference:
+                var memberReference = metadataReader.GetMemberReference((MemberReferenceHandle)attribute.Constructor);
+                switch (memberReference.Parent.Kind)
+                {
+                    case HandleKind.TypeReference:
+                        var typeReference = metadataReader.GetTypeReference((TypeReferenceHandle)memberReference.Parent);
+                        typeNameHandle = typeReference.Name;
+                        typeNamespaceHandle = typeReference.Namespace;
+                        break;
+                    case HandleKind.TypeDefinition:
+                        var typeDefinition = metadataReader.GetTypeDefinition((TypeDefinitionHandle)memberReference.Parent);
+                        typeNameHandle = typeDefinition.Name;
+                        typeNamespaceHandle = typeDefinition.Namespace;
+                        break;
+                    default:
+                        return false;
+                }
+
+                break;
+
+            case HandleKind.MethodDefinition:
+                var methodDefinition = metadataReader.GetMethodDefinition((MethodDefinitionHandle)attribute.Constructor);
+                var declaringType = metadataReader.GetTypeDefinition(methodDefinition.GetDeclaringType());
+                typeNameHandle = declaringType.Name;
+                typeNamespaceHandle = declaringType.Namespace;
+                break;
+
+            default:
+                return false;
+        }
+
+        return string.Equals(metadataReader.GetString(typeNameHandle), "AssemblyMetadataAttribute", StringComparison.Ordinal)
+            && string.Equals(metadataReader.GetString(typeNamespaceHandle), "System.Reflection", StringComparison.Ordinal);
+    }
+}

src/Microsoft.TestPlatform.CrossPlatEngine/Client/IProxyManagerFactory.cs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
+
+namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client;
+
+/// <summary>
+/// Implemented by a runtime provider (an <see cref="ObjectModel.Host.ITestRuntimeProvider"/>) that hosts a
+/// run over its own protocol instead of the vstest testhost protocol, and therefore supplies its own
+/// discovery/execution proxy managers.
+/// </summary>
+/// <remarks>
+/// The standard flow creates a <c>ProxyDiscoveryManager</c>/<c>ProxyExecutionManager</c> that wrap an
+/// <see cref="ObjectModel.Host.ITestRuntimeProvider"/> launching a vstest testhost. Some providers — such as
+/// the Microsoft.Testing.Platform provider — are fundamentally a different shape: the test application is its
+/// own host and speaks its own JSON-RPC protocol, so it needs a different proxy manager entirely. Rather than
+/// teaching <see cref="TestEngine"/> about each such protocol with inline branches, the resolved provider that
+/// implements this interface is asked to produce its own proxy managers. This keeps protocol-specific wiring
+/// in the provider and out of the engine.
+/// <para>
+/// This interface is public because the runtime providers that implement it live in a separate assembly
+/// (<c>Microsoft.TestPlatform.TestHostRuntimeProvider</c>). The concrete proxy managers stay internal to this
+/// assembly; providers create them through the public <see cref="MTP.MtpProxyManagerFactory"/> helper.
+/// </para>
+/// </remarks>
+public interface IProxyManagerFactory
+{
+    /// <summary>
+    /// Creates the discovery manager used to drive discovery for this provider's sources.
+    /// </summary>
+    IProxyDiscoveryManager CreateDiscoveryManager();
+
+    /// <summary>
+    /// Creates the execution manager used to drive execution for this provider's sources.
+    /// </summary>
+    /// <param name="dataCollectionManager">
+    /// The data collection manager to wire in when data collectors are enabled, or <see langword="null"/>
+    /// when data collection is off.
+    /// </param>
+    IProxyExecutionManager CreateExecutionManager(IProxyDataCollectionManager? dataCollectionManager);
+}