Test adapter loading strategy

TestPlatform.sln

@@ -170,7 +170,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachVS", "src\AttachVS\At
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "playground", "playground", "{6CE2F530-582B-4695-A209-41065E103426}"
 	ProjectSection(SolutionItems) = preProject
-		README.md = README.md
+		playground\README.md = playground\README.md
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestPlatform.Playground", "playground\TestPlatform.Playground\TestPlatform.Playground.csproj", "{545A88D3-1AE2-4D39-9B7C-C691768AD17F}"

src/Microsoft.TestPlatform.Client/TestPlatform.cs

@@ -12,30 +12,31 @@ namespace Microsoft.VisualStudio.TestPlatform.Client;
 using System.Linq;
 using System.Reflection;
 
-using Discovery;
-using Execution;
-using Common;
-using Common.ExtensionFramework;
-using Common.Hosting;
-using Common.Logging;
+using Microsoft.VisualStudio.TestPlatform.Client.Discovery;
+using Microsoft.VisualStudio.TestPlatform.Client.Execution;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;
+using Microsoft.VisualStudio.TestPlatform.Common.Hosting;
+using Microsoft.VisualStudio.TestPlatform.Common.Logging;
 using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
-using CrossPlatEngine;
-using ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
-using ObjectModel.Engine;
-using ObjectModel.Host;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
-using PlatformAbstractions;
-using Utilities.Helpers;
-using Utilities.Helpers.Interfaces;
+using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
+using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
+using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
 
-using ClientResources = Resources.Resources;
+using ClientResources = Microsoft.VisualStudio.TestPlatform.Client.Resources.Resources;
 
 /// <summary>
 /// Implementation for TestPlatform.
 /// </summary>
 internal class TestPlatform : ITestPlatform
 {
+
     private readonly TestRuntimeProviderManager _testHostProviderManager;
 
     private readonly IFileHelper _fileHelper;
@@ -87,16 +88,7 @@ public IDiscoveryRequest CreateDiscoveryRequest(
         DiscoveryCriteria discoveryCriteria!!,
         TestPlatformOptions options)
     {
-
-        // Update cache with Extension folder's files.
-        AddExtensionAssemblies(discoveryCriteria.RunSettings);
-
-        // Update extension assemblies from source when design mode is false.
-        var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(discoveryCriteria.RunSettings);
-        if (!runConfiguration.DesignMode)
-        {
-            AddExtensionAssembliesFromSource(discoveryCriteria.Sources);
-        }
+        PopulateExtensions(discoveryCriteria.RunSettings, discoveryCriteria.Sources);
 
         // Initialize loggers.
         var loggerManager = TestEngine.GetLoggerManager(requestData);
@@ -119,15 +111,8 @@ public ITestRunRequest CreateTestRunRequest(
         TestRunCriteria testRunCriteria!!,
         TestPlatformOptions options)
     {
-        AddExtensionAssemblies(testRunCriteria.TestRunSettings);
-
-        var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);
-
-        // Update extension assemblies from source when design mode is false.
-        if (!runConfiguration.DesignMode)
-        {
-            AddExtensionAssembliesFromSource(testRunCriteria);
-        }
+        var sources = GetSources(testRunCriteria);
+        PopulateExtensions(testRunCriteria.TestRunSettings, sources);
 
         // Initialize loggers.
         var loggerManager = TestEngine.GetLoggerManager(requestData);
@@ -156,9 +141,11 @@ public bool StartTestSession(
         StartTestSessionCriteria testSessionCriteria!!,
         ITestSessionEventsHandler eventsHandler)
     {
-        AddExtensionAssemblies(testSessionCriteria.RunSettings);
-
         var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testSessionCriteria.RunSettings);
+        var strategy = runConfiguration.TestAdapterLoadingStrategy;
+
+        AddExtensionAssemblies(testSessionCriteria.RunSettings, strategy);
+
         if (!runConfiguration.DesignMode)
         {
             return false;
@@ -177,6 +164,20 @@ public bool StartTestSession(
 
         return testSessionManager.StartSession(eventsHandler, requestData);
     }
+    private void PopulateExtensions(string runSettings, IEnumerable<string> sources)
+    {
+        var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
+        var strategy = runConfiguration.TestAdapterLoadingStrategy;
+
+        // Update cache with Extension folder's files.
+        AddExtensionAssemblies(runSettings, strategy);
+
+        // Update extension assemblies from source when design mode is false.
+        if (!runConfiguration.DesignMode)
+        {
+            AddExtensionAssembliesFromSource(sources, strategy);
+        }
+    }
 
     /// <summary>
     /// The dispose.
@@ -211,81 +212,54 @@ private void ThrowExceptionIfTestHostManagerIsNull(
         }
     }
 
-    /// <summary>
-    /// Updates the test adapter paths provided through run settings to be used by the test
-    /// service.
-    /// </summary>
-    ///
-    /// <param name="runSettings">The run settings.</param>
-    private void AddExtensionAssemblies(string runSettings)
+
+    private void AddExtensionAssemblies(string runSettings, TestAdapterLoadingStrategy adapterLoadingStrategy)
     {
         IEnumerable<string> customTestAdaptersPaths = RunSettingsUtilities.GetTestAdaptersPaths(runSettings);
 
         if (customTestAdaptersPaths != null)
         {
             foreach (string customTestAdaptersPath in customTestAdaptersPaths)
             {
-                var adapterPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(customTestAdaptersPath));
-                if (!Directory.Exists(adapterPath))
-                {
-                    EqtTrace.Warning($"AdapterPath Not Found: {adapterPath}");
-                    continue;
-                }
+                var extensionAssemblies = ExpandTestAdapterPaths(customTestAdaptersPath, _fileHelper, adapterLoadingStrategy);
 
-                var extensionAssemblies = new List<string>(
-                    _fileHelper.EnumerateFiles(
-                        adapterPath,
-                        SearchOption.AllDirectories,
-                        TestPlatformConstants.TestAdapterEndsWithPattern,
-                        TestPlatformConstants.TestLoggerEndsWithPattern,
-                        TestPlatformConstants.DataCollectorEndsWithPattern,
-                        TestPlatformConstants.RunTimeEndsWithPattern));
-
-                if (extensionAssemblies.Count > 0)
+                if (extensionAssemblies.Any())
                 {
                     UpdateExtensions(extensionAssemblies, skipExtensionFilters: false);
                 }
+
             }
         }
     }
 
     /// <summary>
-    /// Updates the extension assemblies from source directory.
+    /// Updates the test logger paths from source directory.
     /// </summary>
     ///
-    /// <param name="testRunCriteria">The test run criteria.</param>
-    private void AddExtensionAssembliesFromSource(TestRunCriteria testRunCriteria)
+    /// <param name="sources">The list of sources.</param>
+    private void AddExtensionAssembliesFromSource(IEnumerable<string> sources, TestAdapterLoadingStrategy strategy)
     {
-        IEnumerable<string> sources = testRunCriteria.Sources;
-        if (testRunCriteria.HasSpecificTests)
+        // Skip discovery unless we're using the default behavior, or NextToSource is specified.
+        if (strategy != TestAdapterLoadingStrategy.Default 
+            && (strategy & TestAdapterLoadingStrategy.NextToSource) != TestAdapterLoadingStrategy.NextToSource)
         {
-            // If the test execution is with a test filter, group them by sources.
-            sources = testRunCriteria.Tests.Select(tc => tc.Source).Distinct();
+            return;
         }
 
-        AddExtensionAssembliesFromSource(sources);
-    }
-
-    /// <summary>
-    /// Updates the test logger paths from source directory.
-    /// </summary>
-    ///
-    /// <param name="sources">The list of sources.</param>
-    private void AddExtensionAssembliesFromSource(IEnumerable<string> sources)
-    {
         // Currently we support discovering loggers only from Source directory.
         var loggersToUpdate = new List<string>();
 
         foreach (var source in sources)
         {
             var sourceDirectory = Path.GetDirectoryName(source);
-            if (!string.IsNullOrEmpty(sourceDirectory)
-                && _fileHelper.DirectoryExists(sourceDirectory))
+            if (!string.IsNullOrEmpty(sourceDirectory) && _fileHelper.DirectoryExists(sourceDirectory))
             {
+                var searchOption = GetSearchOption(strategy, SearchOption.TopDirectoryOnly);
+
                 loggersToUpdate.AddRange(
                     _fileHelper.EnumerateFiles(
                         sourceDirectory,
-                        SearchOption.TopDirectoryOnly,
+                        searchOption,
                         TestPlatformConstants.TestLoggerEndsWithPattern));
             }
         }
@@ -303,21 +277,133 @@ private void AddExtensionAssembliesFromSource(IEnumerable<string> sources)
     /// </summary>
     private static void AddExtensionAssembliesFromExtensionDirectory()
     {
+        // This method runs before adapter initialization path, ideally we should replace this mechanism
+        // this is currently required because we need TestHostProvider to be able to resolve.
+        var runSettings = RunSettingsManager.Instance.ActiveRunSettings.SettingsXml;
+        var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
+        var strategy = runConfiguration.TestAdapterLoadingStrategy;
+
         var fileHelper = new FileHelper();
-        var extensionsFolder = Path.Combine(
-            Path.GetDirectoryName(
-                typeof(TestPlatform).GetTypeInfo().Assembly.GetAssemblyLocation()),
-            "Extensions");
+        var defaultExtensionPaths = Enumerable.Empty<string>();
 
+        // Explicit adapter loading
+        if ((strategy & TestAdapterLoadingStrategy.Explicit) == TestAdapterLoadingStrategy.Explicit)
+        {
+            defaultExtensionPaths = RunSettingsUtilities.GetTestAdaptersPaths(runSettings)
+                .SelectMany(path => ExpandTestAdapterPaths(path, fileHelper, strategy))
+                .Union(defaultExtensionPaths);
+        }
+
+        var extensionsFolder = Path.Combine(Path.GetDirectoryName(typeof(TestPlatform).GetTypeInfo().Assembly.GetAssemblyLocation()), "Extensions");
         if (fileHelper.DirectoryExists(extensionsFolder))
         {
-            var defaultExtensionPaths = fileHelper.EnumerateFiles(
-                extensionsFolder,
-                SearchOption.TopDirectoryOnly,
-                ".dll",
-                ".exe");
+            // Load default runtime providers
+            if ((strategy & TestAdapterLoadingStrategy.DefaultRuntimeProviders) == TestAdapterLoadingStrategy.DefaultRuntimeProviders)
+            {
+                defaultExtensionPaths = fileHelper
+                    .EnumerateFiles(extensionsFolder, SearchOption.TopDirectoryOnly, TestPlatformConstants.RunTimeEndsWithPattern)
+                    .Union(defaultExtensionPaths);
+            }
+
+            // Default extension loader
+            if (strategy == TestAdapterLoadingStrategy.Default
+                || (strategy & TestAdapterLoadingStrategy.ExtensionsDirectory) == TestAdapterLoadingStrategy.ExtensionsDirectory)
+            {
+                defaultExtensionPaths = fileHelper
+                    .EnumerateFiles(extensionsFolder, SearchOption.TopDirectoryOnly, ".dll", ".exe")
+                    .Union(defaultExtensionPaths);
+            }
+        }
+
+        TestPluginCache.Instance.DefaultExtensionPaths = defaultExtensionPaths.Distinct();
+    }
+
+    private static SearchOption GetSearchOption(TestAdapterLoadingStrategy strategy, SearchOption defaultStrategyOption) {
+        if (strategy == TestAdapterLoadingStrategy.Default) {
+            return defaultStrategyOption;
+        }
+
+        var searchOption = SearchOption.TopDirectoryOnly;
+        if ((strategy & TestAdapterLoadingStrategy.Recursive) == TestAdapterLoadingStrategy.Recursive)
+        {
+            searchOption = SearchOption.AllDirectories;
+        }
+
+        return searchOption;     
+    }
 
-            TestPluginCache.Instance.DefaultExtensionPaths = defaultExtensionPaths;
+    private static IEnumerable<string> ExpandTestAdapterPaths(string path, IFileHelper fileHelper, TestAdapterLoadingStrategy strategy)
+    {
+        var adapterPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));
+
+        // Default behavior is to only accept directories!
+        if (strategy == TestAdapterLoadingStrategy.Default)
+        {
+            return ExpandAdaptersWithDefaultStrategy(adapterPath, fileHelper);
+        }
+
+        var adapters = ExpandAdaptersWithExplicitStrategy(adapterPath, fileHelper, strategy);
+
+        return adapters.Distinct();
+    }
+
+    private static IEnumerable<string> ExpandAdaptersWithExplicitStrategy(string path, IFileHelper fileHelper, TestAdapterLoadingStrategy strategy)
+    {
+        if ((strategy & TestAdapterLoadingStrategy.Explicit) != TestAdapterLoadingStrategy.Explicit)
+        {
+            return Enumerable.Empty<string>();
         }
+
+        if (fileHelper.Exists(path))
+        {
+            return new[] { path };
+        }
+        else if (fileHelper.DirectoryExists(path))
+        {
+            var searchOption = GetSearchOption(strategy, SearchOption.TopDirectoryOnly);
+
+            var adapterPaths = fileHelper.EnumerateFiles(
+                path,
+                searchOption,
+                TestPlatformConstants.TestAdapterEndsWithPattern,
+                TestPlatformConstants.TestLoggerEndsWithPattern,
+                TestPlatformConstants.DataCollectorEndsWithPattern,
+                TestPlatformConstants.RunTimeEndsWithPattern);
+
+            return adapterPaths;
+        }
+
+        EqtTrace.Warning(string.Format(CultureInfo.InvariantCulture, "AdapterPath Not Found: {0}", path));
+        return Enumerable.Empty<string>();
+    }
+
+    private static IEnumerable<string> ExpandAdaptersWithDefaultStrategy(string path, IFileHelper fileHelper)
+    {
+        if (!fileHelper.DirectoryExists(path))
+        {
+            EqtTrace.Warning(string.Format("AdapterPath Not Found: {0}", path));
+
+            return Enumerable.Empty<string>();
+        }
+
+        return fileHelper.EnumerateFiles(
+                path,
+                SearchOption.AllDirectories,
+                TestPlatformConstants.TestAdapterEndsWithPattern,
+                TestPlatformConstants.TestLoggerEndsWithPattern,
+                TestPlatformConstants.DataCollectorEndsWithPattern,
+                TestPlatformConstants.RunTimeEndsWithPattern);
+    }
+
+    private static IEnumerable<string> GetSources(TestRunCriteria testRunCriteria)
+    {
+        IEnumerable<string> sources = testRunCriteria.Sources;
+        if (testRunCriteria.HasSpecificTests)
+        {
+            // If the test execution is with a test filter, group them by sources.
+            sources = testRunCriteria.Tests.Select(tc => tc.Source).Distinct();
+        }
+
+        return sources;
     }
 }

src/Microsoft.TestPlatform.Common/ExtensionFramework/TestExtensionManager.cs

@@ -128,6 +128,7 @@ public LazyExtension<TExtension, TMetadata> TryGetTestExtension(string extension
                 break;
             }
         }
+
         return testExtension;
     }
 

src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs

@@ -190,4 +190,14 @@ public static IEnumerable<string> GetTestAdaptersPaths(string runSettings)
         return testAdaptersPaths;
     }
 
+    /// <summary>
+    /// Gets the test adapter loading strategy
+    /// </summary>
+    /// <param name="runSettings">Test run settings</param>
+    /// <returns>Test adapter loading strategy</returns>
+    internal static TestAdapterLoadingStrategy GetLoadingStrategy(string runSettings) {
+        var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
+
+        return runConfiguration?.TestAdapterLoadingStrategy ?? TestAdapterLoadingStrategy.Default;
+    }
 }

src/Microsoft.TestPlatform.ObjectModel/Friends.cs

@@ -8,6 +8,8 @@
 [assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Extensions.MSPhoneAdapter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("datacollector, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Common, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
+[assembly: InternalsVisibleTo("vstest.console, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
+[assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Client, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 
 [assembly: InternalsVisibleTo("Microsoft.TestPlatform.ObjectModel.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
 [assembly: InternalsVisibleTo("datacollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]