Fixed dotnet resolution for in-process vstest.console scenarios

src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetHostHelper.cs

@@ -118,8 +118,14 @@ private bool TryGetExecutablePath(string executableBaseName, out string executab
         return false;
     }
 
-    public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitecture, [NotNullWhen(true)] out string? muxerPath)
+    public bool TryGetDotnetPathByArchitecture(
+        PlatformArchitecture targetArchitecture,
+        DotnetMuxerResolution dotnetMuxerResolution,
+        [NotNullWhen(true)] out string? muxerPath)
     {
+        muxerPath = null;
+        EqtTrace.Verbose($"Using dotnet muxer resolution: {dotnetMuxerResolution}");
+
         // If current process is the same as the target architecture we return the current process filename.
         if (_processHelper.GetCurrentProcessArchitecture() == targetArchitecture)
         {
@@ -140,29 +146,37 @@ public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitectu
         bool isWinOs = _environment.OperatingSystem == PlatformOperatingSystem.Windows;
         EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Searching for muxer named '{_muxerName}'");
 
-        // Try to search using env vars in the order
-        // DOTNET_ROOT_{arch}
-        // DOTNET_ROOT(x86) if X86 on Win (here we cannot check if current process is WOW64 because this is SDK process arch and not real host arch so it's irrelevant)
-        //                  "DOTNET_ROOT(x86) is used instead when running a 32-bit executable on a 64-bit OS."
-        // DOTNET_ROOT
-        string envKey = $"DOTNET_ROOT_{targetArchitecture.ToString().ToUpperInvariant()}";
-
-        // Try on arch specific env var
-        string? envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
-
-        // Try on non virtualized x86 var(should happen only on non-x86 architecture)
-        if ((envVar == null || !_fileHelper.DirectoryExists(envVar)) &&
-            targetArchitecture == PlatformArchitecture.X86 && _environment.OperatingSystem == PlatformOperatingSystem.Windows)
+        string? envKey = null;
+        string? envVar = null;
+        if (dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.DotnetRootArchitecture))
         {
-            envKey = $"DOTNET_ROOT(x86)";
+            // Try to search using env vars in the order
+            // DOTNET_ROOT_{arch}
+            // DOTNET_ROOT(x86) if X86 on Win (here we cannot check if current process is WOW64 because this is SDK process arch and not real host arch so it's irrelevant)
+            //                  "DOTNET_ROOT(x86) is used instead when running a 32-bit executable on a 64-bit OS."
+            // DOTNET_ROOT
+            envKey = $"DOTNET_ROOT_{targetArchitecture.ToString().ToUpperInvariant()}";
+
+            // Try on arch specific env var
             envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
         }
 
-        // Try on default DOTNET_ROOT
-        if (envVar == null || !_fileHelper.DirectoryExists(envVar))
+        if (dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.DotnetRootArchitectureLess))
         {
-            envKey = "DOTNET_ROOT";
-            envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            // Try on non virtualized x86 var(should happen only on non-x86 architecture)
+            if ((envVar == null || !_fileHelper.DirectoryExists(envVar)) &&
+                targetArchitecture == PlatformArchitecture.X86 && _environment.OperatingSystem == PlatformOperatingSystem.Windows)
+            {
+                envKey = $"DOTNET_ROOT(x86)";
+                envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            }
+
+            // Try on default DOTNET_ROOT
+            if (envVar == null || !_fileHelper.DirectoryExists(envVar))
+            {
+                envKey = "DOTNET_ROOT";
+                envVar = _environmentVariableHelper.GetEnvironmentVariable(envKey);
+            }
         }
 
         if (envVar != null)
@@ -196,64 +210,74 @@ public bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitectu
             }
         }
 
-        EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer was not found using DOTNET_ROOT* env variables.");
-
-        // Try to search for global registration
-        muxerPath = isWinOs ? GetMuxerFromGlobalRegistrationWin(targetArchitecture) : GetMuxerFromGlobalRegistrationOnUnix(targetArchitecture);
+        if (dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.DotnetRootArchitecture)
+            || dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.DotnetRootArchitectureLess))
+        {
+            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer was not found using DOTNET_ROOT* env variables.");
+        }
 
-        if (muxerPath != null)
+        if (dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.GlobalInstallation))
         {
-            if (!_fileHelper.Exists(muxerPath))
-            {
-                // If muxer doesn't exists or it's wrong we stop the search
-                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer file not found for global registration '{muxerPath}'");
-                muxerPath = null;
-                return false;
-            }
+            // Try to search for global registration
+            muxerPath = isWinOs ? GetMuxerFromGlobalRegistrationWin(targetArchitecture) : GetMuxerFromGlobalRegistrationOnUnix(targetArchitecture);
 
-            if (!IsValidArchitectureMuxer(targetArchitecture, muxerPath))
+            if (muxerPath != null)
             {
-                // If muxer is wrong we stop the search
-                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer resolved using global registration is not compatible with the target architecture: '{muxerPath}'");
-                muxerPath = null;
-                return false;
-            }
-
-            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer compatible with '{targetArchitecture}' resolved from global registration: '{muxerPath}'");
-            return true;
-        }
+                if (!_fileHelper.Exists(muxerPath))
+                {
+                    // If muxer doesn't exists or it's wrong we stop the search
+                    EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer file not found for global registration '{muxerPath}'");
+                    muxerPath = null;
+                    return false;
+                }
 
-        EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer not found using global registrations");
+                if (!IsValidArchitectureMuxer(targetArchitecture, muxerPath))
+                {
+                    // If muxer is wrong we stop the search
+                    EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer resolved using global registration is not compatible with the target architecture: '{muxerPath}'");
+                    muxerPath = null;
+                    return false;
+                }
 
-        // Try searching in default installation location if it exists
-        if (isWinOs)
-        {
-            // If we're on x64/arm64 SDK and target is x86 we need to search on non virtualized windows folder
-            if ((_environment.Architecture == PlatformArchitecture.X64 || _environment.Architecture == PlatformArchitecture.ARM64) &&
-                 targetArchitecture == PlatformArchitecture.X86)
-            {
-                muxerPath = Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles(x86)")!, "dotnet", _muxerName);
-            }
-            else
-            {
-                // If we're on ARM and target is x64 we expect correct installation inside x64 folder
-                muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
-                    ? Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", "x64", _muxerName)
-                    : Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", _muxerName);
+                EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer compatible with '{targetArchitecture}' resolved from global registration: '{muxerPath}'");
+                return true;
             }
+
+            EqtTrace.Verbose($"DotnetHostHelper.TryGetDotnetPathByArchitecture: Muxer not found using global registrations");
         }
-        else
+
+        if (dotnetMuxerResolution.HasFlag(DotnetMuxerResolution.DefaultInstallation))
         {
-            if (_environment.OperatingSystem == PlatformOperatingSystem.OSX)
+            // Try searching in default installation location if it exists
+            if (isWinOs)
             {
-                // If we're on ARM and target is x64 we expect correct installation inside x64 folder
-                muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
-                    ? Path.Combine("/usr/local/share/dotnet/x64", _muxerName)
-                    : Path.Combine("/usr/local/share/dotnet", _muxerName);
+                // If we're on x64/arm64 SDK and target is x86 we need to search on non virtualized windows folder
+                if ((_environment.Architecture == PlatformArchitecture.X64 || _environment.Architecture == PlatformArchitecture.ARM64) &&
+                     targetArchitecture == PlatformArchitecture.X86)
+                {
+                    muxerPath = Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles(x86)")!, "dotnet", _muxerName);
+                }
+                else
+                {
+                    // If we're on ARM and target is x64 we expect correct installation inside x64 folder
+                    muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
+                        ? Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", "x64", _muxerName)
+                        : Path.Combine(_environmentVariableHelper.GetEnvironmentVariable("ProgramFiles")!, "dotnet", _muxerName);
+                }
             }
             else
             {
-                muxerPath = Path.Combine("/usr/share/dotnet", _muxerName);
+                if (_environment.OperatingSystem == PlatformOperatingSystem.OSX)
+                {
+                    // If we're on ARM and target is x64 we expect correct installation inside x64 folder
+                    muxerPath = _environment.Architecture == PlatformArchitecture.ARM64 && targetArchitecture == PlatformArchitecture.X64
+                        ? Path.Combine("/usr/local/share/dotnet/x64", _muxerName)
+                        : Path.Combine("/usr/local/share/dotnet", _muxerName);
+                }
+                else
+                {
+                    muxerPath = Path.Combine("/usr/share/dotnet", _muxerName);
+                }
             }
         }
 

src/Microsoft.TestPlatform.CoreUtilities/Helpers/DotnetMuxerResolution.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
+
+/// <summary>
+/// An enum representing the dotnet muxer resolution.
+/// </summary>
+[Flags]
+public enum DotnetMuxerResolution
+{
+    /// <summary>
+    /// Indicates if the muxer resolution process should take dotnet root into account.
+    /// </summary>
+    DotnetRootArchitecture = 1,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should take arch independent dotnet root into account.
+    /// </summary>
+    DotnetRootArchitectureLess = 2,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should look in the global installation location.
+    /// </summary>
+    GlobalInstallation = 4,
+
+    /// <summary>
+    /// Indicates if the muxer resolution process should look in the default installation location.
+    /// </summary>
+    DefaultInstallation = 8,
+
+    /// <summary>
+    /// All muxer resolution options should be taken into account.
+    /// </summary>
+    All = DotnetRootArchitecture | DotnetRootArchitectureLess | GlobalInstallation | DefaultInstallation,
+}

src/Microsoft.TestPlatform.CoreUtilities/Helpers/Interfaces/IDotnetHostHelper.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers;
 using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
 
 namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces;
@@ -27,7 +28,11 @@ public interface IDotnetHostHelper
     /// Try to locate muxer of specific architecture
     /// </summary>
     /// <param name="targetArchitecture">Specific architecture</param>
+    /// <param name="dotnetMuxerResolution">The dotnet muxer resolution.</param>
     /// <param name="muxerPath">Path to the muxer</param>
     /// <returns>True if native muxer is found</returns>
-    bool TryGetDotnetPathByArchitecture(PlatformArchitecture targetArchitecture, out string? muxerPath);
+    bool TryGetDotnetPathByArchitecture(
+        PlatformArchitecture targetArchitecture,
+        DotnetMuxerResolution dotnetMuxerResolution,
+        out string? muxerPath);
 }

src/Microsoft.TestPlatform.CoreUtilities/PublicAPI/PublicAPI.Shipped.txt

@@ -63,7 +63,13 @@ Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces.ITestPlatfo
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.GetDotnetPath() -> string!
 Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.GetMonoPath() -> string!
-Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, out string? muxerPath) -> bool
+Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Helpers.Interfaces.IDotnetHostHelper.TryGetDotnetPathByArchitecture(Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.PlatformArchitecture targetArchitecture, Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution dotnetMuxerResolution, out string? muxerPath) -> bool
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.All = Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DotnetRootArchitecture | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DotnetRootArchitectureLess | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.GlobalInstallation | Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DefaultInstallation -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DefaultInstallation = 8 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DotnetRootArchitecture = 1 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.DotnetRootArchitectureLess = 2 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
+Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution.GlobalInstallation = 4 -> Microsoft.VisualStudio.TestPlatform.CoreUtilities.Helpers.DotnetMuxerResolution
 Microsoft.VisualStudio.TestPlatform.ObjectModel.EqtTrace
 Microsoft.VisualStudio.TestPlatform.ObjectModel.ValidateArg
 Microsoft.VisualStudio.TestPlatform.ObjectModel.ValidateArgProperty

src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs

@@ -431,8 +431,15 @@ public virtual TestProcessStartInfo GetTestHostProcessStartInfo(
                     EqtTrace.Verbose($"DotnetTestHostmanager: Forcing the search to x64 architecure, IsDefaultTargetArchitecture '{_runsettingHelper.IsDefaultTargetArchitecture}' OS '{_platformEnvironment.OperatingSystem}' framework '{_targetFramework}'");
                 }
 
+                // Check if DOTNET_ROOT resolution should be bypassed.
+                var shouldIgnoreDotnetRoot = _environmentVariableHelper.GetEnvironmentVariable("VSTEST_IGNORE_DOTNET_ROOT");
+                var muxerResolution = (!StringUtils.IsNullOrEmpty(shouldIgnoreDotnetRoot)
+                    && shouldIgnoreDotnetRoot.Equals("1", StringComparison.Ordinal))
+                        ? (DotnetMuxerResolution.GlobalInstallation | DotnetMuxerResolution.DefaultInstallation)
+                        : DotnetMuxerResolution.All;
+
                 PlatformArchitecture finalTargetArchitecture = forceToX64 ? PlatformArchitecture.X64 : targetArchitecture;
-                if (!_dotnetHostHelper.TryGetDotnetPathByArchitecture(finalTargetArchitecture, out string? muxerPath))
+                if (!_dotnetHostHelper.TryGetDotnetPathByArchitecture(finalTargetArchitecture, muxerResolution, out string? muxerPath))
                 {
                     string message = string.Format(CultureInfo.CurrentCulture, Resources.NoDotnetMuxerFoundForArchitecture, $"dotnet{(_platformEnvironment.OperatingSystem == PlatformOperatingSystem.Windows ? ".exe" : string.Empty)}", finalTargetArchitecture.ToString());
                     EqtTrace.Error(message);