Add App Host Support for MSBuild

.github/copilot-instructions.md

@@ -9,6 +9,13 @@ The code is written in C# and should follow the .NET coding conventions. Use the
 You should generally match the style of surrounding code when making edits, but if making a substantial change, you can modernize more aggressively.
 New files should use nullable types but don't refactor aggressively existing code.
 
+### Nullable Reference Types
+- **New files**: Always use nullable reference types (do NOT add `#nullable disable`)
+- **Existing files with `#nullable disable`**: Match the existing style; don't add nullable annotations (`?`) to types
+- **Existing files with nullable enabled**: Use proper nullable annotations
+- When a file has `#nullable disable`, parameters like `IDictionary<string, string>` should NOT have `?` suffix
+- Shared files (in `src/Shared/`) may be compiled into multiple assemblies with different nullable contexts; be careful with nullable annotations in these files
+
 Generate tests for new codepaths, and add tests for any bugs you fix. Use the existing test framework, which is xUnit with Shouldly assertions. Use Shouldly assertions for all assertions in modified code, even if the file is predominantly using xUnit assertions.
 
 When making changes, check if related documentation exists in the `documentation/` folder (including `documentation/specs/`) and update it to reflect your changes. Keep documentation in sync with code changes, especially for telemetry, APIs, and architectural decisions.

src/Build.UnitTests/BackEnd/AppHostSupport_Tests.cs

@@ -0,0 +1,159 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.Build.BackEnd;
+using Microsoft.Build.Internal;
+using Microsoft.Build.Shared;
+using Microsoft.Build.UnitTests;
+using Shouldly;
+using Xunit;
+using Xunit.Abstractions;
+
+#nullable disable
+
+namespace Microsoft.Build.Engine.UnitTests.BackEnd
+{
+    /// <summary>
+    /// Tests for MSBuild App Host support functionality.
+    /// Tests the DOTNET_ROOT environment variable handling for app host bootstrap.
+    /// </summary>
+    public sealed class AppHostSupport_Tests
+    {
+        private readonly ITestOutputHelper _output;
+
+        private readonly string _dotnetHostPath = NativeMethodsShared.IsWindows
+            ? @"C:\Program Files\dotnet\dotnet.exe"
+            : "/usr/share/dotnet/dotnet";
+
+        public AppHostSupport_Tests(ITestOutputHelper output)
+        {
+            _output = output;
+        }
+
+        [Fact]
+        public void CreateDotnetRootEnvironmentOverrides_SetsDotnetRootFromHostPath()
+        {
+            var overrides = DotnetHostEnvironmentHelper.CreateDotnetRootEnvironmentOverrides(_dotnetHostPath);
+
+            overrides.ShouldNotBeNull();
+            overrides.ShouldContainKey("DOTNET_ROOT");
+
+            string expectedDotnetRoot = Path.GetDirectoryName(_dotnetHostPath);
+            overrides["DOTNET_ROOT"].ShouldBe(expectedDotnetRoot);
+        }
+
+        [Fact]
+        public void CreateDotnetRootEnvironmentOverrides_ClearsArchitectureSpecificVariables()
+        {
+            var overrides = DotnetHostEnvironmentHelper.CreateDotnetRootEnvironmentOverrides(_dotnetHostPath);
+
+            // Assert - architecture-specific variables should be set to null (to be cleared)
+            overrides.ShouldContainKey("DOTNET_ROOT_X64");
+            overrides["DOTNET_ROOT_X64"].ShouldBeNull();
+
+            overrides.ShouldContainKey("DOTNET_ROOT_X86");
+            overrides["DOTNET_ROOT_X86"].ShouldBeNull();
+
+            overrides.ShouldContainKey("DOTNET_ROOT_ARM64");
+            overrides["DOTNET_ROOT_ARM64"].ShouldBeNull();
+        }
+
+
+        [WindowsOnlyTheory]
+        [InlineData(@"C:\custom\sdk\dotnet.exe", @"C:\custom\sdk")]
+        [InlineData(@"D:\tools\dotnet\dotnet.exe", @"D:\tools\dotnet")]
+        public void CreateDotnetRootEnvironmentOverrides_HandlesVariousPaths_Windows(string hostPath, string expectedRoot)
+        {
+            var overrides = DotnetHostEnvironmentHelper.CreateDotnetRootEnvironmentOverrides(hostPath);
+
+            overrides["DOTNET_ROOT"].ShouldBe(expectedRoot);
+        }
+
+        [UnixOnlyTheory]
+        [InlineData("/usr/local/share/dotnet/dotnet", "/usr/local/share/dotnet")]
+        [InlineData("/home/user/.dotnet/dotnet", "/home/user/.dotnet")]
+        public void CreateDotnetRootEnvironmentOverrides_HandlesVariousPaths_Unix(string hostPath, string expectedRoot)
+        {
+            var overrides = DotnetHostEnvironmentHelper.CreateDotnetRootEnvironmentOverrides(hostPath);
+
+            overrides["DOTNET_ROOT"].ShouldBe(expectedRoot);
+        }
+
+        [Fact]
+        public void ClearBootstrapDotnetRootEnvironment_ClearsVariablesNotInOriginalEnvironment()
+        {
+            using (TestEnvironment env = TestEnvironment.Create(_output))
+            {
+                // Arrange - set DOTNET_ROOT variants that simulate app host bootstrap
+                env.SetEnvironmentVariable("DOTNET_ROOT", @"C:\TestDotnet");
+                env.SetEnvironmentVariable("DOTNET_ROOT_X64", @"C:\TestDotnetX64");
+
+                // Original environment does NOT have these variables
+                var originalEnvironment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+                DotnetHostEnvironmentHelper.ClearBootstrapDotnetRootEnvironment(originalEnvironment);
+
+                Environment.GetEnvironmentVariable("DOTNET_ROOT").ShouldBeNull();
+                Environment.GetEnvironmentVariable("DOTNET_ROOT_X64").ShouldBeNull();
+            }
+        }
+
+        [Fact]
+        public void ClearBootstrapDotnetRootEnvironment_PreservesVariablesInOriginalEnvironment()
+        {
+            using (TestEnvironment env = TestEnvironment.Create(_output))
+            {
+                // Arrange - set DOTNET_ROOT that was in the original environment
+                string originalValue = @"C:\OriginalDotnet";
+                env.SetEnvironmentVariable("DOTNET_ROOT", originalValue);
+
+                // Register other DOTNET_ROOT variants with TestEnvironment so cleanup works correctly.
+                // These will be cleared by the helper if not in originalEnvironment.
+                env.SetEnvironmentVariable("DOTNET_ROOT_X64", null);
+                env.SetEnvironmentVariable("DOTNET_ROOT_X86", null);
+                env.SetEnvironmentVariable("DOTNET_ROOT_ARM64", null);
+
+                // Original environment HAS DOTNET_ROOT
+                var originalEnvironment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["DOTNET_ROOT"] = originalValue
+                };
+
+                DotnetHostEnvironmentHelper.ClearBootstrapDotnetRootEnvironment(originalEnvironment);
+
+                // Assert - DOTNET_ROOT should be preserved since it was in original environment
+                Environment.GetEnvironmentVariable("DOTNET_ROOT").ShouldBe(originalValue);
+            }
+        }
+
+        [Fact]
+        public void ClearBootstrapDotnetRootEnvironment_HandlesMixedScenario()
+        {
+            using (TestEnvironment env = TestEnvironment.Create(_output))
+            {
+                string originalDotnetRoot = @"C:\OriginalDotnet";
+                string bootstrapX64 = @"C:\BootstrapX64";
+
+                env.SetEnvironmentVariable("DOTNET_ROOT", originalDotnetRoot);
+                env.SetEnvironmentVariable("DOTNET_ROOT_X64", bootstrapX64);
+                env.SetEnvironmentVariable("DOTNET_ROOT_X86", @"C:\BootstrapX86");
+
+                // Original environment has DOTNET_ROOT but not the architecture-specific ones
+                var originalEnvironment = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["DOTNET_ROOT"] = originalDotnetRoot
+                };
+
+                DotnetHostEnvironmentHelper.ClearBootstrapDotnetRootEnvironment(originalEnvironment);
+
+                Environment.GetEnvironmentVariable("DOTNET_ROOT").ShouldBe(originalDotnetRoot); // Preserved
+                Environment.GetEnvironmentVariable("DOTNET_ROOT_X64").ShouldBeNull(); // Cleared
+                Environment.GetEnvironmentVariable("DOTNET_ROOT_X86").ShouldBeNull(); // Cleared
+                Environment.GetEnvironmentVariable("DOTNET_ROOT_ARM64").ShouldBeNull(); // Was already null
+            }
+        }
+    }
+}

src/Build.UnitTests/NetTaskHost_E2E_Tests.cs

@@ -15,8 +15,6 @@ namespace Microsoft.Build.Engine.UnitTests
 {
     public class NetTaskHost_E2E_Tests
     {
-        private const string LatestDotNetCoreForMSBuild = "net10.0";
-
         private static string AssemblyLocation { get; } = Path.Combine(Path.GetDirectoryName(typeof(NetTaskHost_E2E_Tests).Assembly.Location) ?? AppContext.BaseDirectory);
 
         private static string TestAssetsRootPath { get; } = Path.Combine(AssemblyLocation, "TestAssets");
@@ -29,8 +27,10 @@ public NetTaskHost_E2E_Tests(ITestOutputHelper output)
         }
 
         [WindowsFullFrameworkOnlyFact]
-        public void NetTaskHostTest()
+        public void NetTaskHostTest_FallbackToDotnet()
         {
+            // This test verifies the fallback behavior when app host is not used.
+            // When DOTNET_HOST_PATH points to system dotnet, it uses dotnet.exe + MSBuild.dll.
             using TestEnvironment env = TestEnvironment.Create(_output);
             var dotnetPath = env.GetEnvironmentVariable("DOTNET_ROOT");
 
@@ -47,7 +47,7 @@ public void NetTaskHostTest()
             testTaskOutput.ShouldContain($"The task is executed in process: dotnet");
             testTaskOutput.ShouldContain($"Process path: {dotnetPath}", customMessage: testTaskOutput);
 
-            var customTaskAssemblyLocation = Path.GetFullPath(Path.Combine(AssemblyLocation, "..", LatestDotNetCoreForMSBuild, "ExampleTask.dll"));
+            var customTaskAssemblyLocation = Path.GetFullPath(Path.Combine(AssemblyLocation, "..", RunnerUtilities.LatestDotNetCoreForMSBuild, "ExampleTask.dll"));
             var resource = ResourceUtilities.FormatResourceStringIgnoreCodeAndKeyword(
                "TaskAssemblyLocationMismatch",
 
@@ -57,6 +57,46 @@ public void NetTaskHostTest()
             testTaskOutput.ShouldNotContain(resource);
         }
 
+        [WindowsFullFrameworkOnlyFact] // Verifies that when MSBuild.exe app host is available in the SDK, it is used instead of dotnet.exe + MSBuild.dll.
+        public void NetTaskHostTest_AppHostUsedWhenAvailable()
+        {
+            using TestEnvironment env = TestEnvironment.Create(_output, setupDotnetEnvVars: true);
+            string testProjectPath = Path.Combine(TestAssetsRootPath, "ExampleNetTask", "TestNetTask", "TestNetTask.csproj");
+
+            string testTaskOutput = RunnerUtilities.ExecBootstrapedMSBuild($"{testProjectPath} -restore -v:n -p:LatestDotNetCoreForMSBuild={RunnerUtilities.LatestDotNetCoreForMSBuild}", out bool successTestTask);
+
+            successTestTask.ShouldBeTrue();
+
+            // When app host is used, the process name should be "MSBuild" not "dotnet"
+            testTaskOutput.ShouldContain("The task is executed in process: MSBuild");
+
+            // The process path should point to MSBuild.exe, not dotnet.exe
+            testTaskOutput.ShouldContain("MSBuild.exe", customMessage: "Expected MSBuild.exe app host to be used");
+            testTaskOutput.ShouldNotContain("Process path: " + Path.Combine(env.GetEnvironmentVariable("DOTNET_ROOT") ?? "", "dotnet.exe"));
+        }
+
+        [WindowsFullFrameworkOnlyFact] // Verifies that when using the app host, DOTNET_ROOT is properly set for child processes to find the runtime.
+        public void NetTaskHostTest_AppHostSetsDotnetRoot()
+        {
+            using TestEnvironment env = TestEnvironment.Create(_output);
+
+            // Clear DOTNET_ROOT to ensure app host sets it
+            env.SetEnvironmentVariable("DOTNET_ROOT", null);
+            env.SetEnvironmentVariable("DOTNET_ROOT_X64", null);
+            env.SetEnvironmentVariable("DOTNET_ROOT_X86", null);
+            env.SetEnvironmentVariable("DOTNET_ROOT_ARM64", null);
+
+            string testProjectPath = Path.Combine(TestAssetsRootPath, "ExampleNetTask", "TestNetTask", "TestNetTask.csproj");
+
+            string testTaskOutput = RunnerUtilities.ExecBootstrapedMSBuild($"{testProjectPath} -restore -v:n -p:LatestDotNetCoreForMSBuild={RunnerUtilities.LatestDotNetCoreForMSBuild}", out bool successTestTask);
+
+            _output.WriteLine(testTaskOutput);
+
+            // The build should succeed - this proves DOTNET_ROOT was properly set for the task host
+            // to find the runtime, even though we cleared it from the parent environment
+            successTestTask.ShouldBeTrue(customMessage: "Build should succeed with app host setting DOTNET_ROOT");
+        }
+
         [WindowsFullFrameworkOnlyFact]
         public void NetTaskHost_CorrectPathsEscapingTest()
         {
@@ -75,8 +115,8 @@ public void NetTaskHost_CorrectPathsEscapingTest()
 
             // Explicitly validate escaping for paths with spaces
             var msBuildDllPathMatch = System.Text.RegularExpressions.Regex.Match(
-                testTaskOutput, 
-                @"Arg\[0\]:\s*(.+)", 
+                testTaskOutput,
+                @"Arg\[0\]:\s*(.+)",
                 System.Text.RegularExpressions.RegexOptions.CultureInvariant);
 
             msBuildDllPathMatch.Success.ShouldBeTrue();
@@ -90,10 +130,10 @@ public void NetTaskHost_CorrectPathsEscapingTest()
                 {
                     // Check that later path segments don't appear as separate command line arguments
                     var arg1Match = System.Text.RegularExpressions.Regex.Match(
-                        testTaskOutput, 
-                        @"Arg\[1\]:\s*(.+)", 
+                        testTaskOutput,
+                        @"Arg\[1\]:\s*(.+)",
                         System.Text.RegularExpressions.RegexOptions.CultureInvariant);
-                        
+
                     if (arg1Match.Success)
                     {
                         string arg1Value = arg1Match.Groups[1].Value.Trim();
@@ -108,7 +148,7 @@ public void NetTaskHost_CorrectPathsEscapingTest()
                 testTaskOutput,
                 @"Command line:\s*(.+)",
                 System.Text.RegularExpressions.RegexOptions.CultureInvariant);
-                
+
             if (cmdLineMatch.Success)
             {
                 string cmdLine = cmdLineMatch.Groups[1].Value.Trim();
@@ -123,7 +163,6 @@ public void MSBuildTaskInNetHostTest()
             using TestEnvironment env = TestEnvironment.Create(_output);
 
             string testProjectPath = Path.Combine(TestAssetsRootPath, "ExampleNetTask", "TestMSBuildTaskInNet", "TestMSBuildTaskInNet.csproj");
-
             string testTaskOutput = RunnerUtilities.ExecBootstrapedMSBuild($"{testProjectPath} -restore  -v:n -p:LatestDotNetCoreForMSBuild={RunnerUtilities.LatestDotNetCoreForMSBuild}", out bool successTestTask);
 
             if (!successTestTask)
@@ -135,8 +174,8 @@ public void MSBuildTaskInNetHostTest()
             testTaskOutput.ShouldContain($"Hello TEST");
         }
 
-        [WindowsFullFrameworkOnlyFact]
-        public void NetTaskWithImplicitHostParamsTest()
+        [WindowsFullFrameworkOnlyFact] // This test verifies the fallback behavior with implicit host parameters.
+        public void NetTaskWithImplicitHostParamsTest_FallbackToDotnet()
         {
             using TestEnvironment env = TestEnvironment.Create(_output);
             var dotnetPath = env.GetEnvironmentVariable("DOTNET_ROOT");
@@ -163,5 +202,21 @@ public void NetTaskWithImplicitHostParamsTest()
             // Output from the task where neither TaskHost nor Runtime were specified
             testTaskOutput.ShouldContain("Found item: Banana");
         }
+
+        [WindowsFullFrameworkOnlyFact] // This test verifies app host behavior with implicit host parameters.
+        public void NetTaskWithImplicitHostParamsTest_AppHost()
+        {
+            using TestEnvironment env = TestEnvironment.Create(_output, setupDotnetEnvVars: true);
+            string testProjectPath = Path.Combine(TestAssetsRootPath, "ExampleNetTask", "TestNetTaskWithImplicitParams", "TestNetTaskWithImplicitParams.csproj");
+
+            string testTaskOutput = RunnerUtilities.ExecBootstrapedMSBuild($"{testProjectPath} -restore -v:n -p:LatestDotNetCoreForMSBuild={RunnerUtilities.LatestDotNetCoreForMSBuild}", out bool successTestTask);
+
+            _output.WriteLine(testTaskOutput);
+
+            successTestTask.ShouldBeTrue();
+
+            testTaskOutput.ShouldContain("The task is executed in process: MSBuild");
+            testTaskOutput.ShouldContain("/nodereuse:True");
+        }
     }
 }

src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs

@@ -23,8 +23,6 @@ internal sealed class DetouredNodeLauncher : INodeLauncher, IBuildComponent
     {
         private readonly List<ISandboxedProcess> _sandboxedProcesses = new();
 
-        private readonly BuildParameters.IBuildParameters _environmentVariables = CreateEnvironmentVariables();
-
         private IFileAccessManager _fileAccessManager;
 
         public static IBuildComponent CreateComponent(BuildComponentType type)
@@ -51,9 +49,9 @@ public void ShutdownComponent()
         }
 
         /// <summary>
-        /// Creates a new MSBuild process
+        /// Creates a new MSBuild process.
         /// </summary>
-        public Process Start(string msbuildLocation, string commandLineArgs, int nodeId)
+        public Process Start(string msbuildLocation, string commandLineArgs, int nodeId, IDictionary<string, string> environmentOverrides = null)
         {
             // Should always have been set already.
             ErrorUtilities.VerifyThrowInternalLength(msbuildLocation, nameof(msbuildLocation));
@@ -92,7 +90,7 @@ public Process Start(string msbuildLocation, string commandLineArgs, int nodeId)
                 PipDescription = "MSBuild",
                 PipSemiStableHash = 0,
                 Arguments = commandLineArgs,
-                EnvironmentVariables = _environmentVariables,
+                EnvironmentVariables = CreateEnvironmentVariables(environmentOverrides),
                 MaxLengthInMemory = 0, // Don't buffer any output
             };
 
@@ -140,14 +138,26 @@ public Process Start(string msbuildLocation, string commandLineArgs, int nodeId)
             return Process.GetProcessById(sp.ProcessId);
         }
 
-        private static BuildParameters.IBuildParameters CreateEnvironmentVariables()
+        /// <summary>
+        /// Creates environment variables with optional overrides for app host bootstrap.
+        /// </summary>
+        /// <param name="environmentOverrides">
+        /// Environment variables to set or remove in the child process.
+        /// A non-null value sets or overrides that variable. A null value removes the variable
+        /// from the child process environment - this is used to clear architecture-specific
+        /// DOTNET_ROOT variants (e.g., DOTNET_ROOT_X64) that would otherwise take precedence
+        /// over DOTNET_ROOT when launching an app host.
+        /// </param>
+        private static BuildParameters.IBuildParameters CreateEnvironmentVariables(IDictionary<string, string> environmentOverrides)
         {
             var envVars = new Dictionary<string, string>();
             foreach (DictionaryEntry baseVar in Environment.GetEnvironmentVariables())
             {
                 envVars.Add((string)baseVar.Key, (string)baseVar.Value);
             }
 
+            DotnetHostEnvironmentHelper.ApplyEnvironmentOverrides(envVars, environmentOverrides);
+
             return BuildParameters.GetFactory().PopulateFromDictionary(envVars);
         }