Add MAUI device deployment to dotnet test

documentation/specs/dotnet-run-for-maui.md

@@ -140,6 +140,37 @@ device selection:
   single target framework. If that framework provides
   `ComputeAvailableDevices`, the user may be prompted for a device.
 
+* **Build per target framework** — the selected `$(Device)` and any
+  `$(RuntimeIdentifier)` supplied by `ComputeAvailableDevices` are passed
+  to the build for that target framework.
+
+* **Deploy per target framework** — after a successful build and before
+  computing the test application's run arguments, `dotnet test` calls
+  `DeployToDevice` when the target exists.
+
+  * `DeployToDevice` receives the selected `$(Device)`,
+    `$(TargetFramework)`, and `$(RuntimeIdentifier)` as global
+    properties.
+
+  * Deployment still runs with `--no-build`, because the user may select
+    a different device for an existing build.
+
+  * If `DeployToDevice` does not exist, deployment is skipped.
+
+  * If `DeployToDevice` fails, `dotnet test` exits with an error and does
+    not invoke `ComputeRunArguments` or start the test application.
+
+* **`ComputeRunArguments` per target framework** — after deployment,
+  `dotnet test` calls `ComputeRunArguments` with the same device,
+  target framework, and runtime identifier. The resulting
+  `$(RunCommand)` and `$(RunArguments)` are used to start the test
+  application.
+
+* **Solutions** — automatic device selection and deployment happen for
+  each test project and target framework in a solution. An explicit
+  `--device` is rejected with `--solution`; use `--project` because a
+  device identifier is project- and platform-specific.
+
 * **`--list-devices`** works the same as with `dotnet run`.
 
 * **`-e` / `--environment`** — `dotnet test` already supports `-e` to
@@ -271,17 +302,25 @@ invoking the target.
 
 ## Binary Logs for Device Selection
 
-When using `-bl` with `dotnet run`, all MSBuild operations are logged to a single
-binlog file: device selection, build, deploy, and run argument computation.
+When using `-bl` with `dotnet run`, all in-process MSBuild operations are
+logged to a single binlog file: device selection, deploy, and run argument
+computation.
 
 File naming for `dotnet run` binlogs:
 
 * `-bl:filename.binlog` creates `filename-dotnet-run.binlog`
 * `-bl` creates `msbuild-dotnet-run.binlog`
 
-Note: The build step may also create `msbuild.binlog` separately. Use
-`--no-build` with `-bl` to only capture run-specific MSBuild
-operations.
+`dotnet test` uses the same behavior for its in-process operations,
+including per-project and per-target-framework device selection,
+deployment, and run argument computation:
+
+* `-bl:filename.binlog` creates `filename-dotnet-test.binlog`
+* `-bl` creates `msbuild-dotnet-test.binlog`
+
+The out-of-process build step may also create `msbuild.binlog`
+separately. Use `--no-build` with `-bl` to capture only the in-process
+run or test operations.
 
 ## What about Launch Profiles?
 

src/Cli/dotnet/Commands/Test/MTP/MSBuildHandler.cs

@@ -9,9 +9,10 @@
 namespace Microsoft.DotNet.Cli.Commands.Test;
 
 [RequiresDynamicCode("Uses MSBuild Object Model types, which are not AOT-safe")]
-internal sealed class MSBuildHandler(BuildOptions buildOptions) : ITestHandler
+internal sealed class MSBuildHandler(BuildOptions buildOptions, FacadeLogger? logger) : ITestHandler
 {
     private readonly BuildOptions _buildOptions = buildOptions;
+    private readonly FacadeLogger? _logger = logger;
 
     private readonly ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules> _testApplications = [];
 
@@ -26,8 +27,8 @@ public bool Initialize()
         }
 
         (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects, int buildExitCode) = isSolution ?
-            MSBuildUtility.GetProjectsFromSolution(projectOrSolutionFilePath, _buildOptions) :
-            MSBuildUtility.GetProjectsFromProject(projectOrSolutionFilePath, _buildOptions);
+            MSBuildUtility.GetProjectsFromSolution(projectOrSolutionFilePath, _buildOptions, _logger) :
+            MSBuildUtility.GetProjectsFromProject(projectOrSolutionFilePath, _buildOptions, _logger);
 
         LogProjectProperties(projects);
 

src/Cli/dotnet/Commands/Test/MTP/MSBuildUtility.cs

@@ -21,15 +21,16 @@ namespace Microsoft.DotNet.Cli.Commands.Test;
 
 internal static class MSBuildUtility
 {
-    private const string dotnetTestVerb = "dotnet-test";
-
     // Related: https://github.com/dotnet/msbuild/pull/7992
     // Related: https://github.com/dotnet/msbuild/issues/12711
     [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ProjectShouldBuild")]
     static extern bool ProjectShouldBuild(SolutionFile solutionFile, string projectFile);
 
     [RequiresDynamicCode("Uses MSBuild Object Model types, which are not AOT-safe")]
-    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, int BuildExitCode) GetProjectsFromSolution(string solutionFilePath, BuildOptions buildOptions)
+    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, int BuildExitCode) GetProjectsFromSolution(
+        string solutionFilePath,
+        BuildOptions buildOptions,
+        FacadeLogger? logger)
     {
         int buildExitCode = BuildOrRestoreProjectOrSolution(solutionFilePath, buildOptions);
 
@@ -68,27 +69,30 @@ public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModul
             .Where(p => p.Item1.IncludeInBuild)
             .Select(p => (p.AbsolutePath, (string?)p.Item1.ConfigurationName, (string?)p.Item1.PlatformName));
 
-        FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. buildOptions.MSBuildArgs], dotnetTestVerb);
-
         using var collection = new ProjectCollection(globalProperties, loggers: logger is null ? null : [logger], toolsetDefinitionLocations: ToolsetDefinitionLocations.Default);
         var evaluationContext = EvaluationContext.Create(EvaluationContext.SharingPolicy.Shared);
-        var (projects, deviceBuildExitCode) = GetProjectsProperties(collection, evaluationContext, projectPaths, buildOptions);
-        logger?.ReallyShutdown();
+        var (projects, deviceBuildExitCode) = GetProjectsProperties(collection, evaluationContext, projectPaths, buildOptions, logger);
         collection.UnloadAllProjects();
 
         return (projects, deviceBuildExitCode != 0 ? deviceBuildExitCode : buildExitCode);
     }
 
     [RequiresDynamicCode("Uses MSBuild Object Model types, which are not AOT-safe")]
-    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, int BuildExitCode) GetProjectsFromProject(string projectFilePath, BuildOptions buildOptions)
+    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, int BuildExitCode) GetProjectsFromProject(
+        string projectFilePath,
+        BuildOptions buildOptions,
+        FacadeLogger? logger)
     {
         // Pre-build device selection: evaluate the project to select devices BEFORE building,
         // so that device-provided RuntimeIdentifiers are included in the build.
-        var deviceSelection = SolutionAndProjectUtility.SelectDevicesBeforeBuild(projectFilePath, buildOptions);
+        var deviceSelection = SolutionAndProjectUtility.SelectDevicesBeforeBuild(
+            projectFilePath,
+            buildOptions,
+            logger: logger);
 
         if (deviceSelection is not null)
         {
-            return BuildPerTfmWithDevices(projectFilePath, buildOptions, deviceSelection);
+            return BuildPerTfmWithDevices(projectFilePath, buildOptions, deviceSelection, logger);
         }
 
         int buildExitCode = BuildOrRestoreProjectOrSolution(projectFilePath, buildOptions);
@@ -98,14 +102,11 @@ public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModul
             return (Array.Empty<ParallelizableTestModuleGroupWithSequentialInnerModules>(), buildExitCode);
         }
 
-        FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. buildOptions.MSBuildArgs], dotnetTestVerb);
-
         var msbuildArgs = MSBuildArgs.AnalyzeMSBuildArguments(buildOptions.MSBuildArgs, CommonOptions.CreatePropertyOption(), CommonOptions.CreateRestorePropertyOption(), CommonOptions.CreateMSBuildTargetOption(), CommonOptions.CreateVerbosityOption(), CommonOptions.CreateNoLogoOption());
 
         using var collection = new ProjectCollection(globalProperties: CommonRunHelpers.GetGlobalPropertiesFromArgs(msbuildArgs), logger is null ? null : [logger], toolsetDefinitionLocations: ToolsetDefinitionLocations.Default);
         var evaluationContext = EvaluationContext.Create(EvaluationContext.SharingPolicy.Shared);
-        IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects = SolutionAndProjectUtility.GetProjectProperties(projectFilePath, collection, evaluationContext, buildOptions, configuration: null, platform: null);
-        logger?.ReallyShutdown();
+        IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects = SolutionAndProjectUtility.GetProjectProperties(projectFilePath, collection, evaluationContext, buildOptions, logger, configuration: null, platform: null);
         collection.UnloadAllProjects();
         return (projects, buildExitCode);
     }
@@ -119,6 +120,7 @@ private static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModu
         string projectFilePath,
         BuildOptions buildOptions,
         SolutionAndProjectUtility.DeviceSelectionResult deviceSelection,
+        FacadeLogger? logger,
         string? configuration = null,
         string? platform = null)
     {
@@ -164,8 +166,6 @@ private static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModu
                 return (Array.Empty<ParallelizableTestModuleGroupWithSequentialInnerModules>(), exitCode);
             }
 
-            FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. perTfmBuildOptions.MSBuildArgs], dotnetTestVerb);
-
             var msbuildArgs = SolutionAndProjectUtility.AnalyzeStandardTestMSBuildArgs(perTfmBuildOptions.MSBuildArgs);
 
             using var collection = new ProjectCollection(
@@ -174,8 +174,7 @@ private static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModu
                 toolsetDefinitionLocations: ToolsetDefinitionLocations.Default);
             var evaluationContext = EvaluationContext.Create(EvaluationContext.SharingPolicy.Shared);
             IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> modules = SolutionAndProjectUtility.GetProjectProperties(
-                projectFilePath, collection, evaluationContext, perTfmBuildOptions, configuration, platform);
-            logger?.ReallyShutdown();
+                projectFilePath, collection, evaluationContext, perTfmBuildOptions, logger, configuration, platform);
 
             allGroups.AddRange(modules);
         }
@@ -369,7 +368,8 @@ private static (ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerMo
         ProjectCollection projectCollection,
         EvaluationContext evaluationContext,
         IEnumerable<(string ProjectFilePath, string? Configuration, string? Platform)> projects,
-        BuildOptions buildOptions)
+        BuildOptions buildOptions,
+        FacadeLogger? logger)
     {
         var allProjects = new ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules>();
         var nonDeviceProjects = new List<(string ProjectFilePath, string? Configuration, string? Platform)>();
@@ -378,11 +378,22 @@ private static (ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerMo
         // (BuildManager.DefaultBuildManager), which is a process-wide singleton and cannot run concurrently.
         foreach (var project in projects)
         {
-            var deviceSelection = SolutionAndProjectUtility.SelectDevicesBeforeBuild(project.ProjectFilePath, buildOptions, projectCollection, evaluationContext);
+            var deviceSelection = SolutionAndProjectUtility.SelectDevicesBeforeBuild(
+                project.ProjectFilePath,
+                buildOptions,
+                projectCollection,
+                evaluationContext,
+                logger);
 
             if (deviceSelection is not null)
             {
-                var (modules, exitCode) = BuildPerTfmWithDevices(project.ProjectFilePath, buildOptions, deviceSelection, project.Configuration, project.Platform);
+                var (modules, exitCode) = BuildPerTfmWithDevices(
+                    project.ProjectFilePath,
+                    buildOptions,
+                    deviceSelection,
+                    logger,
+                    project.Configuration,
+                    project.Platform);
                 if (exitCode != 0)
                 {
                     return (allProjects, exitCode);
@@ -400,20 +411,33 @@ private static (ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerMo
         }
 
         // Phase 2: Handle non-device projects in parallel (existing behavior).
+        var gracefulExceptions = new ConcurrentQueue<GracefulException>();
         Parallel.ForEach(
             nonDeviceProjects,
             // We don't use --max-parallel-test-modules here.
             // If user wants to limit the test applications run in parallel, we don't want to punish them and force the evaluation to also be limited.
             new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount },
             (project) =>
             {
-                IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projectsMetadata = SolutionAndProjectUtility.GetProjectProperties(project.ProjectFilePath, projectCollection, evaluationContext, buildOptions, project.Configuration, project.Platform);
-                foreach (var projectMetadata in projectsMetadata)
+                try
+                {
+                    IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projectsMetadata = SolutionAndProjectUtility.GetProjectProperties(project.ProjectFilePath, projectCollection, evaluationContext, buildOptions, logger, project.Configuration, project.Platform);
+                    foreach (var projectMetadata in projectsMetadata)
+                    {
+                        allProjects.Add(projectMetadata);
+                    }
+                }
+                catch (GracefulException ex)
                 {
-                    allProjects.Add(projectMetadata);
+                    gracefulExceptions.Enqueue(ex);
                 }
             });
 
+        if (gracefulExceptions.TryDequeue(out GracefulException? gracefulException))
+        {
+            throw gracefulException;
+        }
+
         return (allProjects, 0);
     }
 }

src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs

@@ -40,28 +40,37 @@ public int Run(ParseResult parseResult, bool isHelp)
             throw new GracefulException(CliCommandStrings.CmdDeviceOptionsRequireProject);
         }
 
-        // --list-devices: list available devices for the project and exit early.
-        // Never builds, deploys, or runs tests.
-        if (buildOptions.ListDevices)
+        FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. buildOptions.MSBuildArgs], "dotnet-test");
+        ITestHandler testHandler;
+        try
         {
-            return HandleListDevices(buildOptions);
-        }
+            // --list-devices: list available devices for the project and exit early.
+            // Never builds, deploys, or runs tests.
+            if (buildOptions.ListDevices)
+            {
+                return HandleListDevices(buildOptions, logger);
+            }
 
-        // When --device is specified, force single target framework selection because
-        // a device is platform-specific and we need to know which TFM was intended.
-        if (!string.IsNullOrWhiteSpace(buildOptions.Device))
-        {
-            buildOptions = HandleDeviceWithTargetFrameworkSelection(buildOptions);
-        }
+            // When --device is specified, force single target framework selection because
+            // a device is platform-specific and we need to know which TFM was intended.
+            if (!string.IsNullOrWhiteSpace(buildOptions.Device))
+            {
+                buildOptions = HandleDeviceWithTargetFrameworkSelection(buildOptions, logger);
+            }
 
-        ITestHandler testHandler = buildOptions.PathOptions.TestModules is { } testModules
-            ? new TestModulesFilterHandler(testModules, parseResult)
-            : RuntimeFeature.IsDynamicCodeSupported ? new MSBuildHandler(buildOptions)
-                : throw new PlatformNotSupportedException("Dynamic code is not supported on this platform.");
+            testHandler = buildOptions.PathOptions.TestModules is { } testModules
+                ? new TestModulesFilterHandler(testModules, parseResult)
+                : RuntimeFeature.IsDynamicCodeSupported ? new MSBuildHandler(buildOptions, logger)
+                    : throw new PlatformNotSupportedException("Dynamic code is not supported on this platform.");
 
-        if (!testHandler.Initialize())
+            if (!testHandler.Initialize())
+            {
+                return ExitCode.GenericFailure;
+            }
+        }
+        finally
         {
-            return ExitCode.GenericFailure;
+            logger?.ReallyShutdown();
         }
 
         int degreeOfParallelism = GetDegreeOfParallelism(parseResult);
@@ -167,7 +176,7 @@ private static int GetDegreeOfParallelism(ParseResult parseResult)
     /// Solutions are rejected because each project may have its own device list, so
     /// applying a single --device value across a solution is ambiguous.
     /// </summary>
-    private static BuildOptions HandleDeviceWithTargetFrameworkSelection(BuildOptions buildOptions)
+    private static BuildOptions HandleDeviceWithTargetFrameworkSelection(BuildOptions buildOptions, FacadeLogger? logger)
     {
         var msbuildArgs = SolutionAndProjectUtility.AnalyzeStandardTestMSBuildArgs(buildOptions.MSBuildArgs);
 
@@ -191,7 +200,10 @@ private static BuildOptions HandleDeviceWithTargetFrameworkSelection(BuildOption
         if (!globalProperties.ContainsKey(ProjectProperties.TargetFramework))
         {
             // Evaluate the project to get TargetFrameworks
-            using var collection = new ProjectCollection(globalProperties);
+            using var collection = new ProjectCollection(
+                globalProperties,
+                logger is null ? null : [logger],
+                ToolsetDefinitionLocations.Default);
             var projectInstance = ProjectInstance.FromFile(projectPath, new ProjectOptions
             {
                 GlobalProperties = globalProperties,
@@ -241,7 +253,7 @@ private static BuildOptions HandleDeviceWithTargetFrameworkSelection(BuildOption
     /// <see cref="RunCommandSelector.TrySelectDevice"/>, and exits without
     /// building, deploying, or running tests.
     /// </summary>
-    private static int HandleListDevices(BuildOptions buildOptions)
+    private static int HandleListDevices(BuildOptions buildOptions, FacadeLogger? logger)
     {
         if (!ValidationUtility.ValidateBuildPathOptions(buildOptions.PathOptions, out var projectPath, out bool isSolution))
         {
@@ -258,7 +270,6 @@ private static int HandleListDevices(BuildOptions buildOptions)
         bool isInteractive = !Console.IsOutputRedirected && !new CIEnvironmentDetectorForTelemetry().IsCIEnvironment();
 
         var standardArgs = SolutionAndProjectUtility.AnalyzeStandardTestMSBuildArgs(buildOptions.MSBuildArgs);
-
         // Mirror the `dotnet run --list-devices` flow: a single RunCommandSelector
         // handles both target framework selection and device listing, with
         // InvalidateGlobalProperties between steps so the device list is computed
@@ -268,7 +279,8 @@ private static int HandleListDevices(BuildOptions buildOptions)
             isInteractive,
             standardArgs,
             ImmutableDictionary<string, string>.Empty,
-            commandName: "dotnet test");
+            commandName: "dotnet test",
+            logger);
 
         // Step 1: Prompt for TargetFramework if the project is multi-targeted and -f wasn't provided.
         if (!selector.TrySelectTargetFramework(out string? selectedFramework))