[release/11.0.1xx-rc1] Honor MTP zero-tests and exit-code policies in dotnet test

src/Cli/dotnet/Commands/Test/MTP/Options.cs

@@ -63,3 +63,7 @@ internal record BuildOptions(
     string? Device,
     bool ListDevices,
     IReadOnlyDictionary<string, string> EnvironmentVariables);
+
+internal readonly record struct TestApplicationPolicy(
+    bool FailOnAllSkippedTests,
+    string? IgnoredExitCodes);

src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs

@@ -284,7 +284,6 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode)
         int totalRetriedTests = 0;
         int totalRetriedExecutions = 0;
         int totalFlakyTests = 0;
-        bool anyAssemblyUnsuccessful = false;
         int failedAssembliesWithoutFailedTests = 0;
 
         foreach (TestProgressState assembly in assemblies)
@@ -298,57 +297,40 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode)
             totalFlakyTests += assembly.FlakyTests;
             if (!assembly.Success)
             {
-                anyAssemblyUnsuccessful = true;
                 if (assembly.FailedTests == 0)
                 {
                     failedAssembliesWithoutFailedTests++;
                 }
             }
         }
 
-        bool notEnoughTests = totalTests < _options.MinimumExpectedTests;
-        bool allTestsWereSkipped = (totalTests == 0 && !_options.AllowZeroTests)
-            || (totalTests > 0 && totalTests == totalSkippedTests);
-        bool anyTestFailed = totalFailedTests > 0;
-        bool anyAssemblyFailed = anyAssemblyUnsuccessful || HasHandshakeFailure;
-        bool unexpectedNonZeroExitCode = exitCode is not null
-            && exitCode != ExitCode.Success
-            && exitCode != ExitCode.ZeroTests
-            && exitCode != ExitCode.MinimumExpectedTestsPolicyViolation;
-        bool runFailed = anyAssemblyFailed || anyTestFailed || notEnoughTests || allTestsWereSkipped || unexpectedNonZeroExitCode || _wasCancelled;
+        bool runFailed = exitCode is null || exitCode != ExitCode.Success;
+        bool zeroTestsFailure = exitCode == ExitCode.ZeroTests;
+        bool minimumExpectedTestsFailure = exitCode == ExitCode.MinimumExpectedTestsPolicyViolation;
         terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen);
 
         terminal.Append(CliCommandStrings.TestRunSummary);
         terminal.Append(' ');
 
-        if (_wasCancelled)
+        if (!runFailed)
         {
-            terminal.Append(CliCommandStrings.Aborted);
+            terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed));
         }
-        else if (notEnoughTests)
+        else if (_wasCancelled)
         {
-            terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests));
+            terminal.Append(CliCommandStrings.Aborted);
         }
-        else if (anyTestFailed || HasHandshakeFailure || unexpectedNonZeroExitCode)
+        else if (minimumExpectedTestsFailure && _options.MinimumExpectedTests > 0)
         {
-            // Handshake failures take precedence over "Zero tests ran": when an assembly failed to
-            // hand-shake we want the headline to reflect that the run failed, not that no tests ran
-            // (which would imply a benign empty run). We intentionally do NOT escalate the broader
-            // anyAssemblyFailed here, because a project that legitimately contains zero tests exits
-            // with ExitCodes.ZeroTests (non-zero) and would otherwise be misclassified as a failure.
-            terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
+            terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests));
         }
-        else if (allTestsWereSkipped)
+        else if (zeroTestsFailure)
         {
             terminal.Append(CliCommandStrings.ZeroTestsRan);
         }
-        else if (anyAssemblyFailed)
-        {
-            terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
-        }
         else
         {
-            terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed));
+            terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
         }
 
         if (!_options.ShowAssembly && assemblies.Count == 1)

src/Cli/dotnet/Commands/Test/MTP/TestApplicationActionQueue.cs

@@ -13,8 +13,11 @@ internal class TestApplicationActionQueue
     private readonly Channel<ParallelizableTestModuleGroupWithSequentialInnerModules> _channel;
     private readonly Task[] _readers;
     private readonly CancellationToken _cancellationToken;
+    private readonly TestApplicationPolicy _fallbackTestApplicationPolicy;
+    private readonly BuildOptions _buildOptions;
 
     private int? _aggregateExitCode;
+    private readonly List<TestApplicationPolicy> _effectivePolicies = [];
 
     private readonly Lock _lock = new();
 
@@ -28,11 +31,14 @@ public TestApplicationActionQueue(
         CtrlCCancellationManager ctrlC,
         ArtifactPostProcessingManager artifactPostProcessingManager,
         TestRunPolicy testRunPolicy,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        TestApplicationPolicy fallbackTestApplicationPolicy = default)
     {
         _channel = Channel.CreateUnbounded<ParallelizableTestModuleGroupWithSequentialInnerModules>(new UnboundedChannelOptions { SingleReader = false, SingleWriter = false });
         _readers = new Task[degreeOfParallelism];
         _cancellationToken = cancellationToken;
+        _fallbackTestApplicationPolicy = fallbackTestApplicationPolicy;
+        _buildOptions = buildOptions;
 
         for (int i = 0; i < degreeOfParallelism; i++)
         {
@@ -50,6 +56,17 @@ public TestApplicationActionQueue(
 
     public void Enqueue(ParallelizableTestModuleGroupWithSequentialInnerModules testApplication)
     {
+        lock (_lock)
+        {
+            foreach (TestModule module in testApplication)
+            {
+                _effectivePolicies.Add(
+                    MicrosoftTestingPlatformTestCommand.GetEffectiveTestApplicationPolicy(
+                        module,
+                        _buildOptions));
+            }
+        }
+
         if (_cancellationToken.IsCancellationRequested)
         {
             return;
@@ -165,6 +182,49 @@ private async Task Read(
         }
     }
 
+    internal bool FailOnAllSkippedTests
+    {
+        get
+        {
+            lock (_lock)
+            {
+                IReadOnlyList<TestApplicationPolicy> effectivePolicies = GetEffectivePolicies();
+                return ShouldFailOnAllSkippedTests(effectivePolicies);
+            }
+        }
+    }
+
+    internal int ApplyExitCodeIgnorePolicy(int exitCode)
+    {
+        lock (_lock)
+        {
+            IReadOnlyList<TestApplicationPolicy> effectivePolicies = GetEffectivePolicies();
+            return ApplyExitCodeIgnorePolicy(exitCode, effectivePolicies);
+        }
+    }
+
+    internal static bool ShouldFailOnAllSkippedTests(IReadOnlyList<TestApplicationPolicy> effectivePolicies)
+        => effectivePolicies.Any(policy =>
+            policy.FailOnAllSkippedTests &&
+            MicrosoftTestingPlatformTestCommand.ApplyExitCodeIgnorePolicy(
+                ExitCode.ZeroTests,
+                policy.IgnoredExitCodes) != ExitCode.Success);
+
+    internal static int ApplyExitCodeIgnorePolicy(
+        int exitCode,
+        IReadOnlyList<TestApplicationPolicy> effectivePolicies)
+        => effectivePolicies.All(policy =>
+            MicrosoftTestingPlatformTestCommand.ApplyExitCodeIgnorePolicy(
+                exitCode,
+                policy.IgnoredExitCodes) == ExitCode.Success)
+                ? ExitCode.Success
+                : exitCode;
+
+    private IReadOnlyList<TestApplicationPolicy> GetEffectivePolicies()
+        => _effectivePolicies.Count == 0
+            ? [_fallbackTestApplicationPolicy]
+            : _effectivePolicies;
+
     internal static int NormalizeExitCode(int result, bool hasFailureDuringDispose)
     {
         if (result == ExitCode.ZeroTests)