Skip to content

[release/11.0.1xx-rc1] Honor MTP zero-tests and exit-code policies in dotnet test #56296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Evangelink merged 2 commits into from
Sep 15, 2026
Merged

[release/11.0.1xx-rc1] Honor MTP zero-tests and exit-code policies in dotnet test #56296

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7509da8
Honor MTP zero-tests and exit-code policies in dotnet test (#56219)
Evangelink Sep 14, 2026
e08c3f4
Honor configured MTP exit policies
Evangelink Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 205 additions & 4 deletions src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@
using Microsoft.DotNet.Cli.Extensions;
using Microsoft.DotNet.Cli.Telemetry;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.ProjectTools;

namespace Microsoft.DotNet.Cli.Commands.Test;

internal partial class MicrosoftTestingPlatformTestCommand
{
private const string IgnoreExitCodeOptionName = "--ignore-exit-code";
private const string MinimumExpectedTestsOptionName = "--minimum-expected-tests";
private const string TestingPlatformExitCodeIgnoreEnvironmentVariable = "TESTINGPLATFORM_EXITCODE_IGNORE";
private const string ZeroTestsPolicyOptionName = "--zero-tests-policy";
private const string ZeroTestsPolicyStrictArgument = "strict";

public int Run(ParseResult parseResult, bool isHelp)
{
Expand All @@ -32,6 +37,13 @@ public int Run(ParseResult parseResult, bool isHelp)
bool forwardedMinimumExpectedTests = HasForwardedOption(
buildOptions.TestApplicationArguments,
MinimumExpectedTestsOptionName);
var fallbackTestApplicationPolicy = new TestApplicationPolicy(
IsStrictZeroTestsPolicy(buildOptions.TestApplicationArguments),
GetEffectiveIgnoredExitCodes(
buildOptions.TestApplicationArguments,
buildOptions.EnvironmentVariables.TryGetValue(TestingPlatformExitCodeIgnoreEnvironmentVariable, out string? value)
? value
: Environment.GetEnvironmentVariable(TestingPlatformExitCodeIgnoreEnvironmentVariable)));

bool collectTestMap = parseResult.HasOption(definition.CollectTestMapOption) || forwardedCollectTestMap;
bool affectedTests = parseResult.HasOption(definition.AffectedTestsOption) || forwardedAffectedTests;
Expand Down Expand Up @@ -179,7 +191,8 @@ public int Run(ParseResult parseResult, bool isHelp)
ctrlC,
artifactPostProcessingManager,
testRunPolicy,
queueCancellation.Token);
queueCancellation.Token,
fallbackTestApplicationPolicy);
exitCode = testHandler.RunTestApplications(actionQueue);
TestRunCancellationReason cancellationReason = testRunPolicy.Complete();

Expand Down Expand Up @@ -217,7 +230,11 @@ public int Run(ParseResult parseResult, bool isHelp)
else if (exitCode == ExitCode.Success &&
!isHelp &&
!parseResult.HasOption(definition.MinimumExpectedTestsOption) &&
ShouldFailForNoExecutedTests(testOptions.IsAffectedTestsMode, output.TotalTests, output.SkippedTests))
ShouldFailForNoExecutedTests(
testOptions.IsAffectedTestsMode,
actionQueue.FailOnAllSkippedTests,
output.TotalTests,
output.SkippedTests))
{
// Whole-run "zero tests ran" verdict. Individual modules that matched no tests return exit
// code 8, but TestApplicationActionQueue normalizes that to success so a single empty module
Expand All @@ -228,6 +245,7 @@ public int Run(ParseResult parseResult, bool isHelp)
exitCode = ExitCode.ZeroTests;
}

exitCode = actionQueue.ApplyExitCodeIgnorePolicy(exitCode.Value);
return exitCode.Value;
}
finally
Expand Down Expand Up @@ -564,9 +582,192 @@ public ForwardedOptionState Merge(ForwardedOptionState other)
MinimumExpectedTests || other.MinimumExpectedTests);
}

internal static bool ShouldFailForNoExecutedTests(bool isAffectedTestsMode, int totalTests, int skippedTests)
internal static bool ShouldFailForNoExecutedTests(
bool isAffectedTestsMode,
bool failOnAllSkippedTests,
int totalTests,
int skippedTests)
=> (!isAffectedTestsMode && totalTests == 0) ||
(totalTests > 0 && totalTests == skippedTests);
(failOnAllSkippedTests && totalTests > 0 && totalTests == skippedTests);

internal static bool IsStrictZeroTestsPolicy(IReadOnlyList<string> arguments)
=> string.Equals(
GetForwardedOptionValue(arguments, ZeroTestsPolicyOptionName),
ZeroTestsPolicyStrictArgument,
StringComparison.OrdinalIgnoreCase);
Comment thread
Evangelink marked this conversation as resolved.

internal static string? GetEffectiveIgnoredExitCodes(
IReadOnlyList<string> arguments,
string? environmentValue)
=> !string.IsNullOrEmpty(environmentValue)
? environmentValue
: GetForwardedOptionValue(arguments, IgnoreExitCodeOptionName);

internal static int ApplyExitCodeIgnorePolicy(int exitCode, string? ignoredExitCodes)
{
if (ignoredExitCodes is null)
{
return exitCode;
}

foreach (string ignoredExitCode in ignoredExitCodes.Split(';'))
{
if (int.TryParse(
ignoredExitCode,
NumberStyles.Integer,
CultureInfo.InvariantCulture,
out int parsedExitCode) &&
parsedExitCode == exitCode)
{
return ExitCode.Success;
}
}

return exitCode;
}

internal static TestApplicationPolicy GetEffectiveTestApplicationPolicy(
TestModule module,
BuildOptions buildOptions)
{
var parsedArguments = new List<string>();
AddParsedArguments(module.RunProperties.Arguments, parsedArguments);
parsedArguments.AddRange(buildOptions.TestApplicationArguments);
ProjectLaunchProfile? launchProfile = module.LaunchSettings as ProjectLaunchProfile;
if (launchProfile is not null && !buildOptions.NoLaunchProfileArguments)
{
AddParsedArguments(launchProfile.CommandLineArgs, parsedArguments);
}

string workingDirectory = string.IsNullOrEmpty(module.RunProperties.WorkingDirectory)
? Directory.GetCurrentDirectory()
: module.RunProperties.WorkingDirectory;
var expandedArguments = new List<string>();
IReadOnlyList<string> effectiveArguments = TryExpandResponseFileArguments(
parsedArguments,
workingDirectory,
new HashSet<string>(FileUtilities.PathComparer),
expandedArguments)
? expandedArguments
: parsedArguments;

string? environmentValue = Environment.GetEnvironmentVariable(TestingPlatformExitCodeIgnoreEnvironmentVariable);
if (launchProfile is not null &&
launchProfile.EnvironmentVariables.TryGetValue(
TestingPlatformExitCodeIgnoreEnvironmentVariable,
out string? launchProfileValue))
Comment thread
Evangelink marked this conversation as resolved.
Outdated
{
environmentValue = launchProfileValue;
}

if (module.EnvironmentVariables.TryGetValue(
TestingPlatformExitCodeIgnoreEnvironmentVariable,
out string? moduleValue))
{
environmentValue = moduleValue;
}

return new TestApplicationPolicy(
IsStrictZeroTestsPolicy(effectiveArguments),
GetEffectiveIgnoredExitCodes(effectiveArguments, environmentValue));
Comment thread
Evangelink marked this conversation as resolved.
Outdated
}

private static void AddParsedArguments(string? arguments, List<string> parsedArguments)
{
if (string.IsNullOrEmpty(arguments))
{
return;
}

try
{
parsedArguments.AddRange(SplitResponseFileLine(arguments));
}
catch (FormatException)
{
// MTP reports malformed command lines. Ignore this source when reconstructing policy so
// the SDK does not replace the child parser's diagnostic with a synthesized verdict.
}
}

private static string? GetForwardedOptionValue(IReadOnlyList<string> arguments, string canonicalOption)
{
for (int i = 0; i < arguments.Count; i++)
{
string argument = arguments[i];
if (!IsOption(argument, canonicalOption, allowValue: true))
{
continue;
}

int separatorIndex = argument.IndexOfAny('=', ':');
if (separatorIndex >= 0)
{
return Unquote(argument[(separatorIndex + 1)..]);
}

return i + 1 < arguments.Count ? Unquote(arguments[i + 1]) : null;
}

return null;
}

private static string Unquote(string value)
=> value.Length >= 2 &&
((value[0] == '\'' && value[^1] == '\'') ||
(value[0] == '"' && value[^1] == '"'))
? value[1..^1]
: value;

private static bool TryExpandResponseFileArguments(
IReadOnlyList<string> arguments,
string workingDirectory,
HashSet<string> recursionStack,
List<string> expandedArguments)
{
foreach (string argument in arguments)
{
if (argument.Length <= 1 || argument[0] != '@')
{
expandedArguments.Add(argument);
continue;
}

string fullPath = Path.GetFullPath(argument[1..], workingDirectory);
if (!recursionStack.Add(fullPath) || !File.Exists(fullPath))
{
return false;
}

try
{
string[] responseFileArguments = [..
File.ReadAllLines(fullPath)
.Select(static line => line.Trim())
.Where(static line => line.Length > 0 && line[0] != '#')
.SelectMany(SplitResponseFileLine)];

if (!TryExpandResponseFileArguments(
responseFileArguments,
workingDirectory,
recursionStack,
expandedArguments))
{
return false;
}
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or FormatException)
{
return false;
}
finally
{
recursionStack.Remove(fullPath);
}
}

return true;
}

private static TestListFormat GetListTestsFormat(ParseResult parseResult, TestCommandDefinition.MicrosoftTestingPlatform definition)
{
Expand Down
4 changes: 4 additions & 0 deletions src/Cli/dotnet/Commands/Test/MTP/Options.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,7 @@ internal record BuildOptions(
string? Device,
bool ListDevices,
IReadOnlyDictionary<string, string> EnvironmentVariables);

internal readonly record struct TestApplicationPolicy(
bool FailOnAllSkippedTests,
string? IgnoredExitCodes);
40 changes: 11 additions & 29 deletions src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,6 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode)
int totalRetriedTests = 0;
int totalRetriedExecutions = 0;
int totalFlakyTests = 0;
bool anyAssemblyUnsuccessful = false;
int failedAssembliesWithoutFailedTests = 0;

foreach (TestProgressState assembly in assemblies)
Expand All @@ -298,57 +297,40 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode)
totalFlakyTests += assembly.FlakyTests;
if (!assembly.Success)
{
anyAssemblyUnsuccessful = true;
if (assembly.FailedTests == 0)
{
failedAssembliesWithoutFailedTests++;
}
}
}

bool notEnoughTests = totalTests < _options.MinimumExpectedTests;
bool allTestsWereSkipped = (totalTests == 0 && !_options.AllowZeroTests)
|| (totalTests > 0 && totalTests == totalSkippedTests);
bool anyTestFailed = totalFailedTests > 0;
bool anyAssemblyFailed = anyAssemblyUnsuccessful || HasHandshakeFailure;
bool unexpectedNonZeroExitCode = exitCode is not null
&& exitCode != ExitCode.Success
&& exitCode != ExitCode.ZeroTests
&& exitCode != ExitCode.MinimumExpectedTestsPolicyViolation;
bool runFailed = anyAssemblyFailed || anyTestFailed || notEnoughTests || allTestsWereSkipped || unexpectedNonZeroExitCode || _wasCancelled;
bool runFailed = exitCode is null || exitCode != ExitCode.Success;
bool zeroTestsFailure = exitCode == ExitCode.ZeroTests;
bool minimumExpectedTestsFailure = exitCode == ExitCode.MinimumExpectedTestsPolicyViolation;
terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen);

terminal.Append(CliCommandStrings.TestRunSummary);
terminal.Append(' ');

if (_wasCancelled)
if (!runFailed)
{
terminal.Append(CliCommandStrings.Aborted);
terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed));
}
else if (notEnoughTests)
else if (_wasCancelled)
{
terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests));
terminal.Append(CliCommandStrings.Aborted);
}
else if (anyTestFailed || HasHandshakeFailure || unexpectedNonZeroExitCode)
else if (minimumExpectedTestsFailure && _options.MinimumExpectedTests > 0)
{
// Handshake failures take precedence over "Zero tests ran": when an assembly failed to
// hand-shake we want the headline to reflect that the run failed, not that no tests ran
// (which would imply a benign empty run). We intentionally do NOT escalate the broader
// anyAssemblyFailed here, because a project that legitimately contains zero tests exits
// with ExitCodes.ZeroTests (non-zero) and would otherwise be misclassified as a failure.
terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests));
}
else if (allTestsWereSkipped)
else if (zeroTestsFailure)
{
terminal.Append(CliCommandStrings.ZeroTestsRan);
}
else if (anyAssemblyFailed)
{
terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
}
else
{
terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed));
terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed));
}

if (!_options.ShowAssembly && assemblies.Count == 1)
Expand Down
Loading
Loading