diff --git a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs index bbee70706829..e26f29b29d3b 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/MicrosoftTestingPlatformTestCommand.cs @@ -5,6 +5,7 @@ using System.CommandLine; using System.Globalization; using System.Runtime.CompilerServices; +using System.Text.Json; using Microsoft.Build.Definition; using Microsoft.Build.Evaluation; using Microsoft.Build.Execution; @@ -14,12 +15,19 @@ using Microsoft.DotNet.Cli.Extensions; using Microsoft.DotNet.Cli.Telemetry; using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.ProjectTools; namespace Microsoft.DotNet.Cli.Commands.Test; internal partial class MicrosoftTestingPlatformTestCommand { + private const string CommandLineOptionsConfigurationPropertyName = "commandLineOptions"; + private const string IgnoreExitCodeOptionName = "--ignore-exit-code"; private const string MinimumExpectedTestsOptionName = "--minimum-expected-tests"; + private const string TestConfigurationFileSuffix = ".testconfig.json"; + private const string TestingPlatformExitCodeIgnoreEnvironmentVariable = "TESTINGPLATFORM_EXITCODE_IGNORE"; + private const string ZeroTestsPolicyOptionName = "--zero-tests-policy"; + private const string ZeroTestsPolicyStrictArgument = "strict"; public int Run(ParseResult parseResult, bool isHelp) { @@ -32,6 +40,8 @@ public int Run(ParseResult parseResult, bool isHelp) bool forwardedMinimumExpectedTests = HasForwardedOption( buildOptions.TestApplicationArguments, MinimumExpectedTestsOptionName); + TestApplicationPolicy fallbackTestApplicationPolicy = + GetEffectiveTestApplicationPolicy(module: null, buildOptions); bool collectTestMap = parseResult.HasOption(definition.CollectTestMapOption) || forwardedCollectTestMap; bool affectedTests = parseResult.HasOption(definition.AffectedTestsOption) || forwardedAffectedTests; @@ -179,7 +189,8 @@ public int Run(ParseResult parseResult, bool isHelp) ctrlC, artifactPostProcessingManager, testRunPolicy, - queueCancellation.Token); + queueCancellation.Token, + fallbackTestApplicationPolicy); exitCode = testHandler.RunTestApplications(actionQueue); TestRunCancellationReason cancellationReason = testRunPolicy.Complete(); @@ -217,7 +228,11 @@ public int Run(ParseResult parseResult, bool isHelp) else if (exitCode == ExitCode.Success && !isHelp && !parseResult.HasOption(definition.MinimumExpectedTestsOption) && - ShouldFailForNoExecutedTests(testOptions.IsAffectedTestsMode, output.TotalTests, output.SkippedTests)) + ShouldFailForNoExecutedTests( + testOptions.IsAffectedTestsMode, + actionQueue.FailOnAllSkippedTests, + output.TotalTests, + output.SkippedTests)) { // Whole-run "zero tests ran" verdict. Individual modules that matched no tests return exit // code 8, but TestApplicationActionQueue normalizes that to success so a single empty module @@ -228,6 +243,7 @@ public int Run(ParseResult parseResult, bool isHelp) exitCode = ExitCode.ZeroTests; } + exitCode = actionQueue.ApplyExitCodeIgnorePolicy(exitCode.Value); return exitCode.Value; } finally @@ -564,9 +580,388 @@ public ForwardedOptionState Merge(ForwardedOptionState other) MinimumExpectedTests || other.MinimumExpectedTests); } - internal static bool ShouldFailForNoExecutedTests(bool isAffectedTestsMode, int totalTests, int skippedTests) + internal static bool ShouldFailForNoExecutedTests( + bool isAffectedTestsMode, + bool failOnAllSkippedTests, + int totalTests, + int skippedTests) => (!isAffectedTestsMode && totalTests == 0) || - (totalTests > 0 && totalTests == skippedTests); + (failOnAllSkippedTests && totalTests > 0 && totalTests == skippedTests); + + internal static bool IsStrictZeroTestsPolicy(IReadOnlyList arguments) + => string.Equals( + GetForwardedOptionValue(arguments, ZeroTestsPolicyOptionName), + ZeroTestsPolicyStrictArgument, + StringComparison.OrdinalIgnoreCase); + + internal static string? GetEffectiveIgnoredExitCodes( + IReadOnlyList arguments, + string? environmentValue, + string? configurationValue = null) + => !string.IsNullOrEmpty(environmentValue) + ? environmentValue + : GetForwardedOptionValue(arguments, IgnoreExitCodeOptionName) ?? configurationValue; + + internal static int ApplyExitCodeIgnorePolicy(int exitCode, string? ignoredExitCodes) + { + if (ignoredExitCodes is null) + { + return exitCode; + } + + foreach (string ignoredExitCode in ignoredExitCodes.Split(';')) + { + if (int.TryParse( + ignoredExitCode, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int parsedExitCode) && + parsedExitCode == exitCode) + { + return ExitCode.Success; + } + } + + return exitCode; + } + + internal static TestApplicationPolicy GetEffectiveTestApplicationPolicy( + TestModule? module, + BuildOptions buildOptions) + { + var parsedArguments = new List(); + if (module is not null) + { + AddParsedArguments(module.RunProperties.Arguments, parsedArguments); + } + + parsedArguments.AddRange(buildOptions.TestApplicationArguments); + ProjectLaunchProfile? launchProfile = module?.LaunchSettings as ProjectLaunchProfile; + if (launchProfile is not null && !buildOptions.NoLaunchProfileArguments) + { + AddParsedArguments(launchProfile.CommandLineArgs, parsedArguments); + } + + string workingDirectory = string.IsNullOrEmpty(module?.RunProperties.WorkingDirectory) + ? Directory.GetCurrentDirectory() + : module.RunProperties.WorkingDirectory; + var expandedArguments = new List(); + IReadOnlyList effectiveArguments = TryExpandResponseFileArguments( + parsedArguments, + workingDirectory, + new HashSet(FileUtilities.PathComparer), + expandedArguments) + ? expandedArguments + : parsedArguments; + + string? environmentValue = Environment.GetEnvironmentVariable(TestingPlatformExitCodeIgnoreEnvironmentVariable); + if (launchProfile is not null) + { + environmentValue = ApplyEnvironmentVariableOverrides( + environmentValue, + launchProfile.EnvironmentVariables); + } + + environmentValue = ApplyEnvironmentVariableOverrides( + environmentValue, + module?.EnvironmentVariables ?? buildOptions.EnvironmentVariables); + + TestApplicationPolicy configurationPolicy = + GetConfigurationFilePolicy(module, buildOptions, effectiveArguments, workingDirectory); + string? commandLineZeroTestsPolicy = + GetForwardedOptionValue(effectiveArguments, ZeroTestsPolicyOptionName); + + return new TestApplicationPolicy( + commandLineZeroTestsPolicy is null + ? configurationPolicy.FailOnAllSkippedTests + : string.Equals( + commandLineZeroTestsPolicy, + ZeroTestsPolicyStrictArgument, + StringComparison.OrdinalIgnoreCase), + GetEffectiveIgnoredExitCodes( + effectiveArguments, + environmentValue, + configurationPolicy.IgnoredExitCodes)); + } + + private static string? ApplyEnvironmentVariableOverrides( + string? currentValue, + IEnumerable> environmentVariables) + { + StringComparison comparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + foreach ((string name, string value) in environmentVariables) + { + if (string.Equals(name, TestingPlatformExitCodeIgnoreEnvironmentVariable, comparison)) + { + currentValue = value; + } + } + + return currentValue; + } + + private static TestApplicationPolicy GetConfigurationFilePolicy( + TestModule? module, + BuildOptions buildOptions, + IReadOnlyList effectiveArguments, + string workingDirectory) + { + int forwardedConfigurationFileCount = effectiveArguments.Count(argument => + IsOption( + argument, + TestCommandDefinition.MicrosoftTestingPlatform.ConfigFileOptionName, + allowValue: true)); + int configurationFileCount = + forwardedConfigurationFileCount + (buildOptions.PathOptions.ConfigFilePath is null ? 0 : 1); + if (configurationFileCount > 1) + { + return default; + } + + string? configurationFilePath = forwardedConfigurationFileCount == 1 + ? GetForwardedOptionValue( + effectiveArguments, + TestCommandDefinition.MicrosoftTestingPlatform.ConfigFileOptionName) + : buildOptions.PathOptions.ConfigFilePath; + if (configurationFilePath is not null && !Path.IsPathFullyQualified(configurationFilePath)) + { + configurationFilePath = Path.GetFullPath(configurationFilePath, workingDirectory); + } + + if (configurationFilePath is null && module is not null && !string.IsNullOrEmpty(module.TargetPath)) + { + configurationFilePath = Path.ChangeExtension( + Path.GetFullPath(module.TargetPath, workingDirectory), + TestConfigurationFileSuffix); + } + + if (configurationFilePath is null || !File.Exists(configurationFilePath)) + { + return default; + } + + try + { + using FileStream stream = File.OpenRead(configurationFilePath); + using JsonDocument document = JsonDocument.Parse( + stream, + new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + }); + + if (document.RootElement.ValueKind != JsonValueKind.Object || + HasDuplicatePropertyNames(document.RootElement) || + !TryGetProperty( + document.RootElement, + CommandLineOptionsConfigurationPropertyName, + out JsonElement commandLineOptions) || + commandLineOptions.ValueKind != JsonValueKind.Object) + { + return default; + } + + string? zeroTestsPolicy = + GetConfigurationOptionValue(commandLineOptions, ZeroTestsPolicyOptionName); + return new TestApplicationPolicy( + string.Equals( + zeroTestsPolicy, + ZeroTestsPolicyStrictArgument, + StringComparison.OrdinalIgnoreCase), + GetConfigurationOptionValue(commandLineOptions, IgnoreExitCodeOptionName)); + } + catch (Exception exception) when ( + exception is IOException or UnauthorizedAccessException or JsonException or ArgumentException or NotSupportedException) + { + // The test application reports configuration errors. Ignore this source so the SDK does not + // replace that diagnostic with a synthesized aggregate verdict. + Logger.LogTrace($"Failed to reconstruct test policy from '{configurationFilePath}': {exception}"); + return default; + } + } + + private static string? GetConfigurationOptionValue( + JsonElement commandLineOptions, + string canonicalOption) + { + if (!TryGetProperty(commandLineOptions, canonicalOption[2..], out JsonElement value)) + { + return null; + } + + if (value.ValueKind == JsonValueKind.Array) + { + JsonElement.ArrayEnumerator values = value.EnumerateArray(); + if (!values.MoveNext()) + { + return null; + } + + JsonElement singleValue = values.Current; + return values.MoveNext() ? null : GetConfigurationScalarValue(singleValue); + } + + return GetConfigurationScalarValue(value); + } + + private static string? GetConfigurationScalarValue(JsonElement value) + => value.ValueKind switch + { + JsonValueKind.Number or JsonValueKind.String or JsonValueKind.True or JsonValueKind.False + => value.ToString(), + _ => null, + }; + + private static bool TryGetProperty( + JsonElement element, + string propertyName, + out JsonElement value) + { + foreach (JsonProperty property in element.EnumerateObject()) + { + if (string.Equals(property.Name, propertyName, StringComparison.OrdinalIgnoreCase)) + { + value = property.Value; + return true; + } + } + + value = default; + return false; + } + + private static bool HasDuplicatePropertyNames(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement item in element.EnumerateArray()) + { + if (HasDuplicatePropertyNames(item)) + { + return true; + } + } + + return false; + } + + if (element.ValueKind != JsonValueKind.Object) + { + return false; + } + + var propertyNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (JsonProperty property in element.EnumerateObject()) + { + if (!propertyNames.Add(property.Name) || HasDuplicatePropertyNames(property.Value)) + { + return true; + } + } + + return false; + } + + private static void AddParsedArguments(string? arguments, List parsedArguments) + { + if (string.IsNullOrEmpty(arguments)) + { + return; + } + + try + { + parsedArguments.AddRange(SplitResponseFileLine(arguments)); + } + catch (FormatException) + { + // MTP reports malformed command lines. Ignore this source when reconstructing policy so + // the SDK does not replace the child parser's diagnostic with a synthesized verdict. + } + } + + private static string? GetForwardedOptionValue(IReadOnlyList arguments, string canonicalOption) + { + for (int i = 0; i < arguments.Count; i++) + { + string argument = arguments[i]; + if (!IsOption(argument, canonicalOption, allowValue: true)) + { + continue; + } + + int separatorIndex = argument.IndexOfAny('=', ':'); + if (separatorIndex >= 0) + { + return Unquote(argument[(separatorIndex + 1)..]); + } + + return i + 1 < arguments.Count ? Unquote(arguments[i + 1]) : null; + } + + return null; + } + + private static string Unquote(string value) + => value.Length >= 2 && + ((value[0] == '\'' && value[^1] == '\'') || + (value[0] == '"' && value[^1] == '"')) + ? value[1..^1] + : value; + + private static bool TryExpandResponseFileArguments( + IReadOnlyList arguments, + string workingDirectory, + HashSet recursionStack, + List expandedArguments) + { + foreach (string argument in arguments) + { + if (argument.Length <= 1 || argument[0] != '@') + { + expandedArguments.Add(argument); + continue; + } + + string fullPath = Path.GetFullPath(argument[1..], workingDirectory); + if (!recursionStack.Add(fullPath) || !File.Exists(fullPath)) + { + return false; + } + + try + { + string[] responseFileArguments = [.. + File.ReadAllLines(fullPath) + .Select(static line => line.Trim()) + .Where(static line => line.Length > 0 && line[0] != '#') + .SelectMany(SplitResponseFileLine)]; + + if (!TryExpandResponseFileArguments( + responseFileArguments, + workingDirectory, + recursionStack, + expandedArguments)) + { + return false; + } + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or FormatException) + { + return false; + } + finally + { + recursionStack.Remove(fullPath); + } + } + + return true; + } private static TestListFormat GetListTestsFormat(ParseResult parseResult, TestCommandDefinition.MicrosoftTestingPlatform definition) { diff --git a/src/Cli/dotnet/Commands/Test/MTP/Options.cs b/src/Cli/dotnet/Commands/Test/MTP/Options.cs index af1d571c01d0..d92b191fc0be 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Options.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Options.cs @@ -63,3 +63,7 @@ internal record BuildOptions( string? Device, bool ListDevices, IReadOnlyDictionary EnvironmentVariables); + +internal readonly record struct TestApplicationPolicy( + bool FailOnAllSkippedTests, + string? IgnoredExitCodes); diff --git a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs index 180288f826cf..fcca1d14b16d 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs @@ -294,7 +294,6 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode) int totalRetriedTests = 0; int totalRetriedExecutions = 0; int totalFlakyTests = 0; - bool anyAssemblyUnsuccessful = false; int failedAssembliesWithoutFailedTests = 0; foreach (TestProgressState assembly in assemblies) @@ -308,7 +307,6 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode) totalFlakyTests += assembly.FlakyTests; if (!assembly.Success) { - anyAssemblyUnsuccessful = true; if (assembly.FailedTests == 0) { failedAssembliesWithoutFailedTests++; @@ -316,49 +314,33 @@ private void AppendTestRunSummary(ITerminal terminal, int? exitCode) } } - bool notEnoughTests = totalTests < _options.MinimumExpectedTests; - bool allTestsWereSkipped = (totalTests == 0 && !_options.AllowZeroTests) - || (totalTests > 0 && totalTests == totalSkippedTests); - bool anyTestFailed = totalFailedTests > 0; - bool anyAssemblyFailed = anyAssemblyUnsuccessful || HasHandshakeFailure; - bool unexpectedNonZeroExitCode = exitCode is not null - && exitCode != ExitCode.Success - && exitCode != ExitCode.ZeroTests - && exitCode != ExitCode.MinimumExpectedTestsPolicyViolation; - bool runFailed = anyAssemblyFailed || anyTestFailed || notEnoughTests || allTestsWereSkipped || unexpectedNonZeroExitCode || _wasCancelled; + bool runFailed = exitCode is null || exitCode != ExitCode.Success; + bool zeroTestsFailure = exitCode == ExitCode.ZeroTests; + bool minimumExpectedTestsFailure = exitCode == ExitCode.MinimumExpectedTestsPolicyViolation; terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen); terminal.Append(CliCommandStrings.TestRunSummary); terminal.Append(' '); - if (_wasCancelled) + if (!runFailed) { - terminal.Append(CliCommandStrings.Aborted); + terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed)); } - else if (notEnoughTests) + else if (_wasCancelled) { - terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests)); + terminal.Append(CliCommandStrings.Aborted); } - else if (anyTestFailed || HasHandshakeFailure || unexpectedNonZeroExitCode) + else if (minimumExpectedTestsFailure && _options.MinimumExpectedTests > 0) { - // Handshake failures take precedence over "Zero tests ran": when an assembly failed to - // hand-shake we want the headline to reflect that the run failed, not that no tests ran - // (which would imply a benign empty run). We intentionally do NOT escalate the broader - // anyAssemblyFailed here, because a project that legitimately contains zero tests exits - // with ExitCodes.ZeroTests (non-zero) and would otherwise be misclassified as a failure. - terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed)); + terminal.Append(string.Format(CultureInfo.CurrentCulture, CliCommandStrings.MinimumExpectedTestsPolicyViolation, totalTests, _options.MinimumExpectedTests)); } - else if (allTestsWereSkipped) + else if (zeroTestsFailure) { terminal.Append(CliCommandStrings.ZeroTestsRan); } - else if (anyAssemblyFailed) - { - terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed)); - } else { - terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Passed)); + terminal.Append(string.Format(CultureInfo.CurrentCulture, "{0}!", CliCommandStrings.Failed)); } if (!_options.ShowAssembly && assemblies.Count == 1) diff --git a/src/Cli/dotnet/Commands/Test/MTP/TestApplicationActionQueue.cs b/src/Cli/dotnet/Commands/Test/MTP/TestApplicationActionQueue.cs index 3bbc60edac5b..0c5033a31e08 100644 --- a/src/Cli/dotnet/Commands/Test/MTP/TestApplicationActionQueue.cs +++ b/src/Cli/dotnet/Commands/Test/MTP/TestApplicationActionQueue.cs @@ -13,8 +13,11 @@ internal class TestApplicationActionQueue private readonly Channel _channel; private readonly Task[] _readers; private readonly CancellationToken _cancellationToken; + private readonly TestApplicationPolicy _fallbackTestApplicationPolicy; + private readonly BuildOptions _buildOptions; private int? _aggregateExitCode; + private readonly List _effectivePolicies = []; private readonly Lock _lock = new(); @@ -28,11 +31,14 @@ public TestApplicationActionQueue( CtrlCCancellationManager ctrlC, ArtifactPostProcessingManager artifactPostProcessingManager, TestRunPolicy testRunPolicy, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + TestApplicationPolicy fallbackTestApplicationPolicy = default) { _channel = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = false, SingleWriter = false }); _readers = new Task[degreeOfParallelism]; _cancellationToken = cancellationToken; + _fallbackTestApplicationPolicy = fallbackTestApplicationPolicy; + _buildOptions = buildOptions; for (int i = 0; i < degreeOfParallelism; i++) { @@ -50,6 +56,17 @@ public TestApplicationActionQueue( public void Enqueue(ParallelizableTestModuleGroupWithSequentialInnerModules testApplication) { + lock (_lock) + { + foreach (TestModule module in testApplication) + { + _effectivePolicies.Add( + MicrosoftTestingPlatformTestCommand.GetEffectiveTestApplicationPolicy( + module, + _buildOptions)); + } + } + if (_cancellationToken.IsCancellationRequested) { return; @@ -165,6 +182,49 @@ private async Task Read( } } + internal bool FailOnAllSkippedTests + { + get + { + lock (_lock) + { + IReadOnlyList effectivePolicies = GetEffectivePolicies(); + return ShouldFailOnAllSkippedTests(effectivePolicies); + } + } + } + + internal int ApplyExitCodeIgnorePolicy(int exitCode) + { + lock (_lock) + { + IReadOnlyList effectivePolicies = GetEffectivePolicies(); + return ApplyExitCodeIgnorePolicy(exitCode, effectivePolicies); + } + } + + internal static bool ShouldFailOnAllSkippedTests(IReadOnlyList effectivePolicies) + => effectivePolicies.Any(policy => + policy.FailOnAllSkippedTests && + MicrosoftTestingPlatformTestCommand.ApplyExitCodeIgnorePolicy( + ExitCode.ZeroTests, + policy.IgnoredExitCodes) != ExitCode.Success); + + internal static int ApplyExitCodeIgnorePolicy( + int exitCode, + IReadOnlyList effectivePolicies) + => effectivePolicies.All(policy => + MicrosoftTestingPlatformTestCommand.ApplyExitCodeIgnorePolicy( + exitCode, + policy.IgnoredExitCodes) == ExitCode.Success) + ? ExitCode.Success + : exitCode; + + private IReadOnlyList GetEffectivePolicies() + => _effectivePolicies.Count == 0 + ? [_fallbackTestApplicationPolicy] + : _effectivePolicies; + internal static int NormalizeExitCode(int result, bool hasFailureDuringDispose) { if (result == ExitCode.ZeroTests) diff --git a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs index 0403810d61d6..8eb1a76f5ce0 100644 --- a/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/GivenDotnetTestBuildsAndRunsTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Text.RegularExpressions; +using System.Xml.Linq; using Microsoft.DotNet.Cli.Commands; using Microsoft.DotNet.Cli.Commands.Test; using Microsoft.DotNet.Cli.Utils; @@ -21,6 +22,16 @@ public GivenDotnetTestBuildsAndRunsTests() private static string EnvironmentVariableReference(string name) => $"%{name}%"; + private static void MakeAllTestsSkipped(TestAsset testInstance) + { + string programPath = Path.Join(testInstance.Path, "Program.cs"); + File.WriteAllText( + programPath, + File.ReadAllText(programPath).Replace( + """new PassedTestNodeStateProperty("OK")""", + """new SkippedTestNodeStateProperty("OK skipped too!")""")); + } + [DataRow(TestingConstants.Debug)] [DataRow(TestingConstants.Release)] [TestMethod] @@ -46,6 +57,307 @@ public void RunTestProjectWithNoTests_ShouldReturnExitCodeZeroTests(string confi result.ExitCode.Should().Be(ExitCodes.ZeroTests); } + [TestMethod] + [DataRow("", false, ExitCodes.Success, "Passed!")] + [DataRow("allow-skipped", false, ExitCodes.Success, "Passed!")] + [DataRow("strict", false, ExitCodes.ZeroTests, "Zero tests ran")] + [DataRow("strict", true, ExitCodes.Success, "Passed!")] + public void RunAllSkippedTestsHonorsZeroTestsAndIgnoreExitCodePolicies( + string zeroTestsPolicy, + bool ignoreZeroTestsExitCode, + int expectedExitCode, + string expectedSummary) + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + + var arguments = new List { "-c", TestingConstants.Debug }; + if (zeroTestsPolicy.Length > 0) + { + arguments.Add("--zero-tests-policy"); + arguments.Add(zeroTestsPolicy); + } + + if (ignoreZeroTestsExitCode) + { + arguments.Add("--ignore-exit-code"); + arguments.Add(ExitCodes.ZeroTests.ToString()); + } + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute([.. arguments]); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain($"Test run summary: {expectedSummary}") + .And.Contain("total: 2") + .And.Contain("succeeded: 0") + .And.Contain("failed: 0") + .And.Contain("skipped: 2"); + + if (expectedExitCode == ExitCodes.Success) + { + result.StdOut.Should().NotContain("Test run completed with non-success exit code"); + } + else + { + result.StdOut.Should().Contain($"Test run completed with non-success exit code: {expectedExitCode}"); + } + } + + result.ExitCode.Should().Be(expectedExitCode); + } + + [TestMethod] + [DataRow("--zero-tests-policy\n'strict'", ExitCodes.ZeroTests, "Zero tests ran")] + [DataRow("--zero-tests-policy\n'strict'\n--ignore-exit-code\n'8'", ExitCodes.Success, "Passed!")] + public void RunAllSkippedTestsHonorsPoliciesFromResponseFile( + string responseFileContents, + int expectedExitCode, + string expectedSummary) + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + string responseFilePath = Path.Join(testInstance.Path, "options.rsp"); + File.WriteAllText(responseFilePath, responseFileContents); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug, $"@{responseFilePath}"); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut.Should().Contain($"Test run summary: {expectedSummary}"); + } + + result.ExitCode.Should().Be(expectedExitCode); + } + + [TestMethod] + public void RunAllSkippedTestsHonorsStrictPolicyFromExplicitConfigurationFile() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + string configurationFilePath = Path.Join(testInstance.Path, "strict.testconfig.json"); + File.WriteAllText( + configurationFilePath, + """{"commandLineOptions":{"zero-tests-policy":"strict"}}"""); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug, "--config-file", configurationFilePath); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut.Should().Contain("Test run summary: Zero tests ran"); + } + + result.ExitCode.Should().Be(ExitCodes.ZeroTests); + } + + [TestMethod] + public void RunNoTestsHonorsIgnoredExitCodeFromExplicitConfigurationFile() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectSolution", Guid.NewGuid().ToString()) + .WithSource(); + string configurationFilePath = Path.Join(testInstance.Path, "ignore.testconfig.json"); + File.WriteAllText( + configurationFilePath, + """{"commandLineOptions":{"ignore-exit-code":"8"}}"""); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug, "--config-file", configurationFilePath); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut.Should().Contain("Test run summary: Passed!"); + } + + result.ExitCode.Should().Be(ExitCodes.Success); + } + + [TestMethod] + public void RunAllSkippedTestsHonorsStrictPolicyFromDefaultConfigurationFile() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource() + .WithProjectChanges(project => + { + var ns = project.Root!.Name.Namespace; + project.Root.Add( + new XElement( + ns + "ItemGroup", + new XElement( + ns + "None", + new XAttribute("Update", "TestProject.testconfig.json"), + new XElement(ns + "CopyToOutputDirectory", "PreserveNewest")))); + }); + MakeAllTestsSkipped(testInstance); + File.WriteAllText( + Path.Join(testInstance.Path, "TestProject.testconfig.json"), + """{"commandLineOptions":{"zero-tests-policy":"strict"}}"""); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut.Should().Contain("Test run summary: Zero tests ran"); + } + + result.ExitCode.Should().Be(ExitCodes.ZeroTests); + } + + [TestMethod] + public void RunAllSkippedTestsHonorsLaunchProfilePolicyAndEnvironmentPrecedence() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + string propertiesDirectory = Path.Join(testInstance.Path, "Properties"); + Directory.CreateDirectory(propertiesDirectory); + File.WriteAllText( + Path.Join(propertiesDirectory, "launchSettings.json"), + """ + { + "profiles": { + "TestProject": { + "commandName": "Project", + "commandLineArgs": "--zero-tests-policy strict --ignore-exit-code 8", + "environmentVariables": { + "TESTINGPLATFORM_EXITCODE_IGNORE": "9" + } + } + } + } + """); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Zero tests ran") + .And.Contain($"Test run completed with non-success exit code: {ExitCodes.ZeroTests}"); + } + + result.ExitCode.Should().Be(ExitCodes.ZeroTests); + } + + [TestMethod] + [OSCondition(OperatingSystems.Windows)] + public void RunAllSkippedTestsHonorsCaseInsensitiveLaunchProfileEnvironmentOnWindows() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + string propertiesDirectory = Path.Join(testInstance.Path, "Properties"); + Directory.CreateDirectory(propertiesDirectory); + File.WriteAllText( + Path.Join(propertiesDirectory, "launchSettings.json"), + """ + { + "profiles": { + "TestProject": { + "commandName": "Project", + "commandLineArgs": "--zero-tests-policy strict --ignore-exit-code 8", + "environmentVariables": { + "testingplatform_exitcode_ignore": "9" + } + } + } + } + """); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute("-c", TestingConstants.Debug); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Zero tests ran") + .And.Contain($"Test run completed with non-success exit code: {ExitCodes.ZeroTests}"); + } + + result.ExitCode.Should().Be(ExitCodes.ZeroTests); + } + + [TestMethod] + [OSCondition(OperatingSystems.Linux | OperatingSystems.OSX)] + public void RunAllSkippedTestsHonorsCaseSensitiveLaunchProfileEnvironmentOnUnix() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + MakeAllTestsSkipped(testInstance); + string propertiesDirectory = Path.Join(testInstance.Path, "Properties"); + Directory.CreateDirectory(propertiesDirectory); + File.WriteAllText( + Path.Join(propertiesDirectory, "launchSettings.json"), + """ + { + "profiles": { + "TestProject": { + "commandName": "Project", + "commandLineArgs": "--zero-tests-policy strict", + "environmentVariables": { + "testingplatform_exitcode_ignore": "9" + } + } + } + } + """); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable("TESTINGPLATFORM_EXITCODE_IGNORE", "8") + .Execute("-c", TestingConstants.Debug); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut.Should().Contain("Test run summary: Passed!"); + } + + result.ExitCode.Should().Be(ExitCodes.Success); + } + + [TestMethod] + public void RunMinimumExpectedTestsViolationHonorsIgnoreExitCodePolicy() + { + TestAsset testInstance = TestAssetsManager.CopyTestAsset("TestProjectWithTests", Guid.NewGuid().ToString()) + .WithSource(); + + CommandResult result = new DotnetTestCommand(Log, disableNewOutput: false) + .WithWorkingDirectory(testInstance.Path) + .Execute( + "-c", TestingConstants.Debug, + "--minimum-expected-tests", "3", + "--ignore-exit-code", ExitCodes.MinimumExpectedTestsPolicyViolation.ToString()); + + if (!SdkTestContext.IsLocalized()) + { + result.StdOut + .Should().Contain("Test run summary: Passed!") + .And.Contain("total: 2") + .And.Contain("succeeded: 1") + .And.Contain("failed: 0") + .And.Contain("skipped: 1") + .And.NotContain("Minimum expected tests policy violation") + .And.NotContain("Test run completed with non-success exit code"); + } + + result.ExitCode.Should().Be(ExitCodes.Success); + } + [DataRow(TestingConstants.Debug)] [DataRow(TestingConstants.Release)] [TestMethod] diff --git a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs index 0e2ee7bf09bd..05ba1fb5bbfd 100644 --- a/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TerminalTestReporterTests.cs @@ -232,12 +232,15 @@ public void TestExecutionCompleted_WithAllowedZeroTests_PrintsPassingAssemblyAnd } [TestMethod] - public void TestExecutionCompleted_WithAllowedZeroTestsAndAllSelectedTestsSkipped_RemainsZeroTests() + [DataRow(TestExitCode.Success, "Passed!")] + [DataRow(TestExitCode.ZeroTests, "Zero tests ran")] + public void TestExecutionCompleted_WithAllSelectedTestsSkipped_MatchesFinalExitCode( + int exitCode, + string expectedSummary) { var capturingConsole = new CapturingConsole(); var options = new TerminalTestReporterOptions { - AllowZeroTests = true, AnsiMode = AnsiMode.SimpleAnsi, ShowProgress = false, ShowAssembly = true, @@ -255,9 +258,9 @@ public void TestExecutionCompleted_WithAllowedZeroTestsAndAllSelectedTestsSkippe exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.Success, outputData: null, errorData: null); - reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.Success); + reporter.TestExecutionCompleted(DateTimeOffset.UtcNow, exitCode); - StripAnsi(capturingConsole.GetOutput()).Should().Contain("Zero tests ran"); + StripAnsi(capturingConsole.GetOutput()).Should().Contain($"Test run summary: {expectedSummary}"); } [TestMethod] @@ -280,6 +283,27 @@ public void TestExecutionCompleted_WithAllowedZeroTestsAndUnexpectedNonZeroExit_ StripAnsi(capturingConsole.GetOutput()).Should().Contain("Test run summary: Failed!"); } + [TestMethod] + public void TestExecutionCompleted_WithForwardedMinimumExpectedTestsViolation_PrintsGenericFailure() + { + var capturingConsole = new CapturingConsole(); + var options = new TerminalTestReporterOptions + { + AnsiMode = AnsiMode.SimpleAnsi, + ShowProgress = false, + }; + + using var reporter = new TerminalTestReporter(capturingConsole, options); + reporter.TestExecutionStarted(DateTimeOffset.UtcNow, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + reporter.TestExecutionCompleted( + DateTimeOffset.UtcNow, + exitCode: Microsoft.DotNet.Cli.Commands.Test.ExitCode.MinimumExpectedTestsPolicyViolation); + + string output = StripAnsi(capturingConsole.GetOutput()); + output.Should().Contain("Test run summary: Failed!"); + output.Should().NotContain("minimum expected 0"); + } + [TestMethod] [DataRow(false, "Test run completed with non-success exit code: 5. The command-line arguments are invalid.")] [DataRow(true, "Test discovery completed with non-success exit code: 5. The command-line arguments are invalid.")] diff --git a/test/dotnet.Tests/CommandTests/Test/TestApplicationActionQueueTests.cs b/test/dotnet.Tests/CommandTests/Test/TestApplicationActionQueueTests.cs index 2fe3f28b016e..e82cae10fce4 100644 --- a/test/dotnet.Tests/CommandTests/Test/TestApplicationActionQueueTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TestApplicationActionQueueTests.cs @@ -18,4 +18,38 @@ public void NormalizeExitCode_ZeroTestsWithDisposeFailure_ReturnsGenericFailure( result.Should().Be(TestExitCode.GenericFailure); } + + [TestMethod] + public void ShouldFailOnAllSkippedTests_UsesStrictPoliciesThatDoNotIgnoreZeroTests() + { + TestApplicationActionQueue.ShouldFailOnAllSkippedTests( + [ + new(FailOnAllSkippedTests: false, IgnoredExitCodes: null), + new(FailOnAllSkippedTests: true, IgnoredExitCodes: "8"), + ]).Should().BeFalse(); + + TestApplicationActionQueue.ShouldFailOnAllSkippedTests( + [ + new(FailOnAllSkippedTests: false, IgnoredExitCodes: null), + new(FailOnAllSkippedTests: true, IgnoredExitCodes: null), + ]).Should().BeTrue(); + } + + [TestMethod] + public void ApplyExitCodeIgnorePolicy_RequiresEveryModuleToIgnoreTheExitCode() + { + TestApplicationActionQueue.ApplyExitCodeIgnorePolicy( + TestExitCode.MinimumExpectedTestsPolicyViolation, + [ + new(FailOnAllSkippedTests: false, IgnoredExitCodes: "9"), + new(FailOnAllSkippedTests: false, IgnoredExitCodes: "8;9"), + ]).Should().Be(TestExitCode.Success); + + TestApplicationActionQueue.ApplyExitCodeIgnorePolicy( + TestExitCode.MinimumExpectedTestsPolicyViolation, + [ + new(FailOnAllSkippedTests: false, IgnoredExitCodes: "9"), + new(FailOnAllSkippedTests: false, IgnoredExitCodes: null), + ]).Should().Be(TestExitCode.MinimumExpectedTestsPolicyViolation); + } } diff --git a/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs b/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs index 3a0eb72cff07..97b6436bef95 100644 --- a/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs +++ b/test/dotnet.Tests/CommandTests/Test/TestCommandParserTests.cs @@ -612,23 +612,78 @@ public void MTPCommandDetectsMinimumExpectedTestsInResponseFile() } [TestMethod] - [DataRow(false, 0, 0, true)] - [DataRow(true, 0, 0, false)] - [DataRow(false, 2, 2, true)] - [DataRow(true, 2, 2, true)] - [DataRow(true, 2, 1, false)] + [DataRow(false, false, 0, 0, true)] + [DataRow(true, false, 0, 0, false)] + [DataRow(false, false, 2, 2, false)] + [DataRow(false, true, 2, 2, true)] + [DataRow(true, true, 2, 2, true)] + [DataRow(true, false, 2, 2, false)] + [DataRow(true, true, 2, 1, false)] public void MTPCommandFailsOnlyForDisallowedEmptyOrAllSkippedRuns( bool isAffectedTestsMode, + bool failOnAllSkippedTests, int totalTests, int skippedTests, bool expectedFailure) { MicrosoftTestingPlatformTestCommand.ShouldFailForNoExecutedTests( isAffectedTestsMode, + failOnAllSkippedTests, totalTests, skippedTests).Should().Be(expectedFailure); } + [TestMethod] + public void MTPCommandReadsForwardedTestPolicies() + { + MicrosoftTestingPlatformTestCommand.IsStrictZeroTestsPolicy( + ["--zero-tests-policy", "strict"]).Should().BeTrue(); + MicrosoftTestingPlatformTestCommand.IsStrictZeroTestsPolicy( + ["--zero-tests-policy=STRICT"]).Should().BeTrue(); + MicrosoftTestingPlatformTestCommand.IsStrictZeroTestsPolicy( + ["-zero-tests-policy:strict"]).Should().BeTrue(); + MicrosoftTestingPlatformTestCommand.IsStrictZeroTestsPolicy( + ["--zero-tests-policy", "allow-skipped"]).Should().BeFalse(); + + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + ["--ignore-exit-code", "8;9"], + environmentValue: null).Should().Be("8;9"); + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + ["--ignore-exit-code", "8"], + environmentValue: "9").Should().Be("9"); + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + ["-ignore-exit-code:8"], + environmentValue: "").Should().Be("8"); + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + [], + environmentValue: null, + configurationValue: "8").Should().Be("8"); + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + ["--ignore-exit-code", "9"], + environmentValue: null, + configurationValue: "8").Should().Be("9"); + MicrosoftTestingPlatformTestCommand.GetEffectiveIgnoredExitCodes( + ["--ignore-exit-code", "9"], + environmentValue: "10", + configurationValue: "8").Should().Be("10"); + } + + [TestMethod] + [DataRow(8, null, 8)] + [DataRow(8, "8", 0)] + [DataRow(8, "2; +8 ;9", 0)] + [DataRow(8, "2;9", 8)] + [DataRow(9, "8", 9)] + public void MTPCommandAppliesIgnoreExitCodePolicy( + int exitCode, + string? ignoredExitCodes, + int expectedExitCode) + { + MicrosoftTestingPlatformTestCommand.ApplyExitCodeIgnorePolicy( + exitCode, + ignoredExitCodes).Should().Be(expectedExitCode); + } + [TestMethod] public void MTPCommandRejectsCollectTestMapWithMinimumExpectedTests() {