From 3987a6b4c4f39f0db6e0ef6b48cba9bbcc51eaa5 Mon Sep 17 00:00:00 2001 From: Bert Date: Fri, 28 Aug 2026 16:24:32 +0200 Subject: [PATCH 01/12] add command line options and configuration names --- global.json | 2 +- .../CoverletCommandLineOptionDefinitions.cs | 3 ++ .../CoverletExtensionCommandLineProvider.cs | 50 +++++++++++++++++++ .../CommandLine/CoverletOptionNames.cs | 3 ++ .../CoverletExtensionConfiguration.cs | 14 ++++++ .../Configuration/CoverletMTPConstants.cs | 3 ++ 6 files changed, 74 insertions(+), 1 deletion(-) diff --git a/global.json b/global.json index 7325398c5..a0703e84d 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.302", + "version": "10.0.400", "rollForward": "latestFeature" }, "test": { diff --git a/src/coverlet.MTP/CommandLine/CoverletCommandLineOptionDefinitions.cs b/src/coverlet.MTP/CommandLine/CoverletCommandLineOptionDefinitions.cs index 185a774ed..f0c5312ff 100644 --- a/src/coverlet.MTP/CommandLine/CoverletCommandLineOptionDefinitions.cs +++ b/src/coverlet.MTP/CommandLine/CoverletCommandLineOptionDefinitions.cs @@ -30,6 +30,9 @@ public static IReadOnlyCollection GetAllOptions() new CommandLineOption(CoverletOptionNames.SkipAutoProps, "Skip auto-implemented properties.", ArgumentArity.Zero, isHidden: false), new CommandLineOption(CoverletOptionNames.DoesNotReturnAttribute, "Attributes that mark methods as not returning.", ArgumentArity.ZeroOrMore, isHidden: false), new CommandLineOption(CoverletOptionNames.ExcludeAssembliesWithoutSources, "Exclude assemblies without source code.", ArgumentArity.ZeroOrOne, isHidden: false), + new CommandLineOption(CoverletOptionNames.Threshold, "Coverage threshold percentage.", ArgumentArity.ExactlyOne, isHidden: false), + new CommandLineOption(CoverletOptionNames.ThresholdType, "Type of coverage threshold (line, branch, method).", ArgumentArity.OneOrMore, isHidden: false), + new CommandLineOption(CoverletOptionNames.ThresholdStat, "Statistic for coverage threshold (total, average, minimum).", ArgumentArity.ExactlyOne, isHidden: false), ]; } } diff --git a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs index 0afac6380..fb5fce063 100644 --- a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs +++ b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs @@ -72,6 +72,56 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'.")); } } + + if (commandOption.Name == CoverletOptionNames.Threshold) + { + if (arguments.Length == 0) + { + return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'.")); + } + if (arguments.Length > 1) + { + return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'.")); + } + if (!int.TryParse(arguments[0], out int thresholdValue) || thresholdValue < 0 || thresholdValue > 100) + { + return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'. It must be an integer between 0 and 100.")); + } + } + + // Validate ThresholdType option to ensure it has a valid value. + if (commandOption.Name == CoverletOptionNames.ThresholdType) + { + if (arguments.Length == 0) + { + return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'.")); + } + if (arguments.Length > 1) + { + return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'.")); + } + if (!arguments[0].Contains("line") && !arguments[0].Contains("branch") && !arguments[0].Contains("method")) + { + return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}' (line, branch, method).")); + } + } + + // Validate the ThresholdStat option to ensure it has a valid value + if (commandOption.Name == CoverletOptionNames.ThresholdStat) + { + if (arguments.Length == 0) + { + return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'.")); + } + if (arguments.Length > 1) + { + return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'.")); + } + if (!arguments[0].Contains("total") && !arguments[0].Contains("average") && !arguments[0].Contains("minimum")) + { + return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'(total, average, minimum).")); + } + } return ValidationResult.ValidTask; } diff --git a/src/coverlet.MTP/CommandLine/CoverletOptionNames.cs b/src/coverlet.MTP/CommandLine/CoverletOptionNames.cs index d6c2d0557..9d0b22e43 100644 --- a/src/coverlet.MTP/CommandLine/CoverletOptionNames.cs +++ b/src/coverlet.MTP/CommandLine/CoverletOptionNames.cs @@ -18,4 +18,7 @@ internal static class CoverletOptionNames public const string SkipAutoProps = "coverlet-skip-auto-props"; public const string DoesNotReturnAttribute = "coverlet-does-not-return-attribute"; public const string ExcludeAssembliesWithoutSources = "coverlet-exclude-assemblies-without-sources"; + public const string Threshold = "coverlet-threshold"; + public const string ThresholdType = "coverlet-threshold-type"; + public const string ThresholdStat = "coverlet-threshold-stat"; } diff --git a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs index 4584f3535..5217448e1 100644 --- a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs +++ b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs @@ -1,6 +1,8 @@ // Copyright (c) Toni Solarin-Sodara // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Coverlet.Core.Enums; + namespace Coverlet.MTP.Configuration; /// @@ -17,6 +19,18 @@ internal class CoverletExtensionConfiguration /// Test module /// public string? TestModule { get; set; } + /// + /// Threshold for coverage percentage. If the coverage is below this threshold, the build will fail. + /// + public int? threshold { get; set; } + + /// + /// define which coverage statistic (Average|Minimum|Total) to use for threshold comparison. If not specified, the default is Total. If the coverage is below this threshold, the build will fail. + /// + public ThresholdStatistic thresholdStat { get; set; } + + // Coverage type thresholds - array of coverage types to apply the threshold to. If not specified, the default is line coverage. If the coverage is below this threshold, the build will fail. + public List thresholdTypeFlagValues { get; set; } = ["line"]; // Coverage parameters public string[]? IncludeFilters { get; set; } diff --git a/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs b/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs index f80e5edbe..beb5aae1e 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs @@ -31,4 +31,7 @@ internal static class CoverletMTPConstants public const string DeterministicReportKey = "DeterministicReport"; public const string ExcludeAssembliesWithoutSourcesKey = "ExcludeAssembliesWithoutSources"; public const string DisableManagedInstrumentationRestoreKey = "DisableManagedInstrumentationRestore"; + public const string ThresholdKey = "Threshold"; + public const string ThresholdStatKey = "ThresholdStat"; + public const string ThresholdTypeFlagValuesKey = "ThresholdType"; } From ea52c1c1ff31753cc5db3058d2a1d74fec8580f8 Mon Sep 17 00:00:00 2001 From: Bert Date: Sat, 29 Aug 2026 11:25:28 +0200 Subject: [PATCH 02/12] update documentation and parser --- Documentation/Coverlet.MTP.Integration.md | 36 +++++++++++-------- Documentation/DriversFeatures.md | 4 +-- .../CoverletExtensionConfiguration.cs | 6 ++-- .../Configuration/CoverletMTPConstants.cs | 8 ++++- .../Configuration/CoverletMTPSettings.cs | 8 ++++- .../CoverletMTPSettingsParser.cs | 29 +++++++++++++++ .../CoverletMTPCommandLineTests.cs | 3 ++ 7 files changed, 72 insertions(+), 22 deletions(-) diff --git a/Documentation/Coverlet.MTP.Integration.md b/Documentation/Coverlet.MTP.Integration.md index 358dd7ab7..cfe605e28 100644 --- a/Documentation/Coverlet.MTP.Integration.md +++ b/Documentation/Coverlet.MTP.Integration.md @@ -87,6 +87,9 @@ dotnet exec --help | `--coverlet-skip-auto-props` | Skip auto-implemented properties. (default: `false`) | | `--coverlet-does-not-return-attribute ` | Attributes that mark methods as not returning. Can be specified multiple times. (default: `none`) | | `--coverlet-exclude-assemblies-without-sources ` | Exclude assemblies without source code. Values: `MissingAll`, `MissingAny`, `None`. (default: `None`) | +| `--threshold ` | Exits with error if the coverage % is below value | +| `--threshold-type ` | Coverage type to apply the threshold to. [default: `line`, `branch`, `method`] | +| `--threshold-stat ` | Coverage statistic used to enforce the threshold value. [default: Minimum] | > [!TIP] > If you encounter instrumentation failures like "The process cannot access the file ... because it is being used by another process", try setting `--coverlet-exclude-assemblies-without-sources MissingAll` (or in a config file: `"ExcludeAssembliesWithoutSources": "MissingAll"`) to skip assemblies without sources and reduce access conflicts. @@ -167,21 +170,21 @@ The `testconfig.json` format is the standard configuration file for Microsoft Te | Key | Type | Description | | :--- | :--- | :---------- | -| `include` | string | Comma-separated include filters (e.g., `[MyApp.*]*`) | -| `includeDirectory` | string | Comma-separated additional directories for sources | -| `exclude` | string | Comma-separated exclude filters (e.g., `[*.Tests]*`) | -| `excludeByFile` | string | Comma-separated glob patterns for source file exclusion | -| `excludeByAttribute` | string | Comma-separated attributes to exclude | -| `format` | string | Comma-separated output formats (default: `cobertura`) | -| `useSourceLink` | bool | Enable SourceLink support | -| `singleHit` | bool | Limit hits to one per location | -| `includeTestAssembly` | bool | Include test assembly in coverage | -| `skipAutoProps` | bool | Skip auto-implemented properties | -| `doesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods | -| `deterministicReport` | bool | Generate deterministic reports | -| `excludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` | -| `disableManagedInstrumentationRestore` | bool | Disable managed instrumentation restore | -| `mergeWith` | string | Path to existing coverage file to merge with | +| `Include` | string | Comma-separated include filters (e.g., `[MyApp.*]*`) | +| `IncludeDirectory` | string | Comma-separated additional directories for sources | +| `Exclude` | string | Comma-separated exclude filters (e.g., `[*.Tests]*`) | +| `ExcludeByFile` | string | Comma-separated glob patterns for source file exclusion | +| `ExcludeByAttribute` | string | Comma-separated attributes to exclude | +| `Format` | string | Comma-separated output formats (default: `cobertura`) | +| `UseSourceLink` | bool | Enable SourceLink support | +| `SingleHit` | bool | Limit hits to one per location | +| `IncludeTestAssembly` | bool | Include test assembly in coverage | +| `SkipAutoProps` | bool | Skip auto-implemented properties | +| `DoesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods | +| `DeterministicReport` | bool | Generate deterministic reports | +| `ExcludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` | +| `DisableManagedInstrumentationRestore` | bool | Disable managed instrumentation restore | +| `MergeWith` | string | Path to existing coverage file to merge with | > [!NOTE] > Keys in `testconfig.json` use **camelCase** (e.g., `excludeByAttribute`), following the Microsoft Testing Platform convention. @@ -236,6 +239,9 @@ The legacy `coverlet.mtp.appsettings.json` format is still supported for backwar | `DoesNotReturnAttribute` | string | Comma-separated attributes marking non-returning methods | | `DeterministicReport` | bool | Generate deterministic reports | | `ExcludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` (default: `MissingAll`) | +| `Threshold` | int | Exits with error if the code coverage [0..100%] is below value | +| `ThresholdType` | string | Comma-separated coverage type to apply the Threshold to. [default: `line`, `branch`, `method`] | +| `ThresholdStat` | string | Coverage statistic used to enforce the threshold value. [default: `Minimum`, `Average`, `Total`] | **Example `coverlet.mtp.appsettings.json`:** diff --git a/Documentation/DriversFeatures.md b/Documentation/DriversFeatures.md index e77429482..cfae0bd99 100644 --- a/Documentation/DriversFeatures.md +++ b/Documentation/DriversFeatures.md @@ -9,11 +9,11 @@ In the table below we keep track of main differences: | Feature | MSBuild | .NET Tool | VS DataCollector | MTP Extension | |:-----------------------------------|:--------------|:-------------|:------------------|:--------------| | .NET Core support(>= 8.0) | Yes | Yes | Yes | Yes | -| .NET Framework support(>= 4.7.2) | Yes | Yes | Yes(since 3.0.0) | No | +| .NET Framework support(>= 4.7.2) | Yes | Yes | Yes(since 3.0.0) | Yes | | Show result on console | Yes | Yes | No | Yes | | Deterministic reports output folder| Yes | Yes | No | No | | Merge reports | Yes | Yes | No | No | -| Coverage threshold validation | Yes | Yes | No | No | +| Coverage threshold validation | Yes | Yes | No | Yes | | Deterministic build support | Yes | No | Yes | No | > [!TIP] diff --git a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs index 5217448e1..6524d83fa 100644 --- a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs +++ b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs @@ -22,15 +22,15 @@ internal class CoverletExtensionConfiguration /// /// Threshold for coverage percentage. If the coverage is below this threshold, the build will fail. /// - public int? threshold { get; set; } + public int? Threshold { get; set; } /// /// define which coverage statistic (Average|Minimum|Total) to use for threshold comparison. If not specified, the default is Total. If the coverage is below this threshold, the build will fail. /// - public ThresholdStatistic thresholdStat { get; set; } + public ThresholdStatistic ThresholdStat { get; set; } // Coverage type thresholds - array of coverage types to apply the threshold to. If not specified, the default is line coverage. If the coverage is below this threshold, the build will fail. - public List thresholdTypeFlagValues { get; set; } = ["line"]; + public List ThresholdType { get; set; } = ["line"]; // Coverage parameters public string[]? IncludeFilters { get; set; } diff --git a/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs b/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs index beb5aae1e..d5b7d7542 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPConstants.cs @@ -1,6 +1,8 @@ // Copyright (c) Toni Solarin-Sodara // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Coverlet.Core.Enums; + namespace Coverlet.MTP.Configuration; /// @@ -12,6 +14,10 @@ internal static class CoverletMTPConstants public const string DefaultReportFormat = "cobertura"; public const string DefaultFileName = "coverage"; public const string DefaultExcludeFilter = "[coverlet.*]*"; + + public const string DefaultThresholdType = "line"; + + public const ThresholdStatistic DefaultThresholdStat = ThresholdStatistic.Minimum; public const string ConfigFileName = "coverlet.mtp.appsettings.json"; // Configuration keys for IConfiguration binding @@ -33,5 +39,5 @@ internal static class CoverletMTPConstants public const string DisableManagedInstrumentationRestoreKey = "DisableManagedInstrumentationRestore"; public const string ThresholdKey = "Threshold"; public const string ThresholdStatKey = "ThresholdStat"; - public const string ThresholdTypeFlagValuesKey = "ThresholdType"; + public const string ThresholdTypeKey = "ThresholdType"; } diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs index 1c1568a6a..ca2541d2b 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Text; +using Coverlet.Core.Enums; namespace Coverlet.MTP.Configuration; @@ -33,7 +34,9 @@ public class CoverletMTPSettings public bool DeterministicReport { get; set; } public string ExcludeAssembliesWithoutSources { get; set; } = "MissingAll"; public bool DisableManagedInstrumentationRestore { get; set; } - + public int Threshold { get; set; } + internal ThresholdStatistic ThresholdStat { get; set; } = ThresholdStatistic.Minimum; + public List ThresholdType { get; set; } = ["line"]; public override string ToString() { var builder = new StringBuilder(); @@ -51,6 +54,9 @@ public override string ToString() builder.AppendFormat("DoesNotReturnAttributes: '{0}', ", string.Join(",", DoesNotReturnAttributes)); builder.AppendFormat("DeterministicReport: '{0}', ", DeterministicReport); builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}'", ExcludeAssembliesWithoutSources); + builder.AppendFormat("Threshold: '{0}', ", Threshold); + builder.AppendFormat("ThresholdStat: '{0}', ", ThresholdStat); + builder.AppendFormat("ThresholdType: '{0}'", string.Join(",", ThresholdType)); return builder.ToString(); } } diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs index 44b91364f..de2dbc179 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs @@ -1,6 +1,7 @@ // Copyright (c) Toni Solarin-Sodara // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Coverlet.Core.Enums; using Microsoft.Extensions.Configuration; namespace Coverlet.MTP.Configuration; @@ -45,6 +46,8 @@ public static CoverletMTPSettings Parse(IConfiguration? configuration, string te settings.ExcludeAssembliesWithoutSources = section[CoverletMTPConstants.ExcludeAssembliesWithoutSourcesKey] ?? "MissingAll"; settings.DisableManagedInstrumentationRestore = ParseBoolValue(section, CoverletMTPConstants.DisableManagedInstrumentationRestoreKey); settings.ReportFormats = ParseReportFormats(section); + settings.ThresholdStat = ParseThresholdStat(section); + settings.ThresholdType = ParseThresholdType(section); return settings; } @@ -62,6 +65,32 @@ private static string[] ParseExcludeFilters(IConfigurationSection section) return [CoverletMTPConstants.DefaultExcludeFilter, .. filters]; } + private static List ParseThresholdType(IConfigurationSection section) + { + List types = []; + types.AddRange(ParseArrayValue(section, CoverletMTPConstants.ThresholdTypeKey)); + return types.Count == 0 ? [CoverletMTPConstants.DefaultThresholdType] : types; + } + + private static ThresholdStatistic ParseThresholdStat(IConfigurationSection section) + { + ThresholdStatistic type = CoverletMTPConstants.DefaultThresholdStat; + + switch (section[CoverletMTPConstants.ThresholdStatKey]?.ToLowerInvariant()) + { + case "minimum": + type = ThresholdStatistic.Minimum; + break; + case "average": + type = ThresholdStatistic.Average; + break; + case "total": + type = ThresholdStatistic.Total; + break; + } + return type; + } + private static string[] ParseArrayValue(IConfigurationSection section, string key) { string? value = section[key]; diff --git a/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs b/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs index 1602c47c7..edf934271 100644 --- a/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs +++ b/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs @@ -276,6 +276,9 @@ public void GetCommandLineOptionsReturnsAllExpectedOptions() CoverletOptionNames.SkipAutoProps, CoverletOptionNames.DoesNotReturnAttribute, CoverletOptionNames.ExcludeAssembliesWithoutSources, + CoverletOptionNames.Threshold, + CoverletOptionNames.ThresholdType, + CoverletOptionNames.ThresholdStat }; Assert.Equal(expectedOptions.Length, options.Count); From 9c0c435f617fa7728e8929edde68b6fff6496774 Mon Sep 17 00:00:00 2001 From: Bert Date: Sun, 30 Aug 2026 10:10:19 +0200 Subject: [PATCH 03/12] add DisplayThresholdSummaryAsync and CoverageThresholdFailed --- .../Collector/CollectorExtension.cs | 134 +++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/src/coverlet.MTP/Collector/CollectorExtension.cs b/src/coverlet.MTP/Collector/CollectorExtension.cs index 3ff0c3044..65a358f1b 100644 --- a/src/coverlet.MTP/Collector/CollectorExtension.cs +++ b/src/coverlet.MTP/Collector/CollectorExtension.cs @@ -4,9 +4,11 @@ #if NETSTANDARD2_0 using System.Diagnostics; #endif +using System.Globalization; using System.Text; using Coverlet.Core; using Coverlet.Core.Abstractions; +using Coverlet.Core.Enums; using Coverlet.Core.Helpers; using Coverlet.Core.Symbols; using Coverlet.MTP.CommandLine; @@ -487,6 +489,117 @@ await _outputDisplay.DisplayAsync( cancellation).ConfigureAwait(false); } + /// + /// Displays the configured coverage thresholds and their evaluation results. + /// + private async Task DisplayThresholdSummaryAsync(CoverageResult result, CancellationToken cancellation) + { + if (!_configuration.Threshold.HasValue) + { + return; + } + + ThresholdStatistic thresholdStat = _configuration.ThresholdStat; + Dictionary thresholdValues = BuildThresholdValues( + _configuration.ThresholdType, + _configuration.Threshold.Value); + ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat); + + var summary = new StringBuilder(); + summary.AppendLine(); + summary.AppendLine(" Coverage Threshold Results:"); + foreach (KeyValuePair thresholdValue in thresholdValues) + { + ThresholdTypeFlags type = thresholdValue.Key; + double threshold = thresholdValue.Value; + double coverage = GetThresholdCoverage(result, type, thresholdStat); + string comparison = (belowThreshold & type) == ThresholdTypeFlags.None ? ">=" : "<"; + summary.AppendLine( + $" {thresholdStat} - {type} ({GetThresholdScopeDescription(thresholdStat)}): " + + $"{coverage.ToString("F1", CultureInfo.InvariantCulture)}% {comparison} " + + $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold"); + } + + if (belowThreshold != ThresholdTypeFlags.None) + { + summary.AppendLine(); + foreach (KeyValuePair thresholdValue in thresholdValues.Where(pair => (belowThreshold & pair.Key) != ThresholdTypeFlags.None)) + { + ThresholdTypeFlags type = thresholdValue.Key; + double threshold = thresholdValue.Value; + summary.AppendLine( + $" The {thresholdStat.ToString().ToLowerInvariant()} {type.ToString().ToLowerInvariant()} coverage is below the specified " + + $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold."); + } + } + + await _outputDisplay.DisplayAsync( + this, + new TextOutputDeviceData(summary.ToString()), + cancellation).ConfigureAwait(false); + } + + private static Dictionary BuildThresholdValues(IEnumerable thresholdTypes, double threshold) + { + var values = new Dictionary(); + foreach (string thresholdType in thresholdTypes) + { + values[ParseThresholdType(thresholdType)] = threshold; + } + + return values; + } + + private static ThresholdTypeFlags ParseThresholdType(string thresholdType) => + thresholdType.Trim().ToLowerInvariant() switch + { + "line" => ThresholdTypeFlags.Line, + "branch" => ThresholdTypeFlags.Branch, + "method" => ThresholdTypeFlags.Method, + _ => throw new InvalidOperationException($"Invalid threshold type '{thresholdType}'. Valid values are line, branch, and method.") + }; + + private static double GetThresholdCoverage(CoverageResult result, ThresholdTypeFlags thresholdType, ThresholdStatistic thresholdStat) + { + if (thresholdStat == ThresholdStatistic.Minimum) + { + return result.Modules.Values + .Select(module => GetCoveragePercent(module, thresholdType)) + .DefaultIfEmpty(0) + .Min(); + } + + CoverageDetails coverage = thresholdType switch + { + ThresholdTypeFlags.Line => CoverageSummary.CalculateLineCoverage(result.Modules), + ThresholdTypeFlags.Branch => CoverageSummary.CalculateBranchCoverage(result.Modules), + ThresholdTypeFlags.Method => CoverageSummary.CalculateMethodCoverage(result.Modules), + _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required.") + }; + + return thresholdStat == ThresholdStatistic.Average + ? coverage.AverageModulePercent + : coverage.Percent; + } + + private static double GetCoveragePercent(Documents module, ThresholdTypeFlags thresholdType) => + thresholdType switch + { + ThresholdTypeFlags.Line => CoverageSummary.CalculateLineCoverage(module).Percent, + ThresholdTypeFlags.Branch => CoverageSummary.CalculateBranchCoverage(module).Percent, + ThresholdTypeFlags.Method => CoverageSummary.CalculateMethodCoverage(module).Percent, + _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required.") + }; + + private static string GetThresholdScopeDescription(ThresholdStatistic thresholdStat) => + thresholdStat switch + { + ThresholdStatistic.Total => "Total over Module", + ThresholdStatistic.Average => "Average per Module", + ThresholdStatistic.Minimum => "Minimum per Module", + _ => throw new ArgumentOutOfRangeException(nameof(thresholdStat), thresholdStat, null) + }; + /// /// Displays generated report paths to output device. /// @@ -546,13 +659,30 @@ private async Task GenerateReportsAsync(CoverageResult result, CancellationToken // Display results await DisplayGeneratedReportsAsync(generatedReports, cancellation); - // Display coverage summary table after the file artifacts list + // Display code coverage summary table after the file artifacts list await DisplayCoverageSummaryAsync(result, cancellation); // Display console-type report output (e.g. teamcity) directly to the output device await DisplayConsoleReportOutputsAsync(consoleOutputs, cancellation); - } + // Display threshold metric summary table after the file artifacts list + await DisplayThresholdSummaryAsync(result, cancellation); + + // Exitcode `CoverageThresholdFailed = 14` shall be set if any threshold is not met + if (_configuration.Threshold.HasValue) + { + ThresholdStatistic thresholdStat = _configuration.ThresholdStat; + Dictionary thresholdValues = BuildThresholdValues( + _configuration.ThresholdType, + _configuration.Threshold.Value); + ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat); + if (belowThreshold != ThresholdTypeFlags.None) + { + _logger.LogError("Coverage thresholds not met. Setting exit code for Microsoft Testing Framework to 14."); + Environment.ExitCode = 14; + } + } + } private string GetHitsFilePath() { // The hits file is in the same directory as the instrumented module From 5cdf325292b4b9b1e429609913a083f8ae1a6ada Mon Sep 17 00:00:00 2001 From: Bert Date: Sun, 30 Aug 2026 11:05:50 +0200 Subject: [PATCH 04/12] add unit tests for failure threshold --- .../Collector/CollectorExtension.cs | 3 + .../CoverletExtensionCommandLineProvider.cs | 41 ++++-- .../Configuration/CoverageConfiguration.cs | 36 ++++++ .../Configuration/CoverletMTPSettings.cs | 2 +- .../CoverletMTPSettingsParser.cs | 4 + .../CollectorExtensionThresholdTests.cs | 117 ++++++++++++++++++ .../CoverletMTPCommandLineTests.cs | 47 +++++++ .../CoverletMTPSettingsParserTests.cs | 20 +++ .../Configuration/CoverletMTPSettingsTests.cs | 9 +- 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs diff --git a/src/coverlet.MTP/Collector/CollectorExtension.cs b/src/coverlet.MTP/Collector/CollectorExtension.cs index 65a358f1b..693c4719f 100644 --- a/src/coverlet.MTP/Collector/CollectorExtension.cs +++ b/src/coverlet.MTP/Collector/CollectorExtension.cs @@ -139,6 +139,9 @@ Task ITestHostProcessLifetimeHandler.BeforeTestHostProcessStartAsync(Cancellatio _configuration.SkipAutoProps = config.SkipAutoProps; _configuration.formats = config.GetOutputFormats(); _configuration.FilePrefix = config.GetFilePrefix(); + _configuration.Threshold = config.GetThreshold(); + _configuration.ThresholdStat = config.GetThresholdStatistic(); + _configuration.ThresholdType = config.GetThresholdTypes(); _configuration.UseSourceLink = false; _logger.LogVerbose($"Test module path: {_testModulePath}"); diff --git a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs index fb5fce063..440c458d5 100644 --- a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs +++ b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs @@ -25,6 +25,26 @@ public CoverletExtensionCommandLineProvider(IExtension extension) public IReadOnlyCollection GetCommandLineOptions() => CoverletCommandLineOptionDefinitions.GetAllOptions(); +#if NETSTANDARD2_0 + private static bool TryParseEnum(Type enumType, string value, bool ignoreCase, out object? result) + { + try + { + result = Enum.Parse(enumType, value, ignoreCase); + return Enum.IsDefined(enumType, result); + } + catch (ArgumentException) + { + result = null; + return false; + } + } +#else + private static bool TryParseEnum(Type enumType, string value, bool ignoreCase, out object? result) + { + return Enum.TryParse(enumType, value, ignoreCase, out result) && Enum.IsDefined(enumType, result); + } +#endif public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) { if (commandOption.Name == CoverletOptionNames.Formats) @@ -92,17 +112,9 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com // Validate ThresholdType option to ensure it has a valid value. if (commandOption.Name == CoverletOptionNames.ThresholdType) { - if (arguments.Length == 0) + if (arguments.Length == 0 || arguments.SelectMany(value => value.Split(',')).Any(value => !IsThresholdType(value))) { - return Task.FromResult(ValidationResult.Invalid($"At least one value must be specified for '{commandOption.Name}'.")); - } - if (arguments.Length > 1) - { - return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'.")); - } - if (!arguments[0].Contains("line") && !arguments[0].Contains("branch") && !arguments[0].Contains("method")) - { - return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}' (line, branch, method).")); + return Task.FromResult(ValidationResult.Invalid($"The value for '{commandOption.Name}' must be line, branch, or method.")); } } @@ -117,7 +129,8 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com { return Task.FromResult(ValidationResult.Invalid($"Only one value is allowed for '{commandOption.Name}'.")); } - if (!arguments[0].Contains("total") && !arguments[0].Contains("average") && !arguments[0].Contains("minimum")) + + if (!TryParseEnum(typeof(Coverlet.Core.Enums.ThresholdStatistic), arguments[0], ignoreCase: true, out object? thresholdStatistic)) { return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'(total, average, minimum).")); } @@ -125,6 +138,11 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com return ValidationResult.ValidTask; } + private static bool IsThresholdType(string value) => + value.Trim().Equals("line", StringComparison.OrdinalIgnoreCase) || + value.Trim().Equals("branch", StringComparison.OrdinalIgnoreCase) || + value.Trim().Equals("method", StringComparison.OrdinalIgnoreCase); + /// /// Validates that the file prefix is a safe filename segment without path traversal risks. /// @@ -166,4 +184,3 @@ public Task ValidateCommandLineOptionsAsync(Microsoft.Testing. return ValidationResult.ValidTask; } } - diff --git a/src/coverlet.MTP/Configuration/CoverageConfiguration.cs b/src/coverlet.MTP/Configuration/CoverageConfiguration.cs index c75d0363a..236242fea 100644 --- a/src/coverlet.MTP/Configuration/CoverageConfiguration.cs +++ b/src/coverlet.MTP/Configuration/CoverageConfiguration.cs @@ -1,6 +1,7 @@ // Copyright (c) Toni Solarin-Sodara // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Coverlet.Core.Enums; using Coverlet.MTP.CommandLine; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Logging; @@ -305,6 +306,41 @@ public string[] GetIncludeDirectories() public bool SkipAutoProps => GetBoolOptionWithDefault(CoverletOptionNames.SkipAutoProps, _configFileSettings?.SkipAutoProps ?? false); + public int? GetThreshold() + { + if (_commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.Threshold, out string[]? values)) + { + return int.Parse(values[0]); + } + + return _configFileSettings?.Threshold; + } + + public ThresholdStatistic GetThresholdStatistic() + { +#if NETSTANDARD2_0 +#pragma warning disable CA2263 // Generic overload not available on netstandard2.0 + return _commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.ThresholdStat, out string[]? values) + ? (ThresholdStatistic)Enum.Parse(typeof(ThresholdStatistic), values[0], ignoreCase: true) + : _configFileSettings?.ThresholdStat ?? CoverletMTPConstants.DefaultThresholdStat; +#pragma warning restore CA2263 +#else + return _commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.ThresholdStat, out string[]? values) + ? Enum.Parse(values[0], ignoreCase: true) + : _configFileSettings?.ThresholdStat ?? CoverletMTPConstants.DefaultThresholdStat; +#endif + } + + public List GetThresholdTypes() + { + if (_commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.ThresholdType, out string[]? values)) + { + return [.. values.SelectMany(value => value.Split(',')).Select(value => value.Trim())]; + } + + return _configFileSettings?.ThresholdType ?? [CoverletMTPConstants.DefaultThresholdType]; + } + public string[] GetDoesNotReturnAttributes() { // Priority 1: Explicit command-line option diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs index ca2541d2b..4f3d29180 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs @@ -34,7 +34,7 @@ public class CoverletMTPSettings public bool DeterministicReport { get; set; } public string ExcludeAssembliesWithoutSources { get; set; } = "MissingAll"; public bool DisableManagedInstrumentationRestore { get; set; } - public int Threshold { get; set; } + public int? Threshold { get; set; } internal ThresholdStatistic ThresholdStat { get; set; } = ThresholdStatistic.Minimum; public List ThresholdType { get; set; } = ["line"]; public override string ToString() diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs index de2dbc179..f3af19834 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs @@ -46,6 +46,7 @@ public static CoverletMTPSettings Parse(IConfiguration? configuration, string te settings.ExcludeAssembliesWithoutSources = section[CoverletMTPConstants.ExcludeAssembliesWithoutSourcesKey] ?? "MissingAll"; settings.DisableManagedInstrumentationRestore = ParseBoolValue(section, CoverletMTPConstants.DisableManagedInstrumentationRestoreKey); settings.ReportFormats = ParseReportFormats(section); + settings.Threshold = ParseThreshold(section); settings.ThresholdStat = ParseThresholdStat(section); settings.ThresholdType = ParseThresholdType(section); @@ -102,6 +103,9 @@ private static string[] ParseArrayValue(IConfigurationSection section, string ke .Where(v => !string.IsNullOrWhiteSpace(v))]; } + private static int? ParseThreshold(IConfigurationSection section) => + int.TryParse(section[CoverletMTPConstants.ThresholdKey], out int threshold) ? threshold : null; + private static bool ParseBoolValue(IConfigurationSection section, string key) { return bool.TryParse(section[key], out bool result) && result; diff --git a/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs b/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs new file mode 100644 index 000000000..c0705b68b --- /dev/null +++ b/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs @@ -0,0 +1,117 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Coverlet.Core; +using Coverlet.Core.Abstractions; +using Coverlet.Core.Enums; +using Coverlet.MTP.Configuration; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Moq; +using Xunit; + +namespace Coverlet.MTP.Collector.Tests; + +public class CollectorExtensionThresholdTests +{ + [Fact] + public async Task DisplayThresholdSummaryAsyncDisplaysConfiguredPassingMetrics() + { + var outputDevice = new Mock(); + CollectorExtension collector = CreateCollector(outputDevice); + ConfigureThreshold(collector, 70, ThresholdStatistic.Total, ["line", "branch", "method"]); + + await DisplayThresholdSummaryAsync(collector, CreateCoverageResult(hits: 1)); + + outputDevice.Verify(x => x.DisplayAsync( + It.Is(producer => producer == collector), + It.Is(data => + data.Text.Contains("Coverage Threshold Results:") && + data.Text.Contains("Total - Line (Total over Module): 100.0% >= 70.0% threshold") && + data.Text.Contains("Total - Branch (Total over Module): 100.0% >= 70.0% threshold") && + data.Text.Contains("Total - Method (Total over Module): 100.0% >= 70.0% threshold")), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DisplayThresholdSummaryAsyncDisplaysFailedMetric() + { + var outputDevice = new Mock(); + CollectorExtension collector = CreateCollector(outputDevice); + ConfigureThreshold(collector, 70, ThresholdStatistic.Total, ["line"]); + + await DisplayThresholdSummaryAsync(collector, CreateCoverageResult(hits: 0)); + + outputDevice.Verify(x => x.DisplayAsync( + It.IsAny(), + It.Is(data => + data.Text.Contains("Total - Line (Total over Module): 0.0% < 70.0% threshold") && + data.Text.Contains("The total line coverage is below the specified 70.0% threshold.")), + It.IsAny()), + Times.Once); + } + + private static CollectorExtension CreateCollector(Mock outputDevice) + { + var loggerFactory = new Mock(); + loggerFactory.Setup(factory => factory.CreateLogger(It.IsAny())) + .Returns(new Mock().Object); + + var commandLineOptions = new Mock(); + var configuration = new Mock(); + var fileSystem = new Mock(); + outputDevice.Setup(device => device.DisplayAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())).Returns(Task.CompletedTask); + + return new CollectorExtension( + loggerFactory.Object, + commandLineOptions.Object, + outputDevice.Object, + configuration.Object, + fileSystem.Object); + } + + private static void ConfigureThreshold(CollectorExtension collector, int threshold, ThresholdStatistic thresholdStat, List thresholdTypes) + { + System.Reflection.FieldInfo configurationField = typeof(CollectorExtension) + .GetField("_configuration", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + var configuration = (CoverletExtensionConfiguration)configurationField.GetValue(collector)!; + configuration.Threshold = threshold; + configuration.ThresholdStat = thresholdStat; + configuration.ThresholdType = thresholdTypes; + } + + private static async Task DisplayThresholdSummaryAsync(CollectorExtension collector, CoverageResult result) + { + System.Reflection.MethodInfo method = typeof(CollectorExtension) + .GetMethod("DisplayThresholdSummaryAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + await (Task)method.Invoke(collector, [result, CancellationToken.None])!; + } + + private static CoverageResult CreateCoverageResult(int hits) + { + var methods = new Methods + { + ["System.Void TestClass::TestMethod()"] = new Method + { + Lines = new Lines { { 1, hits } }, + Branches = [new BranchInfo { Line = 1, Hits = hits }] + } + }; + var classes = new Classes { ["TestClass"] = methods }; + var documents = new Documents { ["TestClass.cs"] = classes }; + + return new CoverageResult + { + Identifier = "test-id", + Modules = new Modules { ["test.dll"] = documents }, + Parameters = new CoverageParameters() + }; + } +} diff --git a/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs b/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs index edf934271..5ca3ac52a 100644 --- a/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs +++ b/test/coverlet.MTP.tests/CommandLine/CoverletMTPCommandLineTests.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; +using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.CommandLine; using Xunit; @@ -52,6 +53,52 @@ public async Task IsValidWhenOptionHasValidValue(string optionName, string value Assert.True(string.IsNullOrEmpty(result.ErrorMessage)); } + [Theory] + [InlineData(CoverletOptionNames.Threshold, "0")] + [InlineData(CoverletOptionNames.Threshold, "70")] + [InlineData(CoverletOptionNames.Threshold, "100")] + [InlineData(CoverletOptionNames.ThresholdStat, "minimum")] + [InlineData(CoverletOptionNames.ThresholdStat, "average")] + [InlineData(CoverletOptionNames.ThresholdStat, "total")] + [InlineData(CoverletOptionNames.ThresholdType, "line")] + [InlineData(CoverletOptionNames.ThresholdType, "branch")] + [InlineData(CoverletOptionNames.ThresholdType, "method")] + public async Task IsValidWhenThresholdOptionHasValidValue(string optionName, string value) + { + CommandLineOption option = _provider.GetCommandLineOptions().First(x => x.Name == optionName); + + ValidationResult result = await _provider.ValidateOptionArgumentsAsync(option, [value]); + + Assert.True(result.IsValid); + } + + [Theory] + [InlineData(CoverletOptionNames.Threshold, "-1")] + [InlineData(CoverletOptionNames.Threshold, "101")] + [InlineData(CoverletOptionNames.Threshold, "70.5")] + [InlineData(CoverletOptionNames.ThresholdStat, "totalish")] + [InlineData(CoverletOptionNames.ThresholdType, "outline")] + [InlineData(CoverletOptionNames.ThresholdType, "line,unknown")] + public async Task IsInvalidWhenThresholdOptionHasInvalidValue(string optionName, string value) + { + CommandLineOption option = _provider.GetCommandLineOptions().First(x => x.Name == optionName); + + ValidationResult result = await _provider.ValidateOptionArgumentsAsync(option, [value]); + + Assert.False(result.IsValid); + Assert.False(string.IsNullOrEmpty(result.ErrorMessage)); + } + + [Fact] + public async Task IsValidWhenThresholdTypeHasMultipleValues() + { + CommandLineOption option = _provider.GetCommandLineOptions().First(x => x.Name == CoverletOptionNames.ThresholdType); + + ValidationResult result = await _provider.ValidateOptionArgumentsAsync(option, ["line,branch", "method"]); + + Assert.True(result.IsValid); + } + [Theory] [InlineData(CoverletOptionNames.Exclude)] [InlineData(CoverletOptionNames.Include)] diff --git a/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsParserTests.cs b/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsParserTests.cs index 5617e4f2e..78a13e2fc 100644 --- a/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsParserTests.cs +++ b/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsParserTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Toni Solarin-Sodara // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Coverlet.Core.Enums; using Microsoft.Extensions.Configuration; using Xunit; @@ -175,6 +176,25 @@ public void ParseExcludeFiltersAlwaysIncludesDefault() Assert.Equal("[coverlet.*]*", settings.ExcludeFilters[0]); // Default is first } + [Fact] + public void ParseThresholdSettingsReadsAllThresholdValues() + { + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Coverlet:Threshold"] = "70", + ["Coverlet:ThresholdStat"] = "total", + ["Coverlet:ThresholdType"] = "line, branch,method" + }) + .Build(); + + CoverletMTPSettings settings = CoverletMTPSettingsParser.Parse(configuration, "test.dll"); + + Assert.Equal(70, settings.Threshold); + Assert.Equal(ThresholdStatistic.Total, settings.ThresholdStat); + Assert.Equal(["line", "branch", "method"], settings.ThresholdType); + } + [Fact] public void ParseFromJsonFileLoadsCorrectly() { diff --git a/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsTests.cs b/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsTests.cs index 6a83a7a8f..75c778761 100644 --- a/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsTests.cs +++ b/test/coverlet.MTP.tests/Configuration/CoverletMTPSettingsTests.cs @@ -29,6 +29,7 @@ public void DefaultValuesAreCorrect() Assert.False(settings.DeterministicReport); Assert.Equal("MissingAll", settings.ExcludeAssembliesWithoutSources); Assert.False(settings.DisableManagedInstrumentationRestore); + Assert.Null(settings.Threshold); } [Fact] @@ -51,7 +52,10 @@ public void ToStringReturnsCorrectFormat() SkipAutoProps = false, DoesNotReturnAttributes = ["DoesNotReturn"], DeterministicReport = true, - ExcludeAssembliesWithoutSources = "MissingAny" + ExcludeAssembliesWithoutSources = "MissingAny", + Threshold = 70, + ThresholdStat = Coverlet.Core.Enums.ThresholdStatistic.Total, + ThresholdType = ["line", "branch"] }; // Act @@ -62,5 +66,8 @@ public void ToStringReturnsCorrectFormat() Assert.Contains("IncludeFilters: '[*]*'", result); Assert.Contains("UseSourceLink: 'True'", result); Assert.Contains("DeterministicReport: 'True'", result); + Assert.Contains("Threshold: '70'", result); + Assert.Contains("ThresholdStat: 'Total'", result); + Assert.Contains("ThresholdType: 'line,branch'", result); } } From 47231081e8e43a9f96e67778ef646f6d0a31d510 Mon Sep 17 00:00:00 2001 From: Bert Date: Tue, 1 Sep 2026 15:59:23 +0200 Subject: [PATCH 05/12] incorporate review comments --- Documentation/Coverlet.MTP.Integration.md | 10 +++++----- .../CoverletExtensionCommandLineProvider.cs | 2 +- .../Configuration/CoverageConfiguration.cs | 9 ++++++--- .../Configuration/CoverletExtensionConfiguration.cs | 2 +- src/coverlet.MTP/Configuration/CoverletMTPSettings.cs | 2 +- .../Configuration/CoverletMTPSettingsParser.cs | 9 ++++++--- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/Documentation/Coverlet.MTP.Integration.md b/Documentation/Coverlet.MTP.Integration.md index cfe605e28..1898a4750 100644 --- a/Documentation/Coverlet.MTP.Integration.md +++ b/Documentation/Coverlet.MTP.Integration.md @@ -87,9 +87,9 @@ dotnet exec --help | `--coverlet-skip-auto-props` | Skip auto-implemented properties. (default: `false`) | | `--coverlet-does-not-return-attribute ` | Attributes that mark methods as not returning. Can be specified multiple times. (default: `none`) | | `--coverlet-exclude-assemblies-without-sources ` | Exclude assemblies without source code. Values: `MissingAll`, `MissingAny`, `None`. (default: `None`) | -| `--threshold ` | Exits with error if the coverage % is below value | -| `--threshold-type ` | Coverage type to apply the threshold to. [default: `line`, `branch`, `method`] | -| `--threshold-stat ` | Coverage statistic used to enforce the threshold value. [default: Minimum] | +| `--coverlet-threshold ` | Exits with error if the coverage % is below value | +| `--coverlet-threshold-type ` | Coverage type to apply the threshold to. [default: `line`, `branch`, `method`] | +| `--coverlet-threshold-stat ` | Coverage statistic used to enforce the threshold value. [default: Minimum] | > [!TIP] > If you encounter instrumentation failures like "The process cannot access the file ... because it is being used by another process", try setting `--coverlet-exclude-assemblies-without-sources MissingAll` (or in a config file: `"ExcludeAssembliesWithoutSources": "MissingAll"`) to skip assemblies without sources and reduce access conflicts. @@ -240,8 +240,8 @@ The legacy `coverlet.mtp.appsettings.json` format is still supported for backwar | `DeterministicReport` | bool | Generate deterministic reports | | `ExcludeAssembliesWithoutSources` | string | Values: `MissingAll`, `MissingAny`, `None` (default: `MissingAll`) | | `Threshold` | int | Exits with error if the code coverage [0..100%] is below value | -| `ThresholdType` | string | Comma-separated coverage type to apply the Threshold to. [default: `line`, `branch`, `method`] | -| `ThresholdStat` | string | Coverage statistic used to enforce the threshold value. [default: `Minimum`, `Average`, `Total`] | +| `ThresholdType` | string | Comma-separated coverage type to apply the Threshold to. [default: `line`] | +| `ThresholdStat` | string | Coverage statistic used to enforce the threshold value. Values: `Minimum`, `Average`, `Total`. (default: `Minimum`) | **Example `coverlet.mtp.appsettings.json`:** diff --git a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs index 440c458d5..74337075e 100644 --- a/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs +++ b/src/coverlet.MTP/CommandLine/CoverletExtensionCommandLineProvider.cs @@ -132,7 +132,7 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com if (!TryParseEnum(typeof(Coverlet.Core.Enums.ThresholdStatistic), arguments[0], ignoreCase: true, out object? thresholdStatistic)) { - return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}'(total, average, minimum).")); + return Task.FromResult(ValidationResult.Invalid($"The value '{arguments[0]}' is not a valid option for '{commandOption.Name}' (total, average, minimum).")); } } return ValidationResult.ValidTask; diff --git a/src/coverlet.MTP/Configuration/CoverageConfiguration.cs b/src/coverlet.MTP/Configuration/CoverageConfiguration.cs index 236242fea..e5f18a0af 100644 --- a/src/coverlet.MTP/Configuration/CoverageConfiguration.cs +++ b/src/coverlet.MTP/Configuration/CoverageConfiguration.cs @@ -308,12 +308,15 @@ public string[] GetIncludeDirectories() public int? GetThreshold() { - if (_commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.Threshold, out string[]? values)) + if (_commandLineOptions.TryGetOptionArgumentList(CoverletOptionNames.Threshold, out string[]? values) && + values.Length > 0 && + int.TryParse(values[0], out int threshold) && + threshold is >= 0 and <= 100) { - return int.Parse(values[0]); + return threshold; } - return _configFileSettings?.Threshold; + return _configFileSettings?.Threshold is >= 0 and <= 100 ? _configFileSettings.Threshold : null; } public ThresholdStatistic GetThresholdStatistic() diff --git a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs index 6524d83fa..6d11f9332 100644 --- a/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs +++ b/src/coverlet.MTP/Configuration/CoverletExtensionConfiguration.cs @@ -25,7 +25,7 @@ internal class CoverletExtensionConfiguration public int? Threshold { get; set; } /// - /// define which coverage statistic (Average|Minimum|Total) to use for threshold comparison. If not specified, the default is Total. If the coverage is below this threshold, the build will fail. + /// define which coverage statistic (Average|Minimum|Total) to use for threshold comparison. If not specified, the default is Minimum. If the coverage is below this threshold, the build will fail. /// public ThresholdStatistic ThresholdStat { get; set; } diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs index 4f3d29180..56db17be6 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettings.cs @@ -53,7 +53,7 @@ public override string ToString() builder.AppendFormat("SkipAutoProps: '{0}', ", SkipAutoProps); builder.AppendFormat("DoesNotReturnAttributes: '{0}', ", string.Join(",", DoesNotReturnAttributes)); builder.AppendFormat("DeterministicReport: '{0}', ", DeterministicReport); - builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}'", ExcludeAssembliesWithoutSources); + builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}', ", ExcludeAssembliesWithoutSources); builder.AppendFormat("Threshold: '{0}', ", Threshold); builder.AppendFormat("ThresholdStat: '{0}', ", ThresholdStat); builder.AppendFormat("ThresholdType: '{0}'", string.Join(",", ThresholdType)); diff --git a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs index f3af19834..6a62f128d 100644 --- a/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs +++ b/src/coverlet.MTP/Configuration/CoverletMTPSettingsParser.cs @@ -68,8 +68,9 @@ private static string[] ParseExcludeFilters(IConfigurationSection section) private static List ParseThresholdType(IConfigurationSection section) { - List types = []; - types.AddRange(ParseArrayValue(section, CoverletMTPConstants.ThresholdTypeKey)); + List types = [.. ParseArrayValue(section, CoverletMTPConstants.ThresholdTypeKey) + .Select(t => t.Trim().ToLowerInvariant()) + .Where(t => t is "line" or "branch" or "method")]; return types.Count == 0 ? [CoverletMTPConstants.DefaultThresholdType] : types; } @@ -104,7 +105,9 @@ private static string[] ParseArrayValue(IConfigurationSection section, string ke } private static int? ParseThreshold(IConfigurationSection section) => - int.TryParse(section[CoverletMTPConstants.ThresholdKey], out int threshold) ? threshold : null; + int.TryParse(section[CoverletMTPConstants.ThresholdKey], out int threshold) && threshold is >= 0 and <= 100 + ? threshold + : null; private static bool ParseBoolValue(IConfigurationSection section, string key) { From a729e74c918f9c155eee9efd6cc2bcb8260265cf Mon Sep 17 00:00:00 2001 From: Bert Date: Mon, 7 Sep 2026 10:12:23 +0200 Subject: [PATCH 06/12] Refactor MTP collector to publish coverage via message bus - Add CoverletCoverageDataProducer for MTP message creation/publishing - CollectorExtension now publishes coverage, threshold, and report messages to IMessageBus - Refactor exit code logic for threshold failures - Update CollectorExtensionProvider for DI of message bus and producer - Refactor and add tests for message-based reporting and producer logic - Update integration test for new summary output - Bump MicrosoftTestingPlatformVersion to 2.4.0 - Update .github/copilot-instructions.md for test/code standards and netstandard2.0 rules --- .github/copilot-instructions.md | 70 +++---- Directory.Packages.props | 2 +- .../Collector/CollectorExtension.cs | 170 +++++++++------ .../Collector/CoverletCoverageDataProducer.cs | 197 ++++++++++++++++++ src/coverlet.MTP/CoverletExtensionProvider.cs | 3 +- .../CollectorExtensionReportMethodsTests.cs | 36 ++-- .../Collector/CollectorExtensionTests.cs | 6 +- .../CollectorExtensionThresholdTests.cs | 61 +++--- .../CoverletCoverageDataProducerTests.cs | 87 ++++++++ .../CollectCoverageTests.cs | 25 +-- 10 files changed, 497 insertions(+), 160 deletions(-) create mode 100644 src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs create mode 100644 test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8f5fd7955..64ad21f54 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -28,6 +28,7 @@ In addition to the rules enforced by `.editorconfig`, you SHOULD: - SA1316: Tuple element names should use correct casing - SA1518: File is required to end with a single newline character - Any update for pattern matching `or` must not break the already-correct `text == "hello" || text == "world"` operator scenario. +- For multi-targeted projects including netstandard2.0, avoid APIs unavailable on netstandard2.0 such as `ArgumentNullException.ThrowIfNull`. ## Testing Guidelines @@ -58,16 +59,10 @@ In addition to the rules enforced by `.editorconfig`, you SHOULD: **Examples:** ❌ **INCORRECT** - Direct file system usage: - -```cs // BAD - Creates real files and directories File.WriteAllText("report.json", content); Directory.CreateDirectory("reports"); bool exists = File.Exists("test.dll"); -``` - ✅ **CORRECT** - Mock file system: - -```cs // GOOD - Uses mocked abstraction with simulated paths var mockFileSystem = new Mock(); mockFileSystem.Setup(x => x.Exists("/fake/path/test.dll")).Returns(true); @@ -75,8 +70,6 @@ mockFileSystem.Setup(x => x.Exists("/fake/reports")).Returns(true); mockFileSystem.Setup(x => x.WriteAllText(It.IsAny(), It.IsAny())); // Verify the mock was called correctly mockFileSystem.Verify(x => x.WriteAllText(It.Is(path => path.EndsWith("report.json")), It.IsAny()), Times.Once); -``` - ### Moq Testing Rules (Critical - Prevents Runtime Errors) **NEVER use extension methods in Moq `Setup()` or `Verify()` calls.** @@ -93,45 +86,29 @@ Extension methods are static methods that cannot be intercepted by Moq. Using th #### Example: Mocking ILogger ❌ **INCORRECT** - Will throw `NotSupportedException`: - -```cs // This will FAIL at runtime _mockLogger.Verify(x => x.LogInformation(It.IsAny()), Times.Once); _mockLogger.Verify(x => x.LogInformation(It.Is(s => s.Contains("json"))), Times.Once); _mockLogger.Setup(x => x.LogWarning(It.IsAny())); -``` - ✅ **CORRECT** - Mocks the underlying `Log` method: - -```cs // Verify LogInformation was called once _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.Once); // Verify LogInformation was called with a message containing "json" _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), It.Is((v, t) => v.ToString()!.Contains("json")), It.IsAny(), It.IsAny>()), Times.Once); // Setup LogWarning behavior _mockLogger.Setup(x => x.Log(LogLevel.Warning, It.IsAny())); -``` - #### Example: Mocking LogDebug ❌ **INCORRECT**: - -```cs // This will FAIL at runtime _mockLogger.Verify(x => x.LogDebug(It.IsAny()), Times.Once); -``` - ✅ **CORRECT** - Mocks the underlying `Log` method: - -```cs // Verify LogDebug was called once _mockLogger.Verify(x => x.Log(LogLevel.Debug, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.Once); // Verify LogDebug was called with a message containing "xml" _mockLogger.Verify(x => x.Log(LogLevel.Debug, It.IsAny(), It.Is((v, t) => v.ToString()!.Contains("xml")), It.IsAny(), It.IsAny>()), Times.Once); // Setup LogError behavior _mockLogger.Setup(x => x.Log(LogLevel.Error, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())); -``` - **Key Points:** 1. Always use `ILogger.Log()` with the appropriate `LogLevel` instead of extension methods. 2. Use `It.IsAny()` for the state parameter. @@ -168,16 +145,12 @@ This codebase uses **TWO different ILogger interfaces** with different signature **Common Pitfall - Microsoft.Testing.Platform.Logging.ILogger:** ❌ **INCORRECT** - Assumes `EventId` parameter (which doesn't exist in MTP Logger): - -```cs // BAD - Microsoft.Testing.Platform.Logging.ILogger does NOT have EventId _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), // ⚠️ EventId does NOT exist in MTP LOGGER It.IsAny(), It.IsAny(), It.IsAny>()), Times.Once); ``` ✅ **CORRECT** - Uses actual MTP ILogger API signature (async methods): - -```cs // GOOD - Microsoft.Testing.Platform.Logging.ILogger uses simple async methods _mockLogger.Verify(x => x.LogInformationAsync(It.IsAny(), It.IsAny()), Times.Once); _mockLogger.Verify(x => x.LogErrorAsync(It.IsAny(), It.IsAny()), Times.Once); @@ -187,8 +160,6 @@ _mockLogger.Verify(x => x.LogInformationAsync(It.Is(s => s.Contains("exp // For synchronous LoggerExtensions (extension methods): // NOTE: These are extension methods and cannot be verified with Moq; verify the underlying Log(...) call instead. _mockLogger.Verify(x => x.Log(LogLevel.Information, It.Is(s => s.Contains("Coverage reports generated")), It.IsAny(), It.IsAny>()), Times.Once); -``` - **Verification Checklist:** - [ ] I have searched for the interface definition using `get_symbols_by_name`. - [ ] I have reviewed existing usage in the codebase using `code_search`. @@ -208,8 +179,6 @@ When mocking interfaces, **reference actual adapter implementations** in the cod - `src/coverlet.core/Abstractions/ILogger.cs` - Coverlet's internal logger interface. **Example from CoverletLoggerAdapter.cs:** - -```csharp // Shows actual MTP ILogger usage - simple methods, no EventId public void LogInformation(string message, bool important = false) { @@ -221,9 +190,7 @@ public void LogInformation(string message, bool important = false) { _logger.LogInformation(message); } -} -``` -### Test Generation Verification (Critical Rule) +}### Test Generation Verification (Critical Rule) **Before generating any test, you MUST:** @@ -283,6 +250,39 @@ public void LogInformation(string message, bool important = false) **When resolving issues, limit documentation to ONE comprehensive document ONLY.** +**Validation Checklist Before Generating Tests:** +- [ ] I have searched for existing tests using `code_search`. +- [ ] I have reviewed existing test files in the same test project. +- [ ] I have identified which existing tests cover similar scenarios. +- [ ] I have documented which proposed tests are redundant. +- [ ] I can justify why each new test adds unique value. +- [ ] I have considered refactoring existing tests instead of adding duplicates. +- [ ] I have verified the actual API signatures being tested. +- [ ] I have used the correct mocking approach for the specific ILogger interface. +- [ ] I have used mocked file system abstractions instead of real file I/O. + +## Summary of Key Testing Rules + +1. **Always use `IFileSystem` abstraction** - Never use `File`, `Directory`, or `Path` static methods directly in tests. +2. **Always verify API signatures** - Use `get_symbols_by_name` and `code_search` before mocking external APIs. +3. **Know your ILogger** - Microsoft.Testing.Platform.Logging.ILogger ≠ Microsoft.Extensions.Logging.ILogger. +4. **Avoid extension methods in Moq** - They cannot be intercepted and will cause runtime exceptions. +5. **Use simulated paths** - Always use fake paths like `/fake/path/test.dll` in test mocks. +6. **Verify existing tests** - Check for duplicates before adding new test methods. +7. **Use Theory for parameterized tests** - Don't create multiple test methods for different input values. + +## Issue-Specific Guidelines + +- For issue #1965, identify problematic assemblies before instrumentation and skip them, rather than relying on partial-restore/non-fatal restore behavior after failure. +- For assembly-level instrumentation viability, preflight logic should only check lock and resolvability, not PDB/source-based exclusion; PDB/source exclusion remains handled by existing assembly-without-sources filtering via CanInstrument/options. +- **Prefer calling `instrumenter.CanInstrument()` before preflight** so assemblies already excluded by existing coverage filters (no PDB/no local sources) skip preflight probing. + +## Documentation Guidelines for Issue Resolution + +### Documentation Limitation (Critical Rule) + +**When resolving issues, limit documentation to ONE comprehensive document ONLY.** + #### Rule - Create a **single, well-organized proposal or resolution document** per issue - Document location: **`Documentation/Plans/Issue-[IssueNumber]-Resolution.md`** diff --git a/Directory.Packages.props b/Directory.Packages.props index 5f5485ad5..77af13ebf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,7 +14,7 @@ 18.8.1 3.2.2 3.1.5 - 2.3.3 + 2.4.0 4.20.72 diff --git a/src/coverlet.MTP/Collector/CollectorExtension.cs b/src/coverlet.MTP/Collector/CollectorExtension.cs index 693c4719f..d7cd6dab9 100644 --- a/src/coverlet.MTP/Collector/CollectorExtension.cs +++ b/src/coverlet.MTP/Collector/CollectorExtension.cs @@ -4,7 +4,6 @@ #if NETSTANDARD2_0 using System.Diagnostics; #endif -using System.Globalization; using System.Text; using Coverlet.Core; using Coverlet.Core.Abstractions; @@ -20,9 +19,12 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Testing.Platform.Configurations; using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Extensions.TestHostControllers; +using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.TestHost; namespace Coverlet.MTP.Collector; @@ -39,6 +41,8 @@ internal sealed class CollectorExtension : ITestHostProcessLifetimeHandler, ITes private IServiceProvider? _serviceProvider; private readonly Microsoft.Testing.Platform.Configurations.IConfiguration? _platformConfiguration; private readonly Microsoft.Testing.Platform.OutputDevice.IOutputDevice _outputDisplay; + private readonly IMessageBus? _messageBus; + private readonly CoverletCoverageDataProducer _coverageDataProducer; private ICoverage? _coverage; private readonly Microsoft.Testing.Platform.Logging.ILoggerFactory _loggerFactory; private readonly Microsoft.Testing.Platform.CommandLine.ICommandLineOptions _commandLineOptions; @@ -48,6 +52,8 @@ internal sealed class CollectorExtension : ITestHostProcessLifetimeHandler, ITes private bool IsCoverageEnabled => _isCoverageEnabled ??= _commandLineOptions.IsOptionSet(CoverletOptionNames.Coverage); + private static readonly char[] s_ignoredExitCodeSeparators = [',', ';']; + private readonly CoverletExtension _extension = new(); private readonly IReporterFactory _reporterFactory; @@ -62,7 +68,9 @@ public CollectorExtension( Microsoft.Testing.Platform.OutputDevice.IOutputDevice? outputDevice, Microsoft.Testing.Platform.Configurations.IConfiguration? configuration, IFileSystem? fileSystem = null, - IReporterFactory? reporterFactory = null) + IReporterFactory? reporterFactory = null, + IMessageBus? messageBus = null, + CoverletCoverageDataProducer? coverageDataProducer = null) { _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _commandLineOptions = commandLineOptions ?? throw new ArgumentNullException(nameof(commandLineOptions)); @@ -70,6 +78,8 @@ public CollectorExtension( _outputDisplay = outputDevice ?? throw new ArgumentNullException(nameof(outputDevice)); _fileSystem = fileSystem ?? new FileSystem(); // Use provided or create default _reporterFactory = reporterFactory ?? new DefaultReporterFactory(); + _messageBus = messageBus; + _coverageDataProducer = coverageDataProducer ?? new CoverletCoverageDataProducer(); _configuration = new CoverletExtensionConfiguration(); _logger = new CoverletLoggerAdapter(_loggerFactory); @@ -326,7 +336,7 @@ async Task ITestHostProcessLifetimeHandler.OnTestHostProcessExitedAsync( _logger.LogVerbose($"Coverage result modules: {result.Modules?.Count ?? 0}"); // Generate reports - await GenerateReportsAsync(result, cancellation); + await GenerateReportsAsync(result, testHostProcessInformation.ExitCode, cancellation); _logger.LogInformation("Code coverage collection completed"); } @@ -475,28 +485,39 @@ await _outputDisplay.DisplayAsync( } /// - /// Builds a coverage summary table string matching the format used by coverlet.msbuild and coverlet.console. - /// - private static string BuildCoverageSummaryTable(CoverageResult result) => - CoverageSummary.BuildCoverageSummaryTable(result.Modules); - - /// - /// Displays the coverage summary table to the output device. + /// Publishes coverage metrics, threshold evaluations, and report references to the MTP message bus. /// - private async Task DisplayCoverageSummaryAsync(CoverageResult result, CancellationToken cancellation) + private async Task PublishCoverageDataAsync(CoverageResult result, IReadOnlyList generatedReports, CancellationToken cancellation) { - string table = BuildCoverageSummaryTable(result); - await _outputDisplay.DisplayAsync( - this, - new TextOutputDeviceData(Environment.NewLine + table), - cancellation).ConfigureAwait(false); - } + cancellation.ThrowIfCancellationRequested(); + + if (_messageBus is null) + { + _logger.LogVerbose("MTP message bus is unavailable. Skipping structured coverage data publishing."); + return; + } + + SessionUid sessionUid = CreateSessionUid(result); + + foreach (TestCoverageMessage coverageMessage in _coverageDataProducer.CreateCoverageMessages(result, sessionUid)) + { + cancellation.ThrowIfCancellationRequested(); + await _messageBus.PublishAsync(_coverageDataProducer, coverageMessage).ConfigureAwait(false); + } + + foreach (string generatedReport in generatedReports) + { + cancellation.ThrowIfCancellationRequested(); + string reportFormat = Path.GetExtension(generatedReport).TrimStart('.'); + if (string.IsNullOrWhiteSpace(reportFormat)) + { + continue; + } + + TestCoverageReportMessage reportMessage = _coverageDataProducer.CreateReportMessage(sessionUid, generatedReport, reportFormat); + await _messageBus.PublishAsync(_coverageDataProducer, reportMessage).ConfigureAwait(false); + } - /// - /// Displays the configured coverage thresholds and their evaluation results. - /// - private async Task DisplayThresholdSummaryAsync(CoverageResult result, CancellationToken cancellation) - { if (!_configuration.Threshold.HasValue) { return; @@ -508,38 +529,36 @@ private async Task DisplayThresholdSummaryAsync(CoverageResult result, Cancellat _configuration.Threshold.Value); ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat); - var summary = new StringBuilder(); - summary.AppendLine(); - summary.AppendLine(" Coverage Threshold Results:"); - foreach (KeyValuePair thresholdValue in thresholdValues) + foreach (TestCoverageThresholdMessage thresholdMessage in _coverageDataProducer.CreateThresholdMessages( + result, + sessionUid, + thresholdValues, + thresholdStat, + GetThresholdCoverage, + belowThreshold)) { - ThresholdTypeFlags type = thresholdValue.Key; - double threshold = thresholdValue.Value; - double coverage = GetThresholdCoverage(result, type, thresholdStat); - string comparison = (belowThreshold & type) == ThresholdTypeFlags.None ? ">=" : "<"; - summary.AppendLine( - $" {thresholdStat} - {type} ({GetThresholdScopeDescription(thresholdStat)}): " + - $"{coverage.ToString("F1", CultureInfo.InvariantCulture)}% {comparison} " + - $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold"); + cancellation.ThrowIfCancellationRequested(); + await _messageBus.PublishAsync(_coverageDataProducer, thresholdMessage).ConfigureAwait(false); } + } - if (belowThreshold != ThresholdTypeFlags.None) + private SessionUid CreateSessionUid(CoverageResult result) + { + string? coverageIdentifier = _coverageIdentifier; + if (!string.IsNullOrWhiteSpace(coverageIdentifier)) { - summary.AppendLine(); - foreach (KeyValuePair thresholdValue in thresholdValues.Where(pair => (belowThreshold & pair.Key) != ThresholdTypeFlags.None)) - { - ThresholdTypeFlags type = thresholdValue.Key; - double threshold = thresholdValue.Value; - summary.AppendLine( - $" The {thresholdStat.ToString().ToLowerInvariant()} {type.ToString().ToLowerInvariant()} coverage is below the specified " + - $"{threshold.ToString("F1", CultureInfo.InvariantCulture)}% threshold."); - } + return new SessionUid(coverageIdentifier!); } - await _outputDisplay.DisplayAsync( - this, - new TextOutputDeviceData(summary.ToString()), - cancellation).ConfigureAwait(false); + string? resultIdentifier = result.Identifier; + if (!string.IsNullOrWhiteSpace(resultIdentifier)) + { + return new SessionUid(resultIdentifier!); + } + + string fallbackIdentifier = Guid.NewGuid().ToString("N"); + _logger.LogWarning("Coverage identifier is missing. Falling back to a generated session identifier."); + return new SessionUid(fallbackIdentifier); } private static Dictionary BuildThresholdValues(IEnumerable thresholdTypes, double threshold) @@ -594,14 +613,29 @@ private static double GetCoveragePercent(Documents module, ThresholdTypeFlags th _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required.") }; - private static string GetThresholdScopeDescription(ThresholdStatistic thresholdStat) => - thresholdStat switch + private bool IsCoverageThresholdExitCodeIgnored() + { + const string ignoreExitCodeOption = "ignore-exit-code"; + + if (!_commandLineOptions.TryGetOptionArgumentList(ignoreExitCodeOption, out string[]? ignoredExitCodes) || ignoredExitCodes is null) { - ThresholdStatistic.Total => "Total over Module", - ThresholdStatistic.Average => "Average per Module", - ThresholdStatistic.Minimum => "Minimum per Module", - _ => throw new ArgumentOutOfRangeException(nameof(thresholdStat), thresholdStat, null) - }; + return false; + } + + foreach (string ignoredExitCode in ignoredExitCodes) + { + string[] values = ignoredExitCode.Split(s_ignoredExitCodeSeparators, StringSplitOptions.RemoveEmptyEntries); + foreach (string value in values) + { + if (string.Equals(value.Trim(), "14", StringComparison.Ordinal)) + { + return true; + } + } + } + + return false; + } /// /// Displays generated report paths to output device. @@ -634,7 +668,7 @@ await _outputDisplay.DisplayAsync( } // Refactor GenerateReportsAsync to use extracted methods - private async Task GenerateReportsAsync(CoverageResult result, CancellationToken cancellation) + private async Task GenerateReportsAsync(CoverageResult result, int testHostExitCode, CancellationToken cancellation) { string outputDirectory = _platformConfiguration!.GetTestResultDirectory() ?? Path.GetDirectoryName(_testModulePath) + Path.DirectorySeparatorChar; @@ -662,16 +696,13 @@ private async Task GenerateReportsAsync(CoverageResult result, CancellationToken // Display results await DisplayGeneratedReportsAsync(generatedReports, cancellation); - // Display code coverage summary table after the file artifacts list - await DisplayCoverageSummaryAsync(result, cancellation); + // Publish code coverage summary and threshold data to the MTP message bus. + await PublishCoverageDataAsync(result, generatedReports, cancellation); // Display console-type report output (e.g. teamcity) directly to the output device await DisplayConsoleReportOutputsAsync(consoleOutputs, cancellation); - // Display threshold metric summary table after the file artifacts list - await DisplayThresholdSummaryAsync(result, cancellation); - - // Exitcode `CoverageThresholdFailed = 14` shall be set if any threshold is not met + // Exitcode `CoverageThresholdFailed = 14` shall only override otherwise successful test runs. if (_configuration.Threshold.HasValue) { ThresholdStatistic thresholdStat = _configuration.ThresholdStat; @@ -681,8 +712,19 @@ private async Task GenerateReportsAsync(CoverageResult result, CancellationToken ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat); if (belowThreshold != ThresholdTypeFlags.None) { - _logger.LogError("Coverage thresholds not met. Setting exit code for Microsoft Testing Framework to 14."); - Environment.ExitCode = 14; + int currentExitCode = Environment.ExitCode != 0 ? Environment.ExitCode : testHostExitCode; + if (currentExitCode == 0) + { + if (IsCoverageThresholdExitCodeIgnored()) + { + _logger.LogInformation("Coverage thresholds not met, but exit code 14 is ignored by --ignore-exit-code."); + } + else + { + _logger.LogError("Coverage thresholds not met. Setting exit code for Microsoft Testing Framework to 14."); + Environment.ExitCode = 14; + } + } } } } diff --git a/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs new file mode 100644 index 000000000..2d1cf33af --- /dev/null +++ b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs @@ -0,0 +1,197 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Coverlet.Core; +using Coverlet.Core.Enums; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.TestHost; + +namespace Coverlet.MTP.Collector; + +internal sealed class CoverletCoverageDataProducer : IDataProducer +{ + private readonly CoverletExtension _extension = new(); + + public string Uid => _extension.Uid; + + public string Version => _extension.Version; + + public string DisplayName => "Coverlet Coverage Data Producer"; + + public string Description => "Publishes coverlet coverage and threshold data through Microsoft Testing Platform messages."; + + public Type[] DataTypesProduced { get; } = + [ + typeof(TestCoverageMessage), + typeof(TestCoverageThresholdMessage), + typeof(TestCoverageReportMessage), + ]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public IReadOnlyList CreateCoverageMessages(CoverageResult result, SessionUid sessionUid) + { +#if NETSTANDARD2_0 + if (result is null) + { + throw new ArgumentNullException(nameof(result)); + } +#else + ArgumentNullException.ThrowIfNull(result); +#endif + + var messages = new List(); + + CoverageDetails overallLine = CoverageSummary.CalculateLineCoverage(result.Modules); + CoverageDetails overallBranch = CoverageSummary.CalculateBranchCoverage(result.Modules); + messages.Add(CreateCoverageMessage(sessionUid, CoverageScope.Overall, CoverageMetric.Line, overallLine)); + messages.Add(CreateCoverageMessage(sessionUid, CoverageScope.Overall, CoverageMetric.Branch, overallBranch)); + + foreach (KeyValuePair module in result.Modules) + { + var scope = new CoverageScope(CoverageScopeLevel.Module, module.Key); + CoverageDetails line = CoverageSummary.CalculateLineCoverage(module.Value); + CoverageDetails branch = CoverageSummary.CalculateBranchCoverage(module.Value); + messages.Add(CreateCoverageMessage(sessionUid, scope, CoverageMetric.Line, line)); + messages.Add(CreateCoverageMessage(sessionUid, scope, CoverageMetric.Branch, branch)); + } + + return messages; + } + + public IReadOnlyList CreateThresholdMessages( + CoverageResult result, + SessionUid sessionUid, + IReadOnlyDictionary thresholdValues, + ThresholdStatistic thresholdStatistic, + Func thresholdCoverageCalculator, + ThresholdTypeFlags belowThreshold) + { +#if NETSTANDARD2_0 + if (result is null) + { + throw new ArgumentNullException(nameof(result)); + } + + if (thresholdValues is null) + { + throw new ArgumentNullException(nameof(thresholdValues)); + } + + if (thresholdCoverageCalculator is null) + { + throw new ArgumentNullException(nameof(thresholdCoverageCalculator)); + } +#else + ArgumentNullException.ThrowIfNull(result); + ArgumentNullException.ThrowIfNull(thresholdValues); + ArgumentNullException.ThrowIfNull(thresholdCoverageCalculator); +#endif + + _ = belowThreshold; + + var thresholdMessages = new List(); + CoverageAggregation aggregation = MapAggregation(thresholdStatistic); + + foreach (KeyValuePair thresholdValue in thresholdValues) + { + double actualCoverage = thresholdCoverageCalculator(result, thresholdValue.Key, thresholdStatistic); + CoverageMetric metric = MapMetric(thresholdValue.Key); + bool hasCoverableData = HasCoverableDataForMetric(result, thresholdValue.Key, thresholdStatistic); + + thresholdMessages.Add( + new TestCoverageThresholdMessage( + sessionUid, + CoverageScope.Overall, + metric, + aggregation, + actualCoverage, + thresholdValue.Value, + hasCoverableData, + Uid, + aggregatedOver: aggregation == CoverageAggregation.None ? null : CoverageScopeLevel.Module, + treatNoDataAsFailure: true)); + } + + return thresholdMessages; + } + + public TestCoverageReportMessage CreateReportMessage(SessionUid sessionUid, string reportPath, string reportFormat) + { + if (string.IsNullOrWhiteSpace(reportPath)) + { + throw new ArgumentException("Report path must not be null or empty.", nameof(reportPath)); + } + + if (string.IsNullOrWhiteSpace(reportFormat)) + { + throw new ArgumentException("Report format must not be null or empty.", nameof(reportFormat)); + } + + (CoverageReportFormat format, string? customFormat) = MapReportFormat(reportFormat); + return new TestCoverageReportMessage(sessionUid, reportPath, format, Uid, customFormat); + } + + private TestCoverageMessage CreateCoverageMessage( + SessionUid sessionUid, + CoverageScope scope, + CoverageMetric metric, + CoverageDetails details) + { + return new TestCoverageMessage( + sessionUid, + scope, + metric, + coveredCount: checked((long)details.Covered), + coverableCount: details.Total, + producerId: Uid); + } + + private static CoverageAggregation MapAggregation(ThresholdStatistic thresholdStatistic) => + thresholdStatistic switch + { + ThresholdStatistic.Total => CoverageAggregation.Total, + ThresholdStatistic.Average => CoverageAggregation.Average, + ThresholdStatistic.Minimum => CoverageAggregation.Minimum, + _ => throw new ArgumentOutOfRangeException(nameof(thresholdStatistic), thresholdStatistic, null), + }; + + private static CoverageMetric MapMetric(ThresholdTypeFlags thresholdType) => + thresholdType switch + { + ThresholdTypeFlags.Line => CoverageMetric.Line, + ThresholdTypeFlags.Branch => CoverageMetric.Branch, + ThresholdTypeFlags.Method => CoverageMetric.Method, + _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required."), + }; + + private static (CoverageReportFormat Format, string? CustomFormatName) MapReportFormat(string reportFormat) + { + return reportFormat.Trim().ToLowerInvariant() switch + { + "cobertura" => (CoverageReportFormat.Cobertura, null), + "opencover" => (CoverageReportFormat.OpenCover, null), + "lcov" => (CoverageReportFormat.Lcov, null), + "json" => (CoverageReportFormat.Custom, "json"), + _ => (CoverageReportFormat.Custom, reportFormat), + }; + } + + private static bool HasCoverableDataForMetric(CoverageResult result, ThresholdTypeFlags thresholdType, ThresholdStatistic thresholdStatistic) + { + if (thresholdStatistic == ThresholdStatistic.Minimum) + { + return result.Modules.Count > 0; + } + + CoverageDetails coverage = thresholdType switch + { + ThresholdTypeFlags.Line => CoverageSummary.CalculateLineCoverage(result.Modules), + ThresholdTypeFlags.Branch => CoverageSummary.CalculateBranchCoverage(result.Modules), + ThresholdTypeFlags.Method => CoverageSummary.CalculateMethodCoverage(result.Modules), + _ => throw new ArgumentOutOfRangeException(nameof(thresholdType), thresholdType, "A single coverage threshold type is required."), + }; + + return coverage.Total > 0; + } +} diff --git a/src/coverlet.MTP/CoverletExtensionProvider.cs b/src/coverlet.MTP/CoverletExtensionProvider.cs index e7792a451..ba70f4081 100644 --- a/src/coverlet.MTP/CoverletExtensionProvider.cs +++ b/src/coverlet.MTP/CoverletExtensionProvider.cs @@ -36,7 +36,8 @@ public static void AddCoverletExtensionProvider(this ITestApplicationBuilder bui serviceProvider.GetLoggerFactory(), serviceProvider.GetCommandLineOptions(), serviceProvider.GetOutputDevice(), - configuration) as ITestHostProcessLifetimeHandler; + configuration, + messageBus: serviceProvider.GetMessageBus()) as ITestHostProcessLifetimeHandler; }); // ============================================================ diff --git a/test/coverlet.MTP.tests/Collector/CollectorExtensionReportMethodsTests.cs b/test/coverlet.MTP.tests/Collector/CollectorExtensionReportMethodsTests.cs index c8da595d7..b853848c4 100644 --- a/test/coverlet.MTP.tests/Collector/CollectorExtensionReportMethodsTests.cs +++ b/test/coverlet.MTP.tests/Collector/CollectorExtensionReportMethodsTests.cs @@ -6,8 +6,10 @@ using Coverlet.MTP.CommandLine; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.OutputDevice; using Moq; using Xunit; @@ -28,6 +30,7 @@ public class CollectorExtensionReportMethodsTests private readonly Mock _mockCommandLineOptions; private readonly Mock _mockConfiguration; private readonly Mock _mockOutputDevice; + private readonly Mock _mockMessageBus; private readonly Mock _mockFileSystem; private readonly Mock _mockSourceRootTranslator; @@ -56,6 +59,7 @@ public CollectorExtensionReportMethodsTests() _mockConfiguration = new Mock(); _mockFileSystem = new Mock(); _mockOutputDevice = new Mock(); + _mockMessageBus = new Mock(); _mockSourceRootTranslator = new Mock(); _mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())) @@ -78,6 +82,10 @@ private void SetupDefaultMocks() .Setup(x => x[It.IsAny()]) .Returns((string?)null); + _mockMessageBus + .Setup(x => x.PublishAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + _mockFileSystem .Setup(x => x.Exists(s_simulatedTestModulePath)) .Returns(true); @@ -138,7 +146,8 @@ private CollectorExtension CreateCollectorWithCoverageEnabled() _mockCommandLineOptions.Object, _mockOutputDevice.Object, _mockConfiguration.Object, - _mockFileSystem.Object); + _mockFileSystem.Object, + messageBus: _mockMessageBus.Object); } #region GenerateCoverageReportFiles Tests @@ -538,35 +547,30 @@ public async Task DisplayConsoleReportOutputsAsyncWithMultipleOutputsCallsOutput #endregion - #region DisplayCoverageSummaryAsync Tests + #region PublishCoverageDataAsync Tests [Fact] - public async Task DisplayCoverageSummaryAsyncDisplaysCoverageTable() + public async Task PublishCoverageDataAsyncPublishesCoverageMessages() { // Arrange var collector = CreateCollectorWithCoverageEnabled(); CoverageResult result = CreateTestCoverageResult(); - _mockOutputDevice.Setup(x => x.DisplayAsync( - It.IsAny(), - It.IsAny(), - It.IsAny())) - .Returns(Task.CompletedTask); - System.Reflection.MethodInfo? method = typeof(CollectorExtension) - .GetMethod("DisplayCoverageSummaryAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + .GetMethod("PublishCoverageDataAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); Assert.NotNull(method); // Act - await (Task)method.Invoke(collector, [result, CancellationToken.None])!; + await (Task)method.Invoke(collector, [result, new List(), CancellationToken.None])!; // Assert - _mockOutputDevice.Verify( - x => x.DisplayAsync( - It.Is(p => p == collector), - It.Is(data => data.Text.Contains("Module") && data.Text.Contains("Line") && data.Text.Contains("Method")), - It.IsAny()), + _mockMessageBus.Verify( + x => x.PublishAsync( + It.IsAny(), + It.Is(data => data is TestCoverageMessage + && ((TestCoverageMessage)data).Scope.Level == CoverageScopeLevel.Overall + && ((TestCoverageMessage)data).Metric == CoverageMetric.Line)), Times.Once); } diff --git a/test/coverlet.MTP.tests/Collector/CollectorExtensionTests.cs b/test/coverlet.MTP.tests/Collector/CollectorExtensionTests.cs index 713aa5550..be637f7b4 100644 --- a/test/coverlet.MTP.tests/Collector/CollectorExtensionTests.cs +++ b/test/coverlet.MTP.tests/Collector/CollectorExtensionTests.cs @@ -139,13 +139,13 @@ private CollectorExtension CreateCollector() _mockFileSystem.Object); // Inject the mock file system } - private static async Task InvokeGenerateReportsAsync(CollectorExtension collector, CoverageResult coverageResult) + private static async Task InvokeGenerateReportsAsync(CollectorExtension collector, CoverageResult coverageResult, int testHostExitCode = 0) { System.Reflection.MethodInfo? method = typeof(CollectorExtension) .GetMethod("GenerateReportsAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); Assert.NotNull(method); - await (Task)method.Invoke(collector, [coverageResult, CancellationToken.None])!; + await (Task)method.Invoke(collector, [coverageResult, testHostExitCode, CancellationToken.None])!; } private void ConfigureCollectorForGenerateReports( @@ -673,7 +673,7 @@ public async Task OnTestHostProcessExitedAsyncWithFileAndConsoleFormatsDisplaysR _mockOutputDevice.Verify(x => x.DisplayAsync( It.IsAny(), It.IsAny(), - It.IsAny()), Times.AtLeast(3)); + It.IsAny()), Times.AtLeast(2)); } [Fact] diff --git a/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs b/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs index c0705b68b..34a5141a1 100644 --- a/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs +++ b/test/coverlet.MTP.tests/Collector/CollectorExtensionThresholdTests.cs @@ -7,8 +7,10 @@ using Coverlet.MTP.Configuration; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Configurations; +using Microsoft.Testing.Platform.Extensions.Messages; using Microsoft.Testing.Platform.Extensions.OutputDevice; using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.OutputDevice; using Moq; using Xunit; @@ -18,44 +20,44 @@ namespace Coverlet.MTP.Collector.Tests; public class CollectorExtensionThresholdTests { [Fact] - public async Task DisplayThresholdSummaryAsyncDisplaysConfiguredPassingMetrics() + public async Task PublishCoverageDataAsyncPublishesPassingThresholdMetrics() { var outputDevice = new Mock(); - CollectorExtension collector = CreateCollector(outputDevice); + var messageBus = new Mock(); + CollectorExtension collector = CreateCollector(outputDevice, messageBus); ConfigureThreshold(collector, 70, ThresholdStatistic.Total, ["line", "branch", "method"]); - await DisplayThresholdSummaryAsync(collector, CreateCoverageResult(hits: 1)); - - outputDevice.Verify(x => x.DisplayAsync( - It.Is(producer => producer == collector), - It.Is(data => - data.Text.Contains("Coverage Threshold Results:") && - data.Text.Contains("Total - Line (Total over Module): 100.0% >= 70.0% threshold") && - data.Text.Contains("Total - Branch (Total over Module): 100.0% >= 70.0% threshold") && - data.Text.Contains("Total - Method (Total over Module): 100.0% >= 70.0% threshold")), - It.IsAny()), - Times.Once); + await PublishCoverageDataAsync(collector, CreateCoverageResult(hits: 1)); + + messageBus.Verify(x => x.PublishAsync( + It.IsAny(), + It.Is(data => data is TestCoverageThresholdMessage + && ((TestCoverageThresholdMessage)data).Metric == CoverageMetric.Line + && ((TestCoverageThresholdMessage)data).RequiredPercentage == 70 + && ((TestCoverageThresholdMessage)data).ActualPercentage == 100 + && ((TestCoverageThresholdMessage)data).Passed)), Times.Once); } [Fact] - public async Task DisplayThresholdSummaryAsyncDisplaysFailedMetric() + public async Task PublishCoverageDataAsyncPublishesFailedThresholdMetric() { var outputDevice = new Mock(); - CollectorExtension collector = CreateCollector(outputDevice); + var messageBus = new Mock(); + CollectorExtension collector = CreateCollector(outputDevice, messageBus); ConfigureThreshold(collector, 70, ThresholdStatistic.Total, ["line"]); - await DisplayThresholdSummaryAsync(collector, CreateCoverageResult(hits: 0)); + await PublishCoverageDataAsync(collector, CreateCoverageResult(hits: 0)); - outputDevice.Verify(x => x.DisplayAsync( - It.IsAny(), - It.Is(data => - data.Text.Contains("Total - Line (Total over Module): 0.0% < 70.0% threshold") && - data.Text.Contains("The total line coverage is below the specified 70.0% threshold.")), - It.IsAny()), - Times.Once); + messageBus.Verify(x => x.PublishAsync( + It.IsAny(), + It.Is(data => data is TestCoverageThresholdMessage + && ((TestCoverageThresholdMessage)data).Metric == CoverageMetric.Line + && ((TestCoverageThresholdMessage)data).RequiredPercentage == 70 + && ((TestCoverageThresholdMessage)data).ActualPercentage == 0 + && !((TestCoverageThresholdMessage)data).Passed)), Times.Once); } - private static CollectorExtension CreateCollector(Mock outputDevice) + private static CollectorExtension CreateCollector(Mock outputDevice, Mock messageBus) { var loggerFactory = new Mock(); loggerFactory.Setup(factory => factory.CreateLogger(It.IsAny())) @@ -64,17 +66,20 @@ private static CollectorExtension CreateCollector(Mock outputDevi var commandLineOptions = new Mock(); var configuration = new Mock(); var fileSystem = new Mock(); + outputDevice.Setup(device => device.DisplayAsync( It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + messageBus.Setup(bus => bus.PublishAsync(It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); return new CollectorExtension( loggerFactory.Object, commandLineOptions.Object, outputDevice.Object, configuration.Object, - fileSystem.Object); + fileSystem.Object, + messageBus: messageBus.Object); } private static void ConfigureThreshold(CollectorExtension collector, int threshold, ThresholdStatistic thresholdStat, List thresholdTypes) @@ -87,11 +92,11 @@ private static void ConfigureThreshold(CollectorExtension collector, int thresho configuration.ThresholdType = thresholdTypes; } - private static async Task DisplayThresholdSummaryAsync(CollectorExtension collector, CoverageResult result) + private static async Task PublishCoverageDataAsync(CollectorExtension collector, CoverageResult result) { System.Reflection.MethodInfo method = typeof(CollectorExtension) - .GetMethod("DisplayThresholdSummaryAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; - await (Task)method.Invoke(collector, [result, CancellationToken.None])!; + .GetMethod("PublishCoverageDataAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + await (Task)method.Invoke(collector, [result, new List(), CancellationToken.None])!; } private static CoverageResult CreateCoverageResult(int hits) diff --git a/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs new file mode 100644 index 000000000..798522242 --- /dev/null +++ b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs @@ -0,0 +1,87 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Coverlet.Core; +using Coverlet.Core.Enums; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.TestHost; +using Xunit; + +namespace Coverlet.MTP.Collector.Tests; + +public class CoverletCoverageDataProducerTests +{ + [Fact] + public void CreateCoverageMessagesCreatesOverallAndModuleMessagesForLineAndBranch() + { + var producer = new CoverletCoverageDataProducer(); + CoverageResult result = CreateCoverageResult(hits: 1); + + IReadOnlyList messages = producer.CreateCoverageMessages(result, new SessionUid("session")); + + Assert.Equal(4, messages.Count); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Overall && message.Metric == CoverageMetric.Line); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Overall && message.Metric == CoverageMetric.Branch); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Module && message.Metric == CoverageMetric.Line); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Module && message.Metric == CoverageMetric.Branch); + } + + [Fact] + public void CreateThresholdMessagesMapsThresholdStatisticToCoverageAggregation() + { + var producer = new CoverletCoverageDataProducer(); + CoverageResult result = CreateCoverageResult(hits: 1); + var thresholds = new Dictionary + { + [ThresholdTypeFlags.Line] = 70, + }; + + IReadOnlyList messages = producer.CreateThresholdMessages( + result, + new SessionUid("session"), + thresholds, + ThresholdStatistic.Average, + static (_, _, _) => 100, + ThresholdTypeFlags.None); + + TestCoverageThresholdMessage message = Assert.Single(messages); + Assert.Equal(CoverageAggregation.Average, message.Aggregation); + Assert.Equal(CoverageScopeLevel.Module, message.AggregatedOver); + } + + [Theory] + [InlineData("cobertura", CoverageReportFormat.Cobertura, null)] + [InlineData("opencover", CoverageReportFormat.OpenCover, null)] + [InlineData("lcov", CoverageReportFormat.Lcov, null)] + [InlineData("json", CoverageReportFormat.Custom, "json")] + public void CreateReportMessageMapsKnownFormats(string reportFormat, CoverageReportFormat expectedFormat, string? expectedCustomFormat) + { + var producer = new CoverletCoverageDataProducer(); + + TestCoverageReportMessage message = producer.CreateReportMessage(new SessionUid("session"), "/fake/path/report", reportFormat); + + Assert.Equal(expectedFormat, message.Format); + Assert.Equal(expectedCustomFormat, message.CustomFormatName); + } + + private static CoverageResult CreateCoverageResult(int hits) + { + var methods = new Methods + { + ["System.Void TestClass::TestMethod()"] = new Method + { + Lines = new Lines { { 1, hits } }, + Branches = [new BranchInfo { Line = 1, Hits = hits }], + }, + }; + var classes = new Classes { ["TestClass"] = methods }; + var documents = new Documents { ["TestClass.cs"] = classes }; + + return new CoverageResult + { + Identifier = "test-id", + Modules = new Modules { ["test.dll"] = documents }, + Parameters = new CoverageParameters(), + }; + } +} diff --git a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs index eb53db0e3..2cfa4e557 100644 --- a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs +++ b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs @@ -311,7 +311,7 @@ public async Task CoverageWithJsonFormat_SummaryTableAppearsInConsoleOutput() { // Arrange string testName = TestContext.Current.TestCase!.TestMethodName!; - using var testProject = CreateTestProject(testName, includeSimpleTest: true); + using var testProject = CreateTestProject(testName, includeSimpleTest: true, includeBranchTest: true); await BuildProject(testProject.SolutionPath); // Act @@ -325,20 +325,21 @@ public async Task CoverageWithJsonFormat_SummaryTableAppearsInConsoleOutput() // Assert - test run succeeded Assert.True(result.ExitCode == 0, $"Expected successful test run (exit code 0) but got {result.ExitCode} -> '{result.ErrorText}'.\n\n{result.CombinedOutput}"); - // Assert - module table header appears in output - Assert.True(result.StandardOutput.Contains("| Module"), - $"Expected coverage summary module table (| Module |) in standard output.\n\n{result.CombinedOutput}"); + // Assert - MTP summary appears with module and aggregate rows + Assert.True(result.StandardOutput.Contains("Code Coverage Summary:"), + $"Expected code coverage summary in standard output.\n\n{result.CombinedOutput}"); + + Assert.True(result.StandardOutput.Contains("SampleLibrary.dll - Branch:"), + $"Expected module branch coverage for SampleLibrary.dll in standard output.\n\n{result.CombinedOutput}"); - // Assert - SUT module name appears in the table - Assert.True(result.StandardOutput.Contains("SampleLibrary"), - $"Expected SUT module 'SampleLibrary' in coverage summary table.\n\n{result.CombinedOutput}"); + Assert.True(result.StandardOutput.Contains("Total - Branch:"), + $"Expected total branch coverage in standard output.\n\n{result.CombinedOutput}"); - // Assert - total/average summary table appears - Assert.True(result.StandardOutput.Contains("| Total"), - $"Expected '| Total' row in coverage summary table.\n\n{result.CombinedOutput}"); + Assert.False(result.StandardOutput.Contains("Total - Branch: N/A"), + $"Expected total branch coverage to be numeric, but it was N/A.\n\n{result.CombinedOutput}"); - Assert.True(result.StandardOutput.Contains("| Average"), - $"Expected '| Average' row in coverage summary table.\n\n{result.CombinedOutput}"); + Assert.False(result.StandardOutput.Contains("SampleLibrary.dll - Branch: N/A"), + $"Expected module branch coverage to be numeric, but it was N/A.\n\n{result.CombinedOutput}"); } [Fact] From 880c6af046111173346ef59f38cf2895fef44441 Mon Sep 17 00:00:00 2001 From: Bert Date: Mon, 7 Sep 2026 11:40:13 +0200 Subject: [PATCH 07/12] Add method coverage calculation and reporting The code now computes and reports method coverage alongside line and branch coverage. It introduces overall and per-module method coverage metrics using CoverageSummary.CalculateMethodCoverage, and includes these in the TestCoverageMessage output. Method coverage is now always calculated and reported, independent of threshold configuration. --- .../Collector/CoverletCoverageDataProducer.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs index 2d1cf33af..76a29535a 100644 --- a/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs +++ b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs @@ -42,18 +42,26 @@ public IReadOnlyList CreateCoverageMessages(CoverageResult var messages = new List(); + // Calculate overall coverage for all metrics (line, branch, method) + // regardless of ThresholdType configuration. ThresholdType is only for + // determining which metrics to enforce thresholds on, not what metrics to calculate. CoverageDetails overallLine = CoverageSummary.CalculateLineCoverage(result.Modules); CoverageDetails overallBranch = CoverageSummary.CalculateBranchCoverage(result.Modules); + CoverageDetails overallMethod = CoverageSummary.CalculateMethodCoverage(result.Modules); messages.Add(CreateCoverageMessage(sessionUid, CoverageScope.Overall, CoverageMetric.Line, overallLine)); messages.Add(CreateCoverageMessage(sessionUid, CoverageScope.Overall, CoverageMetric.Branch, overallBranch)); + messages.Add(CreateCoverageMessage(sessionUid, CoverageScope.Overall, CoverageMetric.Method, overallMethod)); + // Calculate per-module coverage for all metrics foreach (KeyValuePair module in result.Modules) { var scope = new CoverageScope(CoverageScopeLevel.Module, module.Key); CoverageDetails line = CoverageSummary.CalculateLineCoverage(module.Value); CoverageDetails branch = CoverageSummary.CalculateBranchCoverage(module.Value); + CoverageDetails method = CoverageSummary.CalculateMethodCoverage(module.Value); messages.Add(CreateCoverageMessage(sessionUid, scope, CoverageMetric.Line, line)); messages.Add(CreateCoverageMessage(sessionUid, scope, CoverageMetric.Branch, branch)); + messages.Add(CreateCoverageMessage(sessionUid, scope, CoverageMetric.Method, method)); } return messages; From 3ad31ed9fc739a7e06531ec4574bf654f03c9522 Mon Sep 17 00:00:00 2001 From: Bert Date: Mon, 7 Sep 2026 19:34:16 +0200 Subject: [PATCH 08/12] fix test CreateCoverageMessagesCreatesOverallAndModuleMessagesForLineBranchMethod --- .../Collector/CoverletCoverageDataProducerTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs index 798522242..569c65a4b 100644 --- a/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs +++ b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs @@ -12,18 +12,20 @@ namespace Coverlet.MTP.Collector.Tests; public class CoverletCoverageDataProducerTests { [Fact] - public void CreateCoverageMessagesCreatesOverallAndModuleMessagesForLineAndBranch() + public void CreateCoverageMessagesCreatesOverallAndModuleMessagesForLineBranchMethod() { var producer = new CoverletCoverageDataProducer(); CoverageResult result = CreateCoverageResult(hits: 1); IReadOnlyList messages = producer.CreateCoverageMessages(result, new SessionUid("session")); - Assert.Equal(4, messages.Count); + Assert.Equal(6, messages.Count); Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Overall && message.Metric == CoverageMetric.Line); Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Overall && message.Metric == CoverageMetric.Branch); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Overall && message.Metric == CoverageMetric.Method); Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Module && message.Metric == CoverageMetric.Line); Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Module && message.Metric == CoverageMetric.Branch); + Assert.Contains(messages, message => message.Scope.Level == CoverageScopeLevel.Module && message.Metric == CoverageMetric.Method); } [Fact] From bf6f4cddf1a14641a7edcd78e41960da4e3fe415 Mon Sep 17 00:00:00 2001 From: Bert Date: Wed, 9 Sep 2026 08:32:32 +0200 Subject: [PATCH 09/12] ignore coverlet.core.coverage.test for LINUX OS --- .devcontainer/devcontainer.json | 2 - .github/copilot-instructions.md | 41 ++---- Documentation/Changelog.md | 1 + .../XUnitTestProject1.csproj | 6 +- .../XUnitTestProject1.csproj | 4 +- .../XUnitTestProject2.csproj | 4 +- .../XUnitTestProject3.csproj | 4 +- .../XUnitTestProject1.csproj | 4 +- .../XUnitTestProject1.csproj | 4 +- eng/azure-pipelines.yml | 2 +- eng/build.yml | 7 +- .../LinuxUnsupportedFactAttribute.cs | 22 +++ .../LinuxUnsupportedTheoryAttribute.cs | 22 +++ .../Infrastructure/TestEnvironment.cs | 3 + .../UnhandledExceptionTestContextTracker.cs | 129 ++++++++++++++++++ 15 files changed, 204 insertions(+), 51 deletions(-) create mode 100644 test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedFactAttribute.cs create mode 100644 test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedTheoryAttribute.cs create mode 100644 test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3577067c6..a66dc5f63 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,12 +14,10 @@ "humao.rest-client", "42Crunch.vscode-openapi", "stoplight.spectral", - "ms-dotnettools.dotnet-interactive-vscode", "ms-dotnettools.vscode-dotnet-runtime", "GitHub.vscode-pull-request-github", "ms-vscode.powershell", "ms-azure-devops.azure-pipelines", - "GitHub.copilot-chat", "GitHub.copilot", "github.vscode-github-actions", "bierner.markdown-preview-github-styles", diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 64ad21f54..fc6e0d894 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -70,6 +70,7 @@ mockFileSystem.Setup(x => x.Exists("/fake/reports")).Returns(true); mockFileSystem.Setup(x => x.WriteAllText(It.IsAny(), It.IsAny())); // Verify the mock was called correctly mockFileSystem.Verify(x => x.WriteAllText(It.Is(path => path.EndsWith("report.json")), It.IsAny()), Times.Once); + ### Moq Testing Rules (Critical - Prevents Runtime Errors) **NEVER use extension methods in Moq `Setup()` or `Verify()` calls.** @@ -97,6 +98,7 @@ _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), It.IsAn _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), It.Is((v, t) => v.ToString()!.Contains("json")), It.IsAny(), It.IsAny>()), Times.Once); // Setup LogWarning behavior _mockLogger.Setup(x => x.Log(LogLevel.Warning, It.IsAny())); + #### Example: Mocking LogDebug ❌ **INCORRECT**: @@ -109,6 +111,7 @@ _mockLogger.Verify(x => x.Log(LogLevel.Debug, It.IsAny(), It.IsAny x.Log(LogLevel.Debug, It.IsAny(), It.Is((v, t) => v.ToString()!.Contains("xml")), It.IsAny(), It.IsAny>()), Times.Once); // Setup LogError behavior _mockLogger.Setup(x => x.Log(LogLevel.Error, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())); + **Key Points:** 1. Always use `ILogger.Log()` with the appropriate `LogLevel` instead of extension methods. 2. Use `It.IsAny()` for the state parameter. @@ -148,8 +151,7 @@ This codebase uses **TWO different ILogger interfaces** with different signature // BAD - Microsoft.Testing.Platform.Logging.ILogger does NOT have EventId _mockLogger.Verify(x => x.Log(LogLevel.Information, It.IsAny(), // ⚠️ EventId does NOT exist in MTP LOGGER It.IsAny(), It.IsAny(), It.IsAny>()), Times.Once); - ``` - + ✅ **CORRECT** - Uses actual MTP ILogger API signature (async methods): // GOOD - Microsoft.Testing.Platform.Logging.ILogger uses simple async methods _mockLogger.Verify(x => x.LogInformationAsync(It.IsAny(), It.IsAny()), Times.Once); @@ -160,6 +162,7 @@ _mockLogger.Verify(x => x.LogInformationAsync(It.Is(s => s.Contains("exp // For synchronous LoggerExtensions (extension methods): // NOTE: These are extension methods and cannot be verified with Moq; verify the underlying Log(...) call instead. _mockLogger.Verify(x => x.Log(LogLevel.Information, It.Is(s => s.Contains("Coverage reports generated")), It.IsAny(), It.IsAny>()), Times.Once); + **Verification Checklist:** - [ ] I have searched for the interface definition using `get_symbols_by_name`. - [ ] I have reviewed existing usage in the codebase using `code_search`. @@ -238,38 +241,9 @@ public void LogInformation(string message, bool important = false) 6. **Verify existing tests** - Check for duplicates before adding new test methods. 7. **Use Theory for parameterized tests** - Don't create multiple test methods for different input values. -## Issue-Specific Guidelines +## Testing with xUnit v3 -- For issue #1965, identify problematic assemblies before instrumentation and skip them, rather than relying on partial-restore/non-fatal restore behavior after failure. -- For assembly-level instrumentation viability, preflight logic should only check lock and resolvability, not PDB/source-based exclusion; PDB/source exclusion remains handled by existing assembly-without-sources filtering via CanInstrument/options. -- **Prefer calling `instrumenter.CanInstrument()` before preflight** so assemblies already excluded by existing coverage filters (no PDB/no local sources) skip preflight probing. - -## Documentation Guidelines for Issue Resolution - -### Documentation Limitation (Critical Rule) - -**When resolving issues, limit documentation to ONE comprehensive document ONLY.** - -**Validation Checklist Before Generating Tests:** -- [ ] I have searched for existing tests using `code_search`. -- [ ] I have reviewed existing test files in the same test project. -- [ ] I have identified which existing tests cover similar scenarios. -- [ ] I have documented which proposed tests are redundant. -- [ ] I can justify why each new test adds unique value. -- [ ] I have considered refactoring existing tests instead of adding duplicates. -- [ ] I have verified the actual API signatures being tested. -- [ ] I have used the correct mocking approach for the specific ILogger interface. -- [ ] I have used mocked file system abstractions instead of real file I/O. - -## Summary of Key Testing Rules - -1. **Always use `IFileSystem` abstraction** - Never use `File`, `Directory`, or `Path` static methods directly in tests. -2. **Always verify API signatures** - Use `get_symbols_by_name` and `code_search` before mocking external APIs. -3. **Know your ILogger** - Microsoft.Testing.Platform.Logging.ILogger ≠ Microsoft.Extensions.Logging.ILogger. -4. **Avoid extension methods in Moq** - They cannot be intercepted and will cause runtime exceptions. -5. **Use simulated paths** - Always use fake paths like `/fake/path/test.dll` in test mocks. -6. **Verify existing tests** - Check for duplicates before adding new test methods. -7. **Use Theory for parameterized tests** - Don't create multiple test methods for different input values. +For this repo's xUnit v3 Microsoft.Testing.Platform test apps, use xUnit-specific filters such as `--filter-method`, `--filter-class`, or `--filter-query`; VSTest-style `--filter` is unsupported. ## Issue-Specific Guidelines @@ -374,6 +348,7 @@ The one comprehensive document MUST include: - Build status - Test status - Coverage metrics +``` #### Documentation Creation (Critical Rule - Always Ask First) diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index 8afaa434a..8b14cf30d 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Improvements +- Publish Microsoft.Testing.Platform coverage messages from coverlet.MTP [#2019](https://github.com/coverlet-coverage/coverlet/pull/2019) - Implement dynamic exclusion filters for assemblies (Coverlet.MTP) [#1946](https://github.com/coverlet-coverage/coverlet/pull/1946) - Replace legacy .sln files with modern .slnx format [#1966](https://github.com/coverlet-coverage/coverlet/pull/1966) - coverlet.console: add trace diagnostics and actionable warnings for instrumentation/hit/empty-result failures [#2005](https://github.com/coverlet-coverage/coverlet/pull/2005) diff --git a/Documentation/Examples/MSBuild/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj b/Documentation/Examples/MSBuild/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj index 1d1171b1f..827f12d63 100644 --- a/Documentation/Examples/MSBuild/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj +++ b/Documentation/Examples/MSBuild/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj @@ -1,4 +1,4 @@ - + net9.0 @@ -9,8 +9,8 @@ - - + + all runtime; build; native; contentfiles; analyzers diff --git a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject1/XUnitTestProject1.csproj b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject1/XUnitTestProject1.csproj index 124062ef6..1a2c02012 100644 --- a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject1/XUnitTestProject1.csproj +++ b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject1/XUnitTestProject1.csproj @@ -13,8 +13,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers diff --git a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject2/XUnitTestProject2.csproj b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject2/XUnitTestProject2.csproj index be63e0ed1..743cea240 100644 --- a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject2/XUnitTestProject2.csproj +++ b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject2/XUnitTestProject2.csproj @@ -13,8 +13,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers diff --git a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject3/XUnitTestProject3.csproj b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject3/XUnitTestProject3.csproj index 6645dcb58..cdc007b6c 100644 --- a/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject3/XUnitTestProject3.csproj +++ b/Documentation/Examples/MSBuild/MergeWith/XUnitTestProject3/XUnitTestProject3.csproj @@ -13,8 +13,8 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers diff --git a/Documentation/Examples/VSTest/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj b/Documentation/Examples/VSTest/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj index f8bc49224..ea3847f34 100644 --- a/Documentation/Examples/VSTest/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj +++ b/Documentation/Examples/VSTest/DeterministicBuild/XUnitTestProject1/XUnitTestProject1.csproj @@ -9,8 +9,8 @@ - - + + all runtime; build; native; contentfiles; analyzers diff --git a/Documentation/Examples/VSTest/HelloWorld/XUnitTestProject1/XUnitTestProject1.csproj b/Documentation/Examples/VSTest/HelloWorld/XUnitTestProject1/XUnitTestProject1.csproj index bb9bfffe8..aa4f02ca3 100644 --- a/Documentation/Examples/VSTest/HelloWorld/XUnitTestProject1/XUnitTestProject1.csproj +++ b/Documentation/Examples/VSTest/HelloWorld/XUnitTestProject1/XUnitTestProject1.csproj @@ -10,8 +10,8 @@ - - + + all runtime; build; native; contentfiles; analyzers diff --git a/eng/azure-pipelines.yml b/eng/azure-pipelines.yml index faae50813..436018d85 100644 --- a/eng/azure-pipelines.yml +++ b/eng/azure-pipelines.yml @@ -56,7 +56,7 @@ jobs: - job: Legacy displayName: Legacy Windows continueOnError: 'true' - timeoutInMinutes: 30 + timeoutInMinutes: 40 cancelTimeoutInMinutes: 5 strategy: matrix: diff --git a/eng/build.yml b/eng/build.yml index 9332d3f59..ebd5bdc14 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -45,8 +45,11 @@ steps: dotnet build-server shutdown dotnet exec "$(Build.SourcesDirectory)/artifacts/bin/coverlet.core.tests/$(BuildConfiguration)_${fw}/coverlet.core.tests.dll" --diagnostic --diagnostic-verbosity trace --report-xunit-trx --report-xunit-trx-filename "coverlet.core.tests.${fw}.trx" --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/" --diagnostic-file-prefix "coverlet.core.tests.${fw}_" --results-directory "$(Build.SourcesDirectory)/artifacts/reports/" --no-progress dotnet build-server shutdown - dotnet exec "$(Build.SourcesDirectory)/artifacts/bin/coverlet.core.coverage.tests/$(BuildConfiguration)_${fw}/coverlet.core.coverage.tests.dll" --diagnostic --diagnostic-verbosity trace --report-xunit-trx --report-xunit-trx-filename "coverlet.core.coverage.tests.${fw}.trx" --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/" --diagnostic-file-prefix "coverlet.core.coverage.tests.${fw}_" --results-directory "$(Build.SourcesDirectory)/artifacts/reports/" --no-progress - dotnet build-server shutdown + if (!$IsLinux) + { + dotnet exec "$(Build.SourcesDirectory)/artifacts/bin/coverlet.core.coverage.tests/$(BuildConfiguration)_${fw}/coverlet.core.coverage.tests.dll" --diagnostic --diagnostic-verbosity trace --report-xunit-trx --report-xunit-trx-filename "coverlet.core.coverage.tests.${fw}.trx" --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/" --diagnostic-file-prefix "coverlet.core.coverage.tests.${fw}_" --results-directory "$(Build.SourcesDirectory)/artifacts/reports/" --no-progress + dotnet build-server shutdown + } dotnet exec "$(Build.SourcesDirectory)/artifacts/bin/coverlet.msbuild.tasks.tests/$(BuildConfiguration)_${fw}/coverlet.msbuild.tasks.tests.dll" --diagnostic --diagnostic-verbosity trace --report-xunit-trx --report-xunit-trx-filename "coverlet.msbuild.tasks.tests.${fw}.trx" --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/" --diagnostic-file-prefix "coverlet.msbuild.tasks.tests.${fw}_" --results-directory "$(Build.SourcesDirectory)/artifacts/reports/" --no-progress dotnet build-server shutdown dotnet exec "$(Build.SourcesDirectory)/artifacts/bin/coverlet.integration.tests/$(BuildConfiguration)_${fw}/coverlet.integration.tests.dll" --diagnostic --diagnostic-verbosity trace --report-xunit-trx --report-xunit-trx-filename "coverlet.integration.tests.${fw}.trx" --diagnostic-output-directory "$(Build.SourcesDirectory)/artifacts/log/" --diagnostic-file-prefix "coverlet.integration.tests.${fw}_" --results-directory "$(Build.SourcesDirectory)/artifacts/reports/" --no-progress diff --git a/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedFactAttribute.cs b/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedFactAttribute.cs new file mode 100644 index 000000000..8d96c2fb0 --- /dev/null +++ b/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedFactAttribute.cs @@ -0,0 +1,22 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +namespace Coverlet.Core.Tests.Infrastructure; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public sealed class LinuxUnsupportedFactAttribute : FactAttribute +{ + public LinuxUnsupportedFactAttribute( + [CallerFilePath] string sourceFilePath = "", + [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + SkipType = typeof(TestEnvironment); + SkipWhen = nameof(TestEnvironment.IsLinux); + Skip = "Skipped on Linux due to runtime IL limitations (BadImageFormatException / InvalidProgramException)."; + } +} diff --git a/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedTheoryAttribute.cs b/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedTheoryAttribute.cs new file mode 100644 index 000000000..5609a6dcc --- /dev/null +++ b/test/coverlet.core.coverage.tests/Infrastructure/LinuxUnsupportedTheoryAttribute.cs @@ -0,0 +1,22 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +namespace Coverlet.Core.Tests.Infrastructure; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public sealed class LinuxUnsupportedTheoryAttribute : TheoryAttribute +{ + public LinuxUnsupportedTheoryAttribute( + [CallerFilePath] string sourceFilePath = "", + [CallerLineNumber] int sourceLineNumber = 0) + : base(sourceFilePath, sourceLineNumber) + { + SkipType = typeof(TestEnvironment); + SkipWhen = nameof(TestEnvironment.IsLinux); + Skip = "Skipped on Linux due to runtime IL limitations (BadImageFormatException / InvalidProgramException)."; + } +} diff --git a/test/coverlet.core.coverage.tests/Infrastructure/TestEnvironment.cs b/test/coverlet.core.coverage.tests/Infrastructure/TestEnvironment.cs index 99e33fc63..9abf8f3fe 100644 --- a/test/coverlet.core.coverage.tests/Infrastructure/TestEnvironment.cs +++ b/test/coverlet.core.coverage.tests/Infrastructure/TestEnvironment.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Runtime.InteropServices; namespace Coverlet.Core.Tests.Infrastructure; @@ -24,6 +25,8 @@ public static class TestEnvironment // GitHub Actions - same as $(GITHUB_ACTIONS) in Directory.Build.props string.Equals(Environment.GetEnvironmentVariable("GITHUB_ACTIONS"), "true", StringComparison.OrdinalIgnoreCase); + public static bool IsLinux => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + /// /// Returns true if tests are running inside Visual Studio Test Explorer (interactively). /// This checks for VS-specific environment variables that are ONLY set by VS Test Explorer. diff --git a/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs b/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs new file mode 100644 index 000000000..b3f9bf1ff --- /dev/null +++ b/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs @@ -0,0 +1,129 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Reflection; +using System.Text; +using System.Threading; +using Xunit.v3; + +namespace Coverlet.Core.Tests.Infrastructure; + +internal static class UnhandledExceptionTestContextTracker +{ + private static readonly AsyncLocal s_currentContext = new(); + private static string s_lastKnownContext; + + public static void Set(IXunitTest test) + { + if (test is null) + { + throw new ArgumentNullException(nameof(test)); + } + + var context = new TestExecutionContext( + GetTestDisplayName(test), + test.TestCase.UniqueID, + test.TestCase.TestClassName, + test.TestCase.TestMethodName, + DateTimeOffset.UtcNow); + + s_currentContext.Value = context; + Volatile.Write(ref s_lastKnownContext, context.ToDiagnosticString()); + } + + public static void Clear() + { + s_currentContext.Value = null; + } + + public static string GetCurrentDiagnosticMessage() + { + TestExecutionContext currentContext = s_currentContext.Value; + if (currentContext is not null) + { + return currentContext.ToDiagnosticString(); + } + + string lastKnownContext = Volatile.Read(ref s_lastKnownContext); + return string.IsNullOrWhiteSpace(lastKnownContext) + ? "Last running test: " + : lastKnownContext; + } + + private static string GetTestDisplayName(IXunitTest test) + { + string testCaseDisplayName = test.TestCase.TestCaseDisplayName; + if (!string.IsNullOrWhiteSpace(testCaseDisplayName)) + { + return testCaseDisplayName; + } + + string className = test.TestCase.TestClassName; + string methodName = test.TestCase.TestMethodName; + return string.IsNullOrWhiteSpace(className) || string.IsNullOrWhiteSpace(methodName) + ? test.ToString() ?? "" + : $"{className}.{methodName}"; + } + + private sealed class TestExecutionContext( + string displayName, + string testCaseUniqueId, + string testClassName, + string testMethodName, + DateTimeOffset startedAtUtc) + { + public string ToDiagnosticString() + { + var builder = new StringBuilder(); + builder.Append("Last running test: ").Append(displayName); + + if (!string.IsNullOrWhiteSpace(testCaseUniqueId)) + { + builder.Append(Environment.NewLine) + .Append("Test case ID: ") + .Append(testCaseUniqueId); + } + + if (!string.IsNullOrWhiteSpace(testClassName)) + { + builder.Append(Environment.NewLine) + .Append("Test class: ") + .Append(testClassName); + } + + if (!string.IsNullOrWhiteSpace(testMethodName)) + { + builder.Append(Environment.NewLine) + .Append("Test method: ") + .Append(testMethodName); + } + + builder.Append(Environment.NewLine) + .Append("Started (UTC): ") + .Append(startedAtUtc.ToString("O")); + + return builder.ToString(); + } + } +} + +[AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false, Inherited = true)] +internal sealed class TrackCurrentTestAttribute : Attribute, IBeforeAfterTestAttribute +{ + public void Before(MethodInfo methodUnderTest, IXunitTest test) + { + ArgumentNullException.ThrowIfNull(methodUnderTest); + ArgumentNullException.ThrowIfNull(test); + + UnhandledExceptionTestContextTracker.Set(test); + } + + public void After(MethodInfo methodUnderTest, IXunitTest test) + { + ArgumentNullException.ThrowIfNull(methodUnderTest); + ArgumentNullException.ThrowIfNull(test); + + UnhandledExceptionTestContextTracker.Clear(); + } +} From 78b8b94dd1eb437cdc5f07a46eb72f6f86bed565 Mon Sep 17 00:00:00 2001 From: Bert Date: Wed, 9 Sep 2026 09:17:25 +0200 Subject: [PATCH 10/12] Update to .NET 10.0.12 and align package versions Update global.json SDK to 10.0.401 and introduce $(MicrosoftNetRuntimeVersion) property set to 10.0.12. Replace hardcoded Microsoft.Extensions.* and System.* package versions in project files with the new property. Bump Microsoft.CodeAnalysis to 5.9.0, Microsoft.NET.Test.Sdk to 18.9.0, and xunit.runner.visualstudio to 4.0.0. Update net9.0-specific dependencies in coverlet.core.csproj to 9.0.20 and System.Text.Json in coverlet.integration.template.csproj to 10.0.12. Align conditional references in coverlet.integration.tests.csproj to use the new runtime version property. --- Directory.Packages.props | 49 ++++++++++--------- global.json | 2 +- src/coverlet.core/coverlet.core.csproj | 6 +-- .../coverlet.integration.template.csproj | 2 +- .../coverlet.integration.tests.csproj | 10 ++-- 5 files changed, 37 insertions(+), 32 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 77af13ebf..3bf6315e6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,15 +7,18 @@ - 17.11.48 - 5.6.0 + + 17.11.48 + 5.9.0 + 7.6.0 - 18.8.1 + 18.9.0 3.2.2 - 3.1.5 + 4.0.0 2.4.0 4.20.72 + 10.0.12 @@ -24,21 +27,23 @@ - + - - - - - - - + + + + + + + + + - + @@ -66,20 +71,20 @@ - - - - - + + + + + - + - - - + + + diff --git a/global.json b/global.json index a0703e84d..0777cf772 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.400", + "version": "10.0.401", "rollForward": "latestFeature" }, "test": { diff --git a/src/coverlet.core/coverlet.core.csproj b/src/coverlet.core/coverlet.core.csproj index 6dbd4e8f9..1b763abb9 100644 --- a/src/coverlet.core/coverlet.core.csproj +++ b/src/coverlet.core/coverlet.core.csproj @@ -22,9 +22,9 @@ - - - + + + diff --git a/test/coverlet.integration.template/coverlet.integration.template.csproj b/test/coverlet.integration.template/coverlet.integration.template.csproj index 9359cfa61..803e400d9 100644 --- a/test/coverlet.integration.template/coverlet.integration.template.csproj +++ b/test/coverlet.integration.template/coverlet.integration.template.csproj @@ -21,7 +21,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/test/coverlet.integration.tests/coverlet.integration.tests.csproj b/test/coverlet.integration.tests/coverlet.integration.tests.csproj index f061b4185..a5a8aaa4b 100644 --- a/test/coverlet.integration.tests/coverlet.integration.tests.csproj +++ b/test/coverlet.integration.tests/coverlet.integration.tests.csproj @@ -10,20 +10,20 @@ true false - false + false - + - + - + - + all runtime; build; native; contentfiles; analyzers From d8576bb329422adeb464634872cacddbdfe46695 Mon Sep 17 00:00:00 2001 From: Bert Date: Wed, 9 Sep 2026 11:04:40 +0200 Subject: [PATCH 11/12] add CoverageThresholdFailure_UsesMtpManagedExitCode and log MTP ExitCode status. --- Documentation/Coverlet.MTP.Integration.md | 4 +-- .../Collector/CollectorExtension.cs | 27 ++++++++++--------- .../Collector/CoverletCoverageDataProducer.cs | 2 +- .../CoverletCoverageDataProducerTests.cs | 2 +- .../CollectCoverageTests.cs | 22 +++++++++++++++ .../UnhandledExceptionTestContextTracker.cs | 2 +- 6 files changed, 41 insertions(+), 18 deletions(-) diff --git a/Documentation/Coverlet.MTP.Integration.md b/Documentation/Coverlet.MTP.Integration.md index 1898a4750..4b03134ea 100644 --- a/Documentation/Coverlet.MTP.Integration.md +++ b/Documentation/Coverlet.MTP.Integration.md @@ -88,8 +88,8 @@ dotnet exec --help | `--coverlet-does-not-return-attribute ` | Attributes that mark methods as not returning. Can be specified multiple times. (default: `none`) | | `--coverlet-exclude-assemblies-without-sources ` | Exclude assemblies without source code. Values: `MissingAll`, `MissingAny`, `None`. (default: `None`) | | `--coverlet-threshold ` | Exits with error if the coverage % is below value | -| `--coverlet-threshold-type ` | Coverage type to apply the threshold to. [default: `line`, `branch`, `method`] | -| `--coverlet-threshold-stat ` | Coverage statistic used to enforce the threshold value. [default: Minimum] | +| `--coverlet-threshold-type ` | Coverage type(s) to apply the threshold to. Can be specified multiple times or as a comma-separated list. (default: `line`) | +| `--coverlet-threshold-stat ` | Coverage statistic used to enforce the threshold value. (default: `Minimum`) | > [!TIP] > If you encounter instrumentation failures like "The process cannot access the file ... because it is being used by another process", try setting `--coverlet-exclude-assemblies-without-sources MissingAll` (or in a config file: `"ExcludeAssembliesWithoutSources": "MissingAll"`) to skip assemblies without sources and reduce access conflicts. diff --git a/src/coverlet.MTP/Collector/CollectorExtension.cs b/src/coverlet.MTP/Collector/CollectorExtension.cs index d7cd6dab9..f42690c53 100644 --- a/src/coverlet.MTP/Collector/CollectorExtension.cs +++ b/src/coverlet.MTP/Collector/CollectorExtension.cs @@ -508,7 +508,13 @@ private async Task PublishCoverageDataAsync(CoverageResult result, IReadOnlyList foreach (string generatedReport in generatedReports) { cancellation.ThrowIfCancellationRequested(); - string reportFormat = Path.GetExtension(generatedReport).TrimStart('.'); + string fileName = Path.GetFileName(generatedReport); + string fileNameLower = fileName.ToLowerInvariant(); + string reportFormat = fileNameLower.Contains(".cobertura.") ? "cobertura" + : fileNameLower.Contains(".opencover.") ? "opencover" + : fileNameLower.EndsWith(".info", StringComparison.Ordinal) ? "lcov" + : fileNameLower.EndsWith(".json", StringComparison.Ordinal) ? "json" + : Path.GetExtension(fileNameLower).TrimStart('.'); if (string.IsNullOrWhiteSpace(reportFormat)) { continue; @@ -702,7 +708,7 @@ private async Task GenerateReportsAsync(CoverageResult result, int testHostExitC // Display console-type report output (e.g. teamcity) directly to the output device await DisplayConsoleReportOutputsAsync(consoleOutputs, cancellation); - // Exitcode `CoverageThresholdFailed = 14` shall only override otherwise successful test runs. + // Coverage threshold exit-code behavior is owned by Microsoft Testing Platform. if (_configuration.Threshold.HasValue) { ThresholdStatistic thresholdStat = _configuration.ThresholdStat; @@ -712,18 +718,13 @@ private async Task GenerateReportsAsync(CoverageResult result, int testHostExitC ThresholdTypeFlags belowThreshold = result.GetThresholdTypesBelowThreshold(thresholdValues, thresholdStat); if (belowThreshold != ThresholdTypeFlags.None) { - int currentExitCode = Environment.ExitCode != 0 ? Environment.ExitCode : testHostExitCode; - if (currentExitCode == 0) + if (IsCoverageThresholdExitCodeIgnored()) { - if (IsCoverageThresholdExitCodeIgnored()) - { - _logger.LogInformation("Coverage thresholds not met, but exit code 14 is ignored by --ignore-exit-code."); - } - else - { - _logger.LogError("Coverage thresholds not met. Setting exit code for Microsoft Testing Framework to 14."); - Environment.ExitCode = 14; - } + _logger.LogInformation("Coverage thresholds not met, but exit code 14 is ignored by --ignore-exit-code."); + } + else + { + _logger.LogError("Coverage thresholds not met."); } } } diff --git a/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs index 76a29535a..4c7d745b1 100644 --- a/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs +++ b/src/coverlet.MTP/Collector/CoverletCoverageDataProducer.cs @@ -180,7 +180,7 @@ private static (CoverageReportFormat Format, string? CustomFormatName) MapReport "cobertura" => (CoverageReportFormat.Cobertura, null), "opencover" => (CoverageReportFormat.OpenCover, null), "lcov" => (CoverageReportFormat.Lcov, null), - "json" => (CoverageReportFormat.Custom, "json"), + "json" => (CoverageReportFormat.Custom, "coverlet.json"), _ => (CoverageReportFormat.Custom, reportFormat), }; } diff --git a/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs index 569c65a4b..b5da94bc8 100644 --- a/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs +++ b/test/coverlet.MTP.tests/Collector/CoverletCoverageDataProducerTests.cs @@ -55,7 +55,7 @@ public void CreateThresholdMessagesMapsThresholdStatisticToCoverageAggregation() [InlineData("cobertura", CoverageReportFormat.Cobertura, null)] [InlineData("opencover", CoverageReportFormat.OpenCover, null)] [InlineData("lcov", CoverageReportFormat.Lcov, null)] - [InlineData("json", CoverageReportFormat.Custom, "json")] + [InlineData("json", CoverageReportFormat.Custom, "coverlet.json")] public void CreateReportMessageMapsKnownFormats(string reportFormat, CoverageReportFormat expectedFormat, string? expectedCustomFormat) { var producer = new CoverletCoverageDataProducer(); diff --git a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs index 2cfa4e557..dd405d32f 100644 --- a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs +++ b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs @@ -65,6 +65,27 @@ public async Task BasicCoverage_CollectsDataForCoveredLines() CheckCoverageResult(testProject, result, CoverageJsonFileName); } + [Fact] + public async Task CoverageThresholdFailure_UsesMtpManagedExitCode() + { + // Arrange + string testName = TestContext.Current.TestCase!.TestMethodName!; + using var testProject = CreateTestProject(testName, includeSimpleTest: true, includeMultipleClasses: true); + await BuildProject(testProject.SolutionPath); + + // Act + var result = await RunTestsWithCoverage( + testProject, + "--coverlet --coverlet-output-format json --coverlet-threshold 100 --coverlet-threshold-type line --coverlet-threshold-stat total", + testName); + + TestContext.Current?.AddAttachment("Test Output", result.CombinedOutput); + + // Assert + Assert.True(result.ExitCode == 14, $"Expected threshold failure exit code 14 from Microsoft Testing Platform but got {result.ExitCode} -> '{result.ErrorText}'.\n\n{result.CombinedOutput}"); + Assert.Contains("86.7% < 100.0% threshold", result.CombinedOutput); + } + [Fact] public async Task CoverageWithFormat_GeneratesCorrectOutputFormat() { @@ -1411,6 +1432,7 @@ private static async Task RunTestsWithCoverage(TestProjectInfo testP 11 => "test process will exit if dependent process exits", 12 => "test session was unable to run because the client does not support any of the supported protocol versions", 13 => "exceeded number of maximum failed tests", + 14 => "coverage threshold was not met", _ => "unrecognized exit code" }; diff --git a/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs b/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs index b3f9bf1ff..89696d0be 100644 --- a/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs +++ b/test/coverlet.core.coverage.tests/Infrastructure/UnhandledExceptionTestContextTracker.cs @@ -12,7 +12,7 @@ namespace Coverlet.Core.Tests.Infrastructure; internal static class UnhandledExceptionTestContextTracker { private static readonly AsyncLocal s_currentContext = new(); - private static string s_lastKnownContext; + private static string s_lastKnownContext = string.Empty; public static void Set(IXunitTest test) { From a6de28de4e7140c59f74a5ecb031614938339cf9 Mon Sep 17 00:00:00 2001 From: Bert Date: Wed, 9 Sep 2026 11:28:59 +0200 Subject: [PATCH 12/12] use "coverage threshold was not met" from STDERR for test assert --- test/coverlet.MTP.validation.tests/CollectCoverageTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs index dd405d32f..0bfcc0afa 100644 --- a/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs +++ b/test/coverlet.MTP.validation.tests/CollectCoverageTests.cs @@ -83,7 +83,7 @@ public async Task CoverageThresholdFailure_UsesMtpManagedExitCode() // Assert Assert.True(result.ExitCode == 14, $"Expected threshold failure exit code 14 from Microsoft Testing Platform but got {result.ExitCode} -> '{result.ErrorText}'.\n\n{result.CombinedOutput}"); - Assert.Contains("86.7% < 100.0% threshold", result.CombinedOutput); + Assert.Contains("coverage threshold was not met", result.CombinedOutput); } [Fact]