diff --git a/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs b/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs index 7f0d7e5db5..625e78a816 100644 --- a/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs +++ b/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs @@ -21,7 +21,8 @@ private class Config : ManualConfig private class FastestToSlowestOrderer : IOrderer { - public IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase) => + public IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, + IEnumerable order = null) => from benchmark in benchmarksCase orderby benchmark.Parameters["X"] descending, benchmark.Descriptor.WorkloadMethodDisplayInfo @@ -37,7 +38,8 @@ orderby summary[benchmark].ResultStatistics.Mean public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) => benchmarkCase.Job.DisplayInfo + "_" + benchmarkCase.Parameters.DisplayInfo; - public IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups) => + public IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups, + IEnumerable order = null) => logicalGroups.OrderBy(it => it.Key); public bool SeparateLogicalGroups => true; diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index f7ca6df67d..b1e74668bf 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -30,7 +30,7 @@ public sealed class ImmutableConfig : IConfig private readonly ImmutableHashSet jobs; private readonly ImmutableHashSet hardwareCounters; private readonly ImmutableHashSet filters; - private readonly ImmutableHashSet rules; + private readonly ImmutableArray rules; internal ImmutableConfig( ImmutableArray uniqueColumnProviders, @@ -41,7 +41,7 @@ internal ImmutableConfig( ImmutableHashSet uniqueAnalyzers, ImmutableHashSet uniqueValidators, ImmutableHashSet uniqueFilters, - ImmutableHashSet uniqueRules, + ImmutableArray uniqueRules, ImmutableHashSet uniqueRunnableJobs, ConfigUnionRule unionRule, string artifactsPath, diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 14c41f6e7c..a04f26f804 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -45,7 +45,7 @@ public static ImmutableConfig Create(IConfig source) var uniqueValidators = GetValidators(source.GetValidators(), MandatoryValidators, source.Options); var uniqueFilters = source.GetFilters().ToImmutableHashSet(); - var uniqueRules = source.GetLogicalGroupRules().ToImmutableHashSet(); + var uniqueRules = source.GetLogicalGroupRules().ToImmutableArray(); var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet(); diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 9bdb0d32b7..7a46d6450e 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -29,7 +29,7 @@ public class ManualConfig : IConfig private readonly List jobs = new List(); private readonly HashSet hardwareCounters = new HashSet(); private readonly List filters = new List(); - private readonly HashSet logicalGroupRules = new HashSet(); + private readonly List logicalGroupRules = new List(); public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; @@ -191,7 +191,12 @@ public ManualConfig AddFilter(params IFilter[] newFilters) public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rules) { - logicalGroupRules.AddRange(rules); + foreach (var rule in rules) + { + if (logicalGroupRules.Contains(rule)) + logicalGroupRules.Remove(rule); + logicalGroupRules.Add(rule); + } return this; } diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 27e5ac6be5..0bca3cf3ec 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; @@ -19,8 +20,7 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; private readonly IComparer jobComparer = JobComparer.Instance; - private readonly IComparer benchmarkComparer; - private readonly IComparer> logicalGroupComparer; + private readonly IComparer targetComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } public MethodOrderPolicy MethodOrderPolicy { get; } @@ -31,14 +31,15 @@ public DefaultOrderer( { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; - IComparer targetComparer = new DescriptorComparer(methodOrderPolicy); - benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer); - logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer); + targetComparer = new DescriptorComparer(methodOrderPolicy); } [PublicAPI] - public virtual IEnumerable GetExecutionOrder(ImmutableArray benchmarkCases) + public virtual IEnumerable GetExecutionOrder( + ImmutableArray benchmarkCases, + IEnumerable order = null) { + var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); var list = benchmarkCases.ToList(); list.Sort(benchmarkComparer); return list; @@ -47,7 +48,7 @@ public virtual IEnumerable GetExecutionOrder(ImmutableArray GetSummaryOrder(ImmutableArray benchmarksCases, Summary summary) { var benchmarkLogicalGroups = benchmarksCases.GroupBy(b => GetLogicalGroupKey(benchmarksCases, b)); - foreach (var logicalGroup in GetLogicalGroupOrder(benchmarkLogicalGroups)) + foreach (var logicalGroup in GetLogicalGroupOrder(benchmarkLogicalGroups, benchmarksCases.FirstOrDefault()?.Config.GetLogicalGroupRules())) foreach (var benchmark in GetSummaryOrderForGroup(logicalGroup.ToImmutableArray(), summary)) yield return benchmark; } @@ -65,7 +66,7 @@ protected virtual IEnumerable GetSummaryOrderForGroup(ImmutableAr case SummaryOrderPolicy.Declared: return benchmarksCase; default: - return GetExecutionOrder(benchmarksCase); + return GetExecutionOrder(benchmarksCase, benchmarksCase.FirstOrDefault()?.Config.GetLogicalGroupRules()); } } @@ -84,41 +85,62 @@ public string GetHighlightGroupKey(BenchmarkCase benchmarkCase) public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) { - var rules = new HashSet(benchmarkCase.Config.GetLogicalGroupRules()); + var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList(); + var implicitRules = new List(); bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline); bool hasDescriptorBaselines = allBenchmarksCases.Any(b => b.Descriptor.Baseline); if (hasJobBaselines) { - rules.Add(BenchmarkLogicalGroupRule.ByMethod); - rules.Add(BenchmarkLogicalGroupRule.ByParams); + implicitRules.Add(BenchmarkLogicalGroupRule.ByParams); + implicitRules.Add(BenchmarkLogicalGroupRule.ByMethod); } if (hasDescriptorBaselines) { - rules.Add(BenchmarkLogicalGroupRule.ByJob); - rules.Add(BenchmarkLogicalGroupRule.ByParams); + implicitRules.Add(BenchmarkLogicalGroupRule.ByParams); + implicitRules.Add(BenchmarkLogicalGroupRule.ByJob); } if (hasJobBaselines && hasDescriptorBaselines) { - rules.Remove(BenchmarkLogicalGroupRule.ByMethod); - rules.Remove(BenchmarkLogicalGroupRule.ByJob); + implicitRules.Remove(BenchmarkLogicalGroupRule.ByMethod); + implicitRules.Remove(BenchmarkLogicalGroupRule.ByJob); } + var rules = new List(explicitRules); + foreach (var rule in implicitRules.Where(rule => !rules.Contains(rule))) + rules.Add(rule); + var keys = new List(); - if (rules.Contains(BenchmarkLogicalGroupRule.ByCategory)) - keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories)); - if (rules.Contains(BenchmarkLogicalGroupRule.ByMethod)) - keys.Add(benchmarkCase.Descriptor.DisplayInfo); - if (rules.Contains(BenchmarkLogicalGroupRule.ByJob)) - keys.Add(benchmarkCase.Job.DisplayInfo); - if (rules.Contains(BenchmarkLogicalGroupRule.ByParams)) - keys.Add(benchmarkCase.Parameters.ValueInfo); + foreach (var rule in rules) + { + switch (rule) + { + case BenchmarkLogicalGroupRule.ByMethod: + keys.Add(benchmarkCase.Descriptor.DisplayInfo); + break; + case BenchmarkLogicalGroupRule.ByJob: + keys.Add(benchmarkCase.Job.DisplayInfo); + break; + case BenchmarkLogicalGroupRule.ByParams: + keys.Add(benchmarkCase.Parameters.ValueInfo); + break; + case BenchmarkLogicalGroupRule.ByCategory: + keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(rule), rule, $"Not supported {nameof(BenchmarkLogicalGroupRule)}"); + } + } string logicalGroupKey = string.Join("-", keys.Where(key => key != string.Empty)); return logicalGroupKey == string.Empty ? "*" : logicalGroupKey; } - public virtual IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups) + public virtual IEnumerable> GetLogicalGroupOrder( + IEnumerable> logicalGroups, + IEnumerable order = null) { + var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); + var logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer); var list = logicalGroups.ToList(); list.Sort(logicalGroupComparer); return list; @@ -128,21 +150,36 @@ public virtual IEnumerable> GetLogicalGroupOrde private class BenchmarkComparer : IComparer { + private static readonly BenchmarkLogicalGroupRule[] DefaultOrder = + { + BenchmarkLogicalGroupRule.ByCategory, + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByJob, + BenchmarkLogicalGroupRule.ByMethod + }; + private readonly IComparer categoryComparer; private readonly IComparer paramsComparer; private readonly IComparer jobComparer; private readonly IComparer targetComparer; + private readonly List order; public BenchmarkComparer( IComparer categoryComparer, IComparer paramsComparer, IComparer jobComparer, - IComparer targetComparer) + IComparer targetComparer, + IEnumerable order) { this.categoryComparer = categoryComparer; this.targetComparer = targetComparer; this.jobComparer = jobComparer; this.paramsComparer = paramsComparer; + + this.order = new List(); + foreach (var rule in (order ?? ImmutableArray.Empty).Concat(DefaultOrder)) + if (!this.order.Contains(rule)) + this.order.Add(rule); } public int Compare(BenchmarkCase x, BenchmarkCase y) @@ -150,14 +187,21 @@ public int Compare(BenchmarkCase x, BenchmarkCase y) if (x == null && y == null) return 0; if (x != null && y == null) return 1; if (x == null) return -1; - return new[] + + foreach (var rule in order) { - categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0, - paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0, - jobComparer?.Compare(x.Job, y.Job) ?? 0, - targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0, - string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo) - }.FirstOrDefault(c => c != 0); + int compare = rule switch + { + BenchmarkLogicalGroupRule.ByMethod => targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0, + BenchmarkLogicalGroupRule.ByJob => jobComparer?.Compare(x.Job, y.Job) ?? 0, + BenchmarkLogicalGroupRule.ByParams => paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0, + BenchmarkLogicalGroupRule.ByCategory => categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0, + _ => throw new ArgumentOutOfRangeException() + }; + if (compare != 0) + return compare; + } + return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } } @@ -176,4 +220,4 @@ public int Compare(IGrouping x, IGrouping GetExecutionOrder(ImmutableArray benchmarksCase); + IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, IEnumerable order = null); [PublicAPI, NotNull] IEnumerable GetSummaryOrder(ImmutableArray benchmarksCases, [NotNull] Summary summary); @@ -22,7 +23,8 @@ public interface IOrderer string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, [NotNull] BenchmarkCase benchmarkCase); [PublicAPI, NotNull] - IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups); + IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups, + IEnumerable order = null); [PublicAPI] bool SeparateLogicalGroups { get; } diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt index b676c6ead3..136b7d77b3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt @@ -10,22 +10,22 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Base-[Param=2] | Yes | ^ - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsParamsJobs.Base-[Param=2] | No | + Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Foo-[Param=2] | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsParamsJobs.Foo-[Param=2] | No | + Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Bar-[Param=2] | Yes | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsParamsJobs.Bar-[Param=2] | No | + Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Base-[Param=10] | Yes | ^ - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Base-[Param=10] | No | + Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Foo-[Param=10] | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Foo-[Param=10] | No | + Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Bar-[Param=10] | Yes | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Bar-[Param=10] | No | + Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParams.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParams.approved.txt index 2e1a7f441a..7abc81047b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParams.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParams.approved.txt @@ -9,12 +9,12 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- | - Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob-[Param=2] | Yes | ^ - Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob-[Param=2] | No | - Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | DefaultJob-[Param=2] | No | + Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-DefaultJob | Yes | ^ + Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-DefaultJob | No | + Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-DefaultJob | No | | | | | | | | | | | - Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob-[Param=10] | Yes | ^ - Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | DefaultJob-[Param=10] | No | - Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | DefaultJob-[Param=10] | No | + Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-DefaultJob | Yes | ^ + Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=10]-DefaultJob | No | + Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=10]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParamsJobs.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParamsJobs.approved.txt index d5dbf83b8a..f63f8cd50b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParamsJobs.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParamsJobs.approved.txt @@ -10,20 +10,20 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1-[Param=2] | Yes | ^ - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | Job1-[Param=2] | No | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | Job1-[Param=2] | No | + Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job1 | Yes | ^ + Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-Job1 | No | + Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2-[Param=2] | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | Job2-[Param=2] | No | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | Job2-[Param=2] | No | + Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job2 | Yes | + Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=2]-Job2 | No | + Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=2]-Job2 | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1-[Param=10] | Yes | ^ - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | Job1-[Param=10] | No | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | Job1-[Param=10] | No | + Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job1 | Yes | ^ + Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | [Param=10]-Job1 | No | + Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | [Param=10]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2-[Param=10] | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | Job2-[Param=10] | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | Job2-[Param=10] | No | + Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job2 | Yes | + Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | [Param=10]-Job2 | No | + Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | [Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt index 253a8bcb8b..ba6ec2fa2c 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt @@ -10,36 +10,36 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2] | Yes | ^ + A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2] | No | + A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2] | Yes | + A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2] | No | + A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10] | Yes | ^ + A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10] | No | + A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10] | Yes | + A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10] | No | + A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2] | Yes | ^ + B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2] | No | + B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2] | Yes | + B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2] | No | + B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10] | Yes | ^ + B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10] | No | + B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10] | Yes | + B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10] | No | + B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByCategory.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByCategory.approved.txt index 1b79fd5dee..63a9edf151 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByCategory.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByCategory.approved.txt @@ -10,28 +10,28 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job1-[Param=2] | Yes | ^ - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-Job1-[Param=2] | No | + A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job1 | Yes | ^ + A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-[Param=2]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job2-[Param=2] | Yes | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-Job2-[Param=2] | No | + A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job2 | Yes | + A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-[Param=2]-Job2 | No | | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job1-[Param=10] | Yes | ^ - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-Job1-[Param=10] | No | + A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job1 | Yes | ^ + A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-[Param=10]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job2-[Param=10] | Yes | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-Job2-[Param=10] | No | + A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job2 | Yes | + A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-[Param=10]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job1-[Param=2] | Yes | ^ - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-Job1-[Param=2] | No | + B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job1 | Yes | ^ + B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-[Param=2]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job2-[Param=2] | Yes | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-Job2-[Param=2] | No | + B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job2 | Yes | + B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-[Param=2]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job1-[Param=10] | Yes | ^ - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-Job1-[Param=10] | No | + B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job1 | Yes | ^ + B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-[Param=10]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job2-[Param=10] | Yes | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-Job2-[Param=10] | No | + B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job2 | Yes | + B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-[Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index 126f9b9733..c1f633cfca 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -17,24 +17,25 @@ namespace BenchmarkDotNet.Tests.Order { public class DefaultOrdererTests { - [Fact] - public void CategoriesHasHigherPriorityThanParameters() - { - var summary = new Summary("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), - "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty); + private static Summary CreateMockSummary() => new("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), + "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty); - BenchmarkCase CreateBenchmarkCase(string category, int parameter) => new( - new Descriptor(null, null, categories: new[] { category }), - new Job(), - new ParameterInstances(new[] - { - new ParameterInstance(new ParameterDefinition("P", false, null, false, null, 0), parameter, SummaryStyle.Default) - }), - DefaultConfig.Instance.CreateImmutableConfig() - ); + private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new( + new Descriptor(null, null, categories: new[] { category }), + new Job(), + new ParameterInstances(new[] + { + new ParameterInstance(new ParameterDefinition("P", false, null, false, null, 0), parameter, SummaryStyle.Default) + }), + DefaultConfig.Instance.AddLogicalGroupRules(rules).CreateImmutableConfig() + ); - string GetId(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Categories.First() + benchmarkCase.Parameters.Items.First().Value; + private static string GetId(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Categories.First() + benchmarkCase.Parameters.Items.First().Value; + [Fact] + public void CategoriesHasHigherPriorityThanParameters() + { + var summary = CreateMockSummary(); var benchmarkCases = new List { CreateBenchmarkCase("A", 1), @@ -42,9 +43,28 @@ public void CategoriesHasHigherPriorityThanParameters() CreateBenchmarkCase("A", 2), CreateBenchmarkCase("B", 2) }.ToImmutableArray(); + string[] sortedBenchmarkCases = DefaultOrderer.Instance.GetSummaryOrder(benchmarkCases, summary).Select(GetId).ToArray(); + Assert.Equal(new[] { "A1", "A2", "B1", "B2" }, sortedBenchmarkCases); + } + [Fact] + public void OrderCanBeOverriden() + { + BenchmarkLogicalGroupRule[] rules = + { + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByCategory, + }; + var summary = CreateMockSummary(); + var benchmarkCases = new List + { + CreateBenchmarkCase("A", 1, rules), + CreateBenchmarkCase("B", 1, rules), + CreateBenchmarkCase("A", 2, rules), + CreateBenchmarkCase("B", 2, rules) + }.ToImmutableArray(); string[] sortedBenchmarkCases = DefaultOrderer.Instance.GetSummaryOrder(benchmarkCases, summary).Select(GetId).ToArray(); - Assert.Equal(new[] {"A1", "A2", "B1", "B2"}, sortedBenchmarkCases); + Assert.Equal(new[] { "A1", "B1", "A2", "B2" }, sortedBenchmarkCases); } } } \ No newline at end of file