Compare parameters by object value rather than string representation (Related to #2838)

src/BenchmarkDotNet/Order/DefaultOrderer.cs

@@ -89,6 +89,8 @@ public string GetHighlightGroupKey(BenchmarkCase benchmarkCase)
 
         public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCases, BenchmarkCase benchmarkCase)
         {
+            var paramSets = allBenchmarksCases.Select(benchmarkCase => benchmarkCase.Parameters).Distinct(ParameterEqualityComparer.Instance).ToArray();
+
             var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList();
             var implicitRules = new List<BenchmarkLogicalGroupRule>();
             bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline);
@@ -125,7 +127,7 @@ public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCase
                         keys.Add(benchmarkCase.Job.DisplayInfo);
                         break;
                     case BenchmarkLogicalGroupRule.ByParams:
-                        keys.Add(benchmarkCase.Parameters.ValueInfo);
+                        keys.Add($"Distinct Param Set {Array.FindIndex(paramSets, (paramSet) => ParameterEqualityComparer.Instance.Equals(paramSet, benchmarkCase.Parameters))}");
                         break;
                     case BenchmarkLogicalGroupRule.ByCategory:
                         keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories));

src/BenchmarkDotNet/Parameters/ParameterComparer.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace BenchmarkDotNet.Parameters
 {
@@ -18,25 +20,41 @@ public int Compare(ParameterInstances x, ParameterInstances y)
                 if (compareTo != 0)
                     return compareTo;
             }
+
             return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo);
         }
 
         private int CompareValues(object x, object y)
         {
             // Detect IComparable implementations.
             // This works for all primitive types in addition to user types that implement IComparable.
-            if (x != null && y != null && x.GetType() == y.GetType() &&
-                x is IComparable xComparable)
+            if (x != null && y != null && x.GetType() == y.GetType())
             {
-                try
+                if (x is IComparable xComparable)
                 {
-                    return xComparable.CompareTo(y);
+                    try
+                    {
+                        return xComparable.CompareTo(y);
+                    }
+                    // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable.
+                    // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346
+                    // For now, catch and ignore the exception, and fallback to string comparison below.
+                    catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable."))
+                    {
+                    }
                 }
-                // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable.
-                // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346
-                // For now, catch and ignore the exception, and fallback to string comparison below.
-                catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable."))
+                else if (x is IEnumerable xEnumerable  && y is IEnumerable yEnumerable) // collection equality support
                 {
+                    if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened
+                    {
+                        if (xArr.Rank != yArr.Rank)
+                            return xArr.Rank.CompareTo(yArr.Rank);
+                    }
+
+                    var xFlat = xEnumerable.OfType<object>().ToArray();
+                    var yFlat = yEnumerable.OfType<object>().ToArray();
+
+                    return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat);
                 }
             }
 

src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs

@@ -0,0 +1,20 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace BenchmarkDotNet.Parameters
+{
+    internal class ParameterEqualityComparer : IEqualityComparer<ParameterInstances>
+    {
+        public static readonly ParameterEqualityComparer Instance = new ParameterEqualityComparer();
+
+        public bool Equals(ParameterInstances? x, ParameterInstances? y)
+        {
+            return ParameterComparer.Instance.Compare(x, y) == 0;
+        }
+
+        public int GetHashCode([DisallowNull] ParameterInstances obj)
+        {
+            return obj.ValueInfo.GetHashCode();
+        }
+    }
+}

tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs

@@ -1,4 +1,5 @@
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Tests.XUnit;
 using BenchmarkDotNet.Toolchains;
@@ -298,6 +299,60 @@ public IEnumerable<object[]> Sources()
             public void Any(string name, IEnumerable<int> source) => source.Any();
         }
 
+        [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)]
+        public void IEnumerableArgumentsCanBeGrouped(IToolchain toolchain)
+        {
+            var summary = CanExecute<EnumerableOnDifferentMethods>(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping
+
+            // There should be two logical groups, one for the first argument and one for the second argument
+            // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group
+
+            var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor);
+
+            Assert.True(
+                descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2)
+            );
+        }
+
+        public class EnumerableOnDifferentMethods
+        {
+            public IEnumerable<int> EnumerableOne()
+            {
+                yield return 1;
+                yield return 2;
+                yield return 3;
+            }
+
+            public IEnumerable<int> EnumerableTwo()
+            {
+                yield return 1;
+                yield return 2;
+                yield return 3;
+            }
+
+            public IEnumerable<IEnumerable<int>> GetEnumerables()
+            {
+                yield return EnumerableOne();
+                yield return EnumerableTwo();
+            }
+
+            [Benchmark(Baseline = true)]
+            [ArgumentsSource(nameof(GetEnumerables))]
+            public void AcceptsEnumerables(IEnumerable<int> arr)
+            {
+                if (arr.Count() != 3)
+                    throw new ArgumentException("Incorrect length");
+            }
+
+            [Benchmark]
+            [ArgumentsSource(nameof(GetEnumerables))]
+            public void AcceptsEnumerables2(IEnumerable<int> collection)
+            {
+                if (collection.Count() != 3)
+                    throw new ArgumentException("Incorrect length");
+            }
+        }
+
         [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)]
         public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute<WithJaggedArray>(toolchain);
 
@@ -311,9 +366,9 @@ public void Test(int[][] array)
                     throw new ArgumentNullException(nameof(array));
 
                 for (int i = 0; i < 10; i++)
-                for (int j = 0; j < i; j++)
-                    if (array[i][j] != i)
-                        throw new ArgumentException("Invalid value");
+                    for (int j = 0; j < i; j++)
+                        if (array[i][j] != i)
+                            throw new ArgumentException("Invalid value");
             }
 
             public IEnumerable<object> CreateMatrix()
@@ -543,6 +598,46 @@ public void AcceptsArrays(int[] even, int[] notEven)
             }
         }
 
+        [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)]
+        public void ArrayArgumentsCanBeGrouped(IToolchain toolchain)
+        {
+            var summary = CanExecute<ArrayOnDifferentMethods>(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping
+
+            // There should be two logical groups, one for the first argument and one for the second argument
+            // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group
+
+            var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor);
+
+            Assert.True(
+                descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2)
+            );
+        }
+
+        public class ArrayOnDifferentMethods
+        {
+            public IEnumerable<int[]> GetArrays()
+            {
+                yield return new int[] { 1, 2, 3 };
+                yield return new int[] { 2, 3, 4 };
+            }
+
+            [Benchmark(Baseline = true)]
+            [ArgumentsSource(nameof(GetArrays))]
+            public void AcceptsArrays(int[] arr)
+            {
+                if (arr.Length != 3)
+                    throw new ArgumentException("Incorrect length");
+            }
+
+            [Benchmark]
+            [ArgumentsSource(nameof(GetArrays))]
+            public void AcceptsArrays2(int[] arr)
+            {
+                if (arr.Length != 3)
+                    throw new ArgumentException("Incorrect length");
+            }
+        }
+
         [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)]
         public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute<WithVeryBigInteger>(toolchain);
 
@@ -1026,6 +1121,16 @@ public Disposable(int id)
             }
         }
 
-        private void CanExecute<T>(IToolchain toolchain) => CanExecute<T>(CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain)));
+        private Reports.Summary CanExecute<T>(IToolchain toolchain, Func<IConfig, IConfig> furtherConfigure = null)
+        {
+            var config = CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain));
+
+            if (furtherConfigure is not null)
+            {
+                config = furtherConfigure(config);
+            }
+
+            return CanExecute<T>(config);
+        }
     }
 }