Introduce MockToolchain (#2178)

* Introduce MockToolchain

* simplify the design (#2268)

---------

Co-authored-by: Adam Sitnik <[email protected]>

src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs

@@ -64,6 +64,18 @@ internal ExecuteResult(List<Measurement> measurements, GcStats gcStats, Threadin
             ExceptionFrequency = exceptionFrequency;
         }
 
+        internal ExecuteResult(List<Measurement> measurements)
+        {
+            FoundExecutable = true;
+            ExitCode = 0;
+            errors = new List<string>();
+            PrefixedLines = Array.Empty<string>();
+            this.measurements = measurements;
+            GcStats = GcStats.Empty;
+            ThreadingStats = ThreadingStats.Empty;
+            ExceptionFrequency = 0;
+        }
+
         internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode)
             => exitCode != 0
                 ? CreateFailed(exitCode)
@@ -72,6 +84,13 @@ internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode
         internal static ExecuteResult CreateFailed(int exitCode = -1)
             => new ExecuteResult(false, exitCode, default, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), 0);
 
+        internal static ExecuteResult CreateFailed(string error)
+        {
+            var result = new ExecuteResult(false, -1, default, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), 0);
+            result.errors.Add(error);
+            return result;
+        }
+
         public override string ToString() => "ExecuteResult: " + (FoundExecutable ? "Found executable" : "Executable not found");
 
         public void LogIssues(ILogger logger, BuildResult buildResult)

tests/BenchmarkDotNet.Tests/Columns/RatioColumnTest.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Linq;
+using System.Xml.Linq;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Tests.Mocks;
+using BenchmarkDotNet.Tests.Mocks.Toolchain;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.Tests.Columns
+{
+    public class RatioColumnTest
+    {
+        private readonly ITestOutputHelper output;
+
+        public RatioColumnTest(ITestOutputHelper output)
+        {
+            this.output = output;
+        }
+
+        [Fact]
+        public void RatioColumnTest01()
+        {
+            var summary = MockRunner.Run<BenchmarkClass>(output, name => name switch
+            {
+                "Foo" => new double[] { 2, 2, 2 },
+                "Bar" => new double[] { 4, 4, 4 },
+                _ => throw new InvalidOperationException()
+            });
+
+            var ratioColumn = summary.GetColumns().FirstOrDefault(column => column.ColumnName == "Ratio");
+            Assert.NotNull(ratioColumn);
+
+            var fooCase = summary.BenchmarksCases.First(c => c.Descriptor.WorkloadMethod.Name == "Foo");
+            var barCase = summary.BenchmarksCases.First(c => c.Descriptor.WorkloadMethod.Name == "Bar");
+            Assert.Equal("1.00", ratioColumn.GetValue(summary, fooCase));
+            Assert.Equal("2.00", ratioColumn.GetValue(summary, barCase));
+        }
+
+        public class BenchmarkClass
+        {
+            [Benchmark(Baseline = true)]
+            public void Foo() { }
+
+            [Benchmark]
+            public void Bar() { }
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/Mocks/MockRunner.cs

@@ -0,0 +1,48 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Tests.Mocks.Toolchain;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.Tests.Mocks
+{
+    public static class MockRunner
+    {
+        public static Summary Run<T>(ITestOutputHelper output, Func<string, double[]> measurer)
+            => Run<T>(output, benchmarkCase => measurer(benchmarkCase.Descriptor.WorkloadMethod.Name)
+                        .Select((value, i) => new Measurement(1, IterationMode.Workload, IterationStage.Result, i, 1, value))
+                        .ToList());
+
+        public static Summary Run<T>(ITestOutputHelper output, Func<BenchmarkCase, List<Measurement>> measurer)
+        {
+            var job = new Job("MockJob")
+            {
+                Infrastructure =
+                {
+                    Toolchain = new MockToolchain(measurer)
+                }
+            }.Freeze();
+
+            var logger = new AccumulationLogger();
+
+            var config = DefaultConfig.Instance
+                .WithOptions(ConfigOptions.DisableOptimizationsValidator)
+                .AddJob(job)
+                .AddLogger(logger);
+            var summary = BenchmarkRunner.Run<T>(config);
+
+            var exporter = MarkdownExporter.Mock;
+            exporter.ExportToLog(summary, logger);
+            output.WriteLine(logger.GetLog());
+
+            return summary;
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.Parameters;
+using BenchmarkDotNet.Toolchains.Results;
+using BenchmarkDotNet.Validators;
+
+namespace BenchmarkDotNet.Tests.Mocks.Toolchain
+{
+    public class MockToolchain : IToolchain
+    {
+        public MockToolchain(Func<BenchmarkCase, List<Measurement>> measurer)
+            => Executor = new MockExecutor(measurer);
+
+        public string Name => nameof(MockToolchain);
+        public IGenerator Generator => new MockGenerator();
+        public IBuilder Builder => new MockBuilder();
+        public IExecutor Executor { get; private set; }
+        public bool IsInProcess => false;
+        public IEnumerable<ValidationError> Validate(BenchmarkCase benchmarkCase, IResolver resolver) => ImmutableArray<ValidationError>.Empty;
+
+        public override string ToString() => GetType().Name;
+
+        private class MockGenerator : IGenerator
+        {
+            public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath)
+                => GenerateResult.Success(ArtifactsPaths.Empty, ImmutableArray<string>.Empty);
+        }
+
+        private class MockBuilder : IBuilder
+        {
+            public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) => BuildResult.Success(generateResult);
+        }
+
+        private class MockExecutor : IExecutor
+        {
+            private readonly Func<BenchmarkCase, List<Measurement>> measurer;
+
+            public MockExecutor(Func<BenchmarkCase, List<Measurement>> measurer) => this.measurer = measurer;
+
+            public ExecuteResult Execute(ExecuteParameters executeParameters) => new (measurer(executeParameters.BenchmarkCase));
+        }
+    }
+}