don't perform an extra run to get GC stats for .NET Core, part of dot…

…net#550
alinasmirnova · Sep 22, 2018 · d5e6dd7 · d5e6dd7
1 parent eab4a33
commit d5e6dd7
Show file tree

Hide file tree

Showing 11 changed files with 105 additions and 49 deletions.
diff --git a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs
@@ -31,8 +31,8 @@ public IColumnProvider GetColumnProvider()
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
             => diagnosers.ForEach(diagnoser => diagnoser.Handle(signal, parameters));
 
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report)
-            => diagnosers.ForEach(diagnoser => diagnoser.ProcessResults(benchmark, report));
+        public void ProcessResults(DiagnoserResults results)
+            => diagnosers.ForEach(diagnoser => diagnoser.ProcessResults(results));
 
         public void DisplayResults(ILogger logger)
         {

diff --git a/src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs b/src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs
@@ -0,0 +1,21 @@
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Diagnosers
+{
+    public class DiagnoserResults
+    {
+        public DiagnoserResults(Benchmark benchmark, long totalOperations, GcStats gcStats)
+        {
+            Benchmark = benchmark;
+            TotalOperations = totalOperations;
+            GcStats = gcStats;
+        }
+
+        public Benchmark Benchmark { get; }
+
+        public long TotalOperations { get; }
+
+        public GcStats GcStats { get; }
+    }
+}
diff --git a/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs
@@ -44,6 +44,7 @@ public IEnumerable<IExporter> Exporters
             };
 
         public IColumnProvider GetColumnProvider() => EmptyColumnProvider.Instance;
+        public void ProcessResults(DiagnoserResults _) { }
 
         public RunMode GetRunMode(Benchmark benchmark)
         {
@@ -57,16 +58,11 @@ public RunMode GetRunMode(Benchmark benchmark)
 
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
-            if (signal == HostSignal.AfterAll && ShouldUseWindowsDissasembler(parameters.Benchmark))
-                results.Add(
-                    parameters.Benchmark,
-                    windowsDisassembler.Dissasemble(parameters));
-        }
+            var benchmark = parameters.Benchmark;
 
-        // no need to run benchmarks once again, just do this after all runs
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report)
-        {
-            if (ShouldUseMonoDisassembler(benchmark))
+            if (signal == HostSignal.AfterAll && ShouldUseWindowsDissasembler(benchmark))
+                results.Add(benchmark, windowsDisassembler.Dissasemble(parameters));
+            else if (signal == HostSignal.SeparateLogic && ShouldUseMonoDisassembler(benchmark))
                 results.Add(benchmark, monoDisassembler.Disassemble(benchmark, benchmark.Job.Env.Runtime as MonoRuntime));
         }
 

diff --git a/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs
@@ -21,7 +21,7 @@ public interface IDiagnoser
 
         void Handle(HostSignal signal, DiagnoserActionParameters parameters);
 
-        void ProcessResults(Benchmark benchmark, BenchmarkReport report);
+        void ProcessResults(DiagnoserResults results);
 
         void DisplayResults(ILogger logger);
 

diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs
@@ -9,6 +9,7 @@
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Validators;
 
@@ -23,8 +24,6 @@ public class MemoryDiagnoser : IDiagnoser
 
         private readonly Dictionary<Benchmark, GcStats> results = new Dictionary<Benchmark, GcStats>();
 
-        public RunMode GetRunMode(Benchmark benchmark) => RunMode.ExtraRun;
-
         public IEnumerable<string> Ids => new[] { DiagnoserId };
 
         public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
@@ -41,8 +40,21 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
         public void DisplayResults(ILogger logger) { }
 
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report) 
-            => results.Add(benchmark, report.GcStats);
+        public RunMode GetRunMode(Benchmark benchmark)
+        {
+            // for .NET Core we don't need to enable any kind of monitoring
+            // the allocated memory is available via GC's API
+            // so we don't need to perform any extra run
+            if (benchmark.Job.ResolveValue(EnvMode.RuntimeCharacteristic, EnvResolver.Instance) is CoreRuntime)
+                return RunMode.NoOverhead;
+
+            // for classic .NET we need to enable AppDomain.MonitoringIsEnabled
+            // which may cause overhead, so we perform an extra run to collect stats about allocated memory
+            return RunMode.ExtraRun; 
+        }
+
+        public void ProcessResults(DiagnoserResults results) 
+            => this.results.Add(results.Benchmark, results.GcStats);
 
         public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) 
             => Array.Empty<ValidationError>();

diff --git a/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs
@@ -29,7 +29,7 @@ public class UnresolvedDiagnoser : IDiagnoser
         public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
         public IColumnProvider GetColumnProvider() => EmptyColumnProvider.Instance;
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report) { }
+        public void ProcessResults(DiagnoserResults results) { }
 
         public void DisplayResults(ILogger logger) => logger.WriteLineError(GetErrorMessage());
 

diff --git a/src/BenchmarkDotNet.Core/Engines/HostSignal.cs b/src/BenchmarkDotNet.Core/Engines/HostSignal.cs
@@ -20,6 +20,11 @@ public enum HostSignal
         /// <summary>
         /// after all (the last thing the benchmarking engine does is to fire this signal)
         /// </summary>
-        AfterAll
+        AfterAll,
+
+        /// <summary>
+        /// used to run some code independent to the benchmarked process
+        /// </summary>
+        SeparateLogic
     }
 }
diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs
@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Exporters;
@@ -20,6 +21,7 @@
 using BenchmarkDotNet.Toolchains.Parameters;
 using BenchmarkDotNet.Toolchains.Results;
 using BenchmarkDotNet.Validators;
+using RunMode = BenchmarkDotNet.Jobs.RunMode;
 
 namespace BenchmarkDotNet.Running
 {
@@ -211,7 +213,7 @@ private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, Read
                 if (!buildResult.IsBuildSuccess)
                     return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats));
 
-                var executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver, out GcStats gcStats);
+                var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver);
 
                 var runs = new List<Measurement>();
 
@@ -276,10 +278,11 @@ private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateR
             return buildResult;
         }
 
-        private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver, out GcStats gcStats)
+        private static (List<ExecuteResult> executeResults, GcStats gcStats) Execute(
+            ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver)
         {
             var executeResults = new List<ExecuteResult>();
-            gcStats = default(GcStats);
+            var gcStats = default(GcStats);
 
             logger.WriteLineInfo("// *** Execute ***");
             bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver);
@@ -289,30 +292,39 @@ private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark,
                 1,
                 autoLaunchCount ? defaultValue : benchmark.Job.Run.LaunchCount);
 
-            for (int launchIndex = 0; launchIndex < launchCount; launchIndex++)
+            var noOverheadCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.NoOverhead);
+
+            for (int launchIndex = 1; launchIndex <= launchCount; launchIndex++)
             {
-                string printedLaunchCount = (analyzeRunToRunVariance &&
-                    autoLaunchCount &&
-                    launchIndex < 2)
+                string printedLaunchCount = (analyzeRunToRunVariance && autoLaunchCount && launchIndex <= 2)
                     ? ""
                     : " / " + launchCount;
-                logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}");
+                logger.WriteLineInfo($"// Launch: {launchIndex}{printedLaunchCount}");
+
+                // use diagnoser only for the last run (we need single result, not many)
+                bool useDiagnoser = launchIndex == launchCount && noOverheadCompositeDiagnoser != null;
 
-                var noOverheadDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.NoOverhead);
                 var executeResult = toolchain.Executor.Execute(
-                    new ExecuteParameters(buildResult, benchmark, logger, resolver, config, noOverheadDiagnoser));
+                    new ExecuteParameters(
+                        buildResult,
+                        benchmark,
+                        logger,
+                        resolver,
+                        config,
+                        useDiagnoser ? noOverheadCompositeDiagnoser : null));
 
                 if (!executeResult.FoundExecutable)
                     logger.WriteLineError($"Executable {buildResult.ArtifactsPaths.ExecutablePath} not found");
                 if (executeResult.ExitCode != 0)
                     logger.WriteLineError("ExitCode != 0");
+
                 executeResults.Add(executeResult);
 
                 var measurements = executeResults
-                        .SelectMany(r => r.Data)
-                        .Select(line => Measurement.Parse(logger, line, 0))
-                        .Where(r => r.IterationMode != IterationMode.Unknown)
-                        .ToArray();
+                    .SelectMany(r => r.Data)
+                    .Select(line => Measurement.Parse(logger, line, 0))
+                    .Where(r => r.IterationMode != IterationMode.Unknown)
+                    .ToArray();
 
                 if (!measurements.Any())
                 {
@@ -321,7 +333,15 @@ private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark,
                     break;
                 }
 
-                if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance)
+                if (useDiagnoser)
+                {
+                    gcStats = GcStats.Parse(executeResult.Data.Last());
+
+                    noOverheadCompositeDiagnoser.ProcessResults(
+                        new DiagnoserResults(benchmark, measurements.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations), gcStats));
+                }
+
+                if (autoLaunchCount && launchIndex == 2 && analyzeRunToRunVariance)
                 {
                     // TODO: improve this logic
                     var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
@@ -333,32 +353,34 @@ private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark,
             logger.WriteLine();
 
             // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
-            if (config.GetDiagnosers().Any(diagnoser => diagnoser.GetRunMode(benchmark) == Diagnosers.RunMode.ExtraRun))
+            var extraRunCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.ExtraRun);
+            if (extraRunCompositeDiagnoser != null)
             {
                 logger.WriteLineInfo("// Run, Diagnostic");
-                var compositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.ExtraRun);
 
                 var executeResult = toolchain.Executor.Execute(
-                    new ExecuteParameters(buildResult, benchmark, logger, resolver, config, compositeDiagnoser));
+                    new ExecuteParameters(buildResult, benchmark, logger, resolver, config, extraRunCompositeDiagnoser));
 
                 var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
                 gcStats = GcStats.Parse(executeResult.Data.Last());
-                var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns, gcStats);
-                compositeDiagnoser.ProcessResults(benchmark, report);
+
+                extraRunCompositeDiagnoser.ProcessResults(
+                    new DiagnoserResults(benchmark, allRuns.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations), gcStats));
 
                 if (!executeResult.FoundExecutable)
                     logger.WriteLineError("Executable not found");
                 logger.WriteLine();
             }
 
-            foreach (var diagnoser in config.GetDiagnosers().Where(diagnoser => diagnoser.GetRunMode(benchmark) == Diagnosers.RunMode.SeparateLogic))
+            var separateLogicCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.SeparateLogic);
+            if(separateLogicCompositeDiagnoser != null)
             {
                 logger.WriteLineInfo("// Run, Diagnostic [SeparateLogic]");
 
-                diagnoser.ProcessResults(benchmark, null);
+                separateLogicCompositeDiagnoser.Handle(HostSignal.AfterAll, new DiagnoserActionParameters(null, benchmark, config));
             }
 
-            return executeResults;
+            return (executeResults, gcStats);
         }
 
         private static Benchmark[] GetSupportedBenchmarks(IList<Benchmark> benchmarks, CompositeLogger logger, Func<Job, IToolchain> toolchainProvider, IResolver resolver)

diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs
@@ -29,7 +29,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
                 Stop();
         }
 
-        public virtual void ProcessResults(Benchmark benchmark, BenchmarkReport report) { }
+        public virtual void ProcessResults(DiagnoserResults results) { }
 
         public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => Enumerable.Empty<ValidationError>();
 

diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs
@@ -42,10 +42,10 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
                 Stop();
         }
 
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report)
+        public void ProcessResults(DiagnoserResults results)
         {
-            var stats = ProcessEtwEvents(benchmark, report.AllMeasurements.Sum(m => m.Operations));
-            results.Add(benchmark, stats);
+            var stats = ProcessEtwEvents(results.Benchmark, results.TotalOperations);
+            this.results.Add(results.Benchmark, stats);
         }
 
         public void DisplayResults(ILogger logger) { }

diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs
@@ -63,12 +63,12 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
                 Stop();
         }
 
-        public void ProcessResults(Benchmark benchmark, BenchmarkReport report)
+        public void ProcessResults(DiagnoserResults results)
         {
-            var processId = BenchmarkToProcess[benchmark];
+            var processId = BenchmarkToProcess[results.Benchmark];
             var stats = StatsPerProcess[processId];
-            stats.TotalOperations = report.AllMeasurements.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations);
-            results.Add(benchmark, stats);
+            stats.TotalOperations = results.TotalOperations;
+            this.results.Add(results.Benchmark, stats);
         }
 
         public void DisplayResults(ILogger logger) { }