From d5e6dd73b69628a3cedf257daa1d0e4433225ecc Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 15 Dec 2017 18:49:37 +0100 Subject: [PATCH] don't perform an extra run to get GC stats for .NET Core, part of #550 --- .../Diagnosers/CompositeDiagnoser.cs | 4 +- .../Diagnosers/DiagnoserResults.cs | 21 ++++++ .../Diagnosers/DisassemblyDiagnoser.cs | 14 ++-- .../Diagnosers/IDiagnoser.cs | 2 +- .../Diagnosers/MemoryDiagnoser.cs | 20 ++++-- .../Diagnosers/UnresolvedDiagnoser.cs | 2 +- .../Engines/HostSignal.cs | 7 +- .../Running/BenchmarkRunnerCore.cs | 68 ++++++++++++------- .../JitDiagnoser.cs | 2 +- .../MemoryDiagnoser.cs | 6 +- .../PmcDiagnoser.cs | 8 +-- 11 files changed, 105 insertions(+), 49 deletions(-) create mode 100644 src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs diff --git a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs index bf7a5356e7..0f65466d88 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs @@ -31,8 +31,8 @@ public IColumnProvider GetColumnProvider() public void Handle(HostSignal signal, DiagnoserActionParameters parameters) => diagnosers.ForEach(diagnoser => diagnoser.Handle(signal, parameters)); - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) - => diagnosers.ForEach(diagnoser => diagnoser.ProcessResults(benchmark, report)); + public void ProcessResults(DiagnoserResults results) + => diagnosers.ForEach(diagnoser => diagnoser.ProcessResults(results)); public void DisplayResults(ILogger logger) { diff --git a/src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs b/src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs new file mode 100644 index 0000000000..1a2e59e890 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Diagnosers/DiagnoserResults.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Diagnosers +{ + public class DiagnoserResults + { + public DiagnoserResults(Benchmark benchmark, long totalOperations, GcStats gcStats) + { + Benchmark = benchmark; + TotalOperations = totalOperations; + GcStats = gcStats; + } + + public Benchmark Benchmark { get; } + + public long TotalOperations { get; } + + public GcStats GcStats { get; } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs index bd0ac37f14..8d51aea59c 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/DisassemblyDiagnoser.cs @@ -44,6 +44,7 @@ public IEnumerable Exporters }; public IColumnProvider GetColumnProvider() => EmptyColumnProvider.Instance; + public void ProcessResults(DiagnoserResults _) { } public RunMode GetRunMode(Benchmark benchmark) { @@ -57,16 +58,11 @@ public RunMode GetRunMode(Benchmark benchmark) public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { - if (signal == HostSignal.AfterAll && ShouldUseWindowsDissasembler(parameters.Benchmark)) - results.Add( - parameters.Benchmark, - windowsDisassembler.Dissasemble(parameters)); - } + var benchmark = parameters.Benchmark; - // no need to run benchmarks once again, just do this after all runs - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) - { - if (ShouldUseMonoDisassembler(benchmark)) + if (signal == HostSignal.AfterAll && ShouldUseWindowsDissasembler(benchmark)) + results.Add(benchmark, windowsDisassembler.Dissasemble(parameters)); + else if (signal == HostSignal.SeparateLogic && ShouldUseMonoDisassembler(benchmark)) results.Add(benchmark, monoDisassembler.Disassemble(benchmark, benchmark.Job.Env.Runtime as MonoRuntime)); } diff --git a/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs index e40fbeb8eb..c51ddfce53 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs @@ -21,7 +21,7 @@ public interface IDiagnoser void Handle(HostSignal signal, DiagnoserActionParameters parameters); - void ProcessResults(Benchmark benchmark, BenchmarkReport report); + void ProcessResults(DiagnoserResults results); void DisplayResults(ILogger logger); diff --git a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs index 867f48de3c..c8527c28d3 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Validators; @@ -23,8 +24,6 @@ public class MemoryDiagnoser : IDiagnoser private readonly Dictionary results = new Dictionary(); - public RunMode GetRunMode(Benchmark benchmark) => RunMode.ExtraRun; - public IEnumerable Ids => new[] { DiagnoserId }; public IEnumerable Exporters => Array.Empty(); @@ -41,8 +40,21 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public void DisplayResults(ILogger logger) { } - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) - => results.Add(benchmark, report.GcStats); + public RunMode GetRunMode(Benchmark benchmark) + { + // for .NET Core we don't need to enable any kind of monitoring + // the allocated memory is available via GC's API + // so we don't need to perform any extra run + if (benchmark.Job.ResolveValue(EnvMode.RuntimeCharacteristic, EnvResolver.Instance) is CoreRuntime) + return RunMode.NoOverhead; + + // for classic .NET we need to enable AppDomain.MonitoringIsEnabled + // which may cause overhead, so we perform an extra run to collect stats about allocated memory + return RunMode.ExtraRun; + } + + public void ProcessResults(DiagnoserResults results) + => this.results.Add(results.Benchmark, results.GcStats); public IEnumerable Validate(ValidationParameters validationParameters) => Array.Empty(); diff --git a/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs index aad9db11a3..6e2f37043b 100644 --- a/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs +++ b/src/BenchmarkDotNet.Core/Diagnosers/UnresolvedDiagnoser.cs @@ -29,7 +29,7 @@ public class UnresolvedDiagnoser : IDiagnoser public IEnumerable Exporters => Array.Empty(); public IColumnProvider GetColumnProvider() => EmptyColumnProvider.Instance; public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) { } + public void ProcessResults(DiagnoserResults results) { } public void DisplayResults(ILogger logger) => logger.WriteLineError(GetErrorMessage()); diff --git a/src/BenchmarkDotNet.Core/Engines/HostSignal.cs b/src/BenchmarkDotNet.Core/Engines/HostSignal.cs index 4392aa3009..c10a5eaf3c 100644 --- a/src/BenchmarkDotNet.Core/Engines/HostSignal.cs +++ b/src/BenchmarkDotNet.Core/Engines/HostSignal.cs @@ -20,6 +20,11 @@ public enum HostSignal /// /// after all (the last thing the benchmarking engine does is to fire this signal) /// - AfterAll + AfterAll, + + /// + /// used to run some code independent to the benchmarked process + /// + SeparateLogic } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs index 27232bafba..270f55f347 100644 --- a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs +++ b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; @@ -20,6 +21,7 @@ using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; +using RunMode = BenchmarkDotNet.Jobs.RunMode; namespace BenchmarkDotNet.Running { @@ -211,7 +213,7 @@ private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, Read if (!buildResult.IsBuildSuccess) return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats)); - var executeResults = Execute(logger, benchmark, toolchain, buildResult, config, resolver, out GcStats gcStats); + var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver); var runs = new List(); @@ -276,10 +278,11 @@ private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateR return buildResult; } - private static List Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver, out GcStats gcStats) + private static (List executeResults, GcStats gcStats) Execute( + ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver) { var executeResults = new List(); - gcStats = default(GcStats); + var gcStats = default(GcStats); logger.WriteLineInfo("// *** Execute ***"); bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver); @@ -289,30 +292,39 @@ private static List Execute(ILogger logger, Benchmark benchmark, 1, autoLaunchCount ? defaultValue : benchmark.Job.Run.LaunchCount); - for (int launchIndex = 0; launchIndex < launchCount; launchIndex++) + var noOverheadCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.NoOverhead); + + for (int launchIndex = 1; launchIndex <= launchCount; launchIndex++) { - string printedLaunchCount = (analyzeRunToRunVariance && - autoLaunchCount && - launchIndex < 2) + string printedLaunchCount = (analyzeRunToRunVariance && autoLaunchCount && launchIndex <= 2) ? "" : " / " + launchCount; - logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}"); + logger.WriteLineInfo($"// Launch: {launchIndex}{printedLaunchCount}"); + + // use diagnoser only for the last run (we need single result, not many) + bool useDiagnoser = launchIndex == launchCount && noOverheadCompositeDiagnoser != null; - var noOverheadDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.NoOverhead); var executeResult = toolchain.Executor.Execute( - new ExecuteParameters(buildResult, benchmark, logger, resolver, config, noOverheadDiagnoser)); + new ExecuteParameters( + buildResult, + benchmark, + logger, + resolver, + config, + useDiagnoser ? noOverheadCompositeDiagnoser : null)); if (!executeResult.FoundExecutable) logger.WriteLineError($"Executable {buildResult.ArtifactsPaths.ExecutablePath} not found"); if (executeResult.ExitCode != 0) logger.WriteLineError("ExitCode != 0"); + executeResults.Add(executeResult); var measurements = executeResults - .SelectMany(r => r.Data) - .Select(line => Measurement.Parse(logger, line, 0)) - .Where(r => r.IterationMode != IterationMode.Unknown) - .ToArray(); + .SelectMany(r => r.Data) + .Select(line => Measurement.Parse(logger, line, 0)) + .Where(r => r.IterationMode != IterationMode.Unknown) + .ToArray(); if (!measurements.Any()) { @@ -321,7 +333,15 @@ private static List Execute(ILogger logger, Benchmark benchmark, break; } - if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance) + if (useDiagnoser) + { + gcStats = GcStats.Parse(executeResult.Data.Last()); + + noOverheadCompositeDiagnoser.ProcessResults( + new DiagnoserResults(benchmark, measurements.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations), gcStats)); + } + + if (autoLaunchCount && launchIndex == 2 && analyzeRunToRunVariance) { // TODO: improve this logic var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median; @@ -333,32 +353,34 @@ private static List Execute(ILogger logger, Benchmark benchmark, logger.WriteLine(); // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results - if (config.GetDiagnosers().Any(diagnoser => diagnoser.GetRunMode(benchmark) == Diagnosers.RunMode.ExtraRun)) + var extraRunCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.ExtraRun); + if (extraRunCompositeDiagnoser != null) { logger.WriteLineInfo("// Run, Diagnostic"); - var compositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.ExtraRun); var executeResult = toolchain.Executor.Execute( - new ExecuteParameters(buildResult, benchmark, logger, resolver, config, compositeDiagnoser)); + new ExecuteParameters(buildResult, benchmark, logger, resolver, config, extraRunCompositeDiagnoser)); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); gcStats = GcStats.Parse(executeResult.Data.Last()); - var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns, gcStats); - compositeDiagnoser.ProcessResults(benchmark, report); + + extraRunCompositeDiagnoser.ProcessResults( + new DiagnoserResults(benchmark, allRuns.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations), gcStats)); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); logger.WriteLine(); } - foreach (var diagnoser in config.GetDiagnosers().Where(diagnoser => diagnoser.GetRunMode(benchmark) == Diagnosers.RunMode.SeparateLogic)) + var separateLogicCompositeDiagnoser = config.GetCompositeDiagnoser(benchmark, Diagnosers.RunMode.SeparateLogic); + if(separateLogicCompositeDiagnoser != null) { logger.WriteLineInfo("// Run, Diagnostic [SeparateLogic]"); - diagnoser.ProcessResults(benchmark, null); + separateLogicCompositeDiagnoser.Handle(HostSignal.AfterAll, new DiagnoserActionParameters(null, benchmark, config)); } - return executeResults; + return (executeResults, gcStats); } private static Benchmark[] GetSupportedBenchmarks(IList benchmarks, CompositeLogger logger, Func toolchainProvider, IResolver resolver) diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs index 4401860312..cb176fe8ac 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs @@ -29,7 +29,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) Stop(); } - public virtual void ProcessResults(Benchmark benchmark, BenchmarkReport report) { } + public virtual void ProcessResults(DiagnoserResults results) { } public IEnumerable Validate(ValidationParameters validationParameters) => Enumerable.Empty(); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs index 9ceb3b094c..e3807a30f7 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/MemoryDiagnoser.cs @@ -42,10 +42,10 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) Stop(); } - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) + public void ProcessResults(DiagnoserResults results) { - var stats = ProcessEtwEvents(benchmark, report.AllMeasurements.Sum(m => m.Operations)); - results.Add(benchmark, stats); + var stats = ProcessEtwEvents(results.Benchmark, results.TotalOperations); + this.results.Add(results.Benchmark, stats); } public void DisplayResults(ILogger logger) { } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs index b0ffaaf1a1..85d58298ce 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/PmcDiagnoser.cs @@ -63,12 +63,12 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) Stop(); } - public void ProcessResults(Benchmark benchmark, BenchmarkReport report) + public void ProcessResults(DiagnoserResults results) { - var processId = BenchmarkToProcess[benchmark]; + var processId = BenchmarkToProcess[results.Benchmark]; var stats = StatsPerProcess[processId]; - stats.TotalOperations = report.AllMeasurements.Where(measurement => !measurement.IterationMode.IsIdle()).Sum(m => m.Operations); - results.Add(benchmark, stats); + stats.TotalOperations = results.TotalOperations; + this.results.Add(results.Benchmark, stats); } public void DisplayResults(ILogger logger) { }