From cc5834e6e4bf1b60b807c2b7aa2431284bfeb9b4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 16:07:25 +0200 Subject: [PATCH 01/13] don't execute long operations more than once per iteration, #736 --- src/BenchmarkDotNet/Engines/Engine.cs | 15 +-- src/BenchmarkDotNet/Engines/EngineFactory.cs | 68 +++++++++-- .../Engines/EngineParameters.cs | 6 +- src/BenchmarkDotNet/Engines/IEngine.cs | 8 +- src/BenchmarkDotNet/Engines/IEngineFactory.cs | 2 +- src/BenchmarkDotNet/Engines/IterationMode.cs | 7 +- .../Templates/BenchmarkType.txt | 21 +--- .../Toolchains/InProcess/InProcessRunner.cs | 25 ++-- .../CustomEngineTests.cs | 5 +- .../Engine/EngineFactoryTests.cs | 109 ++++++++++++++++++ .../BenchmarkDotNet.Tests/Mocks/MockEngine.cs | 4 +- 11 files changed, 198 insertions(+), 72 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 45719d8bbd..7c68402ce2 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -41,7 +41,6 @@ public class Engine : IEngine private readonly EngineWarmupStage warmupStage; private readonly EngineTargetStage targetStage; private readonly bool includeMemoryStats; - private bool isJitted; internal Engine( IHost host, @@ -78,22 +77,10 @@ internal Engine( targetStage = new EngineTargetStage(this); } - public void Jitting() - { - // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt - Dummy1Action.Invoke(); - MainAction.Invoke(1); - Dummy2Action.Invoke(); - IdleAction.Invoke(1); - Dummy3Action.Invoke(); - isJitted = true; - } + public void Dispose() => GlobalCleanupAction?.Invoke(); public RunResults Run() { - if (Strategy.NeedsJitting() != isJitted) - throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!"); - long invokeCount = InvocationCount; IReadOnlyList idle = null; diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index a5f74bda49..cd04a64ba7 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,39 +1,85 @@ using System; +using BenchmarkDotNet.Horology; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - // TODO: Default instance? public class EngineFactory : IEngineFactory { - public IEngine Create(EngineParameters engineParameters) + public IEngine CreateReadyToRun(EngineParameters engineParameters) { - if (engineParameters.MainAction == null) - throw new ArgumentNullException(nameof(engineParameters.MainAction)); + if (engineParameters.MainSingleAction == null) + throw new ArgumentNullException(nameof(engineParameters.MainSingleAction)); + if (engineParameters.MainMultiAction == null) + throw new ArgumentNullException(nameof(engineParameters.MainMultiAction)); if (engineParameters.Dummy1Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy1Action)); if (engineParameters.Dummy2Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy2Action)); if (engineParameters.Dummy3Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy3Action)); - if (engineParameters.IdleAction == null) - throw new ArgumentNullException(nameof(engineParameters.IdleAction)); + if (engineParameters.IdleSingleAction == null) + throw new ArgumentNullException(nameof(engineParameters.IdleSingleAction)); + if (engineParameters.IdleMultiAction == null) + throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction)); if(engineParameters.TargetJob == null) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); + + engineParameters.GlobalSetupAction?.Invoke(); - return new Engine( + var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, engineParameters.Resolver).NeedsJitting(); + if (!needsJitting) + { + // whatever it is, we can not interfere + return CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + } + + var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); + if (needsPilot) + { + var singleActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); + + var iterationTime = engineParameters.Resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); + if (ShouldExecuteOncePerIteration(Jit(singleActionEngine), iterationTime)) + { + var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1! + + return CreateEngine(engineParameters, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); + } + } + + // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return + var multiActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine)); + + return multiActionEngine; + } + + /// + /// returns true if it takes longer than the desired iteration time (0,5s by default) to execute benchmark once + /// + private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime) + => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime; + + private static Measurement Jit(Engine engine) + => engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1)); + + private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) + => new Engine( engineParameters.Host, engineParameters.Dummy1Action, engineParameters.Dummy2Action, engineParameters.Dummy3Action, - engineParameters.IdleAction, - engineParameters.MainAction, - engineParameters.TargetJob, + idle, + main, + job, engineParameters.GlobalSetupAction, engineParameters.GlobalCleanupAction, engineParameters.IterationSetupAction, engineParameters.IterationCleanupAction, engineParameters.OperationsPerInvoke, engineParameters.MeasureGcStats); - } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index 49475b2eb0..2087add34f 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -7,11 +7,13 @@ namespace BenchmarkDotNet.Engines public class EngineParameters { public IHost Host { get; set; } - public Action MainAction { get; set; } + public Action MainSingleAction { get; set; } + public Action MainMultiAction { get; set; } public Action Dummy1Action { get; set; } public Action Dummy2Action { get; set; } public Action Dummy3Action { get; set; } - public Action IdleAction { get; set; } + public Action IdleSingleAction { get; set; } + public Action IdleMultiAction { get; set; } public Job TargetJob { get; set; } = Job.Default; public long OperationsPerInvoke { get; set; } = 1; public Action GlobalSetupAction { get; set; } = null; diff --git a/src/BenchmarkDotNet/Engines/IEngine.cs b/src/BenchmarkDotNet/Engines/IEngine.cs index f31c6d74c1..bd4142244c 100644 --- a/src/BenchmarkDotNet/Engines/IEngine.cs +++ b/src/BenchmarkDotNet/Engines/IEngine.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Engines { - public interface IEngine + public interface IEngine : IDisposable { [NotNull] IHost Host { get; } @@ -36,12 +36,6 @@ public interface IEngine Measurement RunIteration(IterationData data); - /// - /// must perform jitting via warmup calls - /// is called after first call to GlobalSetup, from the auto-generated benchmark process - /// - void Jitting(); - RunResults Run(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngineFactory.cs b/src/BenchmarkDotNet/Engines/IEngineFactory.cs index 89dfb14eb3..c518441409 100644 --- a/src/BenchmarkDotNet/Engines/IEngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/IEngineFactory.cs @@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines { public interface IEngineFactory { - IEngine Create(EngineParameters engineParameters); + IEngine CreateReadyToRun(EngineParameters engineParameters); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs index e60002ba68..0e45f01bc0 100644 --- a/src/BenchmarkDotNet/Engines/IterationMode.cs +++ b/src/BenchmarkDotNet/Engines/IterationMode.cs @@ -35,6 +35,11 @@ public enum IterationMode /// /// Unknown /// - Unknown + Unknown, + + /// + /// executing benchmark for the purpose of JIT wamup + /// + Jit } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index 05112f476d..5956b9610c 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -34,23 +34,14 @@ MeasureGcStats = $MeasureGcStats$ }; - var engine = new $EngineFactoryType$().Create(engineParameters); - - instance?.globalSetupAction(); - instance?.iterationSetupAction(); - - if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting()) - engine.Jitting(); // does first call to main action, must be executed after globalSetup() and iterationSetup()! - - instance?.iterationCleanupAction(); - - var results = engine.Run(); - - instance?.globalCleanupAction(); + using (var engine = new $EngineFactoryType$().CreateReadyToRun(engineParameters)) + { + var results = engine.Run(); - host.ReportResults(results); // printing costs memory, do this after runs + host.ReportResults(results); // printing costs memory, do this after runs - instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) + instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) + } } public delegate $IdleMethodReturnTypeName$ IdleDelegate($ArgumentsDefinition$); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs index 814c23e38e..0134c979af 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs @@ -112,11 +112,11 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg var engineParameters = new EngineParameters { Host = host, - MainAction = mainAction.InvokeMultiple, + MainMultiAction = mainAction.InvokeMultiple, Dummy1Action = dummy1.InvokeSingle, Dummy2Action = dummy2.InvokeSingle, Dummy3Action = dummy3.InvokeSingle, - IdleAction = idleAction.InvokeMultiple, + IdleMultiAction = idleAction.InvokeMultiple, GlobalSetupAction = globalSetupAction.InvokeSingle, GlobalCleanupAction = globalCleanupAction.InvokeSingle, IterationSetupAction = iterationSetupAction.InvokeSingle, @@ -126,23 +126,14 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg MeasureGcStats = config.HasMemoryDiagnoser() }; - var engine = job + using (var engine = job .ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance) - .Create(engineParameters); - - globalSetupAction.InvokeSingle(); - iterationSetupAction.InvokeSingle(); - - if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting()) - engine.Jitting(); // does first call to main action, must be executed after setup()! - - iterationCleanupAction.InvokeSingle(); - - var results = engine.Run(); - - globalCleanupAction.InvokeSingle(); + .CreateReadyToRun(engineParameters)) + { + var results = engine.Run(); - host.ReportResults(results); // printing costs memory, do this after runs + host.ReportResults(results); // printing costs memory, do this after runs + } } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index ef0cdd2b92..74cb2090c2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -54,7 +54,7 @@ public void Empty() { } public class CustomFactory : IEngineFactory { - public IEngine Create(EngineParameters engineParameters) + public IEngine CreateReadyToRun(EngineParameters engineParameters) => new CustomEngine { GlobalCleanupAction = engineParameters.GlobalCleanupAction, @@ -75,6 +75,8 @@ public RunResults Run() default); } + public void Dispose() => GlobalCleanupAction?.Invoke(); + public IHost Host { get; } public void WriteLine() { } public void WriteLine(string line) { } @@ -87,7 +89,6 @@ public void WriteLine(string line) { } public IResolver Resolver { get; } public Measurement RunIteration(IterationData data) { throw new NotImplementedException(); } - public void Jitting() { } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs new file mode 100644 index 0000000000..b9ce180eb4 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -0,0 +1,109 @@ +using System; +using System.IO; +using System.Threading; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using Xunit; + +namespace BenchmarkDotNet.Tests.Engine +{ + public class EngineFactoryTests + { + int timesBenchmarkCalled = 0, timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; + + void GlobalSetup() => timesGlobalSetupCalled++; + void IterationSetup() => timesIterationSetupCalled++; + void IterationCleanup() => timesIterationCleanupCalled++; + void GlobalCleanup() => timesGlobalCleanupCalled++; + + void Throwing(long _) => throw new InvalidOperationException("must NOT be called"); + + void VeryTimeConsumingSingle(long _) + { + timesBenchmarkCalled++; + Thread.Sleep(TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds())); + } + + void InstantSingle(long _) => timesBenchmarkCalled++; + + void Instant16(long _) => timesBenchmarkCalled += 16; + + [Fact] + public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSettings() + { + var engineParameters = CreateEngineParameters(singleAction: VeryTimeConsumingSingle, multiAction: Throwing, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(1, timesIterationSetupCalled); + Assert.Equal(1, timesBenchmarkCalled); + Assert.Equal(1, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispode + + Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration + Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor + + engine.Dispose(); // cleanup is called as part of dispode + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() + { + var engineParameters = CreateEngineParameters(singleAction: Throwing, multiAction: Throwing, job: Job.Dry); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(0, timesIterationSetupCalled); + Assert.Equal(0, timesBenchmarkCalled); + Assert.Equal(0, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings() + { + var engineParameters = CreateEngineParameters(singleAction: InstantSingle, multiAction: Instant16, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(2, timesIterationSetupCalled); // once for single and & once for 16 + Assert.Equal(1 + 16, timesBenchmarkCalled); + Assert.Equal(2, timesIterationCleanupCalled); // once for single and & once for 16 + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + private EngineParameters CreateEngineParameters(Action singleAction, Action multiAction, Job job) + => new EngineParameters + { + Dummy1Action = () => { }, + Dummy2Action = () => { }, + Dummy3Action = () => { }, + GlobalSetupAction = GlobalSetup, + GlobalCleanupAction = GlobalCleanup, + Host = new ConsoleHost(TextWriter.Null, TextReader.Null), + IdleMultiAction = _ => { }, + IdleSingleAction = _ => { }, + IterationCleanupAction = IterationCleanup, + IterationSetupAction = IterationSetup, + MainMultiAction = multiAction, + MainSingleAction = singleAction, + Resolver = EngineResolver.Instance, + TargetJob = job + }; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index b5f64d2b79..07e0a6a0b9 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -22,6 +22,8 @@ public MockEngine(ITestOutputHelper output, Job job, Func GlobalSetupAction?.Invoke(); [UsedImplicitly] public IHost Host { get; } @@ -52,8 +54,6 @@ public Measurement RunIteration(IterationData data) return measurement; } - public void Jitting() { } - public RunResults Run() => default; public void WriteLine() => output.WriteLine(""); From d7b06b0218acf77aa248d798adc319f719987516 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 18:07:21 +0200 Subject: [PATCH 02/13] generate the right C# code for #736 --- .../Templates/BenchmarkType.txt | 62 ++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index 5956b9610c..453a10c6d6 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -20,11 +20,13 @@ var engineParameters = new BenchmarkDotNet.Engines.EngineParameters() { Host = host, - MainAction = instance.MainMultiAction, + MainMultiAction = instance.MainMultiAction, + MainSingleAction = instance.MainSingleAction, Dummy1Action = instance.Dummy1, Dummy2Action = instance.Dummy2, Dummy3Action = instance.Dummy3, - IdleAction = instance.IdleMultiAction, + IdleSingleAction = instance.IdleSingleAction, + IdleMultiAction = instance.IdleMultiAction, GlobalSetupAction = instance.globalSetupAction, GlobalCleanupAction = instance.globalCleanupAction, IterationSetupAction = instance.iterationSetupAction, @@ -115,6 +117,12 @@ } } + private void IdleSingleAction(long _) + { + $LoadArguments$ + consumer.Consume(idleDelegate($PassArguments$)); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -124,6 +132,12 @@ } } + private void MainSingleAction(long _) + { + $LoadArguments$ + consumer.Consume(targetDelegate($PassArguments$)$ConsumeField$); + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public $TargetMethodReturnType$ $DiassemblerEntryMethodName$() { @@ -149,6 +163,14 @@ DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); } + private void IdleSingleAction(long _) + { + $LoadArguments$ + $IdleMethodReturnTypeName$ result = default($IdleMethodReturnTypeName$); + result = idleDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -160,6 +182,14 @@ NonGenericKeepAliveWithoutBoxing(result); } + private void MainSingleAction(long _) + { + $LoadArguments$ + $TargetMethodReturnType$ result = default($TargetMethodReturnType$); + result = targetDelegate($PassArguments$); + NonGenericKeepAliveWithoutBoxing(result); + } + // we must not simply use DeadCodeEliminationHelper.KeepAliveWithoutBoxing because it's generic method // and stack-only types like Span can not be generic type arguments http://adamsitnik.com/Span/#span-must-not-be-a-generic-type-argument [MethodImpl(MethodImplOptions.NoInlining)] @@ -189,6 +219,14 @@ } DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } + + private void IdleSingleAction(long _) + { + $LoadArguments$ + $IdleMethodReturnTypeName$ value = default($IdleMethodReturnTypeName$); + value = idleDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); + } private $TargetMethodReturnType$ mainDefaultValueHolder = default($TargetMethodReturnType$); @@ -202,6 +240,14 @@ } DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); } + + private void MainSingleAction(long _) + { + $LoadArguments$ + ref $TargetMethodReturnType$ alias = ref mainDefaultValueHolder; + alias = targetDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); + } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public ref $TargetMethodReturnType$ $DiassemblerEntryMethodName$() @@ -225,6 +271,12 @@ } } + private void IdleSingleAction(long _) + { + $LoadArguments$ + idleDelegate($PassArguments$); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -233,6 +285,12 @@ targetDelegate($PassArguments$);@Unroll@ } } + + private void MainSingleAction(long _) + { + $LoadArguments$ + targetDelegate($PassArguments$); + } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public void $DiassemblerEntryMethodName$() From 6902ad1163648075233da1f71ac7e7886e898bfc Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 18:30:25 +0200 Subject: [PATCH 03/13] EngineParameters.Resolver was always null or ignored ;), #736 --- src/BenchmarkDotNet/Engines/Engine.cs | 3 ++- src/BenchmarkDotNet/Engines/EngineFactory.cs | 19 ++++++++++++------- .../Engines/EngineParameters.cs | 1 - 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 7c68402ce2..ae99cf37cb 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -44,6 +44,7 @@ public class Engine : IEngine internal Engine( IHost host, + IResolver resolver, Action dummy1Action, Action dummy2Action, Action dummy3Action, Action idleAction, Action mainAction, Job targetJob, Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke, bool includeMemoryStats) @@ -63,7 +64,7 @@ internal Engine( OperationsPerInvoke = operationsPerInvoke; this.includeMemoryStats = includeMemoryStats; - Resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + Resolver = resolver; Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver); ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index cd04a64ba7..dc024c7eec 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,7 +1,9 @@ using System; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Engines { @@ -25,32 +27,34 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction)); if(engineParameters.TargetJob == null) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); + + var resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); engineParameters.GlobalSetupAction?.Invoke(); - var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, engineParameters.Resolver).NeedsJitting(); + var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, resolver).NeedsJitting(); if (!needsJitting) { // whatever it is, we can not interfere - return CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + return CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); } var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); if (needsPilot) { - var singleActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); + var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); - var iterationTime = engineParameters.Resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); + var iterationTime = resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); if (ShouldExecuteOncePerIteration(Jit(singleActionEngine), iterationTime)) { var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1! - return CreateEngine(engineParameters, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); + return CreateEngine(engineParameters, resolver, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); } } // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return - var multiActionEngine = CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + var multiActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine)); @@ -66,9 +70,10 @@ private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval private static Measurement Jit(Engine engine) => engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1)); - private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) + private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) => new Engine( engineParameters.Host, + resolver, engineParameters.Dummy1Action, engineParameters.Dummy2Action, engineParameters.Dummy3Action, diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index 2087add34f..f384d32f9e 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -20,7 +20,6 @@ public class EngineParameters public Action GlobalCleanupAction { get; set; } = null; public Action IterationSetupAction { get; set; } = null; public Action IterationCleanupAction { get; set; } = null; - public IResolver Resolver { get; set; } public bool MeasureGcStats { get; set; } } } \ No newline at end of file From 343a60372e7589e9ffa29b3beb0aaa369deacddd Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 18:39:34 +0200 Subject: [PATCH 04/13] don't forget to JIT idle, #736 --- src/BenchmarkDotNet/Engines/EngineFactory.cs | 6 ++++- src/BenchmarkDotNet/Engines/IterationMode.cs | 2 +- .../Engines/IterationModeExtensions.cs | 2 +- .../Engine/EngineFactoryTests.cs | 27 +++++++++++-------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index dc024c7eec..79766243da 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -68,7 +68,11 @@ private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime; private static Measurement Jit(Engine engine) - => engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1)); + { + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJit, index: -1, invokeCount: 1, unrollFactor: 1))); // don't forget to JIT idle + + return engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1)); + } private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) => new Engine( diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs index 0e45f01bc0..7532528abf 100644 --- a/src/BenchmarkDotNet/Engines/IterationMode.cs +++ b/src/BenchmarkDotNet/Engines/IterationMode.cs @@ -40,6 +40,6 @@ public enum IterationMode /// /// executing benchmark for the purpose of JIT wamup /// - Jit + Jit, IdleJit } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs index 88cb925ef9..619360cd90 100644 --- a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs +++ b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs @@ -3,6 +3,6 @@ public static class IterationModeExtensions { public static bool IsIdle(this IterationMode mode) - => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget; + => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJit; } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index b9ce180eb4..6894726ef0 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -9,7 +9,8 @@ namespace BenchmarkDotNet.Tests.Engine { public class EngineFactoryTests { - int timesBenchmarkCalled = 0, timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; + int timesBenchmarkCalled = 0, timesIdleCalled = 0; + int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; void GlobalSetup() => timesGlobalSetupCalled++; void IterationSetup() => timesIterationSetupCalled++; @@ -25,19 +26,22 @@ void VeryTimeConsumingSingle(long _) } void InstantSingle(long _) => timesBenchmarkCalled++; - void Instant16(long _) => timesBenchmarkCalled += 16; + + void IdleSingle(long _) => timesIdleCalled++; + void Idle16(long _) => timesIdleCalled += 16; [Fact] public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSettings() { - var engineParameters = CreateEngineParameters(singleAction: VeryTimeConsumingSingle, multiAction: Throwing, job: Job.Default); + var engineParameters = CreateEngineParameters(mainSingleAction: VeryTimeConsumingSingle, mainMultiAction: Throwing, job: Job.Default); var engine = new EngineFactory().CreateReadyToRun(engineParameters); Assert.Equal(1, timesGlobalSetupCalled); Assert.Equal(1, timesIterationSetupCalled); Assert.Equal(1, timesBenchmarkCalled); + Assert.Equal(1, timesIdleCalled); Assert.Equal(1, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispode @@ -52,13 +56,14 @@ public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSett [Fact] public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() { - var engineParameters = CreateEngineParameters(singleAction: Throwing, multiAction: Throwing, job: Job.Dry); + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: Job.Dry); var engine = new EngineFactory().CreateReadyToRun(engineParameters); Assert.Equal(1, timesGlobalSetupCalled); Assert.Equal(0, timesIterationSetupCalled); Assert.Equal(0, timesBenchmarkCalled); + Assert.Equal(0, timesIdleCalled); Assert.Equal(0, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); @@ -70,13 +75,14 @@ public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() [Fact] public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings() { - var engineParameters = CreateEngineParameters(singleAction: InstantSingle, multiAction: Instant16, job: Job.Default); + var engineParameters = CreateEngineParameters(mainSingleAction: InstantSingle, mainMultiAction: Instant16, job: Job.Default); var engine = new EngineFactory().CreateReadyToRun(engineParameters); Assert.Equal(1, timesGlobalSetupCalled); Assert.Equal(2, timesIterationSetupCalled); // once for single and & once for 16 Assert.Equal(1 + 16, timesBenchmarkCalled); + Assert.Equal(1 + 16, timesIdleCalled); Assert.Equal(2, timesIterationCleanupCalled); // once for single and & once for 16 Assert.Equal(0, timesGlobalCleanupCalled); @@ -87,7 +93,7 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit Assert.Equal(1, timesGlobalCleanupCalled); } - private EngineParameters CreateEngineParameters(Action singleAction, Action multiAction, Job job) + private EngineParameters CreateEngineParameters(Action mainSingleAction, Action mainMultiAction, Job job) => new EngineParameters { Dummy1Action = () => { }, @@ -96,13 +102,12 @@ private EngineParameters CreateEngineParameters(Action singleAction, Actio GlobalSetupAction = GlobalSetup, GlobalCleanupAction = GlobalCleanup, Host = new ConsoleHost(TextWriter.Null, TextReader.Null), - IdleMultiAction = _ => { }, - IdleSingleAction = _ => { }, + IdleMultiAction = Idle16, + IdleSingleAction = IdleSingle, IterationCleanupAction = IterationCleanup, IterationSetupAction = IterationSetup, - MainMultiAction = multiAction, - MainSingleAction = singleAction, - Resolver = EngineResolver.Instance, + MainMultiAction = mainMultiAction, + MainSingleAction = mainSingleAction, TargetJob = job }; } From c1d8ac65166f26a5018d3bf03a31b619da6c9e21 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 19:08:59 +0200 Subject: [PATCH 05/13] do the math right for unroll factor for JIT, #736 --- src/BenchmarkDotNet/Engines/EngineFactory.cs | 11 ++++++----- .../Engine/EngineFactoryTests.cs | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index 79766243da..e8f5cf5111 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -29,6 +29,7 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); var resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + var unrollFactor = engineParameters.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); engineParameters.GlobalSetupAction?.Invoke(); @@ -45,7 +46,7 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); var iterationTime = resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); - if (ShouldExecuteOncePerIteration(Jit(singleActionEngine), iterationTime)) + if (ShouldExecuteOncePerIteration(Jit(singleActionEngine, unrollFactor: 1), iterationTime)) { var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1! @@ -56,7 +57,7 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return var multiActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine)); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, unrollFactor)); return multiActionEngine; } @@ -67,11 +68,11 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime) => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime; - private static Measurement Jit(Engine engine) + private static Measurement Jit(Engine engine, int unrollFactor) { - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJit, index: -1, invokeCount: 1, unrollFactor: 1))); // don't forget to JIT idle + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle - return engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: 1, unrollFactor: 1)); + return engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); } private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 6894726ef0..50399cabfe 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -39,10 +39,10 @@ public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSett var engine = new EngineFactory().CreateReadyToRun(engineParameters); Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(1, timesIterationSetupCalled); + Assert.Equal(1 + 1, timesIterationSetupCalled); // 1x for Idle, 1x for Target Assert.Equal(1, timesBenchmarkCalled); Assert.Equal(1, timesIdleCalled); - Assert.Equal(1, timesIterationCleanupCalled); + Assert.Equal(1 + 1, timesIterationCleanupCalled); // 1x for Idle, 1x for Target Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispode Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration @@ -80,10 +80,10 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit var engine = new EngineFactory().CreateReadyToRun(engineParameters); Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(2, timesIterationSetupCalled); // once for single and & once for 16 + Assert.Equal((1+1) * (1+1), timesIterationSetupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) Assert.Equal(1 + 16, timesBenchmarkCalled); Assert.Equal(1 + 16, timesIdleCalled); - Assert.Equal(2, timesIterationCleanupCalled); // once for single and & once for 16 + Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) Assert.Equal(0, timesGlobalCleanupCalled); Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); From 38d06068a0cb9394258bb795f825d8ef33beda8d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 19:30:42 +0200 Subject: [PATCH 06/13] generate the right IL code, #736 --- src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs index 0134c979af..b045c8651a 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs @@ -112,10 +112,12 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg var engineParameters = new EngineParameters { Host = host, + MainSingleAction = _ => mainAction.InvokeSingle(), MainMultiAction = mainAction.InvokeMultiple, Dummy1Action = dummy1.InvokeSingle, Dummy2Action = dummy2.InvokeSingle, Dummy3Action = dummy3.InvokeSingle, + IdleSingleAction = _ => idleAction.InvokeSingle(), IdleMultiAction = idleAction.InvokeMultiple, GlobalSetupAction = globalSetupAction.InvokeSingle, GlobalCleanupAction = globalCleanupAction.InvokeSingle, From 9ce830c06abd981c4a2817a427e76ac376e83a4b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 19:48:50 +0200 Subject: [PATCH 07/13] Setup and Cleanup are jitted together with benchmark, #736 --- ...upAndCleanupTargetSpecificBenchmarkTest.cs | 47 ++++++++----------- .../AllSetupAndCleanupTest.cs | 23 ++++----- .../CoreRtTests.cs | 3 ++ 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs index ef4185b145..f7e997dcbe 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs @@ -11,7 +11,6 @@ namespace BenchmarkDotNet.IntegrationTests { public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecutor { - private const string FirstPrefix = "// ### First Called: "; private const string FirstGlobalSetupCalled = FirstPrefix + "GlobalSetup"; private const string FirstGlobalCleanupCalled = FirstPrefix + "GlobalCleanup"; @@ -31,25 +30,22 @@ public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecut private readonly string[] firstExpectedLogLines = { "// ### First Called: GlobalSetup", - "// ### First Called: IterationSetup (1)", // IterationSetup Jitting - "// ### First Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### First Called: IterationSetup (2)", // MainWarmup1 + "// ### First Called: IterationSetup (1)", // MainWarmup1 "// ### First Called: Benchmark", // MainWarmup1 - "// ### First Called: IterationCleanup (2)", // MainWarmup1 - "// ### First Called: IterationSetup (3)", // MainWarmup2 + "// ### First Called: IterationCleanup (1)", // MainWarmup1 + "// ### First Called: IterationSetup (2)", // MainWarmup2 "// ### First Called: Benchmark", // MainWarmup2 - "// ### First Called: IterationCleanup (3)", // MainWarmup2 + "// ### First Called: IterationCleanup (2)", // MainWarmup2 - "// ### First Called: IterationSetup (4)", // MainTarget1 + "// ### First Called: IterationSetup (3)", // MainTarget1 "// ### First Called: Benchmark", // MainTarget1 - "// ### First Called: IterationCleanup (4)", // MainTarget1 - "// ### First Called: IterationSetup (5)", // MainTarget2 + "// ### First Called: IterationCleanup (3)", // MainTarget1 + "// ### First Called: IterationSetup (4)", // MainTarget2 "// ### First Called: Benchmark", // MainTarget2 - "// ### First Called: IterationCleanup (5)", // MainTarget2 - "// ### First Called: IterationSetup (6)", // MainTarget3 + "// ### First Called: IterationCleanup (4)", // MainTarget2 + "// ### First Called: IterationSetup (5)", // MainTarget3 "// ### First Called: Benchmark", // MainTarget3 - "// ### First Called: IterationCleanup (6)", // MainTarget3 + "// ### First Called: IterationCleanup (5)", // MainTarget3 "// ### First Called: GlobalCleanup" }; @@ -57,25 +53,22 @@ public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecut private readonly string[] secondExpectedLogLines = { "// ### Second Called: GlobalSetup", - "// ### Second Called: IterationSetup (1)", // IterationSetup Jitting - "// ### Second Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### Second Called: IterationSetup (2)", // MainWarmup1 + "// ### Second Called: IterationSetup (1)", // MainWarmup1 "// ### Second Called: Benchmark", // MainWarmup1 - "// ### Second Called: IterationCleanup (2)", // MainWarmup1 - "// ### Second Called: IterationSetup (3)", // MainWarmup2 + "// ### Second Called: IterationCleanup (1)", // MainWarmup1 + "// ### Second Called: IterationSetup (2)", // MainWarmup2 "// ### Second Called: Benchmark", // MainWarmup2 - "// ### Second Called: IterationCleanup (3)", // MainWarmup2 + "// ### Second Called: IterationCleanup (2)", // MainWarmup2 - "// ### Second Called: IterationSetup (4)", // MainTarget1 + "// ### Second Called: IterationSetup (3)", // MainTarget1 "// ### Second Called: Benchmark", // MainTarget1 - "// ### Second Called: IterationCleanup (4)", // MainTarget1 - "// ### Second Called: IterationSetup (5)", // MainTarget2 + "// ### Second Called: IterationCleanup (3)", // MainTarget1 + "// ### Second Called: IterationSetup (4)", // MainTarget2 "// ### Second Called: Benchmark", // MainTarget2 - "// ### Second Called: IterationCleanup (5)", // MainTarget2 - "// ### Second Called: IterationSetup (6)", // MainTarget3 + "// ### Second Called: IterationCleanup (4)", // MainTarget2 + "// ### Second Called: IterationSetup (5)", // MainTarget3 "// ### Second Called: Benchmark", // MainTarget3 - "// ### Second Called: IterationCleanup (6)", // MainTarget3 + "// ### Second Called: IterationCleanup (5)", // MainTarget3 "// ### Second Called: GlobalCleanup" }; diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 140c9e97ac..4aa998e5ff 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -22,25 +22,22 @@ public class AllSetupAndCleanupTest : BenchmarkTestExecutor private readonly string[] expectedLogLines = { "// ### Called: GlobalSetup", - "// ### Called: IterationSetup (1)", // IterationSetup Jitting - "// ### Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### Called: IterationSetup (2)", // MainWarmup1 + "// ### Called: IterationSetup (1)", // MainWarmup1 "// ### Called: Benchmark", // MainWarmup1 - "// ### Called: IterationCleanup (2)", // MainWarmup1 - "// ### Called: IterationSetup (3)", // MainWarmup2 + "// ### Called: IterationCleanup (1)", // MainWarmup1 + "// ### Called: IterationSetup (2)", // MainWarmup2 "// ### Called: Benchmark", // MainWarmup2 - "// ### Called: IterationCleanup (3)", // MainWarmup2 + "// ### Called: IterationCleanup (2)", // MainWarmup2 - "// ### Called: IterationSetup (4)", // MainTarget1 + "// ### Called: IterationSetup (3)", // MainTarget1 "// ### Called: Benchmark", // MainTarget1 - "// ### Called: IterationCleanup (4)", // MainTarget1 - "// ### Called: IterationSetup (5)", // MainTarget2 + "// ### Called: IterationCleanup (3)", // MainTarget1 + "// ### Called: IterationSetup (4)", // MainTarget2 "// ### Called: Benchmark", // MainTarget2 - "// ### Called: IterationCleanup (5)", // MainTarget2 - "// ### Called: IterationSetup (6)", // MainTarget3 + "// ### Called: IterationCleanup (4)", // MainTarget2 + "// ### Called: IterationSetup (5)", // MainTarget3 "// ### Called: Benchmark", // MainTarget3 - "// ### Called: IterationCleanup (6)", // MainTarget3 + "// ### Called: IterationCleanup (5)", // MainTarget3 "// ### Called: GlobalCleanup" }; diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index 1b5e1f20a0..a477919625 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -17,6 +17,9 @@ public CoreRtTests(ITestOutputHelper outputHelper) : base(outputHelper) { } [Fact] public void CoreRtIsSupported() { + if (IntPtr.Size == sizeof(int)) // CoreRT does not support 32bit yet + return; + var config = ManualConfig.CreateEmpty() .With(Job.Dry .With(Runtime.CoreRT) From 773c47f4ec944767b88cc0393dfb8b7d8ea5904f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 19:51:21 +0200 Subject: [PATCH 08/13] engine factory is now supposed to create an engine which is ready to run (hence the method name change), #736 --- .../CustomEngineTests.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 74cb2090c2..a23cb72ab1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -54,12 +54,18 @@ public void Empty() { } public class CustomFactory : IEngineFactory { - public IEngine CreateReadyToRun(EngineParameters engineParameters) - => new CustomEngine + public IEngine CreateReadyToRun(EngineParameters engineParameters) + { + var engine = new CustomEngine { GlobalCleanupAction = engineParameters.GlobalCleanupAction, GlobalSetupAction = engineParameters.GlobalSetupAction }; + + engine.GlobalSetupAction?.Invoke(); // engine factory is now supposed to create an engine which is ready to run (hence the method name change) + + return engine; + } } public class CustomEngine : IEngine From d67609f6b1bfd737e552ce9aee099dd87d8c98fe Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 22 May 2018 21:36:56 +0200 Subject: [PATCH 09/13] addressing PR feedback, #736 --- .../Intro/IntroInProcessWrongEnv.cs | 3 ++- .../Diagnosers/WindowsDisassembler.cs | 2 +- src/BenchmarkDotNet/Engines/EngineFactory.cs | 12 ++++++--- src/BenchmarkDotNet/Engines/IterationMode.cs | 2 +- .../Engines/IterationModeExtensions.cs | 2 +- .../Extensions/ProcessExtensions.cs | 3 ++- .../Portability/RuntimeInformation.cs | 4 +-- .../CoreRtTests.cs | 2 +- .../Engine/EngineFactoryTests.cs | 26 +++++++++++++++++++ 9 files changed, 45 insertions(+), 11 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs index 9ba212b8ad..8fe83a3829 100644 --- a/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess; namespace BenchmarkDotNet.Samples.Intro @@ -18,7 +19,7 @@ private class Config : ManualConfig { public Config() { - var wrongPlatform = IntPtr.Size == sizeof(int) + var wrongPlatform = RuntimeInformation.GetCurrentPlatform() == Platform.X86 ? Platform.X64 : Platform.X86; diff --git a/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs index f8a4ba3998..a0a2d03b26 100644 --- a/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs @@ -156,7 +156,7 @@ public static bool Is64Bit(Process process) return !isWow64; } - return IntPtr.Size == 8; // todo: find the way to cover all scenarios for .NET Core + return Portability.RuntimeInformation.GetCurrentPlatform() == Platform.X64; // todo: find the way to cover all scenarios for .NET Core } [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)] diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index e8f5cf5111..a6f94efa30 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -41,7 +41,9 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) } var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); - if (needsPilot) + var hasUnrollFactorDefined = engineParameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + + if (needsPilot && !hasUnrollFactorDefined) { var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); @@ -70,9 +72,13 @@ private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval private static Measurement Jit(Engine engine, int unrollFactor) { - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle + + var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); + + engine.WriteLine(); - return engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); + return result; } private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs index 7532528abf..b9ed7e6e58 100644 --- a/src/BenchmarkDotNet/Engines/IterationMode.cs +++ b/src/BenchmarkDotNet/Engines/IterationMode.cs @@ -40,6 +40,6 @@ public enum IterationMode /// /// executing benchmark for the purpose of JIT wamup /// - Jit, IdleJit + MainJitting, IdleJitting } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs index 619360cd90..13287136d1 100644 --- a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs +++ b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs @@ -3,6 +3,6 @@ public static class IterationModeExtensions { public static bool IsIdle(this IterationMode mode) - => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJit; + => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJitting; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 8b52501cdc..9f2c0e1247 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -31,7 +32,7 @@ private static IntPtr FixAffinity(IntPtr processorAffinity) { int cpuMask = (1 << Environment.ProcessorCount) - 1; - return IntPtr.Size == sizeof(Int64) + return RuntimeInformation.GetCurrentPlatform() == Platform.X64 ? new IntPtr(processorAffinity.ToInt64() & cpuMask) : new IntPtr(processorAffinity.ToInt32() & cpuMask); } diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 4412b775d8..f09eb23d18 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -61,7 +61,7 @@ internal static class RuntimeInformation internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; - internal static string GetArchitecture() => IntPtr.Size == 4 ? "32bit" : "64bit"; + internal static string GetArchitecture() => GetCurrentPlatform() == Platform.X86 ? "32bit" : "64bit"; internal static bool IsWindows() { @@ -246,7 +246,7 @@ internal static bool HasRyuJit() if (IsNetCore) return true; - return IntPtr.Size == 8 + return GetCurrentPlatform() == Platform.X64 && GetConfiguration() != DebugConfigurationName && !new JitHelper().IsMsX64(); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index a477919625..471c1ab82e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -17,7 +17,7 @@ public CoreRtTests(ITestOutputHelper outputHelper) : base(outputHelper) { } [Fact] public void CoreRtIsSupported() { - if (IntPtr.Size == sizeof(int)) // CoreRT does not support 32bit yet + if (RuntimeInformation.GetCurrentPlatform() == Platform.X86) // CoreRT does not support 32bit yet return; var config = ManualConfig.CreateEmpty() diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 50399cabfe..0b2e129830 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -71,6 +71,32 @@ public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() Assert.Equal(1, timesGlobalCleanupCalled); } + + [Fact] + public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() + => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithUnrollFactor(16)); + + [Fact] + public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() + => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithInvocationCount(100)); + + public void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) + { + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Instant16, job: job); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(2, timesIterationSetupCalled); + Assert.Equal(16, timesBenchmarkCalled); + Assert.Equal(16, timesIdleCalled); + Assert.Equal(2, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } [Fact] public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings() From e34fc7a55bf9430dd4a0fe07e2ecf1bfb46dfac7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 23 May 2018 09:44:19 +0200 Subject: [PATCH 10/13] bring back the calls to DummyActions, #736 --- src/BenchmarkDotNet/Engines/EngineFactory.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index a6f94efa30..576a196fa2 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -72,10 +72,16 @@ private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval private static Measurement Jit(Engine engine, int unrollFactor) { + engine.Dummy1Action.Invoke(); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle + engine.Dummy2Action.Invoke(); + var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); + engine.Dummy3Action.Invoke(); + engine.WriteLine(); return result; From 8ae23e8487d5c881259cbe9ca4afcd8d4d32d0b3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 23 May 2018 11:58:25 +0200 Subject: [PATCH 11/13] align iteration mode --- src/BenchmarkDotNet/Reports/Measurement.cs | 8 +++++++- tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Reports/Measurement.cs b/src/BenchmarkDotNet/Reports/Measurement.cs index 461d5106b5..f05a310b92 100644 --- a/src/BenchmarkDotNet/Reports/Measurement.cs +++ b/src/BenchmarkDotNet/Reports/Measurement.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -13,6 +14,8 @@ public struct Measurement : IComparable { private static readonly Measurement Error = new Measurement(-1, IterationMode.Unknown, 0, 0, 0); + private static readonly int IterationModeNameMaxWidth = Enum.GetNames(typeof(IterationMode)).Max(text => text.Length); + public IterationMode IterationMode { get; } public int LaunchIndex { get; } @@ -48,10 +51,13 @@ public Measurement(int launchIndex, IterationMode iterationMode, int iterationIn public string ToOutputLine() { + string alignedIterationMode = IterationMode.ToString().PadRight(IterationModeNameMaxWidth, ' '); + // Usually, a benchmarks takes more than 10 iterations (rarely more than 99) // PadLeft(2, ' ') looks like a good trade-off between alignment and amount of characters string alignedIterationIndex = IterationIndex.ToString().PadLeft(2, ' '); - return $"{IterationMode} {alignedIterationIndex}: {GetDisplayValue()}"; + + return $"{alignedIterationMode} {alignedIterationIndex}: {GetDisplayValue()}"; } private string GetDisplayValue() => $"{Operations} op, {Nanoseconds.ToStr("0.00")} ns, {GetAverageTime()}"; diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 0b2e129830..6bbfa81b68 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -80,7 +80,7 @@ public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionC public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithInvocationCount(100)); - public void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) + private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) { var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Instant16, job: job); From 916444c5397729522b5bb95d7daae910504a164b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 24 May 2018 13:56:01 +0200 Subject: [PATCH 12/13] don't measure the overhead for time consuming benchmarks, don't run pilot if jitting gives the answer, #736 --- src/BenchmarkDotNet/Engines/EngineFactory.cs | 83 ++++++++------ .../Engines/EngineParameters.cs | 22 +++- .../Engine/EngineFactoryTests.cs | 108 ++++++++++++++---- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index 576a196fa2..18719ad8d0 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,5 +1,4 @@ using System; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; @@ -28,69 +27,77 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) if(engineParameters.TargetJob == null) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); - var resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); - var unrollFactor = engineParameters.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); + engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose - engineParameters.GlobalSetupAction?.Invoke(); + if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit + return CreateMultiActionEngine(engineParameters); - var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, resolver).NeedsJitting(); - if (!needsJitting) - { - // whatever it is, we can not interfere - return CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); - } - - var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); - var hasUnrollFactorDefined = engineParameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + int jitIndex = 0; - if (needsPilot && !hasUnrollFactorDefined) + if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it { - var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); - - var iterationTime = resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); - if (ShouldExecuteOncePerIteration(Jit(singleActionEngine, unrollFactor: 1), iterationTime)) - { - var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1! + var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters); + + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor)); - return CreateEngine(engineParameters, resolver, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); - } + return warmedUpMultiActionEngine; } + + var singleActionEngine = CreateSingleActionEngine(engineParameters); + if (Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1) > engineParameters.IterationTime) + return singleActionEngine; // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead - // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return - var multiActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); - - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, unrollFactor)); + var multiActionEngine = CreateMultiActionEngine(engineParameters); + int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver); + if (Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor) > engineParameters.IterationTime) + { // executing defaultUnrollFactor times takes longer than iteration time => medium running benchmark, needs no pilot and no overhead + var defaultUnrollFactorTimesPerIterationNoPilotNoOverhead = CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, + invocationCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor); // run the benchmark exactly once per iteration + + return CreateEngine(engineParameters, defaultUnrollFactorTimesPerIterationNoPilotNoOverhead, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + } + return multiActionEngine; } - /// - /// returns true if it takes longer than the desired iteration time (0,5s by default) to execute benchmark once - /// - private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime) - => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime; - - private static Measurement Jit(Engine engine, int unrollFactor) + /// the time it took to run the benchmark + private static TimeInterval Jit(Engine engine, int jitIndex, int invokeCount, int unrollFactor) { engine.Dummy1Action.Invoke(); - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, jitIndex, invokeCount, unrollFactor))); // don't forget to JIT idle engine.Dummy2Action.Invoke(); - var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); + var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, jitIndex, invokeCount, unrollFactor)); engine.Dummy3Action.Invoke(); engine.WriteLine(); - return result; + return TimeInterval.FromNanoseconds(result.Nanoseconds); } - private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) + private static Engine CreateMultiActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + + private static Engine CreateSingleActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, + CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, invocationCount: 1, unrollFactor: 1), // run the benchmark exactly once per iteration + engineParameters.IdleSingleAction, + engineParameters.MainSingleAction); + + private static Job CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(Job sourceJob, int invocationCount, int unrollFactor) + => sourceJob + .WithInvocationCount(invocationCount).WithUnrollFactor(unrollFactor) + .WithEvaluateOverhead(false); // it's very time consuming, don't evaluate the overhead which would be 0,000025% of the target run or even less + // todo: consider if we should set the warmup count to 2 + + private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) => new Engine( engineParameters.Host, - resolver, + EngineParameters.DefaultResolver, engineParameters.Dummy1Action, engineParameters.Dummy2Action, engineParameters.Dummy3Action, diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index f384d32f9e..9bf8e12051 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -1,11 +1,15 @@ using System; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Engines { public class EngineParameters { + public static readonly IResolver DefaultResolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + public IHost Host { get; set; } public Action MainSingleAction { get; set; } public Action MainMultiAction { get; set; } @@ -16,10 +20,20 @@ public class EngineParameters public Action IdleMultiAction { get; set; } public Job TargetJob { get; set; } = Job.Default; public long OperationsPerInvoke { get; set; } = 1; - public Action GlobalSetupAction { get; set; } = null; - public Action GlobalCleanupAction { get; set; } = null; - public Action IterationSetupAction { get; set; } = null; - public Action IterationCleanupAction { get; set; } = null; + public Action GlobalSetupAction { get; set; } + public Action GlobalCleanupAction { get; set; } + public Action IterationSetupAction { get; set; } + public Action IterationCleanupAction { get; set; } public bool MeasureGcStats { get; set; } + + public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting(); + + public bool HasInvocationCount => TargetJob.HasValue(RunMode.InvocationCountCharacteristic); + + public bool HasUnrollFactor => TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + + public int UnrollFactor => TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + + public TimeInterval IterationTime => TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, DefaultResolver); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 6bbfa81b68..1316142b4c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; using Xunit; namespace BenchmarkDotNet.Tests.Engine @@ -11,6 +15,10 @@ public class EngineFactoryTests { int timesBenchmarkCalled = 0, timesIdleCalled = 0; int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; + + TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds()); + + IResolver DefaultResolver => BenchmarkRunner.DefaultResolver; void GlobalSetup() => timesGlobalSetupCalled++; void IterationSetup() => timesIterationSetupCalled++; @@ -22,7 +30,7 @@ public class EngineFactoryTests void VeryTimeConsumingSingle(long _) { timesBenchmarkCalled++; - Thread.Sleep(TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds())); + Thread.Sleep(IterationTime); } void InstantSingle(long _) => timesBenchmarkCalled++; @@ -31,8 +39,35 @@ void VeryTimeConsumingSingle(long _) void IdleSingle(long _) => timesIdleCalled++; void Idle16(long _) => timesIdleCalled += 16; + public static IEnumerable JobsWhichDontRequireJitting() + { + yield return new object[]{ Job.Dry }; + yield return new object[]{ Job.Default.With(RunStrategy.ColdStart) }; + yield return new object[]{ Job.Default.With(RunStrategy.Monitoring) }; + } + + [Theory] + [MemberData(nameof(JobsWhichDontRequireJitting))] + public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(Job job) + { + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: job); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(0, timesIterationSetupCalled); + Assert.Equal(0, timesBenchmarkCalled); + Assert.Equal(0, timesIdleCalled); + Assert.Equal(0, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + [Fact] - public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSettings() + public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerIterationWithoutOverheadDeduction() { var engineParameters = CreateEngineParameters(mainSingleAction: VeryTimeConsumingSingle, mainMultiAction: Throwing, job: Job.Default); @@ -47,30 +82,14 @@ public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSett Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case engine.Dispose(); // cleanup is called as part of dispode Assert.Equal(1, timesGlobalCleanupCalled); } - - [Fact] - public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() - { - var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: Job.Dry); - - var engine = new EngineFactory().CreateReadyToRun(engineParameters); - - Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(0, timesIterationSetupCalled); - Assert.Equal(0, timesBenchmarkCalled); - Assert.Equal(0, timesIdleCalled); - Assert.Equal(0, timesIterationCleanupCalled); - Assert.Equal(0, timesGlobalCleanupCalled); - - engine.Dispose(); - - Assert.Equal(1, timesGlobalCleanupCalled); - } [Fact] public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() @@ -92,6 +111,8 @@ private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) Assert.Equal(16, timesIdleCalled); Assert.Equal(2, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched engine.Dispose(); @@ -111,6 +132,8 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit Assert.Equal(1 + 16, timesIdleCalled); Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); @@ -119,6 +142,49 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit Assert.Equal(1, timesGlobalCleanupCalled); } + [Fact] + public void DontRunThePilotIfThePilotRequirementIsMetDuringWarmup() + { + var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + var mediumTime = TimeSpan.FromMilliseconds(IterationTime.TotalMilliseconds / unrollFactor); + + void MediumSingle(long _) + { + timesBenchmarkCalled++; + + Thread.Sleep(mediumTime); + } + + void MediumMultiple(long _) + { + timesBenchmarkCalled += unrollFactor; + + for (int i = 0; i < unrollFactor; i++) // the real unroll factor obviously does not use loop ;) + Thread.Sleep(mediumTime); + } + + var engineParameters = CreateEngineParameters(mainSingleAction: MediumSingle, mainMultiAction: MediumMultiple, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal((1+1) * (1+1), timesIterationSetupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(1 + unrollFactor, timesBenchmarkCalled); + Assert.Equal(1 + unrollFactor, timesIdleCalled); + Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.Equal(unrollFactor, engine.TargetJob.Run.InvocationCount); // no need to run pilot! + Assert.Equal(unrollFactor, engine.TargetJob.Run.UnrollFactor); // remains the same! + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + private EngineParameters CreateEngineParameters(Action mainSingleAction, Action mainMultiAction, Job job) => new EngineParameters { From 8fb46352b6b3764a055acc0b34125bef7863ae69 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 25 May 2018 15:55:47 +0200 Subject: [PATCH 13/13] fix the Linux build --- tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 1316142b4c..11edb4830c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -146,7 +146,7 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit public void DontRunThePilotIfThePilotRequirementIsMetDuringWarmup() { var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); - var mediumTime = TimeSpan.FromMilliseconds(IterationTime.TotalMilliseconds / unrollFactor); + var mediumTime = TimeSpan.FromMilliseconds((IterationTime.TotalMilliseconds / unrollFactor) * 2); void MediumSingle(long _) {