Skip to content

Commit b6268db

Browse files
djluckdjluck
authored
Benchmarking (#32)
* Deleting unused files * Moving common csproj projects into a dedicated project, `Common.csproj` * Adding ability to replace existing stat configurations per-type * Implementing some basic benchmarks- currently focuses on comparisons between no instrumentation enabled and with default instrumentation enabled. This is just a first draft of the benchmarks- they require further work to be more accurate. Co-authored-by: James Luck <[email protected]>
1 parent 937ffee commit b6268db

24 files changed

+381
-728
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,5 @@ obj/
3636
.vs/
3737

3838
# Rider
39-
.idea/
39+
.idea/
40+
BenchmarkDotNet.Artifacts/

‎examples/AspNetCoreExample/AspNetCoreExample.csproj

+2-17
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,15 @@
11
<Project Sdk="Microsoft.NET.Sdk.Web">
2-
<PropertyGroup Condition=" $(Configuration.EndsWith('V2')) ">
3-
<PromMajorVersion>2</PromMajorVersion>
4-
<DefineConstants>$(DefineConstants);PROMV2</DefineConstants>
5-
</PropertyGroup>
6-
7-
<PropertyGroup Condition=" $(Configuration.EndsWith('V3')) ">
8-
<PromMajorVersion>3</PromMajorVersion>
9-
<DefineConstants>$(DefineConstants);PROMV3</DefineConstants>
10-
</PropertyGroup>
11-
2+
<Import Project="../../src/Common.csproj"/>
123
<PropertyGroup>
13-
<PromMajorVersion Condition="$(PromMajorVersion) == ''">3</PromMajorVersion>
144
<TargetFramework>netcoreapp3.0</TargetFramework>
15-
<Configurations>ReleaseV2;DebugV2;DebugV3;ReleaseV3</Configurations>
16-
</PropertyGroup>
17-
18-
<PropertyGroup Condition=" $(Configuration.StartsWith('Release')) ">
19-
<Optimize Condition=" '$(Optimize)' == '' ">true</Optimize>
205
</PropertyGroup>
216

227
<ItemGroup>
238
<ProjectReference Include="..\..\src\prometheus-net.DotNetRuntime\prometheus-net.DotNetRuntime.csproj" />
249
</ItemGroup>
2510

2611
<ItemGroup Condition="$(PromMajorVersion) == '3'">
27-
<PackageReference Include="prometheus-net.AspNetCore" Version="3.2.1" />
12+
<PackageReference Include="prometheus-net.AspNetCore" Version="3.1.2" />
2813
</ItemGroup>
2914

3015
<ItemGroup Condition="$(PromMajorVersion) == '2'">

‎prometheus-net.DotNetRuntime.sln

+8
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{31
88
EndProject
99
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AspNetCoreExample", "examples\AspNetCoreExample\AspNetCoreExample.csproj", "{D01E9ED3-E35C-4F44-A5AD-5350E43AA636}"
1010
EndProject
11+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmarks", "src\Benchmarks\Benchmarks.csproj", "{DD607E45-45AD-4F9D-9102-82BD99E49BEC}"
12+
EndProject
1113
Global
1214
GlobalSection(SolutionConfigurationPlatforms) = preSolution
1315
ReleaseV2|Any CPU = ReleaseV2|Any CPU
@@ -40,6 +42,12 @@ Global
4042
{D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU
4143
{D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU
4244
{D01E9ED3-E35C-4F44-A5AD-5350E43AA636}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU
45+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV2|Any CPU.ActiveCfg = ReleaseV2|Any CPU
46+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV2|Any CPU.Build.0 = ReleaseV2|Any CPU
47+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV3|Any CPU.ActiveCfg = ReleaseV3|Any CPU
48+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.ReleaseV3|Any CPU.Build.0 = ReleaseV3|Any CPU
49+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.DebugV2|Any CPU.ActiveCfg = DebugV2|Any CPU
50+
{DD607E45-45AD-4F9D-9102-82BD99E49BEC}.DebugV3|Any CPU.ActiveCfg = DebugV3|Any CPU
4351
EndGlobalSection
4452
GlobalSection(NestedProjects) = preSolution
4553
{D01E9ED3-E35C-4F44-A5AD-5350E43AA636} = {31AD912F-A1DC-434A-8C8D-049F4BBD67D4}

‎src/Benchmarks/BenchmarkDotNet.Artifacts/Benchmarks.Program.TestBenchmark.log

-511
This file was deleted.

‎src/Benchmarks/BenchmarkDotNet.Artifacts/results/Benchmarks.Program.TestBenchmark-report-github.md

-18
This file was deleted.

‎src/Benchmarks/BenchmarkDotNet.Artifacts/results/Benchmarks.Program.TestBenchmark-report.csv

-5
This file was deleted.

‎src/Benchmarks/BenchmarkDotNet.Artifacts/results/Benchmarks.Program.TestBenchmark-report.html

-35
This file was deleted.

‎src/Benchmarks/Benchmarks.csproj

+24-26
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,36 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
<Import Project="../Common.csproj" />
23

34
<PropertyGroup>
4-
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp2.2</TargetFramework>
6-
<ServerGarbageCollection>true</ServerGarbageCollection>
7-
<Configurations>ReleaseV2;DebugV2;DebugV3;ReleaseV3</Configurations>
8-
<Platforms>AnyCPU</Platforms>
5+
<TargetFramework>netcoreapp3.0</TargetFramework>
6+
<AspNetCoreHostingModel>InProcess</AspNetCoreHostingModel>
7+
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
98
</PropertyGroup>
109

11-
<PropertyGroup Condition=" $(Configuration.EndsWith('V2')) ">
12-
<PromMajorVersion>2</PromMajorVersion>
13-
<DefineConstants>$(DefineConstants);PROMV2</DefineConstants>
14-
</PropertyGroup>
15-
16-
<PropertyGroup Condition=" $(Configuration.EndsWith('V3')) ">
17-
<PromMajorVersion>3</PromMajorVersion>
18-
<DefineConstants>$(DefineConstants);PROMV3</DefineConstants>
19-
</PropertyGroup>
10+
<ItemGroup>
11+
<PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
12+
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.0" />
13+
<PackageReference Include="Microsoft.AspNetCore.App" />
14+
<PackageReference Include="Microsoft.AspNetCore.Razor.Design" Version="2.2.0" PrivateAssets="All" />
15+
</ItemGroup>
2016

21-
<PropertyGroup Condition=" $(Configuration.StartsWith('Release')) ">
22-
<Optimize Condition=" '$(Optimize)' == '' ">true</Optimize>
23-
</PropertyGroup>
17+
<ItemGroup>
18+
<ProjectReference Include="..\prometheus-net.DotNetRuntime\prometheus-net.DotNetRuntime.csproj" />
19+
</ItemGroup>
2420

25-
<PropertyGroup Condition=" $(Configuration.StartsWith('Debug')) ">
26-
<DebugSymbols Condition=" '$(DebugSymbols)' == '' ">true</DebugSymbols>
27-
<Optimize Condition=" '$(Optimize)' == '' ">false</Optimize>
28-
</PropertyGroup>
21+
<ItemGroup Condition="$(PromMajorVersion) == '3'">
22+
<PackageReference Include="prometheus-net.AspNetCore" Version="3.1.2" />
23+
</ItemGroup>
2924

30-
<ItemGroup>
31-
<ProjectReference Include="..\prometheus-net.DotNetRuntime\prometheus-net.DotNetRuntime.csproj" />
25+
<ItemGroup Condition="$(PromMajorVersion) == '2'">
26+
<PackageReference Include="prometheus-net.AspNetCore" Version="2.1.3" />
3227
</ItemGroup>
3328

3429
<ItemGroup>
35-
<PackageReference Include="BenchmarkDotNet" Version="0.11.3" />
30+
<Content Update="Properties\launchSettings.json">
31+
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
32+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
33+
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
34+
</Content>
3635
</ItemGroup>
37-
3836
</Project>

‎src/Benchmarks/Benchmarks/AspNetBenchmarkBase.cs

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Net.Http;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using BenchmarkDotNet.Attributes;
8+
using Microsoft.AspNetCore;
9+
using Microsoft.AspNetCore.Hosting;
10+
using Prometheus.DotNetRuntime;
11+
12+
namespace Benchmarks.Benchmarks
13+
{
14+
[BenchmarkCategory("aspnet")]
15+
public abstract class AspNetBenchmarkBase
16+
{
17+
public const int NumRequests = 10_000;
18+
19+
private IWebHost webHost;
20+
private CancellationTokenSource ctSource = new CancellationTokenSource();
21+
private HttpClient client;
22+
private byte[][] buffers;
23+
private Task webHostTask;
24+
25+
public int NumHttpConnections { get; set; } = 50;
26+
27+
[GlobalSetup]
28+
public void GlobalSetup()
29+
{
30+
PreGlobalSetup();
31+
webHost = WebHost.CreateDefaultBuilder()
32+
.UseStartup<Startup>()
33+
.ConfigureKestrel(cfg => { cfg.ListenLocalhost(5000); })
34+
.Build();
35+
webHostTask = webHost.RunAsync(ctSource.Token);
36+
37+
// preallocate buffers to avoid having them counted as part of each benchmark run
38+
buffers = new byte[NumHttpConnections][];
39+
for (int i = 0; i < buffers.Length; i++)
40+
buffers[i] = new byte[1024 * 64];
41+
42+
client = new HttpClient(new SocketsHttpHandler() { MaxConnectionsPerServer = NumHttpConnections });
43+
}
44+
45+
protected virtual void PreGlobalSetup()
46+
{
47+
}
48+
49+
protected virtual void PostGlobalCleanup()
50+
{
51+
}
52+
53+
[GlobalCleanup]
54+
public void GlobalCleanup()
55+
{
56+
ctSource.Cancel();
57+
client.Dispose();
58+
webHostTask.Wait();
59+
PostGlobalCleanup();
60+
}
61+
62+
protected async Task MakeHttpRequests()
63+
{
64+
var requestsRemaining = NumRequests;
65+
var tasks = Enumerable.Range(0, NumHttpConnections)
66+
.Select(async n =>
67+
{
68+
var buffer = buffers[n];
69+
while (Interlocked.Decrement(ref requestsRemaining) > 0)
70+
{
71+
await using var response = await client.GetStreamAsync("http://localhost:5000/api/benchmark");
72+
while (await response.ReadAsync(buffer, 0, buffer.Length) > 0)
73+
{
74+
}
75+
}
76+
});
77+
78+
await Task.WhenAll(tasks);
79+
}
80+
}
81+
}

‎src/Benchmarks/Benchmarks/BaselineBenchmark.cs

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using System.Collections.Generic;
2+
using System.Threading.Tasks;
3+
using BenchmarkDotNet.Attributes;
4+
using Prometheus.DotNetRuntime;
5+
6+
namespace Benchmarks.Benchmarks
7+
{
8+
public class BaselineBenchmark : AspNetBenchmarkBase
9+
{
10+
[Benchmark(Description = "No stats collectors enabled", Baseline = true, OperationsPerInvoke = NumRequests)]
11+
public async Task Make_Requests()
12+
{
13+
await MakeHttpRequests();
14+
}
15+
}
16+
}

‎src/Benchmarks/Benchmarks/DefaultBenchmark.cs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using Prometheus.DotNetRuntime;
2+
3+
namespace Benchmarks.Benchmarks
4+
{
5+
public class DefaultBenchmark : DotNetRuntimeStatsBenchmarkBase
6+
{
7+
protected override DotNetRuntimeStatsBuilder.Builder GetStatsBuilder()
8+
{
9+
return DotNetRuntimeStatsBuilder.Default();
10+
}
11+
}
12+
}

‎src/Benchmarks/Benchmarks/DotNetRuntimeStatsBenchmarkBase.cs

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Threading.Tasks;
4+
using BenchmarkDotNet.Attributes;
5+
using Prometheus.DotNetRuntime;
6+
7+
namespace Benchmarks.Benchmarks
8+
{
9+
public abstract class DotNetRuntimeStatsBenchmarkBase : AspNetBenchmarkBase
10+
{
11+
private IDisposable collector;
12+
13+
protected override void PreGlobalSetup()
14+
{
15+
collector = GetStatsBuilder().StartCollecting();
16+
}
17+
18+
protected override void PostGlobalCleanup()
19+
{
20+
collector.Dispose();
21+
}
22+
23+
[Benchmark(Baseline = false, OperationsPerInvoke = NumRequests)]
24+
public async Task Make_Requests()
25+
{
26+
await MakeHttpRequests();
27+
}
28+
29+
protected abstract DotNetRuntimeStatsBuilder.Builder GetStatsBuilder();
30+
}
31+
}

‎src/Benchmarks/Benchmarks/NoSamplingBenchmark.cs

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using BenchmarkDotNet.Attributes;
2+
using Prometheus.DotNetRuntime;
3+
4+
namespace Benchmarks.Benchmarks
5+
{
6+
public class NoSamplingBenchmark : DotNetRuntimeStatsBenchmarkBase
7+
{
8+
protected override DotNetRuntimeStatsBuilder.Builder GetStatsBuilder()
9+
{
10+
return DotNetRuntimeStatsBuilder.Default()
11+
.WithThreadPoolSchedulingStats(sampleRate: SampleEvery.OneEvent)
12+
.WithJitStats(SampleEvery.OneEvent)
13+
.WithContentionStats(SampleEvery.OneEvent);
14+
}
15+
}
16+
}

‎src/Benchmarks/Controllers/BenchmarkController.cs

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Linq.Expressions;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Microsoft.AspNetCore.Mvc;
8+
9+
namespace Benchmarks.Controllers
10+
{
11+
[Route("api/[controller]")]
12+
[ApiController]
13+
public class BenchmarkController : ControllerBase
14+
{
15+
[HttpGet]
16+
public async Task<ActionResult<IEnumerable<string>>> Get()
17+
{
18+
var r = new Random();
19+
// assign some SOH memory
20+
var soh = new char[1024];
21+
22+
// assign some LOH memory
23+
var loh = new char[1024 * 100];
24+
25+
// Compile a method
26+
var result = CompileMe(() => r.Next());
27+
28+
return new string[] {"value1" + soh[^1] + loh[^1] + result };
29+
}
30+
31+
private int CompileMe(Expression<Func<int>> func)
32+
{
33+
return func.Compile()();
34+
}
35+
}
36+
}

0 commit comments

Comments
 (0)