BenchmarkDotNet init (#2238)

* install BenchmarkDotNet from CI feed. remove Microsoft.NET.Test.Sdk which was generating Program.Main * port simple type to BenchmarkDotNet * don't run Benchmarks as part of test run * port one benchmark with `[Params]` and `[GlobalSetup]` as an example * don't run Benchmarks as part of test run * post code review changes * remove the cast
dotnet · May 2, 2018 · d48f2c0 · d48f2c0
1 parent b8b23c3
commit d48f2c0
Show file tree

Hide file tree

Showing 9 changed files with 150 additions and 201 deletions.
diff --git a/NuGet.Config b/NuGet.Config
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageRestore>
     <add key="enabled" value="True" />
@@ -20,5 +20,6 @@
     <add key="AspNetCIDev" value="https://www.myget.org/F/aspnetcidev/api/v3/index.json" />
     <add key="nugetbuild" value="https://www.myget.org/F/nugetbuild/api/v3/index.json" />
     <add key="roslyn" value="https://dotnet.myget.org/F/roslyn/" />
+    <add key="benchmarkdotnet-ci" value="https://ci.appveyor.com/nuget/benchmarkdotnet" />
   </packageSources>
 </configuration>
diff --git a/build.sh b/build.sh
@@ -137,7 +137,7 @@ do
     projectsFailed[${#projectsFailed[@]}]="$testFile"
 	((errorsEncountered=errorsEncountered+1))
   fi
-done < <(find tests -name "*.csproj")
+done < <(find tests -name "*Tests*.csproj")
 
 RED='\033[0;31m'
 GREEN='\033[0;32m'

diff --git a/scripts/build.ps1 b/scripts/build.ps1
@@ -96,7 +96,7 @@ $errorsEncountered = 0
 $projectsFailed = New-Object System.Collections.Generic.List[String]
 
 if ($SkipTests -ne "true") {
-    foreach ($testFile in [System.IO.Directory]::EnumerateFiles("$PSScriptRoot\..\tests", "*.csproj", "AllDirectories")) {
+    foreach ($testFile in [System.IO.Directory]::EnumerateFiles("$PSScriptRoot\..\tests", "*Tests*.csproj", "AllDirectories")) {
         Write-Host "Building and running tests for project $testFile..."
         Invoke-Expression "$dotnetExePath test $testFile -c $Configuration --no-build -- -notrait category=performance -notrait category=outerloop"
 

diff --git a/tests/Benchmarks/Benchmarks.csproj b/tests/Benchmarks/Benchmarks.csproj
@@ -2,6 +2,7 @@
   <Import Project="..\..\tools\common.props" />
   <PropertyGroup>
     <TargetFramework>netcoreapp2.1</TargetFramework>
+    <OutputType>Exe</OutputType>
     <AllowUnsafeBlocks>False</AllowUnsafeBlocks>
     <AssemblyOriginatorKeyFile>../../tools/test_key.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
@@ -20,10 +21,10 @@
     <NoWarn>$(NoWarn);CS0618</NoWarn>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="$(BenchmarkDotNetVersion)" />
     <PackageReference Include="System.Text.Encodings.Web" Version="$(CoreFxStableVersion)" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(TestSdkVersion)" />
     <PackageReference Include="xunit" Version="$(XunitVersion)" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="$(XunitVersion)" />
     <PackageReference Include="xunit.performance.core" Version="$(XunitPerformanceVersion)" />
   </ItemGroup>
   <!-- Project references -->
@@ -37,8 +38,4 @@
     <ProjectReference Include="..\..\src\System.Text.Http\System.Text.Http.csproj" />
     <ProjectReference Include="..\..\src\System.Text.Json\System.Text.Json.csproj" />
   </ItemGroup>
-  <!-- register for test discovery in Visual Studio -->
-  <ItemGroup>
-    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
-  </ItemGroup>
 </Project>
diff --git a/tests/Benchmarks/BufferWriterBench.cs → tests/Benchmarks/Buffers/Writer.cs b/tests/Benchmarks/BufferWriterBench.cs → tests/Benchmarks/Buffers/Writer.cs
@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Microsoft.Xunit.Performance;
+using BenchmarkDotNet.Attributes;
 using System;
 using System.Buffers;
 using System.Buffers.Text;
@@ -11,9 +11,8 @@
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Text.Utf8;
-using System.Threading;
 
-public class BufferWriterBench
+public class Writer
 {
     private static AsciiString s_crlf = "\r\n";
     private static AsciiString s_eoh = "\r\n\r\n"; // End Of Headers
@@ -35,162 +34,124 @@ public class BufferWriterBench
 
     private static Sink s_sink = new Sink(4096);
 
-    const int InnerIterations = 1000000;
-
-    [Benchmark(InnerIterationCount = InnerIterations)]
-    static void PlatfromBenchmarkPlaintext()
+    [Benchmark]
+    public void PlatfromBenchmarkPlaintext()
     {
-        foreach (var iteration in Benchmark.Iterations)
-        {
-            using (iteration.StartMeasurement())
-            {
-                for (int i = 0; i < InnerIterations; i++)
-                {
-                    s_sink.Reset();
-                    var writer = new PlatfromBenchmark.BufferWriter<Sink>(s_sink);
+        s_sink.Reset();
+        var writer = new PlatfromBenchmark.BufferWriter<Sink>(s_sink);
 
-                    // HTTP 1.1 OK
-                    writer.Write(s_http11OK);
+        // HTTP 1.1 OK
+        writer.Write(s_http11OK);
 
-                    // Server headers
-                    writer.Write(s_headerServer);
+        // Server headers
+        writer.Write(s_headerServer);
 
-                    // Date header
-                    writer.Write(DateHeader.HeaderBytes);
+        // Date header
+        writer.Write(DateHeader.HeaderBytes);
 
-                    // Content-Type header
-                    writer.Write(s_headerContentTypeText);
+        // Content-Type header
+        writer.Write(s_headerContentTypeText);
 
-                    // Content-Length header
-                    writer.Write(s_headerContentLength);
-                    writer.Write((ulong)s_plainTextBody.Length);
+        // Content-Length header
+        writer.Write(s_headerContentLength);
+        writer.Write((ulong)s_plainTextBody.Length);
 
-                    // End of headers
-                    writer.Write(s_eoh);
+        // End of headers
+        writer.Write(s_eoh);
 
-                    // Body
-                    writer.Write(s_plainTextBody);
-                    writer.Commit();
-                }
-            }
-        }
+        // Body
+        writer.Write(s_plainTextBody);
+        writer.Commit();
     }
 
-    [Benchmark(InnerIterationCount = InnerIterations)]
-    static void BufferWriterPlaintext()
+    [Benchmark]
+    public void BufferWriterPlaintext()
     {
-        foreach (var iteration in Benchmark.Iterations)
-        {
-            using (iteration.StartMeasurement())
-            {
-                for (int i = 0; i < InnerIterations; i++)
-                {
-                    s_sink.Reset();
-                    var writer = BufferWriter.Create(s_sink);
+        s_sink.Reset();
+        var writer = BufferWriter.Create(s_sink);
 
-                    // HTTP 1.1 OK
-                    writer.Write(s_http11OK);
+        // HTTP 1.1 OK
+        writer.Write(s_http11OK);
 
-                    // Server headers
-                    writer.Write(s_headerServer);
+        // Server headers
+        writer.Write(s_headerServer);
 
-                    // Date header
-                    writer.Write(DateHeader.HeaderBytes);
+        // Date header
+        writer.Write(DateHeader.HeaderBytes);
 
-                    // Content-Type header
-                    writer.Write(s_headerContentTypeText);
+        // Content-Type header
+        writer.Write(s_headerContentTypeText);
 
-                    // Content-Length header
-                    writer.Write(s_headerContentLength);
-                    writer.Write((ulong)s_plainTextBody.Length);
+        // Content-Length header
+        writer.Write(s_headerContentLength);
+        writer.Write((ulong)s_plainTextBody.Length);
 
-                    // End of headers
-                    writer.Write(s_eoh);
+        // End of headers
+        writer.Write(s_eoh);
 
-                    // Body
-                    writer.Write(s_plainTextBody);
-                    writer.Flush();
-                }
-            }
-        }
+        // Body
+        writer.Write(s_plainTextBody);
+        writer.Flush();
     }
 
-    [Benchmark(InnerIterationCount = InnerIterations)]
-    static void BufferWriterPlaintextUtf8()
+    [Benchmark]
+    public void BufferWriterPlaintextUtf8()
     {
-        foreach (var iteration in Benchmark.Iterations)
-        {
-            using (iteration.StartMeasurement())
-            {
-                for (int i = 0; i < InnerIterations; i++)
-                {
-                    s_sink.Reset();
-                    var writer = BufferWriter.Create(s_sink);
+        s_sink.Reset();
+        var writer = BufferWriter.Create(s_sink);
 
-                    // HTTP 1.1 OK
-                    writer.Write(s_http11OKU8);
+        // HTTP 1.1 OK
+        writer.Write(s_http11OKU8);
 
-                    // Server headers
-                    writer.Write(s_headerServerU8);
+        // Server headers
+        writer.Write(s_headerServerU8);
 
-                    // Date header
-                    writer.Write(DateHeader.HeaderBytes);
+        // Date header
+        writer.Write(DateHeader.HeaderBytes);
 
-                    // Content-Type header
-                    writer.Write(s_headerContentTypeTextU8);
+        // Content-Type header
+        writer.Write(s_headerContentTypeTextU8);
 
-                    // Content-Length header
-                    writer.Write(s_headerContentLengthU8);
-                    writer.Write((ulong)s_plainTextBody.Length);
+        // Content-Length header
+        writer.Write(s_headerContentLengthU8);
+        writer.Write((ulong)s_plainTextBody.Length);
 
-                    // End of headers
-                    writer.Write(s_eohU8);
+        // End of headers
+        writer.Write(s_eohU8);
 
-                    // Body
-                    writer.Write(s_plainTextBodyU8);
-                    writer.Flush();
-                }
-            }
-        }
+        // Body
+        writer.Write(s_plainTextBodyU8);
+        writer.Flush();
     }
 
-    [Benchmark(InnerIterationCount = InnerIterations)]
-    static void BufferWriterCopyPlaintext()
+    [Benchmark]
+    public void BufferWriterCopyPlaintext()
     {
-        foreach (var iteration in Benchmark.Iterations)
-        {
-            using (iteration.StartMeasurement())
-            {
-                for (int i = 0; i < InnerIterations; i++)
-                {
-                    s_sink.Reset();
-                    var writer = new SystemBuffers.BufferWriter<Sink>(s_sink);
+        s_sink.Reset();
+        var writer = new SystemBuffers.BufferWriter<Sink>(s_sink);
 
-                    // HTTP 1.1 OK
-                    writer.Write(s_http11OK);
+        // HTTP 1.1 OK
+        writer.Write(s_http11OK);
 
-                    // Server headers
-                    writer.Write(s_headerServer);
+        // Server headers
+        writer.Write(s_headerServer);
 
-                    // Date header
-                    writer.Write(DateHeader.HeaderBytes);
+        // Date header
+        writer.Write(DateHeader.HeaderBytes);
 
-                    // Content-Type header
-                    writer.Write(s_headerContentTypeText);
+        // Content-Type header
+        writer.Write(s_headerContentTypeText);
 
-                    // Content-Length header
-                    writer.Write(s_headerContentLength);
-                    writer.Write((ulong)s_plainTextBody.Length);
+        // Content-Length header
+        writer.Write(s_headerContentLength);
+        writer.Write((ulong)s_plainTextBody.Length);
 
-                    // End of headers
-                    writer.Write(s_eoh);
+        // End of headers
+        writer.Write(s_eoh);
 
-                    // Body
-                    writer.Write(s_plainTextBody);
-                    writer.Flush();
-                }
-            }
-        }
+        // Body
+        writer.Write(s_plainTextBody);
+        writer.Flush();
     }
 }
 

diff --git a/tests/Benchmarks/Encodings/AsciiDecoding.cs b/tests/Benchmarks/Encodings/AsciiDecoding.cs
@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Text;
+using System.Text.Utf8;
+using BenchmarkDotNet.Attributes;
+
+namespace Benchmarks.Encodings
+{
+    public class AsciiDecoding
+    {
+        private byte[] bytes;
+
+        [Params("/plaintext", "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7")]
+        public string Text;
+
+        [GlobalSetup]
+        public void Setup() => bytes = Encoding.ASCII.GetBytes(Text);
+
+        [Benchmark]
+        public string AsciiToStringPrimitives() => System.Buffers.Text.Encodings.Ascii.ToUtf16String(bytes);
+
+        [Benchmark(Baseline = true)]
+        public string AsciiToStringClr() => Encoding.ASCII.GetString(bytes);
+
+        [Benchmark]
+        public string Utf8ToStringTextEncoder() => new Utf8Span(bytes).ToString();
+    }
+}
+