-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Add performance benchmarks for dataframe arithmetic operations #6827
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
JakeRadMSFT
merged 13 commits into
dotnet:main
from
asmirnov82:dataframe_performance_tests
Sep 26, 2023
Merged
Changes from 8 commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
a70f986
Add performance tests
asmirnov82 69fc3d7
Add extra tests
asmirnov82 2789c7b
Fix
asmirnov82 d3944d7
Fix typo
asmirnov82 8edbae4
Fix Divide_Int16 and Divide_Int32_Int16 benchmarks
asmirnov82 ec06bd8
Fix
asmirnov82 bf9ea8a
Merge remote-tracking branch 'origin/main' into dataframe_performance…
asmirnov82 9f14fce
Change csproj file
asmirnov82 06a349f
Update BenchmarkDotNetVersion to 0.13.5
asmirnov82 95622a3
Fix
asmirnov82 c271b15
Change to 0.13.1 because that is what is latest version in our nuget …
JakeRadMSFT 819a4e4
Merge remote-tracking branch 'origin/main' into dataframe_performance…
asmirnov82 bf8a372
Merge branch 'dataframe_performance_tests' of https://github.com/asmi…
asmirnov82 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
.../Microsoft.Data.Analysis.PerformanceTests/Microsoft.Data.Analysis.PerformanceTests.csproj
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <ImplicitUsings>disable</ImplicitUsings> | ||
| <TargetFrameworks>net6.0</TargetFrameworks> | ||
| <GenerateProgramFile>false</GenerateProgramFile> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="BenchmarkDotNet" Version="0.13.5" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\Microsoft.Data.Analysis\Microsoft.Data.Analysis.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> | ||
202 changes: 202 additions & 0 deletions
202
test/Microsoft.Data.Analysis.PerformanceTests/PerformanceTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
| using BenchmarkDotNet.Attributes; | ||
|
|
||
| namespace Microsoft.Data.Analysis.PerformanceTests | ||
| { | ||
| public class PerformanceTests | ||
| { | ||
| private const int ItemsCount = 1000000; | ||
|
|
||
| private Int32DataFrameColumn _int32Column1; | ||
| private Int32DataFrameColumn _int32Column2; | ||
|
|
||
| private Int16DataFrameColumn _int16Column1; | ||
| private Int16DataFrameColumn _int16Column2; | ||
|
|
||
| private DoubleDataFrameColumn _doubleColumn1; | ||
| private DoubleDataFrameColumn _doubleColumn2; | ||
|
|
||
| private SingleDataFrameColumn _floatColumn1; | ||
| private SingleDataFrameColumn _floatColumn2; | ||
|
|
||
| [GlobalSetup] | ||
| public void SetUp() | ||
| { | ||
| var values = Enumerable.Range(1, ItemsCount).ToArray(); | ||
|
|
||
| _int32Column1 = new Int32DataFrameColumn("Column1", values); | ||
| _int32Column2 = new Int32DataFrameColumn("Column2", values); | ||
|
|
||
| var shortValues = values.Select(v => (short)(v % short.MaxValue + 1)).ToArray(); | ||
| _int16Column1 = new Int16DataFrameColumn("Column1", shortValues); | ||
| _int16Column2 = new Int16DataFrameColumn("Column2", shortValues); | ||
|
|
||
| _doubleColumn1 = new DoubleDataFrameColumn("Column1", values.Select(v => (double)v)); | ||
| _doubleColumn2 = new DoubleDataFrameColumn("Column2", values.Select(v => (double)v)); | ||
|
|
||
| _floatColumn1 = new SingleDataFrameColumn("Column1", values.Select(v => (float)v)); | ||
| _floatColumn2 = new SingleDataFrameColumn("Column2", values.Select(v => (float)v)); | ||
| } | ||
|
|
||
| #region Addition | ||
|
|
||
| [Benchmark] | ||
| public void Add_Int32() | ||
| { | ||
| var column = _int32Column1 + _int32Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Add_Int16() | ||
| { | ||
| var column = _int16Column1 + _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Add_Double() | ||
| { | ||
| var column = _doubleColumn1 + _doubleColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Add_Float() | ||
| { | ||
| var column = _floatColumn1 + _floatColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Add_Int32_Int16() | ||
| { | ||
| var column = _int32Column1 + _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Add_Double_Float() | ||
| { | ||
| var column = _doubleColumn1 + _floatColumn2; | ||
| } | ||
| #endregion | ||
|
|
||
| #region Subtract | ||
| [Benchmark] | ||
| public void Subtract_Int32() | ||
| { | ||
| var column = _int32Column1 - _int32Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Subtract_Int16() | ||
| { | ||
| var column = _int16Column1 - _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Subtract_Double() | ||
| { | ||
| var column = _doubleColumn1 - _doubleColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Subtract_Float() | ||
| { | ||
| var column = _floatColumn1 - _floatColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Subtract_Int32_Int16() | ||
| { | ||
| var column = _int32Column1 - _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Subtract_Double_Float() | ||
| { | ||
| var column = _doubleColumn1 - _floatColumn2; | ||
| } | ||
| #endregion | ||
|
|
||
| #region Multiply | ||
| [Benchmark] | ||
| public void Multiply_Int32() | ||
| { | ||
| var column = _int32Column1 * _int32Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Multiply_Int16() | ||
| { | ||
| var column = _int16Column1 * _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Multiply_Double() | ||
| { | ||
| var column = _doubleColumn1 * _doubleColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Multiply_Float() | ||
| { | ||
| var column = _floatColumn1 * _floatColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Multiply_Int32_Int16() | ||
| { | ||
| var column = _int32Column1 * _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Multiply_Double_Float() | ||
| { | ||
| var column = _doubleColumn1 * _floatColumn2; | ||
| } | ||
| #endregion | ||
|
|
||
| #region Divide | ||
| [Benchmark] | ||
| public void Divide_Int32() | ||
| { | ||
| var column = _int32Column1 / _int32Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Divide_Int16() | ||
| { | ||
| var column = _int16Column1 / _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Divide_Double() | ||
| { | ||
| var column = _doubleColumn1 / _doubleColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Divide_Float() | ||
| { | ||
| var column = _floatColumn1 / _floatColumn2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Divide_Int32_Int16() | ||
| { | ||
| var column = _int32Column1 / _int16Column2; | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void Divide_Double_Float() | ||
| { | ||
| var column = _doubleColumn1 / _floatColumn2; | ||
| } | ||
| #endregion | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| using BenchmarkDotNet.Running; | ||
|
|
||
|
|
||
| namespace Microsoft.Data.Analysis.PerformanceTests | ||
| { | ||
| class Program | ||
| { | ||
| public static void Main(string[] args) | ||
| { | ||
| var summary = BenchmarkRunner.Run<PerformanceTests>(); | ||
| } | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should use the same version as the other performance projects - $(BenchmarkDotNetVersion)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
See:
https://github.com/dotnet/machinelearning/blob/main/test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj#L14C58-L14C83
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@JakeRadMSFT there is an issue with the DotNetBenchmark version 0.12.0, that is currently used in ML .Net. It crashes with NotSupportedException("Unknown .NET Runtime") on .Net 6.0.
However. upgrading solution to the latest 0.13.5 leads to build error as Obsolete methods from this library are used in Microsoft.ML.CpuMath.PerformanceTests.
I am not able to build Native part of ML .Net on my laptop ( due to #6823 ), so I can't verify my changes in the CpuMath.PerformanceTests project (or check if it's working correctly on .Net 6.0). That's why I used the latest version only for the DataFrame project.
Could you please help me with building native dlls for the project?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah let me take a look!