-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Fix creation of dataviews inferred with .NET types with sparse vectors #587
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 14 commits
edba472
5df6fba
ca76231
ea730aa
95e3646
96cae17
e55882d
719adc4
0255942
5d55dfa
d5a419e
11aea96
3409b9f
63fa647
bebf262
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.ML.Runtime.Command; | ||
| using Microsoft.ML.Runtime.Api; | ||
| using Microsoft.ML.Runtime.Data; | ||
| using Microsoft.ML.Runtime.Internal.Utilities; | ||
| using Microsoft.ML.Runtime.Model; | ||
| using Microsoft.ML.Runtime.Tools; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Microsoft.ML.Runtime.RunTests | ||
| { | ||
| public sealed partial class TestSparseDataView : TestDataViewBase | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Why partial?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Copy paste without thinking :) |
||
| { | ||
| const string Cat = "DataView"; | ||
|
|
||
| public TestSparseDataView(ITestOutputHelper obj) : base(obj) | ||
| { | ||
| } | ||
|
|
||
| class DenseExample<T> | ||
| { | ||
| [VectorType(2)] | ||
| public T[] X; | ||
| } | ||
|
|
||
| class SparseExample<T> | ||
| { | ||
| [VectorType(5)] | ||
| public VBuffer<T> X; | ||
| } | ||
|
|
||
| [Fact] | ||
| [TestCategory(Cat)] | ||
| public void SparseDataView() | ||
| { | ||
| GenericSparseDataView(new[] { 1f, 2f, 3f }, new[] { 1f, 10f, 100f }); | ||
| GenericSparseDataView(new DvInt4[] { 1, 2, 3 }, new DvInt4[] { 1, 10, 100 }); | ||
| GenericSparseDataView(new DvBool[] { true, true, true }, new DvBool[] { false, false, false }); | ||
| GenericSparseDataView(new double[] { 1, 2, 3 }, new double[] { 1, 10, 100 }); | ||
| GenericSparseDataView(new DvText[] { new DvText("a"), new DvText("b"), new DvText("c") }, | ||
| new DvText[] { new DvText("aa"), new DvText("bb"), new DvText("cc") }); | ||
| Done(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
You don't have to call Done if you don't do any basefiles comparison #Resolved |
||
| } | ||
|
|
||
| void GenericSparseDataView<T>(T[] v1, T[] v2) | ||
| { | ||
| var inputs = new[] { | ||
| new SparseExample<T>() { X = new VBuffer<T> (5, 3, v1, new int[] { 0, 2, 4 }) }, | ||
| new SparseExample<T>() { X = new VBuffer<T> (5, 3, v2, new int[] { 0, 1, 3 }) } | ||
| }; | ||
| var host = new TlcEnvironment(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
can you wrap it in using? #Resolved |
||
| var data = host.CreateStreamingDataView(inputs); | ||
| var value = new VBuffer<T>(); | ||
| int n = 0; | ||
| using (var cur = data.GetRowCursor(i => true)) | ||
| { | ||
| var getter = cur.GetGetter<VBuffer<T>>(0); | ||
| while (cur.MoveNext()) | ||
| { | ||
| getter(ref value); | ||
| Assert.True(value.Count == 3); | ||
| ++n; | ||
| } | ||
| } | ||
| Assert.True(n == 2); | ||
| } | ||
|
|
||
| [Fact] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Thank you for fixing my mess! In order to invoke second case you need to do something like:
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll do it. #Resolved |
||
| [TestCategory(Cat)] | ||
| public void DenseDataView() | ||
| { | ||
| GenericDenseDataView(new[] { 1f, 2f, 3f }, new[] { 1f, 10f, 100f }); | ||
| GenericDenseDataView(new DvInt4[] { 1, 2, 3 }, new DvInt4[] { 1, 10, 100 }); | ||
| GenericDenseDataView(new DvBool[] { true, true, true }, new DvBool[] { false, false, false }); | ||
| GenericDenseDataView(new double[] { 1, 2, 3 }, new double[] { 1, 10, 100 }); | ||
| GenericDenseDataView(new DvText[] { new DvText("a"), new DvText("b"), new DvText("c") }, | ||
| new DvText[] { new DvText("aa"), new DvText("bb"), new DvText("cc") }); | ||
| Done(); | ||
| } | ||
|
|
||
| void GenericDenseDataView<T>(T[] v1, T[] v2) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Explicit access modifier please. Here and everywhere. |
||
| { | ||
| var inputs = new[] { | ||
| new DenseExample<T>() { X = v1 }, | ||
| new DenseExample<T>() { X = v2 } | ||
| }; | ||
| var host = new TlcEnvironment(); | ||
| var data = host.CreateStreamingDataView(inputs); | ||
| var value = new VBuffer<T>(); | ||
| int n = 0; | ||
| using (var cur = data.GetRowCursor(i => true)) | ||
| { | ||
| var getter = cur.GetGetter<VBuffer<T>>(0); | ||
| while (cur.MoveNext()) | ||
| { | ||
| getter(ref value); | ||
| Assert.True(value.Count == 3); | ||
| ++n; | ||
| } | ||
| } | ||
| Assert.True(n == 2); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm fairly certain not all of these are used.