Skip to content

Support more types for HashEstimator #5104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gh-yewang merged 17 commits into from
May 19, 2020
Merged

Support more types for HashEstimator #5104

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
fd8b9ac
support more types but tests failed
gh-yewang May 6, 2020
2eea30a
fix bugs
gh-yewang May 7, 2020
6517056
bump to ort1.3 pre-release
gh-yewang May 14, 2020
c29c7cb
correct/skip some tests
gh-yewang May 14, 2020
af3146a
refactor tests
gh-yewang May 14, 2020
576d890
fix HashKey8V2
gh-yewang May 14, 2020
ea2ba3a
This reverts commit e0c
gh-yewang May 14, 2020
e0a4d14
add comments
gh-yewang May 14, 2020
973b0b5
revert changes on V1
gh-yewang May 15, 2020
a569327
upgrade to ORT1.3 official
gh-yewang May 19, 2020
586dba5
Merge branch 'master' into wangye/hash
gh-yewang May 19, 2020
792703d
fix part of conflits
gh-yewang May 19, 2020
d947cb7
update
gh-yewang May 19, 2020
4d1f0b9
update
gh-yewang May 19, 2020
a4a4647
update
gh-yewang May 19, 2020
87f5f92
fix test failure
gh-yewang May 19, 2020
7e28276
fix another test failure
gh-yewang May 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
https://dotnet.myget.org/F/roslyn-analyzers/api/v3/index.json;
https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json;
https://pkgs.dev.azure.com/dnceng/public/_packaging/machinelearning-testdata/nuget/v3/index.json;
https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly%40Local/nuget/v3/index.json;
</RestoreSources>
</PropertyGroup>

Expand All @@ -47,7 +48,7 @@
<OutputPath Condition="'$(OutputPath)'==''">$(BaseOutputPath)$(PlatformConfig)\$(MSBuildProjectName)\</OutputPath>

<PackageAssetsPath>$(ObjDir)/packages/</PackageAssetsPath>

<PackageOutputPath Condition="'$(PackageOutputPath)'=='' and '$(NonShippingPackage)' == 'true'">$(BinDir)packages_noship/</PackageOutputPath>
<PackageOutputPath Condition="'$(PackageOutputPath)'==''">$(BinDir)packages/</PackageOutputPath>

Expand All @@ -60,7 +61,7 @@
<ToolsDir Condition="'$(ToolsDir)'==''">$(RepoRoot)Tools/</ToolsDir>
</PropertyGroup>

<Import Project="$(ToolsDir)BuildVersion.targets"
<Import Project="$(ToolsDir)BuildVersion.targets"
Condition="Exists('$(ToolsDir)BuildVersion.targets')" />

<!-- Version properties -->
Expand Down Expand Up @@ -92,16 +93,16 @@
<SourceRevisionId>$(LatestCommit)</SourceRevisionId>
</PropertyGroup>

<Import Project="$(ToolsDir)versioning.props"
<Import Project="$(ToolsDir)versioning.props"
Condition="Exists('$(ToolsDir)versioning.props') and '$(DisableImportVersioningProps)' != 'true'" />

<!-- Language configuration -->
<PropertyGroup>
<LangVersion Condition="'$(MSBuildProjectExtension)'=='.csproj'">8.0</LangVersion>
<LangVersion Condition="'$(MSBuildProjectExtension)'=='.fsproj'">4.7</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>

<PropertyGroup>
<RunningOnUnix Condition="'$(OS)'!='Windows_NT'">true</RunningOnUnix>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion build/Dependencies.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<GoogleProtobufPackageVersion>3.10.1</GoogleProtobufPackageVersion>
<LightGBMPackageVersion>2.2.3</LightGBMPackageVersion>
<MicrosoftExtensionsPackageVersion>2.1.0</MicrosoftExtensionsPackageVersion>
<MicrosoftMLOnnxRuntimePackageVersion>1.2</MicrosoftMLOnnxRuntimePackageVersion>
<MicrosoftMLOnnxRuntimePackageVersion>1.3.0-dev-20200513-0702-eab61e87c</MicrosoftMLOnnxRuntimePackageVersion>
<MlNetMklDepsPackageVersion>0.0.0.9</MlNetMklDepsPackageVersion>
<ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
<SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
Expand Down
29 changes: 12 additions & 17 deletions src/Microsoft.ML.Data/Transforms/Hashing.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,7 +584,7 @@ public uint HashCore(uint seed, uint mask, in float value)
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public uint HashCore(uint seed, uint mask, in float value)
=> float.IsNaN(value) ? 0 : (Hashing.MixHashV2(Hashing.MurmurRound(seed, FloatUtils.GetBits(value == 0 ? 0 : value)), sizeof(uint)) & mask) + 1;
=> float.IsNaN(value) ? 0 : (Hashing.MixHashV2(Hashing.MurmurRound(seed, FloatUtils.GetBits(value == 0 ? 0 : value)), sizeof(float)) & mask) + 1;
}

private readonly struct HashDouble : IHasher<double>
Expand All @@ -599,8 +599,7 @@ public uint HashCore(uint seed, uint mask, in double value)
ulong v = FloatUtils.GetBits(value == 0 ? 0 : value);
var hash = Hashing.MurmurRound(seed, Utils.GetLo(v));
var hi = Utils.GetHi(v);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
hash = Hashing.MurmurRound(hash, hi);
Comment thread
gh-yewang marked this conversation as resolved.
Outdated
return (Hashing.MixHash(hash) & mask) + 1;
}
}
Expand All @@ -616,9 +615,8 @@ public uint HashCore(uint seed, uint mask, in double value)
ulong v = FloatUtils.GetBits(value == 0 ? 0 : value);
var hash = Hashing.MurmurRound(seed, Utils.GetLo(v));
var hi = Utils.GetHi(v);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(uint)) & mask) + 1;
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(double)) & mask) + 1;
}
}

Expand Down Expand Up @@ -757,8 +755,7 @@ public uint HashCore(uint seed, uint mask, in ulong value)
{
var hash = Hashing.MurmurRound(seed, Utils.GetLo(value));
var hi = Utils.GetHi(value);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHash(hash) & mask) + 1;
}
}
Expand All @@ -770,9 +767,8 @@ public uint HashCore(uint seed, uint mask, in ulong value)
{
var hash = Hashing.MurmurRound(seed, Utils.GetLo(value));
var hi = Utils.GetHi(value);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(uint)) & mask) + 1;
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(ulong)) & mask) + 1;
}
}

Expand Down Expand Up @@ -879,8 +875,7 @@ public uint HashCore(uint seed, uint mask, in long value)
{
var hash = Hashing.MurmurRound(seed, Utils.GetLo((ulong)value));
var hi = Utils.GetHi((ulong)value);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHash(hash) & mask) + 1;
}
}
Expand All @@ -892,9 +887,8 @@ public uint HashCore(uint seed, uint mask, in long value)
{
var hash = Hashing.MurmurRound(seed, Utils.GetLo((ulong)value));
var hi = Utils.GetHi((ulong)value);
if (hi != 0)
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(uint)) & mask) + 1;
hash = Hashing.MurmurRound(hash, hi);
return (Hashing.MixHashV2(hash, sizeof(long)) & mask) + 1;
}
}

Expand Down Expand Up @@ -1133,7 +1127,8 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, string srcVariable, stri
castNode.AddAttribute("to", NumberDataViewType.UInt32.RawType);
murmurNode = ctx.CreateNode(opType, castOutput, murmurOutput, ctx.GetNodeName(opType), "com.microsoft");
}
else if (srcType == typeof(uint) || srcType == typeof(int) || srcType == typeof(ReadOnlyMemory<char>))
else if (srcType == typeof(uint) || srcType == typeof(int) || srcType == typeof(ulong) ||
srcType == typeof(long) || srcType == typeof(float) || srcType == typeof(double) || srcType == typeof(ReadOnlyMemory<char>))
{
murmurNode = ctx.CreateNode(opType, srcVariable, murmurOutput, ctx.GetNodeName(opType), "com.microsoft");
}
Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ public sealed partial class TestDataPipe : TestDataPipeBase

private static Double[] _dataDouble = new Double[] { -0.0, 0, 1, -1, 2, -2, Double.NaN, Double.MinValue,
Double.MaxValue, Double.Epsilon, Double.NegativeInfinity, Double.PositiveInfinity };
private static uint[] _resultsDouble = new uint[] { 16, 16, 25, 27, 12, 2, 0, 6, 17, 4, 11, 30 };
private static uint[] _resultsDouble = new uint[] { 30, 30, 19, 24, 32, 25, 0, 2, 7, 30, 5, 3 };

private static VBuffer<Double> _dataDoubleSparse = new VBuffer<Double>(5, 3, new double[] { -0.0, 0, 1 }, new[] { 0, 3, 4 });
private static uint[] _resultsDoubleSparse = new uint[] { 16,16,16,16, 25 };
private static uint[] _resultsDoubleSparse = new uint[] { 30, 30, 30, 30, 19 };

[Fact()]
public void SavePipeLabelParsers()
Expand Down
28 changes: 20 additions & 8 deletions test/Microsoft.ML.Tests/OnnxConversionTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1201,8 +1201,8 @@ public void OneHotHashEncodingOnnxConversionTest()
// An InvalidOperationException stating that the onnx pipeline can't be fully converted is thrown
// when users try to convert the items mentioned above.
public void MurmurHashScalarTest(
[CombinatorialValues(DataKind.SByte, DataKind.Int16, DataKind.Int32, DataKind.Byte,
DataKind.UInt16, DataKind.UInt32, DataKind.String, DataKind.Boolean)] DataKind type,
[CombinatorialValues(DataKind.SByte, DataKind.Int16, DataKind.Int32, DataKind.Int64, DataKind.Byte,
DataKind.UInt16, DataKind.UInt32, DataKind.UInt64, DataKind.Single, DataKind.Double, DataKind.String, DataKind.Boolean)] DataKind type,
[CombinatorialValues(1, 5, 31)] int numberOfBits, bool useOrderedHashing)
{

Expand All @@ -1215,7 +1215,11 @@ public void MurmurHashScalarTest(
(type == DataKind.UInt16) ? 6 :
(type == DataKind.Int32) ? 8 :
(type == DataKind.UInt32) ? 10 :
(type == DataKind.String) ? 12 : 14;
(type == DataKind.Int64) ? 12 :
(type == DataKind.UInt64) ? 14 :
(type == DataKind.Single) ? 16 :
(type == DataKind.Double) ? 18 :
(type == DataKind.String) ? 20 : 22;

var dataView = mlContext.Data.LoadFromTextFile(dataPath, new[] {
new TextLoader.Column("Value", type, column),
Expand Down Expand Up @@ -1252,9 +1256,9 @@ public void MurmurHashScalarTest(
// An InvalidOperationException stating that the onnx pipeline can't be fully converted is thrown
// when users try to convert the items mentioned above.
public void MurmurHashVectorTest(
[CombinatorialValues(DataKind.SByte, DataKind.Int16, DataKind.Int32, DataKind.Byte,
DataKind.UInt16, DataKind.UInt32, DataKind.String, DataKind.Boolean)] DataKind type,
[CombinatorialValues(1, 5, 31)] int numberOfBits)
[CombinatorialValues(DataKind.SByte, DataKind.Int16, DataKind.Int32, DataKind.Int64, DataKind.Byte,
DataKind.UInt16, DataKind.UInt32, DataKind.UInt64, DataKind.Single, DataKind.Double, DataKind.String, DataKind.Boolean)] DataKind type,
[CombinatorialValues(1, 5, 31)] int numberOfBits)
{

var mlContext = new MLContext();
Expand All @@ -1266,15 +1270,23 @@ public void MurmurHashVectorTest(
(type == DataKind.UInt16) ? 6 :
(type == DataKind.Int32) ? 8 :
(type == DataKind.UInt32) ? 10 :
(type == DataKind.String) ? 12 : 14;
(type == DataKind.Int64) ? 12 :
(type == DataKind.UInt64) ? 14 :
(type == DataKind.Single) ? 16 :
(type == DataKind.Double) ? 18 :
(type == DataKind.String) ? 20 : 22;

var columnEnd = (type == DataKind.SByte) ? 1 :
(type == DataKind.Byte) ? 3 :
(type == DataKind.Int16) ? 5 :
(type == DataKind.UInt16) ? 7 :
(type == DataKind.Int32) ? 9 :
(type == DataKind.UInt32) ? 11 :
(type == DataKind.String) ? 13 : 15;
(type == DataKind.Int64) ? 13 :
(type == DataKind.UInt64) ? 15 :
(type == DataKind.Single) ? 17 :
(type == DataKind.Double) ? 19 :
(type == DataKind.String) ? 21 : 23;

var dataView = mlContext.Data.LoadFromTextFile(dataPath, new[] {
new TextLoader.Column("Value", type, columnStart, columnEnd),
Expand Down
36 changes: 27 additions & 9 deletions test/Microsoft.ML.Tests/Transformers/HashTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ ValueGetter<TType> hashGetter<TType>(HashingEstimator.ColumnOptionsInternal colI
Assert.Equal(expectedOrdered3, vecResult.GetItemOrDefault(3));
}

private void HashTestPositiveIntegerCore(ulong value, uint expected, uint expectedOrdered, uint expectedOrdered3)
private void HashTestPositiveInteger32BitCore(ulong value, uint expected, uint expectedOrdered, uint expectedOrdered3)
{
uint eKey = value == 0 ? 0 : expected;
uint eoKey = value == 0 ? 0 : expectedOrdered;
Expand All @@ -228,11 +228,12 @@ private void HashTestPositiveIntegerCore(ulong value, uint expected, uint expect
HashTestCore((uint)value, NumberDataViewType.UInt32, expected, expectedOrdered, expectedOrdered3);
HashTestCore((uint)value, new KeyDataViewType(typeof(uint), int.MaxValue - 1), eKey, eoKey, e3Key);
}
HashTestCore(value, NumberDataViewType.UInt64, expected, expectedOrdered, expectedOrdered3);
HashTestCore((ulong)value, new KeyDataViewType(typeof(ulong), int.MaxValue - 1), eKey, eoKey, e3Key);

HashTestCore(new DataViewRowId(value, 0), RowIdDataViewType.Instance, expected, expectedOrdered, expectedOrdered3);

//This test calls HashKey8V2 which implemented the same way as 32bit
HashTestCore((ulong)value, new KeyDataViewType(typeof(ulong), ulong.MaxValue - 1), eKey, eoKey, e3Key);

// Next let's check signed numbers.

if (value <= (ulong)sbyte.MaxValue)
Expand All @@ -241,16 +242,33 @@ private void HashTestPositiveIntegerCore(ulong value, uint expected, uint expect
HashTestCore((short)value, NumberDataViewType.Int16, expected, expectedOrdered, expectedOrdered3);
if (value <= int.MaxValue)
HashTestCore((int)value, NumberDataViewType.Int32, expected, expectedOrdered, expectedOrdered3);
}

private void HashTestPositiveInteger64BitCore(ulong value, uint expected, uint expectedOrdered, uint expectedOrdered3)
{
uint eKey = value == 0 ? 0 : expected;
uint eoKey = value == 0 ? 0 : expectedOrdered;
uint e3Key = value == 0 ? 0 : expectedOrdered3;

HashTestCore(value, NumberDataViewType.UInt64, expected, expectedOrdered, expectedOrdered3);

// Next let's check signed numbers.
if (value <= long.MaxValue)
HashTestCore((long)value, NumberDataViewType.Int64, expected, expectedOrdered, expectedOrdered3);
}

[Fact]
public void TestHashIntegerNumbers()
{
HashTestPositiveIntegerCore(0, 842, 358, 20);
HashTestPositiveIntegerCore(1, 502, 537, 746);
HashTestPositiveIntegerCore(2, 407, 801, 652);
//32bit
HashTestPositiveInteger32BitCore(0, 842, 358, 20);
HashTestPositiveInteger32BitCore(1, 502, 537, 746);
HashTestPositiveInteger32BitCore(2, 407, 801, 652);

Comment thread
gh-yewang marked this conversation as resolved.
Outdated
//64bit
HashTestPositiveInteger64BitCore(0, 512, 851, 795);
HashTestPositiveInteger64BitCore(1, 329, 190, 574);
HashTestPositiveInteger64BitCore(2, 484, 713, 128);
}

[Fact]
Expand All @@ -267,9 +285,9 @@ public void TestHashFloatingPointNumbers()
HashTestCore(-1f, NumberDataViewType.Single, 252, 612, 780);
HashTestCore(0f, NumberDataViewType.Single, 842, 358, 20);
// Note that while we have the hash for numeric types be equal, the same is not necessarily the case for floating point numbers.
HashTestCore(1d, NumberDataViewType.Double, 937, 667, 424);
HashTestCore(-1d, NumberDataViewType.Double, 930, 78, 813);
HashTestCore(0d, NumberDataViewType.Double, 842, 358, 20);
HashTestCore(1d, NumberDataViewType.Double, 188, 57, 690);
HashTestCore(-1d, NumberDataViewType.Double, 885, 804, 22);
HashTestCore(0d, NumberDataViewType.Double, 512, 851, 795);
}

[Fact]
Expand Down
12 changes: 6 additions & 6 deletions test/data/type-samples.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
sbyte byte short ushort int uint strings boolean
0 1 0 23 0 4554 0 53 0 25 0 35 0 rain 0 1
2 3 2 13 2 455 2 63 2 63 2 63 djldaoiejffjauhglehdlgh pink 1 0
127 23 127 65 127 93 127 99 127 69 127 91 alibaba bug
-128 24 255 25 32767 325 65535 632 2147483647 34 4294967295 45 to mato monkey
0 2 5 98 -32768 335 78 698 -2147483648 97 3 56 U+12w blue
sbyte byte short ushort int uint long ulong float double strings boolean
0 1 0 23 0 4554 0 53 0 25 0 35 0 -1 0 1 0 -1 0 -1 0 rain 0 1
2 3 2 13 2 455 2 63 2 63 2 63 2 63 2 63 1 2 1 2 djldaoiejffjauhglehdlgh pink 1 0
127 23 127 65 127 93 127 99 127 69 127 91 2147483647 34 2147483647 34 -2 300 -2 300 alibaba bug
-128 24 255 25 32767 325 65535 632 2147483647 34 4294967295 45 9223372036854775807 97 9223372036854775807 97 355 4 355 4 to mato monkey
0 2 5 98 -32768 335 78 698 -2147483648 97 3 56 -9223372036854775808 5 4 5 -4000 5 -4000 5 U+12w blue