From 679d169358c040cec0df2ec859f67c24f82f0b85 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 01:05:23 -0700 Subject: [PATCH 1/5] Update libmf for mftest --- src/Native/MatrixFactorizationNative/libmf | 2 +- .../TrainerEstimators/MatrixFactorizationTests.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Native/MatrixFactorizationNative/libmf b/src/Native/MatrixFactorizationNative/libmf index 298715a4e4..403153ca20 160000 --- a/src/Native/MatrixFactorizationNative/libmf +++ b/src/Native/MatrixFactorizationNative/libmf @@ -1 +1 @@ -Subproject commit 298715a4e458bc09c6a27c8643a58095afbdadf1 +Subproject commit 403153ca204817e2901b2872d977088316360641 diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 09ce334a79..acc1973274 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -53,10 +53,10 @@ public void MatrixFactorization_Estimator() Done(); } - [MatrixFactorizationFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - public void MatrixFactorizationSimpleTrainAndPredict() + [Theory, IterationData(5), TestCategory("RunSpecificTest")] + public void MatrixFactorizationSimpleTrainAndPredict(int iterations) { + Console.WriteLine(iterations); var mlContext = new MLContext(seed: 1); // Specific column names of the considered data set @@ -126,6 +126,7 @@ public void MatrixFactorizationSimpleTrainAndPredict() // Windows tolerance is set at 1e-7, and Linux tolerance is set at 1e-5 double windowsTolerance = Math.Pow(10, -7); double linuxTolerance = Math.Pow(10, -5); + double macTolerance = Math.Pow(10, -5); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case @@ -134,10 +135,9 @@ public void MatrixFactorizationSimpleTrainAndPredict() } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - // The Mac case is just broken. Should be fixed later. Re-enable when done. // Mac case - //var expectedMacL2Error = 0.61192207960271; // Mac baseline - //Assert.InRange(metrices.L2, expectedMacL2Error - 5e-3, expectedMacL2Error + 5e-3); // 1e-7 is too small for Mac so we try 1e-5 + var expectedMacL2Error = 0.593218453126428; + Assert.InRange(metrices.MeanSquaredError, expectedMacL2Error - macTolerance, expectedMacL2Error + macTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { From 49a6b63bb3519e803c4e922071d4bc1659c6bcc5 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 01:08:30 -0700 Subject: [PATCH 2/5] Only run mftest --- .vsts-dotnet-ci.yml | 68 ++++----------------------------------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index dd757c7938..1eea0fc3a8 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,7 +27,8 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -36,7 +37,8 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -44,65 +46,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted macOS - -- template: /build/ci/job-template.yml - parameters: - name: Windows_x64_NetCoreApp31 - buildScript: build.cmd - customMatrixes: - Debug_Build: - _configuration: Debug-netcoreapp3_1 - _config_short: DI - _includeBenchmarkData: false - _targetFramework: netcoreapp3.1 - Release_Build: - _configuration: Release-netcoreapp3_1 - _config_short: RI - _includeBenchmarkData: true - _targetFramework: netcoreapp3.1 - innerLoop: true - vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" - pool: - name: Hosted VS2017 - -- template: /build/ci/job-template.yml - parameters: - name: Windows_x64_NetCoreApp21 - buildScript: build.cmd - innerLoop: true - vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" - pool: - name: Hosted VS2017 - -- template: /build/ci/job-template.yml - parameters: - name: Windows_x64_NetFx461 - buildScript: build.cmd - customMatrixes: - Debug_Build: - _configuration: Debug-netfx - _config_short: DFX - _includeBenchmarkData: false - _targetFramework: win-x64 - Release_Build: - _configuration: Release-netfx - _config_short: RFX - _includeBenchmarkData: false - _targetFramework: win-x64 - innerLoop: true - vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" - pool: - name: Hosted VS2017 - -- template: /build/ci/job-template.yml - parameters: - name: Windows_x86_NetCoreApp21 - architecture: x86 - buildScript: build.cmd - innerLoop: true - vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" - pool: - name: Hosted VS2017 From 2b4f958e0e11cd6aa5c902fc73f02f80e4b30c65 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 14:45:19 -0700 Subject: [PATCH 3/5] Testing libmf changes on OneClassMatrixFactorizationInMemoryDataZeroBaseIndex --- .../TrainerEstimators/MatrixFactorizationTests.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index acc1973274..33c6694269 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -53,10 +53,9 @@ public void MatrixFactorization_Estimator() Done(); } - [Theory, IterationData(5), TestCategory("RunSpecificTest")] - public void MatrixFactorizationSimpleTrainAndPredict(int iterations) + [MatrixFactorizationFact] + public void MatrixFactorizationSimpleTrainAndPredict() { - Console.WriteLine(iterations); var mlContext = new MLContext(seed: 1); // Specific column names of the considered data set @@ -419,9 +418,11 @@ private class OneClassMatrixElementZeroBasedForScore public float Score; } - [MatrixFactorizationFact] - public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() + // [MatrixFactorizationFact] + [Theory, IterationData(10), TestCategory("RunSpecificTest")] + public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex(int iterations) { + Console.WriteLine(String.Format("OneClassMatrixFactorizationInMemoryDataZeroBaseIndex {0}", iterations)); // Create an in-memory matrix as a list of tuples (column index, row index, value). For one-class matrix // factorization problem, unspecified matrix elements are all a constant provided by user. If that constant is 0.15, // the following list means a 3-by-2 training matrix with elements: @@ -487,7 +488,8 @@ public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() var testResults = mlContext.Data.CreateEnumerable(testPrediction, false).ToList(); // TODO TEST_STABILITY: We are seeing lower precision on non-Windows platforms - int precision = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 5 : 3; + // int precision = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 5 : 3; + int precision = 5; // Positive example (i.e., examples can be found in dataMatrix) is close to 1. CompareNumbersWithTolerance(0.982391, testResults[0].Score, digitsOfPrecision: precision); From f5834d38e99929f211ed670a7c020e9178eae667 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 14 May 2020 12:48:03 -0700 Subject: [PATCH 4/5] Test TextNormalizingOnnxConversionTest with new ubuntu build --- .vsts-dotnet-ci.yml | 2 +- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 8 +++++--- .../TrainerEstimators/MatrixFactorizationTests.cs | 5 +---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 1eea0fc3a8..dab6755a74 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -8,7 +8,7 @@ resources: image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033 - container: UbuntuContainer - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200508132555-78cbb55 jobs: - template: /build/ci/job-template.yml diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index aca9d09c00..c97146e607 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -440,9 +440,11 @@ public void PlattCalibratorOnnxConversionTest2() Done(); } - [Fact] - public void TextNormalizingOnnxConversionTest() + // [Fact] + [Theory, IterationData(5), TestCategory("RunSpecificTest")] + public void TextNormalizingOnnxConversionTest(int iterations) { + Console.WriteLine(String.Format("TextNormalizingOnnxConversionTest Iteration {0}", iterations)); var mlContext = new MLContext(seed: 1); var dataPath = GetDataPath("wikipedia-detox-250-line-test.tsv"); var dataView = ML.Data.LoadFromTextFile(dataPath, new[] { @@ -462,7 +464,7 @@ public void TextNormalizingOnnxConversionTest() // Compare model scores produced by ML.NET and ONNX's runtime. // Skipping test in Linux platforms temporarily - if (IsOnnxRuntimeSupported() && !RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (IsOnnxRuntimeSupported()) { // Evaluate the saved ONNX model using the data used to train the ML.NET pipeline. var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(onnxModelPath); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 142784af76..db94176efd 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -128,7 +128,6 @@ public void MatrixFactorizationSimpleTrainAndPredict() // variables that are first obtained with the default random numger generator in libMF C++ libraries. double windowsAndMacTolerance = Math.Pow(10, -7); double linuxTolerance = Math.Pow(10, -5); - double macTolerance = Math.Pow(10, -5); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case @@ -424,7 +423,6 @@ private class OneClassMatrixElementZeroBasedForScore [Fact] public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() { - Console.WriteLine(String.Format("OneClassMatrixFactorizationInMemoryDataZeroBaseIndex {0}", iterations)); // Create an in-memory matrix as a list of tuples (column index, row index, value). For one-class matrix // factorization problem, unspecified matrix elements are all a constant provided by user. If that constant is 0.15, // the following list means a 3-by-2 training matrix with elements: @@ -490,8 +488,7 @@ public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() var testResults = mlContext.Data.CreateEnumerable(testPrediction, false).ToList(); // TODO TEST_STABILITY: We are seeing lower precision on non-Windows platforms - // int precision = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 5 : 3; - int precision = 5; + int precision = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 5 : 3; // Positive example (i.e., examples can be found in dataMatrix) is close to 1. CompareNumbersWithTolerance(0.982391, testResults[0].Score, digitsOfPrecision: precision); From 8757bdf3780fed1dc054cf41e3a468677f1a23a2 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 15 May 2020 13:58:48 -0700 Subject: [PATCH 5/5] Test with new build --- .vsts-dotnet-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index dab6755a74..664ac2b5a8 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -8,7 +8,7 @@ resources: image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033 - container: UbuntuContainer - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200508132555-78cbb55 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8 jobs: - template: /build/ci/job-template.yml