From 9b7eab3152351adc9c582f4364df621c4c81d2dc Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 14:09:36 -0700 Subject: [PATCH 01/15] moving Ols to a separate project --- Microsoft.ML.sln | 11 + .../Microsoft.ML.AdditionalLearners.csproj | 17 ++ .../OlsLinearRegression.cs | 9 +- src/Microsoft.ML.AdditionalLearners/doc.xml | 27 ++ src/Microsoft.ML.PCA/PcaTrainer.cs | 2 +- .../Microsoft.ML.StandardLearners.csproj | 1 - .../LogisticRegression/LogisticRegression.cs | 252 +----------------- .../MulticlassLogisticRegression.cs | 3 +- .../Standard/doc.xml | 21 -- .../WhiteningTransform.cs | 2 +- .../Microsoft.ML.Predictor.Tests.csproj | 1 + .../TestPredictors.cs | 2 +- 12 files changed, 68 insertions(+), 280 deletions(-) create mode 100644 src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj rename src/{Microsoft.ML.StandardLearners/Standard => Microsoft.ML.AdditionalLearners}/OlsLinearRegression.cs (99%) create mode 100644 src/Microsoft.ML.AdditionalLearners/doc.xml diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 58e24041f1..661a7767c3 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,6 +97,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AdditionalLearners", "src\Microsoft.ML.AdditionalLearners\Microsoft.ML.AdditionalLearners.csproj", "{A5C185B3-514F-4594-B5DE-73A65EB47792}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -329,6 +331,14 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release|Any CPU.Build.0 = Release|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -367,6 +377,7 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {A5C185B3-514F-4594-B5DE-73A65EB47792} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj new file mode 100644 index 0000000000..96d82e07f7 --- /dev/null +++ b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj @@ -0,0 +1,17 @@ + + + + netstandard2.0 + Microsoft.ML.AdditionalLearners + true + + + + + + + + + + + diff --git a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs similarity index 99% rename from src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs rename to src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs index 5fe8b62761..d12c1288f4 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs +++ b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs @@ -7,14 +7,15 @@ using System; using System.Collections.Generic; using System.IO; -using Microsoft.ML.Runtime.Internal.Internallearn; using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.AdditionalLearners; +using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.Model; -using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.Training; using System.Runtime.InteropServices; @@ -30,7 +31,7 @@ [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)] -namespace Microsoft.ML.Runtime.Learners +namespace Microsoft.ML.Runtime.AdditionalLearners { /// public sealed class OlsLinearRegressionTrainer : TrainerBase @@ -329,7 +330,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac internal static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "Microsoft.ML.MklImports"; public enum Layout { diff --git a/src/Microsoft.ML.AdditionalLearners/doc.xml b/src/Microsoft.ML.AdditionalLearners/doc.xml new file mode 100644 index 0000000000..d7ec04bb89 --- /dev/null +++ b/src/Microsoft.ML.AdditionalLearners/doc.xml @@ -0,0 +1,27 @@ + + + + + + + Train an OLS regression model. + + + Ordinary least squares (OLS) is a parameterized regression method. + It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. + The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. + + + + new OrdinaryLeastSquaresRegressor + { + L2Weight = 0.1, + PerParameterSignificance = false, + NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes + } + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.PCA/PcaTrainer.cs b/src/Microsoft.ML.PCA/PcaTrainer.cs index 5814d840ca..985f0a443f 100644 --- a/src/Microsoft.ML.PCA/PcaTrainer.cs +++ b/src/Microsoft.ML.PCA/PcaTrainer.cs @@ -179,7 +179,7 @@ private static VBuffer[] GaussianMatrix(int k, int d, int seed) var rng = new SysRandom(seed); // REVIEW: use a faster Gaussian random matrix generator - //MKL has a fast vectorized random number generation. + // MKL has a fast vectorized random number generation. for (var i = 0; i < k; ++i) { for (var j = 0; j < d; ++j) diff --git a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj index 72b0f912d5..6bada43299 100644 --- a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj +++ b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj @@ -11,7 +11,6 @@ - diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 09e2bbbcc4..dd3a689dc2 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -28,8 +28,6 @@ namespace Microsoft.ML.Runtime.Learners { - using Mkl = Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer.Mkl; - /// /// public sealed partial class LogisticRegression : LbfgsTrainerBase @@ -47,7 +45,6 @@ public sealed class Arguments : ArgumentsBase } private Double _posWeight; - private LinearModelStatistics _stats; public LogisticRegression(IHostEnvironment env, Arguments args) : base(args, env, LoadNameValue, Contracts.CheckRef(args, nameof(args)).ShowTrainingStats) @@ -94,252 +91,7 @@ protected override Float AccumulateOneGradient(ref VBuffer feat, Float la protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor.Factory cursorFactory, Float loss, int numParams) { - Contracts.AssertValue(ch); - Contracts.AssertValue(cursorFactory); - Contracts.Assert(NumGoodRows > 0); - Contracts.Assert(WeightSum > 0); - Contracts.Assert(BiasCount == 1); - Contracts.Assert(loss >= 0); - Contracts.Assert(numParams >= BiasCount); - Contracts.Assert(CurrentWeights.IsDense); - - ch.Info("Model trained with {0} training examples.", NumGoodRows); - - // Compute deviance: start with loss function. - Float deviance = (Float)(2 * loss * WeightSum); - - if (L2Weight > 0) - { - // Need to subtract L2 regularization loss. - // The bias term is not regularized. - var regLoss = VectorUtils.NormSquared(CurrentWeights.Values, 1, CurrentWeights.Length - 1) * L2Weight; - deviance -= regLoss; - } - - if (L1Weight > 0) - { - // Need to subtract L1 regularization loss. - // The bias term is not regularized. - Double regLoss = 0; - VBufferUtils.ForEachDefined(ref CurrentWeights, (ind, value) => { if (ind >= BiasCount) regLoss += Math.Abs(value); }); - deviance -= (Float)regLoss * L1Weight * 2; - } - - ch.Info("Residual Deviance: \t{0} (on {1} degrees of freedom)", deviance, Math.Max(NumGoodRows - numParams, 0)); - - // Compute null deviance, i.e., the deviance of null hypothesis. - // Cap the prior positive rate at 1e-15. - Double priorPosRate = _posWeight / WeightSum; - Contracts.Assert(0 <= priorPosRate && priorPosRate <= 1); - Float nullDeviance = (priorPosRate <= 1e-15 || 1 - priorPosRate <= 1e-15) ? - 0f : (Float)(2 * WeightSum * MathUtils.Entropy(priorPosRate, true)); - ch.Info("Null Deviance: \t{0} (on {1} degrees of freedom)", nullDeviance, NumGoodRows - 1); - - // Compute AIC. - ch.Info("AIC: \t{0}", 2 * numParams + deviance); - - // Show the coefficients statistics table. - var featureColIdx = cursorFactory.Data.Schema.Feature.Index; - var schema = cursorFactory.Data.Data.Schema; - var featureLength = CurrentWeights.Length - BiasCount; - var namesSpans = VBufferUtils.CreateEmpty(featureLength); - if (schema.HasSlotNames(featureColIdx, featureLength)) - schema.GetMetadata(MetadataUtils.Kinds.SlotNames, featureColIdx, ref namesSpans); - Host.Assert(namesSpans.Length == featureLength); - - // Inverse mapping of non-zero weight slots. - Dictionary weightIndicesInvMap = null; - - // Indices of bias and non-zero weight slots. - int[] weightIndices = null; - - // Whether all weights are non-zero. - bool denseWeight = numParams == CurrentWeights.Length; - - // Extract non-zero indices of weight. - if (!denseWeight) - { - weightIndices = new int[numParams]; - weightIndicesInvMap = new Dictionary(numParams); - weightIndices[0] = 0; - weightIndicesInvMap[0] = 0; - int j = 1; - for (int i = 1; i < CurrentWeights.Length; i++) - { - if (CurrentWeights.Values[i] != 0) - { - weightIndices[j] = i; - weightIndicesInvMap[i] = j++; - } - } - - Contracts.Assert(j == numParams); - } - - // Compute the standard error of coefficients. - long hessianDimension = (long)numParams * (numParams + 1) / 2; - if (hessianDimension > int.MaxValue) - { - ch.Warning("The number of parameter is too large. Cannot hold the variance-covariance matrix in memory. " + - "Skipping computation of standard errors and z-statistics of coefficients. Consider choosing a larger L1 regularizer" + - "to reduce the number of parameters."); - _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); - return; - } - - // Building the variance-covariance matrix for parameters. - // The layout of this algorithm is a packed row-major lower triangular matrix. - // E.g., layout of indices for 4-by-4: - // 0 - // 1 2 - // 3 4 5 - // 6 7 8 9 - var hessian = new Double[hessianDimension]; - - // Initialize diagonal elements with L2 regularizers except for the first entry (index 0) - // Since bias is not regularized. - if (L2Weight > 0) - { - // i is the array index of the diagonal entry at iRow-th row and iRow-th column. - // iRow is one-based. - int i = 0; - for (int iRow = 2; iRow <= numParams; iRow++) - { - i += iRow; - hessian[i] = L2Weight; - } - - Contracts.Assert(i == hessian.Length - 1); - } - - // Initialize the remaining entries. - var bias = CurrentWeights.Values[0]; - using (var cursor = cursorFactory.Create()) - { - while (cursor.MoveNext()) - { - var label = cursor.Label; - var weight = cursor.Weight; - var score = bias + VectorUtils.DotProductWithOffset(ref CurrentWeights, 1, ref cursor.Features); - // Compute Bernoulli variance n_i * p_i * (1 - p_i) for the i-th training example. - var variance = weight / (2 + 2 * Math.Cosh(score)); - - // Increment the first entry of hessian. - hessian[0] += variance; - - var values = cursor.Features.Values; - if (cursor.Features.IsDense) - { - int ioff = 1; - - // Increment remaining entries of hessian. - for (int i = 1; i < numParams; i++) - { - ch.Assert(ioff == i * (i + 1) / 2); - int wi = weightIndices == null ? i - 1 : weightIndices[i] - 1; - Contracts.Assert(0 <= wi && wi < cursor.Features.Length); - var val = values[wi] * variance; - // Add the implicit first bias term to X'X - hessian[ioff++] += val; - // Add the remainder of X'X - for (int j = 0; j < i; j++) - { - int wj = weightIndices == null ? j : weightIndices[j + 1] - 1; - Contracts.Assert(0 <= wj && wj < cursor.Features.Length); - hessian[ioff++] += val * values[wj]; - } - } - ch.Assert(ioff == hessian.Length); - } - else - { - var indices = cursor.Features.Indices; - for (int ii = 0; ii < cursor.Features.Count; ++ii) - { - int i = indices[ii]; - int wi = i + 1; - if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(i + 1, out wi)) - continue; - - Contracts.Assert(0 < wi && wi <= cursor.Features.Length); - int ioff = wi * (wi + 1) / 2; - var val = values[ii] * variance; - // Add the implicit first bias term to X'X - hessian[ioff] += val; - // Add the remainder of X'X - for (int jj = 0; jj <= ii; jj++) - { - int j = indices[jj]; - int wj = j + 1; - if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(j + 1, out wj)) - continue; - - Contracts.Assert(0 < wj && wj <= cursor.Features.Length); - hessian[ioff + wj] += val * values[jj]; - } - } - } - } - } - - // Apply Cholesky Decomposition to find the inverse of the Hessian. - Double[] invHessian = null; - try - { - // First, find the Cholesky decomposition LL' of the Hessian. - Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // Note that hessian is already modified at this point. It is no longer the original Hessian, - // but instead represents the Cholesky decomposition L. - // Also note that the following routine is supposed to consume the Cholesky decomposition L instead - // of the original information matrix. - Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // At this point, hessian should contain the inverse of the original Hessian matrix. - // Swap hessian with invHessian to avoid confusion in the following context. - Utils.Swap(ref hessian, ref invHessian); - Contracts.Assert(hessian == null); - } - catch (DllNotFoundException) - { - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); - } - - Float[] stdErrorValues = new Float[numParams]; - stdErrorValues[0] = (Float)Math.Sqrt(invHessian[0]); - - for (int i = 1; i < numParams; i++) - { - // Initialize with inverse Hessian. - stdErrorValues[i] = (Single)invHessian[i * (i + 1) / 2 + i]; - } - - if (L2Weight > 0) - { - // Iterate through all entries of inverse Hessian to make adjustment to variance. - // A discussion on ridge regularized LR coefficient covariance matrix can be found here: - // http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3228544/ - // http://www.inf.unibz.it/dis/teaching/DWDM/project2010/LogisticRegression.pdf - int ioffset = 1; - for (int iRow = 1; iRow < numParams; iRow++) - { - for (int iCol = 0; iCol <= iRow; iCol++) - { - var entry = (Single)invHessian[ioffset]; - var adjustment = -L2Weight * entry * entry; - stdErrorValues[iRow] -= adjustment; - if (0 < iCol && iCol < iRow) - stdErrorValues[iCol] -= adjustment; - ioffset++; - } - } - - Contracts.Assert(ioffset == invHessian.Length); - } - - for (int i = 1; i < numParams; i++) - stdErrorValues[i] = (Float)Math.Sqrt(stdErrorValues[i]); - - VBuffer stdErrors = new VBuffer(CurrentWeights.Length, numParams, stdErrorValues, weightIndices); - _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, ref stdErrors); + throw new NotImplementedException(); } protected override void ProcessPriorDistribution(Float label, Float weight) @@ -382,7 +134,7 @@ protected override ParameterMixingCalibratedPredictor CreatePredictor() CurrentWeights.GetItemOrDefault(0, ref bias); CurrentWeights.CopyTo(ref weights, 1, CurrentWeights.Length - 1); return new ParameterMixingCalibratedPredictor(Host, - new LinearBinaryPredictor(Host, ref weights, bias, _stats), + new LinearBinaryPredictor(Host, ref weights, bias), new PlattCalibrator(Host, -1, 0)); } diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 5bf0511540..853a144453 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -42,6 +42,7 @@ public sealed class MulticlassLogisticRegression : LbfgsTrainerBase", diff --git a/src/Microsoft.ML.StandardLearners/Standard/doc.xml b/src/Microsoft.ML.StandardLearners/Standard/doc.xml index 8d06ef285f..eb87605232 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/doc.xml +++ b/src/Microsoft.ML.StandardLearners/Standard/doc.xml @@ -68,27 +68,6 @@ - - - - Train an OLS regression model. - - - Ordinary least squares (OLS) is a parameterized regression method. - It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. - The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. - - - - new OrdinaryLeastSquaresRegressor - { - L2Weight = 0.1, - PerParameterSignificance = false, - NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes - } - - - \ No newline at end of file diff --git a/src/Microsoft.ML.Transforms/WhiteningTransform.cs b/src/Microsoft.ML.Transforms/WhiteningTransform.cs index 6854157f31..0528b7d4aa 100644 --- a/src/Microsoft.ML.Transforms/WhiteningTransform.cs +++ b/src/Microsoft.ML.Transforms/WhiteningTransform.cs @@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices private static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "Microsoft.ML.MklImports"; public enum Layout { diff --git a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj index 7f0a8d05f9..47bd4fc80b 100644 --- a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj +++ b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index b8c8d9b497..60c9b21495 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -888,7 +888,7 @@ public void RegressorOlsTest() /// /// A test for ordinary least squares regression. /// - [Fact(Skip = "Need CoreTLC specific baseline update")] + [Fact] [TestCategory("Regressor")] public void RegressorOlsTestOne() { From fa3370c75c99841545e1501e7284febe082cdb6c Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 14:14:08 -0700 Subject: [PATCH 02/15] Revert "moving Ols to a separate project" This reverts commit 9b7eab3152351adc9c582f4364df621c4c81d2dc. --- Microsoft.ML.sln | 11 - .../Microsoft.ML.AdditionalLearners.csproj | 17 -- src/Microsoft.ML.AdditionalLearners/doc.xml | 27 -- src/Microsoft.ML.PCA/PcaTrainer.cs | 2 +- .../Microsoft.ML.StandardLearners.csproj | 1 + .../LogisticRegression/LogisticRegression.cs | 252 +++++++++++++++++- .../MulticlassLogisticRegression.cs | 3 +- .../Standard}/OlsLinearRegression.cs | 9 +- .../Standard/doc.xml | 21 ++ .../WhiteningTransform.cs | 2 +- .../Microsoft.ML.Predictor.Tests.csproj | 1 - .../TestPredictors.cs | 2 +- 12 files changed, 280 insertions(+), 68 deletions(-) delete mode 100644 src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj delete mode 100644 src/Microsoft.ML.AdditionalLearners/doc.xml rename src/{Microsoft.ML.AdditionalLearners => Microsoft.ML.StandardLearners/Standard}/OlsLinearRegression.cs (99%) diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 661a7767c3..58e24041f1 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,8 +97,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AdditionalLearners", "src\Microsoft.ML.AdditionalLearners\Microsoft.ML.AdditionalLearners.csproj", "{A5C185B3-514F-4594-B5DE-73A65EB47792}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -331,14 +329,6 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release|Any CPU.Build.0 = Release|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU - {A5C185B3-514F-4594-B5DE-73A65EB47792}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -377,7 +367,6 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {A5C185B3-514F-4594-B5DE-73A65EB47792} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj deleted file mode 100644 index 96d82e07f7..0000000000 --- a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - netstandard2.0 - Microsoft.ML.AdditionalLearners - true - - - - - - - - - - - diff --git a/src/Microsoft.ML.AdditionalLearners/doc.xml b/src/Microsoft.ML.AdditionalLearners/doc.xml deleted file mode 100644 index d7ec04bb89..0000000000 --- a/src/Microsoft.ML.AdditionalLearners/doc.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - Train an OLS regression model. - - - Ordinary least squares (OLS) is a parameterized regression method. - It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. - The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. - - - - new OrdinaryLeastSquaresRegressor - { - L2Weight = 0.1, - PerParameterSignificance = false, - NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes - } - - - - - - \ No newline at end of file diff --git a/src/Microsoft.ML.PCA/PcaTrainer.cs b/src/Microsoft.ML.PCA/PcaTrainer.cs index 985f0a443f..5814d840ca 100644 --- a/src/Microsoft.ML.PCA/PcaTrainer.cs +++ b/src/Microsoft.ML.PCA/PcaTrainer.cs @@ -179,7 +179,7 @@ private static VBuffer[] GaussianMatrix(int k, int d, int seed) var rng = new SysRandom(seed); // REVIEW: use a faster Gaussian random matrix generator - // MKL has a fast vectorized random number generation. + //MKL has a fast vectorized random number generation. for (var i = 0; i < k; ++i) { for (var j = 0; j < d; ++j) diff --git a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj index 6bada43299..72b0f912d5 100644 --- a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj +++ b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj @@ -11,6 +11,7 @@ + diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index dd3a689dc2..09e2bbbcc4 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -28,6 +28,8 @@ namespace Microsoft.ML.Runtime.Learners { + using Mkl = Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer.Mkl; + /// /// public sealed partial class LogisticRegression : LbfgsTrainerBase @@ -45,6 +47,7 @@ public sealed class Arguments : ArgumentsBase } private Double _posWeight; + private LinearModelStatistics _stats; public LogisticRegression(IHostEnvironment env, Arguments args) : base(args, env, LoadNameValue, Contracts.CheckRef(args, nameof(args)).ShowTrainingStats) @@ -91,7 +94,252 @@ protected override Float AccumulateOneGradient(ref VBuffer feat, Float la protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor.Factory cursorFactory, Float loss, int numParams) { - throw new NotImplementedException(); + Contracts.AssertValue(ch); + Contracts.AssertValue(cursorFactory); + Contracts.Assert(NumGoodRows > 0); + Contracts.Assert(WeightSum > 0); + Contracts.Assert(BiasCount == 1); + Contracts.Assert(loss >= 0); + Contracts.Assert(numParams >= BiasCount); + Contracts.Assert(CurrentWeights.IsDense); + + ch.Info("Model trained with {0} training examples.", NumGoodRows); + + // Compute deviance: start with loss function. + Float deviance = (Float)(2 * loss * WeightSum); + + if (L2Weight > 0) + { + // Need to subtract L2 regularization loss. + // The bias term is not regularized. + var regLoss = VectorUtils.NormSquared(CurrentWeights.Values, 1, CurrentWeights.Length - 1) * L2Weight; + deviance -= regLoss; + } + + if (L1Weight > 0) + { + // Need to subtract L1 regularization loss. + // The bias term is not regularized. + Double regLoss = 0; + VBufferUtils.ForEachDefined(ref CurrentWeights, (ind, value) => { if (ind >= BiasCount) regLoss += Math.Abs(value); }); + deviance -= (Float)regLoss * L1Weight * 2; + } + + ch.Info("Residual Deviance: \t{0} (on {1} degrees of freedom)", deviance, Math.Max(NumGoodRows - numParams, 0)); + + // Compute null deviance, i.e., the deviance of null hypothesis. + // Cap the prior positive rate at 1e-15. + Double priorPosRate = _posWeight / WeightSum; + Contracts.Assert(0 <= priorPosRate && priorPosRate <= 1); + Float nullDeviance = (priorPosRate <= 1e-15 || 1 - priorPosRate <= 1e-15) ? + 0f : (Float)(2 * WeightSum * MathUtils.Entropy(priorPosRate, true)); + ch.Info("Null Deviance: \t{0} (on {1} degrees of freedom)", nullDeviance, NumGoodRows - 1); + + // Compute AIC. + ch.Info("AIC: \t{0}", 2 * numParams + deviance); + + // Show the coefficients statistics table. + var featureColIdx = cursorFactory.Data.Schema.Feature.Index; + var schema = cursorFactory.Data.Data.Schema; + var featureLength = CurrentWeights.Length - BiasCount; + var namesSpans = VBufferUtils.CreateEmpty(featureLength); + if (schema.HasSlotNames(featureColIdx, featureLength)) + schema.GetMetadata(MetadataUtils.Kinds.SlotNames, featureColIdx, ref namesSpans); + Host.Assert(namesSpans.Length == featureLength); + + // Inverse mapping of non-zero weight slots. + Dictionary weightIndicesInvMap = null; + + // Indices of bias and non-zero weight slots. + int[] weightIndices = null; + + // Whether all weights are non-zero. + bool denseWeight = numParams == CurrentWeights.Length; + + // Extract non-zero indices of weight. + if (!denseWeight) + { + weightIndices = new int[numParams]; + weightIndicesInvMap = new Dictionary(numParams); + weightIndices[0] = 0; + weightIndicesInvMap[0] = 0; + int j = 1; + for (int i = 1; i < CurrentWeights.Length; i++) + { + if (CurrentWeights.Values[i] != 0) + { + weightIndices[j] = i; + weightIndicesInvMap[i] = j++; + } + } + + Contracts.Assert(j == numParams); + } + + // Compute the standard error of coefficients. + long hessianDimension = (long)numParams * (numParams + 1) / 2; + if (hessianDimension > int.MaxValue) + { + ch.Warning("The number of parameter is too large. Cannot hold the variance-covariance matrix in memory. " + + "Skipping computation of standard errors and z-statistics of coefficients. Consider choosing a larger L1 regularizer" + + "to reduce the number of parameters."); + _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); + return; + } + + // Building the variance-covariance matrix for parameters. + // The layout of this algorithm is a packed row-major lower triangular matrix. + // E.g., layout of indices for 4-by-4: + // 0 + // 1 2 + // 3 4 5 + // 6 7 8 9 + var hessian = new Double[hessianDimension]; + + // Initialize diagonal elements with L2 regularizers except for the first entry (index 0) + // Since bias is not regularized. + if (L2Weight > 0) + { + // i is the array index of the diagonal entry at iRow-th row and iRow-th column. + // iRow is one-based. + int i = 0; + for (int iRow = 2; iRow <= numParams; iRow++) + { + i += iRow; + hessian[i] = L2Weight; + } + + Contracts.Assert(i == hessian.Length - 1); + } + + // Initialize the remaining entries. + var bias = CurrentWeights.Values[0]; + using (var cursor = cursorFactory.Create()) + { + while (cursor.MoveNext()) + { + var label = cursor.Label; + var weight = cursor.Weight; + var score = bias + VectorUtils.DotProductWithOffset(ref CurrentWeights, 1, ref cursor.Features); + // Compute Bernoulli variance n_i * p_i * (1 - p_i) for the i-th training example. + var variance = weight / (2 + 2 * Math.Cosh(score)); + + // Increment the first entry of hessian. + hessian[0] += variance; + + var values = cursor.Features.Values; + if (cursor.Features.IsDense) + { + int ioff = 1; + + // Increment remaining entries of hessian. + for (int i = 1; i < numParams; i++) + { + ch.Assert(ioff == i * (i + 1) / 2); + int wi = weightIndices == null ? i - 1 : weightIndices[i] - 1; + Contracts.Assert(0 <= wi && wi < cursor.Features.Length); + var val = values[wi] * variance; + // Add the implicit first bias term to X'X + hessian[ioff++] += val; + // Add the remainder of X'X + for (int j = 0; j < i; j++) + { + int wj = weightIndices == null ? j : weightIndices[j + 1] - 1; + Contracts.Assert(0 <= wj && wj < cursor.Features.Length); + hessian[ioff++] += val * values[wj]; + } + } + ch.Assert(ioff == hessian.Length); + } + else + { + var indices = cursor.Features.Indices; + for (int ii = 0; ii < cursor.Features.Count; ++ii) + { + int i = indices[ii]; + int wi = i + 1; + if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(i + 1, out wi)) + continue; + + Contracts.Assert(0 < wi && wi <= cursor.Features.Length); + int ioff = wi * (wi + 1) / 2; + var val = values[ii] * variance; + // Add the implicit first bias term to X'X + hessian[ioff] += val; + // Add the remainder of X'X + for (int jj = 0; jj <= ii; jj++) + { + int j = indices[jj]; + int wj = j + 1; + if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(j + 1, out wj)) + continue; + + Contracts.Assert(0 < wj && wj <= cursor.Features.Length); + hessian[ioff + wj] += val * values[jj]; + } + } + } + } + } + + // Apply Cholesky Decomposition to find the inverse of the Hessian. + Double[] invHessian = null; + try + { + // First, find the Cholesky decomposition LL' of the Hessian. + Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); + // Note that hessian is already modified at this point. It is no longer the original Hessian, + // but instead represents the Cholesky decomposition L. + // Also note that the following routine is supposed to consume the Cholesky decomposition L instead + // of the original information matrix. + Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); + // At this point, hessian should contain the inverse of the original Hessian matrix. + // Swap hessian with invHessian to avoid confusion in the following context. + Utils.Swap(ref hessian, ref invHessian); + Contracts.Assert(hessian == null); + } + catch (DllNotFoundException) + { + throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); + } + + Float[] stdErrorValues = new Float[numParams]; + stdErrorValues[0] = (Float)Math.Sqrt(invHessian[0]); + + for (int i = 1; i < numParams; i++) + { + // Initialize with inverse Hessian. + stdErrorValues[i] = (Single)invHessian[i * (i + 1) / 2 + i]; + } + + if (L2Weight > 0) + { + // Iterate through all entries of inverse Hessian to make adjustment to variance. + // A discussion on ridge regularized LR coefficient covariance matrix can be found here: + // http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3228544/ + // http://www.inf.unibz.it/dis/teaching/DWDM/project2010/LogisticRegression.pdf + int ioffset = 1; + for (int iRow = 1; iRow < numParams; iRow++) + { + for (int iCol = 0; iCol <= iRow; iCol++) + { + var entry = (Single)invHessian[ioffset]; + var adjustment = -L2Weight * entry * entry; + stdErrorValues[iRow] -= adjustment; + if (0 < iCol && iCol < iRow) + stdErrorValues[iCol] -= adjustment; + ioffset++; + } + } + + Contracts.Assert(ioffset == invHessian.Length); + } + + for (int i = 1; i < numParams; i++) + stdErrorValues[i] = (Float)Math.Sqrt(stdErrorValues[i]); + + VBuffer stdErrors = new VBuffer(CurrentWeights.Length, numParams, stdErrorValues, weightIndices); + _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, ref stdErrors); } protected override void ProcessPriorDistribution(Float label, Float weight) @@ -134,7 +382,7 @@ protected override ParameterMixingCalibratedPredictor CreatePredictor() CurrentWeights.GetItemOrDefault(0, ref bias); CurrentWeights.CopyTo(ref weights, 1, CurrentWeights.Length - 1); return new ParameterMixingCalibratedPredictor(Host, - new LinearBinaryPredictor(Host, ref weights, bias), + new LinearBinaryPredictor(Host, ref weights, bias, _stats), new PlattCalibrator(Host, -1, 0)); } diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 853a144453..5bf0511540 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -42,7 +42,6 @@ public sealed class MulticlassLogisticRegression : LbfgsTrainerBase", diff --git a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs similarity index 99% rename from src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs rename to src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs index d12c1288f4..5fe8b62761 100644 --- a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs @@ -7,15 +7,14 @@ using System; using System.Collections.Generic; using System.IO; -using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.AdditionalLearners; using Microsoft.ML.Runtime.Internal.Internallearn; -using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.Model; +using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.Training; using System.Runtime.InteropServices; @@ -31,7 +30,7 @@ [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)] -namespace Microsoft.ML.Runtime.AdditionalLearners +namespace Microsoft.ML.Runtime.Learners { /// public sealed class OlsLinearRegressionTrainer : TrainerBase @@ -330,7 +329,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac internal static class Mkl { - private const string DllName = "Microsoft.ML.MklImports"; + private const string DllName = "Microsoft.ML.MklImports.dll"; public enum Layout { diff --git a/src/Microsoft.ML.StandardLearners/Standard/doc.xml b/src/Microsoft.ML.StandardLearners/Standard/doc.xml index eb87605232..8d06ef285f 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/doc.xml +++ b/src/Microsoft.ML.StandardLearners/Standard/doc.xml @@ -68,6 +68,27 @@ + + + + Train an OLS regression model. + + + Ordinary least squares (OLS) is a parameterized regression method. + It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. + The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. + + + + new OrdinaryLeastSquaresRegressor + { + L2Weight = 0.1, + PerParameterSignificance = false, + NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes + } + + + \ No newline at end of file diff --git a/src/Microsoft.ML.Transforms/WhiteningTransform.cs b/src/Microsoft.ML.Transforms/WhiteningTransform.cs index 0528b7d4aa..6854157f31 100644 --- a/src/Microsoft.ML.Transforms/WhiteningTransform.cs +++ b/src/Microsoft.ML.Transforms/WhiteningTransform.cs @@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices private static class Mkl { - private const string DllName = "Microsoft.ML.MklImports"; + private const string DllName = "Microsoft.ML.MklImports.dll"; public enum Layout { diff --git a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj index 47bd4fc80b..7f0a8d05f9 100644 --- a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj +++ b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj @@ -17,7 +17,6 @@ - diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 60c9b21495..b8c8d9b497 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -888,7 +888,7 @@ public void RegressorOlsTest() /// /// A test for ordinary least squares regression. /// - [Fact] + [Fact(Skip = "Need CoreTLC specific baseline update")] [TestCategory("Regressor")] public void RegressorOlsTestOne() { From ba40ba46c069f4a507dd6f42fdc1af06ccdca73e Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 15:07:30 -0700 Subject: [PATCH 03/15] separating OLS in its own project --- Microsoft.ML.sln | 11 + .../Microsoft.ML.AdditionalLearners.csproj | 17 ++ .../OlsLinearRegression.cs | 11 +- src/Microsoft.ML.AdditionalLearners/doc.xml | 27 ++ .../Microsoft.ML.StandardLearners.csproj | 1 - .../Standard/LinearPredictor.cs | 2 +- .../LogisticRegression/LogisticRegression.cs | 251 +----------------- .../Common/EntryPoints/core_ep-list.tsv | 1 - .../Common/EntryPoints/core_manifest.json | 143 ---------- .../Microsoft.ML.Predictor.Tests.csproj | 1 + .../TestPredictors.cs | 2 +- test/Microsoft.ML.Tests/CSharpCodeGen.cs | 2 +- 12 files changed, 67 insertions(+), 402 deletions(-) create mode 100644 src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj rename src/{Microsoft.ML.StandardLearners/Standard => Microsoft.ML.AdditionalLearners}/OlsLinearRegression.cs (99%) create mode 100644 src/Microsoft.ML.AdditionalLearners/doc.xml diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 58e24041f1..590f7dfcff 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,6 +97,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AdditionalLearners", "src\Microsoft.ML.AdditionalLearners\Microsoft.ML.AdditionalLearners.csproj", "{E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -329,6 +331,14 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release|Any CPU.Build.0 = Release|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -367,6 +377,7 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj new file mode 100644 index 0000000000..96d82e07f7 --- /dev/null +++ b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj @@ -0,0 +1,17 @@ + + + + netstandard2.0 + Microsoft.ML.AdditionalLearners + true + + + + + + + + + + + diff --git a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs similarity index 99% rename from src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs rename to src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs index 5fe8b62761..7f7f361f54 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs +++ b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs @@ -7,14 +7,15 @@ using System; using System.Collections.Generic; using System.IO; -using Microsoft.ML.Runtime.Internal.Internallearn; using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.AdditionalLearners; +using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.Model; -using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.Training; using System.Runtime.InteropServices; @@ -30,7 +31,7 @@ [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)] -namespace Microsoft.ML.Runtime.Learners +namespace Microsoft.ML.Runtime.AdditionalLearners { /// public sealed class OlsLinearRegressionTrainer : TrainerBase @@ -220,7 +221,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac catch (DllNotFoundException) { // REVIEW: Is there no better way? - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); + throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports) or one of its dependencies is missing."); } // Solve for beta in (LL')beta = X'y: Mkl.Pptrs(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, 1, xtx, xty, 1); @@ -466,7 +467,7 @@ public static void Pptri(Layout layout, UpLo uplo, int n, Double[] ap) Desc = "Train an OLS regression model.", UserName = UserNameValue, ShortName = ShortName, - XmlInclude = new[] { @"" })] + XmlInclude = new[] { @"" })] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.AdditionalLearners/doc.xml b/src/Microsoft.ML.AdditionalLearners/doc.xml new file mode 100644 index 0000000000..d7ec04bb89 --- /dev/null +++ b/src/Microsoft.ML.AdditionalLearners/doc.xml @@ -0,0 +1,27 @@ + + + + + + + Train an OLS regression model. + + + Ordinary least squares (OLS) is a parameterized regression method. + It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. + The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. + + + + new OrdinaryLeastSquaresRegressor + { + L2Weight = 0.1, + PerParameterSignificance = false, + NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes + } + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj index 72b0f912d5..6bada43299 100644 --- a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj +++ b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj @@ -11,7 +11,6 @@ - diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs index 8f81324768..82069d413d 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs @@ -553,7 +553,7 @@ public override void SaveAsIni(TextWriter writer, RoleMappedSchema schema, ICali public abstract class RegressionPredictor : LinearPredictor { - internal RegressionPredictor(IHostEnvironment env, string name, ref VBuffer weights, Float bias) + protected RegressionPredictor(IHostEnvironment env, string name, ref VBuffer weights, Float bias) : base(env, name, ref weights, bias) { } diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 09e2bbbcc4..970361d894 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -28,7 +28,6 @@ namespace Microsoft.ML.Runtime.Learners { - using Mkl = Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer.Mkl; /// /// @@ -47,7 +46,6 @@ public sealed class Arguments : ArgumentsBase } private Double _posWeight; - private LinearModelStatistics _stats; public LogisticRegression(IHostEnvironment env, Arguments args) : base(args, env, LoadNameValue, Contracts.CheckRef(args, nameof(args)).ShowTrainingStats) @@ -94,252 +92,7 @@ protected override Float AccumulateOneGradient(ref VBuffer feat, Float la protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor.Factory cursorFactory, Float loss, int numParams) { - Contracts.AssertValue(ch); - Contracts.AssertValue(cursorFactory); - Contracts.Assert(NumGoodRows > 0); - Contracts.Assert(WeightSum > 0); - Contracts.Assert(BiasCount == 1); - Contracts.Assert(loss >= 0); - Contracts.Assert(numParams >= BiasCount); - Contracts.Assert(CurrentWeights.IsDense); - - ch.Info("Model trained with {0} training examples.", NumGoodRows); - - // Compute deviance: start with loss function. - Float deviance = (Float)(2 * loss * WeightSum); - - if (L2Weight > 0) - { - // Need to subtract L2 regularization loss. - // The bias term is not regularized. - var regLoss = VectorUtils.NormSquared(CurrentWeights.Values, 1, CurrentWeights.Length - 1) * L2Weight; - deviance -= regLoss; - } - - if (L1Weight > 0) - { - // Need to subtract L1 regularization loss. - // The bias term is not regularized. - Double regLoss = 0; - VBufferUtils.ForEachDefined(ref CurrentWeights, (ind, value) => { if (ind >= BiasCount) regLoss += Math.Abs(value); }); - deviance -= (Float)regLoss * L1Weight * 2; - } - - ch.Info("Residual Deviance: \t{0} (on {1} degrees of freedom)", deviance, Math.Max(NumGoodRows - numParams, 0)); - - // Compute null deviance, i.e., the deviance of null hypothesis. - // Cap the prior positive rate at 1e-15. - Double priorPosRate = _posWeight / WeightSum; - Contracts.Assert(0 <= priorPosRate && priorPosRate <= 1); - Float nullDeviance = (priorPosRate <= 1e-15 || 1 - priorPosRate <= 1e-15) ? - 0f : (Float)(2 * WeightSum * MathUtils.Entropy(priorPosRate, true)); - ch.Info("Null Deviance: \t{0} (on {1} degrees of freedom)", nullDeviance, NumGoodRows - 1); - - // Compute AIC. - ch.Info("AIC: \t{0}", 2 * numParams + deviance); - - // Show the coefficients statistics table. - var featureColIdx = cursorFactory.Data.Schema.Feature.Index; - var schema = cursorFactory.Data.Data.Schema; - var featureLength = CurrentWeights.Length - BiasCount; - var namesSpans = VBufferUtils.CreateEmpty(featureLength); - if (schema.HasSlotNames(featureColIdx, featureLength)) - schema.GetMetadata(MetadataUtils.Kinds.SlotNames, featureColIdx, ref namesSpans); - Host.Assert(namesSpans.Length == featureLength); - - // Inverse mapping of non-zero weight slots. - Dictionary weightIndicesInvMap = null; - - // Indices of bias and non-zero weight slots. - int[] weightIndices = null; - - // Whether all weights are non-zero. - bool denseWeight = numParams == CurrentWeights.Length; - - // Extract non-zero indices of weight. - if (!denseWeight) - { - weightIndices = new int[numParams]; - weightIndicesInvMap = new Dictionary(numParams); - weightIndices[0] = 0; - weightIndicesInvMap[0] = 0; - int j = 1; - for (int i = 1; i < CurrentWeights.Length; i++) - { - if (CurrentWeights.Values[i] != 0) - { - weightIndices[j] = i; - weightIndicesInvMap[i] = j++; - } - } - - Contracts.Assert(j == numParams); - } - - // Compute the standard error of coefficients. - long hessianDimension = (long)numParams * (numParams + 1) / 2; - if (hessianDimension > int.MaxValue) - { - ch.Warning("The number of parameter is too large. Cannot hold the variance-covariance matrix in memory. " + - "Skipping computation of standard errors and z-statistics of coefficients. Consider choosing a larger L1 regularizer" + - "to reduce the number of parameters."); - _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); - return; - } - - // Building the variance-covariance matrix for parameters. - // The layout of this algorithm is a packed row-major lower triangular matrix. - // E.g., layout of indices for 4-by-4: - // 0 - // 1 2 - // 3 4 5 - // 6 7 8 9 - var hessian = new Double[hessianDimension]; - - // Initialize diagonal elements with L2 regularizers except for the first entry (index 0) - // Since bias is not regularized. - if (L2Weight > 0) - { - // i is the array index of the diagonal entry at iRow-th row and iRow-th column. - // iRow is one-based. - int i = 0; - for (int iRow = 2; iRow <= numParams; iRow++) - { - i += iRow; - hessian[i] = L2Weight; - } - - Contracts.Assert(i == hessian.Length - 1); - } - - // Initialize the remaining entries. - var bias = CurrentWeights.Values[0]; - using (var cursor = cursorFactory.Create()) - { - while (cursor.MoveNext()) - { - var label = cursor.Label; - var weight = cursor.Weight; - var score = bias + VectorUtils.DotProductWithOffset(ref CurrentWeights, 1, ref cursor.Features); - // Compute Bernoulli variance n_i * p_i * (1 - p_i) for the i-th training example. - var variance = weight / (2 + 2 * Math.Cosh(score)); - - // Increment the first entry of hessian. - hessian[0] += variance; - - var values = cursor.Features.Values; - if (cursor.Features.IsDense) - { - int ioff = 1; - - // Increment remaining entries of hessian. - for (int i = 1; i < numParams; i++) - { - ch.Assert(ioff == i * (i + 1) / 2); - int wi = weightIndices == null ? i - 1 : weightIndices[i] - 1; - Contracts.Assert(0 <= wi && wi < cursor.Features.Length); - var val = values[wi] * variance; - // Add the implicit first bias term to X'X - hessian[ioff++] += val; - // Add the remainder of X'X - for (int j = 0; j < i; j++) - { - int wj = weightIndices == null ? j : weightIndices[j + 1] - 1; - Contracts.Assert(0 <= wj && wj < cursor.Features.Length); - hessian[ioff++] += val * values[wj]; - } - } - ch.Assert(ioff == hessian.Length); - } - else - { - var indices = cursor.Features.Indices; - for (int ii = 0; ii < cursor.Features.Count; ++ii) - { - int i = indices[ii]; - int wi = i + 1; - if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(i + 1, out wi)) - continue; - - Contracts.Assert(0 < wi && wi <= cursor.Features.Length); - int ioff = wi * (wi + 1) / 2; - var val = values[ii] * variance; - // Add the implicit first bias term to X'X - hessian[ioff] += val; - // Add the remainder of X'X - for (int jj = 0; jj <= ii; jj++) - { - int j = indices[jj]; - int wj = j + 1; - if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(j + 1, out wj)) - continue; - - Contracts.Assert(0 < wj && wj <= cursor.Features.Length); - hessian[ioff + wj] += val * values[jj]; - } - } - } - } - } - - // Apply Cholesky Decomposition to find the inverse of the Hessian. - Double[] invHessian = null; - try - { - // First, find the Cholesky decomposition LL' of the Hessian. - Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // Note that hessian is already modified at this point. It is no longer the original Hessian, - // but instead represents the Cholesky decomposition L. - // Also note that the following routine is supposed to consume the Cholesky decomposition L instead - // of the original information matrix. - Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // At this point, hessian should contain the inverse of the original Hessian matrix. - // Swap hessian with invHessian to avoid confusion in the following context. - Utils.Swap(ref hessian, ref invHessian); - Contracts.Assert(hessian == null); - } - catch (DllNotFoundException) - { - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); - } - - Float[] stdErrorValues = new Float[numParams]; - stdErrorValues[0] = (Float)Math.Sqrt(invHessian[0]); - - for (int i = 1; i < numParams; i++) - { - // Initialize with inverse Hessian. - stdErrorValues[i] = (Single)invHessian[i * (i + 1) / 2 + i]; - } - - if (L2Weight > 0) - { - // Iterate through all entries of inverse Hessian to make adjustment to variance. - // A discussion on ridge regularized LR coefficient covariance matrix can be found here: - // http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3228544/ - // http://www.inf.unibz.it/dis/teaching/DWDM/project2010/LogisticRegression.pdf - int ioffset = 1; - for (int iRow = 1; iRow < numParams; iRow++) - { - for (int iCol = 0; iCol <= iRow; iCol++) - { - var entry = (Single)invHessian[ioffset]; - var adjustment = -L2Weight * entry * entry; - stdErrorValues[iRow] -= adjustment; - if (0 < iCol && iCol < iRow) - stdErrorValues[iCol] -= adjustment; - ioffset++; - } - } - - Contracts.Assert(ioffset == invHessian.Length); - } - - for (int i = 1; i < numParams; i++) - stdErrorValues[i] = (Float)Math.Sqrt(stdErrorValues[i]); - - VBuffer stdErrors = new VBuffer(CurrentWeights.Length, numParams, stdErrorValues, weightIndices); - _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, ref stdErrors); + throw new NotImplementedException(); } protected override void ProcessPriorDistribution(Float label, Float weight) @@ -382,7 +135,7 @@ protected override ParameterMixingCalibratedPredictor CreatePredictor() CurrentWeights.GetItemOrDefault(0, ref bias); CurrentWeights.CopyTo(ref weights, 1, CurrentWeights.Length - 1); return new ParameterMixingCalibratedPredictor(Host, - new LinearBinaryPredictor(Host, ref weights, bias, _stats), + new LinearBinaryPredictor(Host, ref weights, bias), new PlattCalibrator(Host, -1, 0)); } diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 4fbdf5045e..fe46ca26e1 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,7 +59,6 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput -Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 5ed47f16a8..36bdd0f62a 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -14050,149 +14050,6 @@ "ITrainerOutput" ] }, - { - "Name": "Trainers.OrdinaryLeastSquaresRegressor", - "Desc": "Train an OLS regression model.", - "FriendlyName": "Ordinary Least Squares (Regression)", - "ShortName": "ols", - "Inputs": [ - { - "Name": "TrainingData", - "Type": "DataView", - "Desc": "The data to be used for training", - "Aliases": [ - "data" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "FeatureColumn", - "Type": "String", - "Desc": "Column to use for features", - "Aliases": [ - "feat" - ], - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": "Features" - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": "Label" - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Column to use for example weight", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 4.0, - "IsNullable": false, - "Default": "Weight" - }, - { - "Name": "NormalizeFeatures", - "Type": { - "Kind": "Enum", - "Values": [ - "No", - "Warn", - "Auto", - "Yes" - ] - }, - "Desc": "Normalize option for the feature column", - "Aliases": [ - "norm" - ], - "Required": false, - "SortOrder": 5.0, - "IsNullable": false, - "Default": "Auto" - }, - { - "Name": "Caching", - "Type": { - "Kind": "Enum", - "Values": [ - "Auto", - "Memory", - "Disk", - "None" - ] - }, - "Desc": "Whether learner should cache input training data", - "Aliases": [ - "cache" - ], - "Required": false, - "SortOrder": 6.0, - "IsNullable": false, - "Default": "Auto" - }, - { - "Name": "L2Weight", - "Type": "Float", - "Desc": "L2 regularization weight", - "Aliases": [ - "l2" - ], - "Required": false, - "SortOrder": 50.0, - "IsNullable": false, - "Default": 1E-06, - "SweepRange": { - "RangeType": "Discrete", - "Values": [ - 1E-06, - 0.1, - 1.0 - ] - } - }, - { - "Name": "PerParameterSignificance", - "Type": "Bool", - "Desc": "Whether to calculate per parameter significance statistics", - "Aliases": [ - "sig" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "The trained model" - } - ], - "InputKind": [ - "ITrainerInputWithWeight", - "ITrainerInputWithLabel", - "ITrainerInput" - ], - "OutputKind": [ - "IRegressionOutput", - "ITrainerOutput" - ] - }, { "Name": "Trainers.PcaAnomalyDetector", "Desc": "Train an PCA Anomaly model.", diff --git a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj index 7f0a8d05f9..47bd4fc80b 100644 --- a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj +++ b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index b8c8d9b497..60c9b21495 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -888,7 +888,7 @@ public void RegressorOlsTest() /// /// A test for ordinary least squares regression. /// - [Fact(Skip = "Need CoreTLC specific baseline update")] + [Fact] [TestCategory("Regressor")] public void RegressorOlsTestOne() { diff --git a/test/Microsoft.ML.Tests/CSharpCodeGen.cs b/test/Microsoft.ML.Tests/CSharpCodeGen.cs index 678edac461..5ee92d1c5e 100644 --- a/test/Microsoft.ML.Tests/CSharpCodeGen.cs +++ b/test/Microsoft.ML.Tests/CSharpCodeGen.cs @@ -16,7 +16,7 @@ public CSharpCodeGen(ITestOutputHelper output) : base(output) { } - [Fact(Skip = "Execute this test if you want to regenerate CSharpApi file")] + [Fact] public void RegenerateCSharpApi() { var basePath = GetDataPath("../../src/Microsoft.ML/CSharpApi.cs"); From 83a737fb407a3b9f59d6c33f6d2ab0ab9f9969bb Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 15:44:05 -0700 Subject: [PATCH 04/15] adding nupkgproj files to create a package for AdditionalLearners adding AdditionalLearners to the core.tests project fixing the core_ep and core_manifest --- .../Microsoft.ML.AdditionalLearners.nupkgproj | 13 ++ ...ft.ML.AdditionalLearners.symbols.nupkgproj | 5 + src/Microsoft.ML/CSharpApi.cs | 2 +- .../Common/EntryPoints/core_ep-list.tsv | 1 + .../Common/EntryPoints/core_manifest.json | 143 ++++++++++++++++++ .../Microsoft.ML.Core.Tests.csproj | 1 + 6 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj create mode 100644 pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj diff --git a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj b/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj new file mode 100644 index 0000000000..cdd1eb273e --- /dev/null +++ b/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj @@ -0,0 +1,13 @@ + + + + netstandard2.0 + ML.NET additional learners. They depend on other packages, like MlNetMklDeps. + + + + + + + + diff --git a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj b/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj new file mode 100644 index 0000000000..d29f958e48 --- /dev/null +++ b/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index 4d22194e60..2b013b26f4 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -8839,7 +8839,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output) namespace Trainers { - /// + /// public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index fe46ca26e1..51f19ec262 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,6 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 36bdd0f62a..5ed47f16a8 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -14050,6 +14050,149 @@ "ITrainerOutput" ] }, + { + "Name": "Trainers.OrdinaryLeastSquaresRegressor", + "Desc": "Train an OLS regression model.", + "FriendlyName": "Ordinary Least Squares (Regression)", + "ShortName": "ols", + "Inputs": [ + { + "Name": "TrainingData", + "Type": "DataView", + "Desc": "The data to be used for training", + "Aliases": [ + "data" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "FeatureColumn", + "Type": "String", + "Desc": "Column to use for features", + "Aliases": [ + "feat" + ], + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": "Features" + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": "Label" + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Column to use for example weight", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 4.0, + "IsNullable": false, + "Default": "Weight" + }, + { + "Name": "NormalizeFeatures", + "Type": { + "Kind": "Enum", + "Values": [ + "No", + "Warn", + "Auto", + "Yes" + ] + }, + "Desc": "Normalize option for the feature column", + "Aliases": [ + "norm" + ], + "Required": false, + "SortOrder": 5.0, + "IsNullable": false, + "Default": "Auto" + }, + { + "Name": "Caching", + "Type": { + "Kind": "Enum", + "Values": [ + "Auto", + "Memory", + "Disk", + "None" + ] + }, + "Desc": "Whether learner should cache input training data", + "Aliases": [ + "cache" + ], + "Required": false, + "SortOrder": 6.0, + "IsNullable": false, + "Default": "Auto" + }, + { + "Name": "L2Weight", + "Type": "Float", + "Desc": "L2 regularization weight", + "Aliases": [ + "l2" + ], + "Required": false, + "SortOrder": 50.0, + "IsNullable": false, + "Default": 1E-06, + "SweepRange": { + "RangeType": "Discrete", + "Values": [ + 1E-06, + 0.1, + 1.0 + ] + } + }, + { + "Name": "PerParameterSignificance", + "Type": "Bool", + "Desc": "Whether to calculate per parameter significance statistics", + "Aliases": [ + "sig" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "The trained model" + } + ], + "InputKind": [ + "ITrainerInputWithWeight", + "ITrainerInputWithLabel", + "ITrainerInput" + ], + "OutputKind": [ + "IRegressionOutput", + "ITrainerOutput" + ] + }, { "Name": "Trainers.PcaAnomalyDetector", "Desc": "Train an PCA Anomaly model.", diff --git a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj index ea92975b35..cdfaa05fd0 100644 --- a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj +++ b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj @@ -15,6 +15,7 @@ + From 61cdb66443346ecdd5c5d10f6e4fbf8a57f3c8ac Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 15:44:33 -0700 Subject: [PATCH 05/15] CSharpApi should not get generated every time. --- test/Microsoft.ML.Tests/CSharpCodeGen.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/CSharpCodeGen.cs b/test/Microsoft.ML.Tests/CSharpCodeGen.cs index 5ee92d1c5e..678edac461 100644 --- a/test/Microsoft.ML.Tests/CSharpCodeGen.cs +++ b/test/Microsoft.ML.Tests/CSharpCodeGen.cs @@ -16,7 +16,7 @@ public CSharpCodeGen(ITestOutputHelper output) : base(output) { } - [Fact] + [Fact(Skip = "Execute this test if you want to regenerate CSharpApi file")] public void RegenerateCSharpApi() { var basePath = GetDataPath("../../src/Microsoft.ML/CSharpApi.cs"); From 188bd4b14dfe56d5d229c74be132807d3c697036 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Mon, 30 Jul 2018 17:07:34 -0700 Subject: [PATCH 06/15] Addressing Ivan's comments --- .../Microsoft.ML.AdditionalLearners.csproj | 2 +- .../LogisticRegression/LogisticRegression.cs | 191 +++++++++++++++++- 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj index 96d82e07f7..46a54a757c 100644 --- a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj +++ b/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj @@ -3,7 +3,7 @@ netstandard2.0 Microsoft.ML.AdditionalLearners - true + true diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 970361d894..47b08c586a 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -46,6 +46,7 @@ public sealed class Arguments : ArgumentsBase } private Double _posWeight; + private LinearModelStatistics _stats; public LogisticRegression(IHostEnvironment env, Arguments args) : base(args, env, LoadNameValue, Contracts.CheckRef(args, nameof(args)).ShowTrainingStats) @@ -92,7 +93,195 @@ protected override Float AccumulateOneGradient(ref VBuffer feat, Float la protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor.Factory cursorFactory, Float loss, int numParams) { - throw new NotImplementedException(); + Contracts.AssertValue(ch); + Contracts.AssertValue(cursorFactory); + Contracts.Assert(NumGoodRows > 0); + Contracts.Assert(WeightSum > 0); + Contracts.Assert(BiasCount == 1); + Contracts.Assert(loss >= 0); + Contracts.Assert(numParams >= BiasCount); + Contracts.Assert(CurrentWeights.IsDense); + + ch.Info("Model trained with {0} training examples.", NumGoodRows); + + // Compute deviance: start with loss function. + Float deviance = (Float)(2 * loss * WeightSum); + + if (L2Weight > 0) + { + // Need to subtract L2 regularization loss. + // The bias term is not regularized. + var regLoss = VectorUtils.NormSquared(CurrentWeights.Values, 1, CurrentWeights.Length - 1) * L2Weight; + deviance -= regLoss; + } + + if (L1Weight > 0) + { + // Need to subtract L1 regularization loss. + // The bias term is not regularized. + Double regLoss = 0; + VBufferUtils.ForEachDefined(ref CurrentWeights, (ind, value) => { if (ind >= BiasCount) regLoss += Math.Abs(value); }); + deviance -= (Float)regLoss * L1Weight * 2; + } + + ch.Info("Residual Deviance: \t{0} (on {1} degrees of freedom)", deviance, Math.Max(NumGoodRows - numParams, 0)); + + // Compute null deviance, i.e., the deviance of null hypothesis. + // Cap the prior positive rate at 1e-15. + Double priorPosRate = _posWeight / WeightSum; + Contracts.Assert(0 <= priorPosRate && priorPosRate <= 1); + Float nullDeviance = (priorPosRate <= 1e-15 || 1 - priorPosRate <= 1e-15) ? + 0f : (Float)(2 * WeightSum * MathUtils.Entropy(priorPosRate, true)); + ch.Info("Null Deviance: \t{0} (on {1} degrees of freedom)", nullDeviance, NumGoodRows - 1); + + // Compute AIC. + ch.Info("AIC: \t{0}", 2 * numParams + deviance); + + // Show the coefficients statistics table. + var featureColIdx = cursorFactory.Data.Schema.Feature.Index; + var schema = cursorFactory.Data.Data.Schema; + var featureLength = CurrentWeights.Length - BiasCount; + var namesSpans = VBufferUtils.CreateEmpty(featureLength); + if (schema.HasSlotNames(featureColIdx, featureLength)) + schema.GetMetadata(MetadataUtils.Kinds.SlotNames, featureColIdx, ref namesSpans); + Host.Assert(namesSpans.Length == featureLength); + + // Inverse mapping of non-zero weight slots. + Dictionary weightIndicesInvMap = null; + + // Indices of bias and non-zero weight slots. + int[] weightIndices = null; + + // Whether all weights are non-zero. + bool denseWeight = numParams == CurrentWeights.Length; + + // Extract non-zero indices of weight. + if (!denseWeight) + { + weightIndices = new int[numParams]; + weightIndicesInvMap = new Dictionary(numParams); + weightIndices[0] = 0; + weightIndicesInvMap[0] = 0; + int j = 1; + for (int i = 1; i < CurrentWeights.Length; i++) + { + if (CurrentWeights.Values[i] != 0) + { + weightIndices[j] = i; + weightIndicesInvMap[i] = j++; + } + } + + Contracts.Assert(j == numParams); + } + + // Compute the standard error of coefficients. + long hessianDimension = (long)numParams * (numParams + 1) / 2; + if (hessianDimension > int.MaxValue) + { + ch.Warning("The number of parameter is too large. Cannot hold the variance-covariance matrix in memory. " + + "Skipping computation of standard errors and z-statistics of coefficients. Consider choosing a larger L1 regularizer" + + "to reduce the number of parameters."); + _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); + return; + } + + // Building the variance-covariance matrix for parameters. + // The layout of this algorithm is a packed row-major lower triangular matrix. + // E.g., layout of indices for 4-by-4: + // 0 + // 1 2 + // 3 4 5 + // 6 7 8 9 + var hessian = new Double[hessianDimension]; + + // Initialize diagonal elements with L2 regularizers except for the first entry (index 0) + // Since bias is not regularized. + if (L2Weight > 0) + { + // i is the array index of the diagonal entry at iRow-th row and iRow-th column. + // iRow is one-based. + int i = 0; + for (int iRow = 2; iRow <= numParams; iRow++) + { + i += iRow; + hessian[i] = L2Weight; + } + + Contracts.Assert(i == hessian.Length - 1); + } + + // Initialize the remaining entries. + var bias = CurrentWeights.Values[0]; + using (var cursor = cursorFactory.Create()) + { + while (cursor.MoveNext()) + { + var label = cursor.Label; + var weight = cursor.Weight; + var score = bias + VectorUtils.DotProductWithOffset(ref CurrentWeights, 1, ref cursor.Features); + // Compute Bernoulli variance n_i * p_i * (1 - p_i) for the i-th training example. + var variance = weight / (2 + 2 * Math.Cosh(score)); + + // Increment the first entry of hessian. + hessian[0] += variance; + + var values = cursor.Features.Values; + if (cursor.Features.IsDense) + { + int ioff = 1; + + // Increment remaining entries of hessian. + for (int i = 1; i < numParams; i++) + { + ch.Assert(ioff == i * (i + 1) / 2); + int wi = weightIndices == null ? i - 1 : weightIndices[i] - 1; + Contracts.Assert(0 <= wi && wi < cursor.Features.Length); + var val = values[wi] * variance; + // Add the implicit first bias term to X'X + hessian[ioff++] += val; + // Add the remainder of X'X + for (int j = 0; j < i; j++) + { + int wj = weightIndices == null ? j : weightIndices[j + 1] - 1; + Contracts.Assert(0 <= wj && wj < cursor.Features.Length); + hessian[ioff++] += val * values[wj]; + } + } + ch.Assert(ioff == hessian.Length); + } + else + { + var indices = cursor.Features.Indices; + for (int ii = 0; ii < cursor.Features.Count; ++ii) + { + int i = indices[ii]; + int wi = i + 1; + if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(i + 1, out wi)) + continue; + + Contracts.Assert(0 < wi && wi <= cursor.Features.Length); + int ioff = wi * (wi + 1) / 2; + var val = values[ii] * variance; + // Add the implicit first bias term to X'X + hessian[ioff] += val; + // Add the remainder of X'X + for (int jj = 0; jj <= ii; jj++) + { + int j = indices[jj]; + int wj = j + 1; + if (weightIndicesInvMap != null && !weightIndicesInvMap.TryGetValue(j + 1, out wj)) + continue; + + Contracts.Assert(0 < wj && wj <= cursor.Features.Length); + hessian[ioff + wj] += val * values[jj]; + } + } + } + } + } + + _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); } protected override void ProcessPriorDistribution(Float label, Float weight) From c2b87e706161aaa8ade63b17203a7c3695db4456 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 10:10:52 -0700 Subject: [PATCH 07/15] referencing package version 0.0.0.4 of MlNetMklDeps that contains new names for the mkl library. --- build/Dependencies.props | 2 +- src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs | 2 +- src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs | 2 +- src/Microsoft.ML.Transforms/WhiteningTransform.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index 5aafe118e5..f48ebaadd4 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -8,6 +8,6 @@ 4.3.0 1.0.0-beta-62824-02 2.1.2.2 - 0.0.0.1 + 0.0.0.4 diff --git a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs index 7f7f361f54..a91895dcb2 100644 --- a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs @@ -330,7 +330,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac internal static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "libMklImports"; public enum Layout { diff --git a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs index 3b56e8bb36..5969e67a4f 100644 --- a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs +++ b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs @@ -343,7 +343,7 @@ private static bool ShouldSkipPath(string path) case "libvw.dll": case "matrixinterf.dll": case "Microsoft.ML.neuralnetworks.gpucuda.dll": - case "Microsoft.ML.mklimports.dll": + case "libMklImports.dll": case "microsoft.research.controls.decisiontrees.dll": case "Microsoft.ML.neuralnetworks.sse.dll": case "neuraltreeevaluator.dll": diff --git a/src/Microsoft.ML.Transforms/WhiteningTransform.cs b/src/Microsoft.ML.Transforms/WhiteningTransform.cs index 6854157f31..0421753b05 100644 --- a/src/Microsoft.ML.Transforms/WhiteningTransform.cs +++ b/src/Microsoft.ML.Transforms/WhiteningTransform.cs @@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices private static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "libMklImports"; public enum Layout { From 2efe7bc757b3d20f4383ff92a83f2d6988be845c Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 10:13:08 -0700 Subject: [PATCH 08/15] Correcting the error message. --- src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs index a91895dcb2..307745dbe8 100644 --- a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs @@ -221,7 +221,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac catch (DllNotFoundException) { // REVIEW: Is there no better way? - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports) or one of its dependencies is missing."); + throw ch.ExceptNotSupp("The MKL library (libMklImports) or one of its dependencies is missing."); } // Solve for beta in (LL')beta = X'y: Mkl.Pptrs(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, 1, xtx, xty, 1); From c666ad7d8dd2245ba792fa48eb863604276101a4 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 13:25:34 -0700 Subject: [PATCH 09/15] renaming AdditionalLearners to HalLearners < - Hardware Accelerated Learners removign unsafe from the Hal csproj. removing the orphaned member section from the doc.xml --- Microsoft.ML.sln | 19 ++++++++--------- ...ft.ML.AdditionalLearners.symbols.nupkgproj | 5 ----- .../Microsoft.ML.HalLearners.nupkgproj} | 2 +- ...Microsoft.ML.HalLearners.symbols.nupkgproj | 5 +++++ .../Microsoft.ML.HalLearners.csproj} | 3 +-- .../OlsLinearRegression.cs | 0 .../doc.xml | 0 .../Standard/doc.xml | 21 ------------------- src/Microsoft.ML/CSharpApi.cs | 2 +- .../Common/EntryPoints/core_ep-list.tsv | 2 +- .../Microsoft.ML.Core.Tests.csproj | 2 +- .../Microsoft.ML.Predictor.Tests.csproj | 2 +- 12 files changed, 20 insertions(+), 43 deletions(-) delete mode 100644 pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj rename pkg/{Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj => Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj} (70%) create mode 100644 pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj rename src/{Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj => Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj} (83%) rename src/{Microsoft.ML.AdditionalLearners => Microsoft.ML.HalLearners}/OlsLinearRegression.cs (100%) rename src/{Microsoft.ML.AdditionalLearners => Microsoft.ML.HalLearners}/doc.xml (100%) diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 590f7dfcff..e4acd0f624 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,7 +97,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AdditionalLearners", "src\Microsoft.ML.AdditionalLearners\Microsoft.ML.AdditionalLearners.csproj", "{E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -331,14 +331,14 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release|Any CPU.Build.0 = Release|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release|Any CPU.Build.0 = Release|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -377,7 +377,6 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {E2169EDF-88F5-4264-BDEA-164E7EBFBE3D} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj b/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj deleted file mode 100644 index d29f958e48..0000000000 --- a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.symbols.nupkgproj +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj similarity index 70% rename from pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj rename to pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj index cdd1eb273e..53112a75e2 100644 --- a/pkg/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.nupkgproj +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj @@ -2,7 +2,7 @@ netstandard2.0 - ML.NET additional learners. They depend on other packages, like MlNetMklDeps. + ML.NET additional learners making use of hardware acceleration. They depend on like MlNetMklDeps. diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj new file mode 100644 index 0000000000..248ae82414 --- /dev/null +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj b/src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj similarity index 83% rename from src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj rename to src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj index 46a54a757c..a5f3c4b748 100644 --- a/src/Microsoft.ML.AdditionalLearners/Microsoft.ML.AdditionalLearners.csproj +++ b/src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj @@ -2,8 +2,7 @@ netstandard2.0 - Microsoft.ML.AdditionalLearners - true + Microsoft.ML.HalLearners diff --git a/src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs similarity index 100% rename from src/Microsoft.ML.AdditionalLearners/OlsLinearRegression.cs rename to src/Microsoft.ML.HalLearners/OlsLinearRegression.cs diff --git a/src/Microsoft.ML.AdditionalLearners/doc.xml b/src/Microsoft.ML.HalLearners/doc.xml similarity index 100% rename from src/Microsoft.ML.AdditionalLearners/doc.xml rename to src/Microsoft.ML.HalLearners/doc.xml diff --git a/src/Microsoft.ML.StandardLearners/Standard/doc.xml b/src/Microsoft.ML.StandardLearners/Standard/doc.xml index 8d06ef285f..eb87605232 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/doc.xml +++ b/src/Microsoft.ML.StandardLearners/Standard/doc.xml @@ -68,27 +68,6 @@ - - - - Train an OLS regression model. - - - Ordinary least squares (OLS) is a parameterized regression method. - It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. - The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. - - - - new OrdinaryLeastSquaresRegressor - { - L2Weight = 0.1, - PerParameterSignificance = false, - NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes - } - - - \ No newline at end of file diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index 2b013b26f4..875b911af1 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -8839,7 +8839,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output) namespace Trainers { - /// + /// public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 51f19ec262..f9dae2625c 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,7 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput -Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput diff --git a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj index cdfaa05fd0..c752663da5 100644 --- a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj +++ b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj @@ -5,6 +5,7 @@ + @@ -15,7 +16,6 @@ - diff --git a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj index 47bd4fc80b..63234b0900 100644 --- a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj +++ b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj @@ -9,6 +9,7 @@ + @@ -17,7 +18,6 @@ - From eeba5773c99f905f909ec385b457eaa019c30177 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 15:10:57 -0700 Subject: [PATCH 10/15] referencing package 0.0.0.5 and updating the name. --- Microsoft.ML.sln | 19 ++++++++++--------- build/Dependencies.props | 2 +- .../ComponentModel/ComponentCatalog.cs | 2 +- .../OlsLinearRegression.cs | 2 +- .../WhiteningTransform.cs | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index e4acd0f624..5d41a375c2 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,7 +97,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -331,14 +331,14 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release|Any CPU.Build.0 = Release|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU - {4084C0CF-8DE4-472D-AF98-9EC96B4C91B1}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.Build.0 = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -377,6 +377,7 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {A7222F41-1CF0-47D9-B80C-B4D77B027A61} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/build/Dependencies.props b/build/Dependencies.props index f48ebaadd4..bc9b1750dc 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -8,6 +8,6 @@ 4.3.0 1.0.0-beta-62824-02 2.1.2.2 - 0.0.0.4 + 0.0.0.5 diff --git a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs index 5969e67a4f..a92c20a4e6 100644 --- a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs +++ b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs @@ -343,7 +343,7 @@ private static bool ShouldSkipPath(string path) case "libvw.dll": case "matrixinterf.dll": case "Microsoft.ML.neuralnetworks.gpucuda.dll": - case "libMklImports.dll": + case "MklImports.dll": case "microsoft.research.controls.decisiontrees.dll": case "Microsoft.ML.neuralnetworks.sse.dll": case "neuraltreeevaluator.dll": diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index 307745dbe8..da5ff4571d 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -330,7 +330,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac internal static class Mkl { - private const string DllName = "libMklImports"; + private const string DllName = "MklImports"; public enum Layout { diff --git a/src/Microsoft.ML.Transforms/WhiteningTransform.cs b/src/Microsoft.ML.Transforms/WhiteningTransform.cs index 0421753b05..a46a764352 100644 --- a/src/Microsoft.ML.Transforms/WhiteningTransform.cs +++ b/src/Microsoft.ML.Transforms/WhiteningTransform.cs @@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices private static class Mkl { - private const string DllName = "libMklImports"; + private const string DllName = "MklImports"; public enum Layout { From dc34085a543967bd11638a7850f4df7deda30cab Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 16:00:56 -0700 Subject: [PATCH 11/15] regenerating the CsharpApi and the eplist post merge --- src/Microsoft.ML/CSharpApi.cs | 2 +- test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index 8bd3ac28c1..fc6d9a4224 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -8911,7 +8911,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output) namespace Trainers { - /// + /// public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 59d168cc3c..ece0248436 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,7 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput -Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput From 3588e9b7fad4866ed593ee9d21619cdb5e5998de Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 16:16:16 -0700 Subject: [PATCH 12/15] regenerate the CSharpApi and the eplists post merge. Fix the namespace post merge. --- src/Microsoft.ML.HalLearners/OlsLinearRegression.cs | 6 +++--- src/Microsoft.ML/CSharpApi.cs | 2 +- test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index da5ff4571d..dbf2999657 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -8,7 +8,7 @@ using System.Collections.Generic; using System.IO; using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.AdditionalLearners; +using Microsoft.ML.Runtime.HalLearners; using Microsoft.ML.Runtime.Internal.Internallearn; using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.CommandLine; @@ -31,7 +31,7 @@ [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)] -namespace Microsoft.ML.Runtime.AdditionalLearners +namespace Microsoft.ML.Runtime.HalLearners { /// public sealed class OlsLinearRegressionTrainer : TrainerBase @@ -467,7 +467,7 @@ public static void Pptri(Layout layout, UpLo uplo, int n, Double[] ap) Desc = "Train an OLS regression model.", UserName = UserNameValue, ShortName = ShortName, - XmlInclude = new[] { @"" })] + XmlInclude = new[] { @"" })] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index fc6d9a4224..bf753fe486 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -8911,7 +8911,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output) namespace Trainers { - /// + /// public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index ece0248436..59d168cc3c 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,7 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput -Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.AdditionalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput From 05909c6c105ad209c0af097d4f39e6a69e46a028 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 16:39:59 -0700 Subject: [PATCH 13/15] typo --- pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj index 53112a75e2..eac06a9893 100644 --- a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj @@ -2,7 +2,7 @@ netstandard2.0 - ML.NET additional learners making use of hardware acceleration. They depend on like MlNetMklDeps. + ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGetpackage. From 0f91d17da5b1ae089c7d931e6ccc28135dcc81ac Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 16:43:06 -0700 Subject: [PATCH 14/15] one shall not space --- pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj index eac06a9893..0fac702783 100644 --- a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj @@ -2,7 +2,7 @@ netstandard2.0 - ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGetpackage. + ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGetpackage. From 7d3bbab6fc06e220bb3c7db86a35a5567a8aac44 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Tue, 31 Jul 2018 16:57:35 -0700 Subject: [PATCH 15/15] spacing --- pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj index 0fac702783..a531c8e403 100644 --- a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj @@ -2,7 +2,7 @@ netstandard2.0 - ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGetpackage. + ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGet package.