diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs index 815e4694b4..41f85c327c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LightGbmWithOptions.cs @@ -1,5 +1,4 @@ -using Microsoft.ML.Trainers.LightGbm; -using static Microsoft.ML.Trainers.LightGbm.Options; +using Microsoft.ML.Trainers.LightGbm; namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification { @@ -19,7 +18,7 @@ public static void Example() // Create the pipeline with LightGbm Estimator using advanced options. var pipeline = mlContext.BinaryClassification.Trainers.LightGbm( - new Options + new LightGbmBinaryTrainer.Options { Booster = new GossBooster.Options { diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs index 5146bf66d8..2b24423e5e 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs @@ -3,7 +3,6 @@ using Microsoft.ML.Data; using Microsoft.ML.SamplesUtils; using Microsoft.ML.Trainers.LightGbm; -using static Microsoft.ML.Trainers.LightGbm.Options; namespace Microsoft.ML.Samples.Dynamic.Trainers.MulticlassClassification { @@ -33,11 +32,11 @@ public static void Example() // - Convert the string labels into key types. // - Apply LightGbm multiclass trainer with advanced options. var pipeline = mlContext.Transforms.Conversion.MapValueToKey("LabelIndex", "Label") - .Append(mlContext.MulticlassClassification.Trainers.LightGbm(new Options + .Append(mlContext.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options { LabelColumnName = "LabelIndex", FeatureColumnName = "Features", - Booster = new DartBooster.Options + Booster = new DartBooster.Options() { TreeDropFraction = 0.15, XgboostDartMode = false diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs index bdc966fc92..1cb039bb18 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/LightGbmWithOptions.cs @@ -1,5 +1,4 @@ using Microsoft.ML.Trainers.LightGbm; -using static Microsoft.ML.Trainers.LightGbm.Options; namespace Microsoft.ML.Samples.Dynamic.Trainers.Ranking { @@ -21,13 +20,13 @@ public static void Example() // Create the Estimator pipeline. For simplicity, we will train a small tree with 4 leaves and 2 boosting iterations. var pipeline = mlContext.Ranking.Trainers.LightGbm( - new Options + new LightGbmRankingTrainer.Options { NumberOfLeaves = 4, MinimumExampleCountPerGroup = 10, LearningRate = 0.1, NumberOfIterations = 2, - Booster = new TreeBooster.Options + Booster = new GradientBooster.Options { FeatureFraction = 0.9 } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs index 4d1eb39a6a..f6d3eeb1f9 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/LightGbmWithOptions.cs @@ -2,7 +2,6 @@ using System.Linq; using Microsoft.ML.Data; using Microsoft.ML.Trainers.LightGbm; -using static Microsoft.ML.Trainers.LightGbm.Options; namespace Microsoft.ML.Samples.Dynamic.Trainers.Regression { @@ -36,13 +35,13 @@ public static void Example() .Where(name => name != labelName) // Drop the Label .ToArray(); var pipeline = mlContext.Transforms.Concatenate("Features", featureNames) - .Append(mlContext.Regression.Trainers.LightGbm(new Options + .Append(mlContext.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options { LabelColumnName = labelName, NumberOfLeaves = 4, MinimumExampleCountPerLeaf = 6, LearningRate = 0.001, - Booster = new GossBooster.Options + Booster = new GossBooster.Options() { TopRate = 0.3, OtherRate = 0.2 diff --git a/src/Microsoft.ML.LightGbm.StaticPipe/LightGbmStaticExtensions.cs b/src/Microsoft.ML.LightGbm.StaticPipe/LightGbmStaticExtensions.cs index 129ab2f593..0fb8bf312b 100644 --- a/src/Microsoft.ML.LightGbm.StaticPipe/LightGbmStaticExtensions.cs +++ b/src/Microsoft.ML.LightGbm.StaticPipe/LightGbmStaticExtensions.cs @@ -42,7 +42,7 @@ public static Scalar LightGbm(this RegressionCatalog.RegressionTrainers c int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations, + int numberOfIterations = Defaults.NumberOfIterations, Action onFit = null) { CheckUserValues(label, features, weights, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations, onFit); @@ -76,10 +76,11 @@ public static Scalar LightGbm(this RegressionCatalog.RegressionTrainers c /// The Score output column indicating the predicted value. public static Scalar LightGbm(this RegressionCatalog.RegressionTrainers catalog, Scalar label, Vector features, Scalar weights, - Options options, + LightGbmRegressionTrainer.Options options, Action onFit = null) { - CheckUserValues(label, features, weights, options, onFit); + Contracts.CheckValue(options, nameof(options)); + CheckUserValues(label, features, weights, onFit); var rec = new TrainerEstimatorReconciler.Regression( (env, labelName, featuresName, weightsName) => @@ -128,7 +129,7 @@ public static (Scalar score, Scalar probability, Scalar pred int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations, + int numberOfIterations = Defaults.NumberOfIterations, Action> onFit = null) { CheckUserValues(label, features, weights, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations, onFit); @@ -165,10 +166,11 @@ public static (Scalar score, Scalar probability, Scalar pred /// from negative to positive infinity), the calibrated prediction (from 0 to 1), and the predicted label. public static (Scalar score, Scalar probability, Scalar predictedLabel) LightGbm(this BinaryClassificationCatalog.BinaryClassificationTrainers catalog, Scalar label, Vector features, Scalar weights, - Options options, + LightGbmBinaryTrainer.Options options, Action> onFit = null) { - CheckUserValues(label, features, weights, options, onFit); + Contracts.CheckValue(options, nameof(options)); + CheckUserValues(label, features, weights, onFit); var rec = new TrainerEstimatorReconciler.BinaryClassifier( (env, labelName, featuresName, weightsName) => @@ -215,7 +217,7 @@ public static Scalar LightGbm(this RankingCatalog.RankingTrainers c int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations, + int numberOfIterations = Defaults.NumberOfIterations, Action onFit = null) { CheckUserValues(label, features, weights, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations, onFit); @@ -253,10 +255,11 @@ public static Scalar LightGbm(this RankingCatalog.RankingTrainers c /// from negative to positive infinity), the calibrated prediction (from 0 to 1), and the predicted label. public static Scalar LightGbm(this RankingCatalog.RankingTrainers catalog, Scalar label, Vector features, Key groupId, Scalar weights, - Options options, + LightGbmRankingTrainer.Options options, Action onFit = null) { - CheckUserValues(label, features, weights, options, onFit); + Contracts.CheckValue(options, nameof(options)); + CheckUserValues(label, features, weights, onFit); Contracts.CheckValue(groupId, nameof(groupId)); var rec = new TrainerEstimatorReconciler.Ranker( @@ -309,7 +312,7 @@ public static (Vector score, Key predictedLabel) int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations, + int numberOfIterations = Defaults.NumberOfIterations, Action onFit = null) { CheckUserValues(label, features, weights, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations, onFit); @@ -347,10 +350,11 @@ public static (Vector score, Key predictedLabel) Key label, Vector features, Scalar weights, - Options options, + LightGbmMulticlassTrainer.Options options, Action onFit = null) { - CheckUserValues(label, features, weights, options, onFit); + Contracts.CheckValue(options, nameof(options)); + CheckUserValues(label, features, weights, onFit); var rec = new TrainerEstimatorReconciler.MulticlassClassificationReconciler( (env, labelName, featuresName, weightsName) => @@ -386,14 +390,11 @@ private static void CheckUserValues(PipelineColumn label, Vector features Contracts.CheckValueOrNull(onFit); } - private static void CheckUserValues(PipelineColumn label, Vector features, Scalar weights, - Options options, - Delegate onFit) + private static void CheckUserValues(PipelineColumn label, Vector features, Scalar weights, Delegate onFit) { Contracts.CheckValue(label, nameof(label)); Contracts.CheckValue(features, nameof(features)); Contracts.CheckValueOrNull(weights); - Contracts.CheckValue(options, nameof(options)); Contracts.CheckValueOrNull(onFit); } } diff --git a/src/Microsoft.ML.LightGbm/LightGbmArguments.cs b/src/Microsoft.ML.LightGbm/LightGbmArguments.cs index cb8a18b384..0ac5a63572 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmArguments.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmArguments.cs @@ -1,10 +1,5 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; +using System.Collections.Generic; using System.Reflection; -using System.Text; using Microsoft.ML; using Microsoft.ML.CommandLine; using Microsoft.ML.EntryPoints; @@ -12,667 +7,344 @@ using Microsoft.ML.Runtime; using Microsoft.ML.Trainers.LightGbm; -[assembly: LoadableClass(typeof(Options.TreeBooster), typeof(Options.TreeBooster.Options), - typeof(SignatureLightGBMBooster), Options.TreeBooster.FriendlyName, Options.TreeBooster.Name)] -[assembly: LoadableClass(typeof(Options.DartBooster), typeof(Options.DartBooster.Options), - typeof(SignatureLightGBMBooster), Options.DartBooster.FriendlyName, Options.DartBooster.Name)] -[assembly: LoadableClass(typeof(Options.GossBooster), typeof(Options.GossBooster.Options), - typeof(SignatureLightGBMBooster), Options.GossBooster.FriendlyName, Options.GossBooster.Name)] +[assembly: LoadableClass(typeof(GradientBooster), typeof(GradientBooster.Options), + typeof(SignatureLightGBMBooster), GradientBooster.FriendlyName, GradientBooster.Name)] +[assembly: LoadableClass(typeof(DartBooster), typeof(DartBooster.Options), + typeof(SignatureLightGBMBooster), DartBooster.FriendlyName, DartBooster.Name)] +[assembly: LoadableClass(typeof(GossBooster), typeof(GossBooster.Options), + typeof(SignatureLightGBMBooster), GossBooster.FriendlyName, GossBooster.Name)] -[assembly: EntryPointModule(typeof(Options.TreeBooster.Options))] -[assembly: EntryPointModule(typeof(Options.DartBooster.Options))] -[assembly: EntryPointModule(typeof(Options.GossBooster.Options))] +[assembly: EntryPointModule(typeof(GradientBooster.Options))] +[assembly: EntryPointModule(typeof(DartBooster.Options))] +[assembly: EntryPointModule(typeof(GossBooster.Options))] namespace Microsoft.ML.Trainers.LightGbm { internal delegate void SignatureLightGBMBooster(); [TlcModule.ComponentKind("BoosterParameterFunction")] - public interface ISupportBoosterParameterFactory : IComponentFactory + internal interface IBoosterParameterFactory : IComponentFactory { + new BoosterParameterBase CreateComponent(IHostEnvironment env); } - public interface IBoosterParameter + public abstract class BoosterParameterBase { - void UpdateParameters(Dictionary res); - } - - /// - /// Options for LightGBM trainer. - /// - /// - /// LightGBM is an external library that's integrated with ML.NET. For detailed information about the parameters - /// please see https://github.com/Microsoft/LightGBM/blob/master/docs/Parameters.rst. - /// - public sealed class Options : TrainerInputBaseWithGroupId - { - public abstract class BoosterParameter : IBoosterParameter - where TOptions : class, new() + private protected static Dictionary NameMapping = new Dictionary() { - private protected TOptions BoosterParameterOptions { get; } - - private protected BoosterParameter(TOptions options) - { - BoosterParameterOptions = options; - } - - /// - /// Update the parameters by specific Booster, will update parameters into "res" directly. - /// - internal virtual void UpdateParameters(Dictionary res) - { - FieldInfo[] fields = BoosterParameterOptions.GetType().GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); - foreach (var field in fields) - { - var attribute = field.GetCustomAttribute(false); - - if (attribute == null) - continue; - - res[GetOptionName(field.Name)] = field.GetValue(BoosterParameterOptions); - } - } - - void IBoosterParameter.UpdateParameters(Dictionary res) => UpdateParameters(res); - } - - private static string GetOptionName(string name) - { - if (_nameMapping.ContainsKey(name)) - return _nameMapping[name]; - - // Otherwise convert the name to the light gbm argument - StringBuilder strBuf = new StringBuilder(); - bool first = true; - foreach (char c in name) - { - if (char.IsUpper(c)) - { - if (first) - first = false; - else - strBuf.Append('_'); - strBuf.Append(char.ToLower(c)); - } - else - strBuf.Append(c); - } - return strBuf.ToString(); - } - - // Static override name map that maps friendly names to lightGBM arguments. - // If an argument is not here, then its name is identical to a lightGBM argument - // and does not require a mapping, for example, Subsample. - private static Dictionary _nameMapping = new Dictionary() - { - {nameof(TreeBooster.Options.MinimumSplitGain), "min_split_gain" }, - {nameof(TreeBooster.Options.MaximumTreeDepth), "max_depth"}, - {nameof(TreeBooster.Options.MinimumChildWeight), "min_child_weight"}, - {nameof(TreeBooster.Options.SubsampleFraction), "subsample"}, - {nameof(TreeBooster.Options.SubsampleFrequency), "subsample_freq"}, - {nameof(TreeBooster.Options.L1Regularization), "reg_alpha"}, - {nameof(TreeBooster.Options.L2Regularization), "reg_lambda"}, - {nameof(TreeBooster.Options.WeightOfPositiveExamples), "scale_pos_weight"}, - {nameof(DartBooster.Options.TreeDropFraction), "drop_rate" }, - {nameof(DartBooster.Options.MaximumNumberOfDroppedTreesPerRound),"max_drop" }, - {nameof(DartBooster.Options.SkipDropFraction), "skip_drop" }, - {nameof(MinimumExampleCountPerLeaf), "min_data_per_leaf"}, - {nameof(NumberOfLeaves), "num_leaves"}, - {nameof(MaximumBinCountPerFeature), "max_bin" }, - {nameof(CustomGains), "label_gain" }, - {nameof(MinimumExampleCountPerGroup), "min_data_per_group" }, - {nameof(MaximumCategoricalSplitPointCount), "max_cat_threshold" }, - {nameof(CategoricalSmoothing), "cat_smooth" }, - {nameof(L2CategoricalRegularization), "cat_l2" } + {nameof(OptionsBase.MinimumSplitGain), "min_split_gain" }, + {nameof(OptionsBase.MaximumTreeDepth), "max_depth"}, + {nameof(OptionsBase.MinimumChildWeight), "min_child_weight"}, + {nameof(OptionsBase.SubsampleFraction), "subsample"}, + {nameof(OptionsBase.SubsampleFrequency), "subsample_freq"}, + {nameof(OptionsBase.L1Regularization), "reg_alpha"}, + {nameof(OptionsBase.L2Regularization), "reg_lambda"}, }; - - [BestFriend] - internal static class Defaults + public BoosterParameterBase(OptionsBase options) { - public const int NumberOfIterations = 100; + Contracts.CheckUserArg(options.MinimumSplitGain >= 0, nameof(OptionsBase.MinimumSplitGain), "must be >= 0."); + Contracts.CheckUserArg(options.MinimumChildWeight >= 0, nameof(OptionsBase.MinimumChildWeight), "must be >= 0."); + Contracts.CheckUserArg(options.SubsampleFraction > 0 && options.SubsampleFraction <= 1, nameof(OptionsBase.SubsampleFraction), "must be in (0,1]."); + Contracts.CheckUserArg(options.FeatureFraction > 0 && options.FeatureFraction <= 1, nameof(OptionsBase.FeatureFraction), "must be in (0,1]."); + Contracts.CheckUserArg(options.L2Regularization >= 0, nameof(OptionsBase.L2Regularization), "must be >= 0."); + Contracts.CheckUserArg(options.L1Regularization >= 0, nameof(OptionsBase.L1Regularization), "must be >= 0."); + BoosterOptions = options; } - /// - /// Gradient boosting decision tree. - /// - /// - /// For details, please see gradient tree boosting. - /// - public sealed class TreeBooster : BoosterParameter + public abstract class OptionsBase : IBoosterParameterFactory { - internal const string Name = "gbdt"; - internal const string FriendlyName = "Tree Booster"; + internal BoosterParameterBase GetBooster() { return null; } /// - /// The options for , used for setting . + /// The minimum loss reduction required to make a further partition on a leaf node of the tree. /// - [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Traditional Gradient Boosting Decision Tree.")] - public class Options : ISupportBoosterParameterFactory - { - /// - /// Whether training data is unbalanced. Used by . - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Use for binary classification when training data is not balanced.", ShortName = "us")] - public bool UnbalancedSets = false; - - /// - /// The minimum loss reduction required to make a further partition on a leaf node of the tree. - /// - /// - /// Larger values make the algorithm more conservative. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Minimum loss reduction required to make a further partition on a leaf node of the tree. the larger, " + - "the more conservative the algorithm will be.")] - [TlcModule.Range(Min = 0.0)] - public double MinimumSplitGain = 0; - - /// - /// The maximum depth of a tree. - /// - /// - /// 0 means no limit. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Maximum depth of a tree. 0 means no limit. However, tree still grows by best-first.")] - [TlcModule.Range(Min = 0, Max = int.MaxValue)] - public int MaximumTreeDepth = 0; - - /// - /// The minimum sum of instance weight needed to form a new node. - /// - /// - /// If the tree partition step results in a leaf node with the sum of instance weight less than , - /// the building process will give up further partitioning. In linear regression mode, this simply corresponds to minimum number - /// of instances needed to be in each node. The larger, the more conservative the algorithm will be. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Minimum sum of instance weight(hessian) needed in a child. If the tree partition step results in a leaf " + - "node with the sum of instance weight less than min_child_weight, then the building process will give up further partitioning. In linear regression mode, " + - "this simply corresponds to minimum number of instances needed to be in each node. The larger, the more conservative the algorithm will be.")] - [TlcModule.Range(Min = 0.0)] - public double MinimumChildWeight = 0.1; - - /// - /// The frequency of performing subsampling (bagging). - /// - /// - /// 0 means disable bagging. N means perform bagging at every N iterations. - /// To enable bagging, should also be set to a value less than 1.0. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Subsample frequency for bagging. 0 means no subsample. " - + "Specifies the frequency at which the bagging occurs, where if this is set to N, the subsampling will happen at every N iterations." + - "This must be set with Subsample as this specifies the amount to subsample.")] - [TlcModule.Range(Min = 0, Max = int.MaxValue)] - public int SubsampleFrequency = 0; - - /// - /// The fraction of training data used for creating trees. - /// - /// - /// Setting it to 0.5 means that LightGBM randomly picks half of the data points to grow trees. - /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Subsample ratio of the training instance. Setting it to 0.5 means that LightGBM randomly collected " + - "half of the data instances to grow trees and this will prevent overfitting. Range: (0,1].")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double SubsampleFraction = 1; - - /// - /// The fraction of features used when creating trees. - /// - /// - /// If is smaller than 1.0, LightGBM will randomly select fraction of features to train each tree. - /// For example, if you set it to 0.8, LightGBM will select 80% of features before training each tree. - /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Subsample ratio of columns when constructing each tree. Range: (0,1].", - ShortName = "ff")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double FeatureFraction = 1; - - /// - /// The L2 regularization term on weights. - /// - /// - /// Increasing this value could help reduce over-fitting. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "L2 regularization term on weights, increasing this value will make model more conservative.", - ShortName = "l2")] - [TlcModule.Range(Min = 0.0)] - [TGUI(Label = "Lambda(L2)", SuggestedSweeps = "0,0.5,1")] - [TlcModule.SweepableDiscreteParam("RegLambda", new object[] { 0f, 0.5f, 1f })] - public double L2Regularization = 0.01; - - /// - /// The L1 regularization term on weights. - /// - /// - /// Increasing this value could help reduce over-fitting. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "L1 regularization term on weights, increase this value will make model more conservative.", - ShortName = "l1")] - [TlcModule.Range(Min = 0.0)] - [TGUI(Label = "Alpha(L1)", SuggestedSweeps = "0,0.5,1")] - [TlcModule.SweepableDiscreteParam("RegAlpha", new object[] { 0f, 0.5f, 1f })] - public double L1Regularization = 0; - - /// - /// Controls the balance of positive and negative weights in . - /// - /// - /// This is useful for training on unbalanced data. A typical value to consider is sum(negative cases) / sum(positive cases). - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Control the balance of positive and negative weights, useful for unbalanced classes." + - " A typical value to consider: sum(negative cases) / sum(positive cases).", - ShortName = "ScalePosWeight")] - public double WeightOfPositiveExamples = 1; - - internal virtual IBoosterParameter CreateComponent(IHostEnvironment env) => new TreeBooster(this); - - IBoosterParameter IComponentFactory.CreateComponent(IHostEnvironment env) => CreateComponent(env); - } + /// + /// Larger values make the algorithm more conservative. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Minimum loss reduction required to make a further partition on a leaf node of the tree. the larger, " + + "the more conservative the algorithm will be.")] + [TlcModule.Range(Min = 0.0)] + public double MinimumSplitGain = 0; - internal TreeBooster(Options options) - : base(options) - { - Contracts.CheckUserArg(BoosterParameterOptions.MinimumSplitGain >= 0, nameof(BoosterParameterOptions.MinimumSplitGain), "must be >= 0."); - Contracts.CheckUserArg(BoosterParameterOptions.MinimumChildWeight >= 0, nameof(BoosterParameterOptions.MinimumChildWeight), "must be >= 0."); - Contracts.CheckUserArg(BoosterParameterOptions.SubsampleFraction > 0 && BoosterParameterOptions.SubsampleFraction <= 1, nameof(BoosterParameterOptions.SubsampleFraction), "must be in (0,1]."); - Contracts.CheckUserArg(BoosterParameterOptions.FeatureFraction > 0 && BoosterParameterOptions.FeatureFraction <= 1, nameof(BoosterParameterOptions.FeatureFraction), "must be in (0,1]."); - Contracts.CheckUserArg(BoosterParameterOptions.L2Regularization >= 0, nameof(BoosterParameterOptions.L2Regularization), "must be >= 0."); - Contracts.CheckUserArg(BoosterParameterOptions.L1Regularization >= 0, nameof(BoosterParameterOptions.L1Regularization), "must be >= 0."); - Contracts.CheckUserArg(BoosterParameterOptions.WeightOfPositiveExamples > 0, nameof(BoosterParameterOptions.WeightOfPositiveExamples), "must be >= 0."); - } + /// + /// The maximum depth of a tree. + /// + /// + /// 0 means no limit. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Maximum depth of a tree. 0 means no limit. However, tree still grows by best-first.")] + [TlcModule.Range(Min = 0, Max = int.MaxValue)] + public int MaximumTreeDepth = 0; - internal override void UpdateParameters(Dictionary res) - { - base.UpdateParameters(res); - res["boosting_type"] = Name; - } - } + /// + /// The minimum sum of instance weight needed to form a new node. + /// + /// + /// If the tree partition step results in a leaf node with the sum of instance weight less than , + /// the building process will give up further partitioning. In linear regression mode, this simply corresponds to minimum number + /// of instances needed to be in each node. The larger, the more conservative the algorithm will be. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Minimum sum of instance weight(hessian) needed in a child. If the tree partition step results in a leaf " + + "node with the sum of instance weight less than min_child_weight, then the building process will give up further partitioning. In linear regression mode, " + + "this simply corresponds to minimum number of instances needed to be in each node. The larger, the more conservative the algorithm will be.")] + [TlcModule.Range(Min = 0.0)] + public double MinimumChildWeight = 0.1; - /// - /// DART booster (Dropouts meet Multiple Additive Regression Trees) - /// - /// - /// For details, please see here. - /// - public sealed class DartBooster : BoosterParameter - { - internal const string Name = "dart"; - internal const string FriendlyName = "Tree Dropout Tree Booster"; + /// + /// The frequency of performing subsampling (bagging). + /// + /// + /// 0 means disable bagging. N means perform bagging at every N iterations. + /// To enable bagging, should also be set to a value less than 1.0. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample frequency for bagging. 0 means no subsample. " + + "Specifies the frequency at which the bagging occurs, where if this is set to N, the subsampling will happen at every N iterations." + + "This must be set with Subsample as this specifies the amount to subsample.")] + [TlcModule.Range(Min = 0, Max = int.MaxValue)] + public int SubsampleFrequency = 0; /// - /// The options for , used for setting . + /// The fraction of training data used for creating trees. /// - [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Dropouts meet Multiple Additive Regression Trees. See https://arxiv.org/abs/1505.01866")] - public sealed class Options : TreeBooster.Options - { - /// - /// The dropout rate, i.e. the fraction of previous trees to drop during the dropout. - /// - /// - /// Valid range is [0,1]. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "The drop ratio for trees. Range:[0,1].")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double TreeDropFraction = 0.1; - - /// - /// The maximum number of dropped trees in a boosting round. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of dropped trees in a boosting round.")] - [TlcModule.Range(Inf = 0, Max = int.MaxValue)] - public int MaximumNumberOfDroppedTreesPerRound = 1; - - /// - /// The probability of skipping the dropout procedure during a boosting iteration. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Probability for not dropping in a boosting round.")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double SkipDropFraction = 0.5; - - /// - /// Whether to enable xgboost dart mode. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "True will enable xgboost dart mode.")] - public bool XgboostDartMode = false; - - /// - /// Whether to enable uniform drop. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "True will enable uniform drop.")] - public bool UniformDrop = false; - - internal override IBoosterParameter CreateComponent(IHostEnvironment env) => new DartBooster(this); - } + /// + /// Setting it to 0.5 means that LightGBM randomly picks half of the data points to grow trees. + /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample ratio of the training instance. Setting it to 0.5 means that LightGBM randomly collected " + + "half of the data instances to grow trees and this will prevent overfitting. Range: (0,1].")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double SubsampleFraction = 1; - internal DartBooster(Options options) - : base(options) - { - Contracts.CheckUserArg(BoosterParameterOptions.TreeDropFraction > 0 && BoosterParameterOptions.TreeDropFraction < 1, nameof(BoosterParameterOptions.TreeDropFraction), "must be in (0,1)."); - Contracts.CheckUserArg(BoosterParameterOptions.SkipDropFraction >= 0 && BoosterParameterOptions.SkipDropFraction < 1, nameof(BoosterParameterOptions.SkipDropFraction), "must be in [0,1)."); - } + /// + /// The fraction of features used when creating trees. + /// + /// + /// If is smaller than 1.0, LightGBM will randomly select fraction of features to train each tree. + /// For example, if you set it to 0.8, LightGBM will select 80% of features before training each tree. + /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample ratio of columns when constructing each tree. Range: (0,1].", + ShortName = "ff")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double FeatureFraction = 1; - internal override void UpdateParameters(Dictionary res) - { - base.UpdateParameters(res); - res["boosting_type"] = Name; - } + /// + /// The L2 regularization term on weights. + /// + /// + /// Increasing this value could help reduce over-fitting. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "L2 regularization term on weights, increasing this value will make model more conservative.", + ShortName = "l2")] + [TlcModule.Range(Min = 0.0)] + [TGUI(Label = "Lambda(L2)", SuggestedSweeps = "0,0.5,1")] + [TlcModule.SweepableDiscreteParam("RegLambda", new object[] { 0f, 0.5f, 1f })] + public double L2Regularization = 0.01; + + /// + /// The L1 regularization term on weights. + /// + /// + /// Increasing this value could help reduce over-fitting. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "L1 regularization term on weights, increase this value will make model more conservative.", + ShortName = "l1")] + [TlcModule.Range(Min = 0.0)] + [TGUI(Label = "Alpha(L1)", SuggestedSweeps = "0,0.5,1")] + [TlcModule.SweepableDiscreteParam("RegAlpha", new object[] { 0f, 0.5f, 1f })] + public double L1Regularization = 0; + + BoosterParameterBase IComponentFactory.CreateComponent(IHostEnvironment env) + => BuildOptions(); + + BoosterParameterBase IBoosterParameterFactory.CreateComponent(IHostEnvironment env) + => BuildOptions(); + + internal abstract BoosterParameterBase BuildOptions(); } - /// - /// Gradient-based One-Side Sampling booster. - /// - /// - /// For details, please see here. - /// - public sealed class GossBooster : BoosterParameter + internal void UpdateParameters(Dictionary res) { - internal const string Name = "goss"; - internal const string FriendlyName = "Gradient-based One-Size Sampling"; - - [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Gradient-based One-Side Sampling.")] - public sealed class Options : TreeBooster.Options + FieldInfo[] fields = BoosterOptions.GetType().GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); + foreach (var field in fields) { - /// - /// The retain ratio of large gradient data. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Retain ratio for large gradient instances.")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double TopRate = 0.2; - - /// - /// The retain ratio of small gradient data. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Retain ratio for small gradient instances.")] - [TlcModule.Range(Inf = 0.0, Max = 1.0)] - public double OtherRate = 0.1; - - internal override IBoosterParameter CreateComponent(IHostEnvironment env) => new GossBooster(this); - } + var attribute = field.GetCustomAttribute(false); - internal GossBooster(Options options) - : base(options) - { - Contracts.CheckUserArg(BoosterParameterOptions.TopRate > 0 && BoosterParameterOptions.TopRate < 1, nameof(BoosterParameterOptions.TopRate), "must be in (0,1)."); - Contracts.CheckUserArg(BoosterParameterOptions.OtherRate >= 0 && BoosterParameterOptions.OtherRate < 1, nameof(BoosterParameterOptions.TopRate), "must be in [0,1)."); - Contracts.Check(BoosterParameterOptions.TopRate + BoosterParameterOptions.OtherRate <= 1, "Sum of topRate and otherRate cannot be larger than 1."); - } + if (attribute == null) + continue; - internal override void UpdateParameters(Dictionary res) - { - base.UpdateParameters(res); - res["boosting_type"] = Name; + var name = NameMapping.ContainsKey(field.Name) ? NameMapping[field.Name] : LightGbmInterfaceUtils.GetOptionName(field.Name); + res[name] = field.GetValue(BoosterOptions); } } /// - /// The evaluation metrics that are available for . - /// - public enum EvalMetricType - { - DefaultMetric, - Rmse, - Mae, - Logloss, - Error, - Merror, - Mlogloss, - Auc, - Ndcg, - Map - }; - - /// - /// The number of boosting iterations. A new tree is created in each iteration, so this is equivalent to the number of trees. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Number of iterations.", SortOrder = 1, ShortName = "iter")] - [TGUI(Label = "Number of boosting iterations", SuggestedSweeps = "10,20,50,100,150,200")] - [TlcModule.SweepableDiscreteParam("NumBoostRound", new object[] { 10, 20, 50, 100, 150, 200 })] - public int NumberOfIterations = Defaults.NumberOfIterations; - - /// - /// The shrinkage rate for trees, used to prevent over-fitting. - /// - /// - /// Valid range is (0,1]. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Shrinkage rate for trees, used to prevent over-fitting. Range: (0,1].", - SortOrder = 2, ShortName = "lr", NullName = "")] - [TGUI(Label = "Learning Rate", SuggestedSweeps = "0.025-0.4;log")] - [TlcModule.SweepableFloatParamAttribute("LearningRate", 0.025f, 0.4f, isLogScale: true)] - public double? LearningRate; - - /// - /// The maximum number of leaves in one tree. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum leaves for trees.", - SortOrder = 2, ShortName = "nl", NullName = "")] - [TGUI(Description = "The maximum number of leaves per tree", SuggestedSweeps = "2-128;log;inc:4")] - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, isLogScale: true, stepSize: 4)] - public int? NumberOfLeaves; - - /// - /// The minimal number of data points required to form a new tree leaf. + /// Create for supporting legacy infra built upon . /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Minimum number of instances needed in a child.", - SortOrder = 2, ShortName = "mil", NullName = "")] - [TGUI(Label = "Min Documents In Leaves", SuggestedSweeps = "1,10,20,50 ")] - [TlcModule.SweepableDiscreteParamAttribute("MinDataPerLeaf", new object[] { 1, 10, 20, 50 })] - public int? MinimumExampleCountPerLeaf; + internal abstract IBoosterParameterFactory BuildFactory(); + internal abstract string BoosterName { get; } - /// - /// The maximum number of bins that feature values will be bucketed in. - /// - /// - /// The small number of bins may reduce training accuracy but may increase general power (deal with over-fitting). - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of bucket bin for features.", ShortName = "mb")] - public int MaximumBinCountPerFeature = 255; + private protected OptionsBase BoosterOptions; + } - /// - /// Determines which booster to use. - /// - /// - /// Available boosters are , , and . - /// - [Argument(ArgumentType.Multiple, HelpText = "Which booster to use, can be gbtree, gblinear or dart. gbtree and dart use tree based model while gblinear uses linear function.", SortOrder = 3)] - public ISupportBoosterParameterFactory Booster = new TreeBooster.Options(); + /// + /// Gradient boosting decision tree. + /// + /// + /// For details, please see gradient tree boosting. + /// + public sealed class GradientBooster : BoosterParameterBase + { + internal const string Name = "gbdt"; + internal const string FriendlyName = "Tree Booster"; /// - /// Determines whether to output progress status during training and evaluation. + /// The options for , used for setting . /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Verbose", ShortName = "v")] - public bool Verbose = false; + [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Traditional Gradient Boosting Decision Tree.")] + public sealed class Options : OptionsBase + { + internal override BoosterParameterBase BuildOptions() => new GradientBooster(this); + } - /// - /// Controls the logging level in LighGBM. - /// - /// - /// means only output Fatal errors. means output Fatal, Warning, and Info level messages. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Printing running messages.")] - public bool Silent = true; + internal GradientBooster(Options options) + : base(options) + { + } - /// - /// Determines the number of threads used to run LightGBM. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Number of parallel threads used to run LightGBM.", ShortName = "nt")] - public int? NumberOfThreads; + internal override IBoosterParameterFactory BuildFactory() => BoosterOptions; - /// - /// Determines what evaluation metric to use. - /// - [Argument(ArgumentType.AtMostOnce, - HelpText = "Evaluation metrics.", - ShortName = "em")] - public EvalMetricType EvaluationMetric = EvalMetricType.DefaultMetric; + internal override string BoosterName => Name; + } - /// - /// Whether to use softmax loss. Used only by . - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Use softmax loss for the multi classification.")] - [TlcModule.SweepableDiscreteParam("UseSoftmax", new object[] { true, false })] - public bool? UseSoftmax; + /// + /// DART booster (Dropouts meet Multiple Additive Regression Trees) + /// + /// + /// For details, please see here. + /// + public sealed class DartBooster : BoosterParameterBase + { + internal const string Name = "dart"; + internal const string FriendlyName = "Tree Dropout Tree Booster"; /// - /// Determines the number of rounds, after which training will stop if validation metric doesn't improve. + /// The options for , used for setting . /// - /// - /// 0 means disable early stopping. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Rounds of early stopping, 0 will disable it.", - ShortName = "es")] - public int EarlyStoppingRound = 0; + [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Dropouts meet Multiple Additive Regresion Trees. See https://arxiv.org/abs/1505.01866")] + public sealed class Options : OptionsBase + { + static Options() + { + // Add additional name mappings + NameMapping.Add(nameof(TreeDropFraction), "drop_rate"); + NameMapping.Add(nameof(MaximumNumberOfDroppedTreesPerRound), "max_drop"); + NameMapping.Add(nameof(SkipDropFraction), "skip_drop"); + } - /// - /// Comma-separated list of gains associated with each relevance label. Used only by . - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Comma separated list of gains associated to each relevance label.", ShortName = "gains")] - [TGUI(Label = "Ranking Label Gain")] - public string CustomGains = "0,3,7,15,31,63,127,255,511,1023,2047,4095"; + /// + /// The dropout rate, i.e. the fraction of previous trees to drop during the dropout. + /// + /// + /// Valid range is [0,1]. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "The drop ratio for trees. Range:(0,1).")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double TreeDropFraction = 0.1; - /// - /// Parameter for the sigmoid function. Used only by , , and . - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Parameter for the sigmoid function. Used only in " + nameof(LightGbmBinaryTrainer) + ", " + nameof(LightGbmMulticlassTrainer) + - " and in " + nameof(LightGbmRankingTrainer) + ".", ShortName = "sigmoid")] - [TGUI(Label = "Sigmoid", SuggestedSweeps = "0.5,1")] - public double Sigmoid = 0.5; + /// + /// The maximum number of dropped trees in a boosting round. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of dropped trees in a boosting round.")] + [TlcModule.Range(Inf = 0, Max = int.MaxValue)] + public int MaximumNumberOfDroppedTreesPerRound = 1; - /// - /// Number of data points per batch, when loading data. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Number of entries in a batch when loading data.", Hide = true)] - public int BatchSize = 1 << 20; + /// + /// The probability of skipping the dropout procedure during a boosting iteration. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Probability for not dropping in a boosting round.")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double SkipDropFraction = 0.5; - /// - /// Whether to enable categorical split or not. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Enable categorical split or not.", ShortName = "cat")] - [TlcModule.SweepableDiscreteParam("UseCat", new object[] { true, false })] - public bool? UseCategoricalSplit; + /// + /// Whether to enable xgboost dart mode. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "True will enable xgboost dart mode.")] + public bool XgboostDartMode = false; - /// - /// Whether to enable special handling of missing value or not. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Enable special handling of missing value or not.")] - [TlcModule.SweepableDiscreteParam("UseMissing", new object[] { true, false })] - public bool HandleMissingValue = false; + /// + /// Whether to enable uniform drop. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "True will enable uniform drop.")] + public bool UniformDrop = false; - /// - /// The minimum number of data points per categorical group. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Minimum number of instances per categorical group.", ShortName = "mdpg")] - [TlcModule.Range(Inf = 0, Max = int.MaxValue)] - [TlcModule.SweepableDiscreteParam("MinDataPerGroup", new object[] { 10, 50, 100, 200 })] - public int MinimumExampleCountPerGroup = 100; + internal override BoosterParameterBase BuildOptions() => new DartBooster(this); + } - /// - /// When the number of categories of one feature is smaller than or equal to , - /// one-vs-other split algorithm will be used. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of categorical thresholds.", ShortName = "maxcat")] - [TlcModule.Range(Inf = 0, Max = int.MaxValue)] - [TlcModule.SweepableDiscreteParam("MaxCatThreshold", new object[] { 8, 16, 32, 64 })] - public int MaximumCategoricalSplitPointCount = 32; + internal DartBooster(Options options) + :base(options) + { + Contracts.CheckUserArg(options.TreeDropFraction > 0 && options.TreeDropFraction < 1, nameof(options.TreeDropFraction), "must be in (0,1)."); + Contracts.CheckUserArg(options.SkipDropFraction >= 0 && options.SkipDropFraction < 1, nameof(options.SkipDropFraction), "must be in [0,1)."); + BoosterOptions = options; + } - /// - /// Laplace smooth term in categorical feature split. - /// This can reduce the effect of noises in categorical features, especially for categories with few data. - /// - /// - /// Constraints: >= 0.0 - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Laplace smooth term in categorical feature split. Avoid the bias of small categories.")] - [TlcModule.Range(Min = 0.0)] - [TlcModule.SweepableDiscreteParam("CatSmooth", new object[] { 1, 10, 20 })] - public double CategoricalSmoothing = 10; + internal override IBoosterParameterFactory BuildFactory() => BoosterOptions; + internal override string BoosterName => Name; + } - /// - /// L2 regularization for categorical split. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "L2 Regularization for categorical split.")] - [TlcModule.Range(Min = 0.0)] - [TlcModule.SweepableDiscreteParam("CatL2", new object[] { 0.1, 0.5, 1, 5, 10 })] - public double L2CategoricalRegularization = 10; + /// + /// Gradient-based One-Side Sampling booster. + /// + /// + /// For details, please see here. + /// + public sealed class GossBooster : BoosterParameterBase + { + internal const string Name = "goss"; + internal const string FriendlyName = "Gradient-based One-Size Sampling"; /// - /// The random seed for LightGBM to use. + /// The options for , used for setting . /// - /// - /// If not specified, will generate a random seed to be used. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Sets the random seed for LightGBM to use.")] - public int? Seed; - - [Argument(ArgumentType.Multiple, HelpText = "Parallel LightGBM Learning Algorithm", ShortName = "parag")] - internal ISupportParallel ParallelTrainer = new SingleTrainerFactory(); - - internal Dictionary ToDictionary(IHost host) + [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Gradient-based One-Side Sampling.")] + public sealed class Options : OptionsBase { - Contracts.CheckValue(host, nameof(host)); - Contracts.CheckUserArg(MaximumBinCountPerFeature > 0, nameof(MaximumBinCountPerFeature), "must be > 0."); - Contracts.CheckUserArg(Sigmoid > 0, nameof(Sigmoid), "must be > 0."); - Dictionary res = new Dictionary(); - - var boosterParams = Booster.CreateComponent(host); - boosterParams.UpdateParameters(res); - - res[GetOptionName(nameof(MaximumBinCountPerFeature))] = MaximumBinCountPerFeature; + /// + /// The retain ratio of large gradient data. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Retain ratio for large gradient instances.")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double TopRate = 0.2; - res["verbose"] = Silent ? "-1" : "1"; - if (NumberOfThreads.HasValue) - res["nthread"] = NumberOfThreads.Value; + /// + /// The retain ratio of small gradient data. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Retain ratio for small gradient instances.")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double OtherRate = 0.1; - res["seed"] = (Seed.HasValue) ? Seed : host.Rand.Next(); + internal override BoosterParameterBase BuildOptions() => new GossBooster(this); + } - string metric = null; - switch (EvaluationMetric) - { - case EvalMetricType.DefaultMetric: - break; - case EvalMetricType.Mae: - metric = "l1"; - break; - case EvalMetricType.Logloss: - metric = "binary_logloss"; - break; - case EvalMetricType.Error: - metric = "binary_error"; - break; - case EvalMetricType.Merror: - metric = "multi_error"; - break; - case EvalMetricType.Mlogloss: - metric = "multi_logloss"; - break; - case EvalMetricType.Rmse: - case EvalMetricType.Auc: - case EvalMetricType.Ndcg: - case EvalMetricType.Map: - metric = EvaluationMetric.ToString().ToLower(); - break; - } - if (!string.IsNullOrEmpty(metric)) - res[GetOptionName(nameof(metric))] = metric; - res[GetOptionName(nameof(Sigmoid))] = Sigmoid; - res[GetOptionName(nameof(CustomGains))] = CustomGains; - res[GetOptionName(nameof(HandleMissingValue))] = HandleMissingValue; - res[GetOptionName(nameof(MinimumExampleCountPerGroup))] = MinimumExampleCountPerGroup; - res[GetOptionName(nameof(MaximumCategoricalSplitPointCount))] = MaximumCategoricalSplitPointCount; - res[GetOptionName(nameof(CategoricalSmoothing))] = CategoricalSmoothing; - res[GetOptionName(nameof(L2CategoricalRegularization))] = L2CategoricalRegularization; - return res; + internal GossBooster(Options options) + :base(options) + { + Contracts.CheckUserArg(options.TopRate > 0 && options.TopRate < 1, nameof(Options.TopRate), "must be in (0,1)."); + Contracts.CheckUserArg(options.OtherRate >= 0 && options.OtherRate < 1, nameof(Options.OtherRate), "must be in [0,1)."); + Contracts.Check(options.TopRate + options.OtherRate <= 1, "Sum of topRate and otherRate cannot be larger than 1."); + BoosterOptions = options; } + + internal override IBoosterParameterFactory BuildFactory() => BoosterOptions; + internal override string BoosterName => Name; } -} +} \ No newline at end of file diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs index 73f47c097d..5796df75de 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs @@ -2,15 +2,18 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using Microsoft.ML; using Microsoft.ML.Calibrators; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; -[assembly: LoadableClass(LightGbmBinaryTrainer.Summary, typeof(LightGbmBinaryTrainer), typeof(Options), +[assembly: LoadableClass(LightGbmBinaryTrainer.Summary, typeof(LightGbmBinaryTrainer), typeof(LightGbmBinaryTrainer.Options), new[] { typeof(SignatureBinaryClassifierTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, LightGbmBinaryTrainer.UserName, LightGbmBinaryTrainer.LoadNameValue, LightGbmBinaryTrainer.ShortName, DocName = "trainer/LightGBM.md")] @@ -82,7 +85,7 @@ private static IPredictorProducing Create(IHostEnvironment env, ModelLoad /// The for training a boosted decision tree binary classification model using LightGBM. /// /// - public sealed class LightGbmBinaryTrainer : LightGbmTrainerBase>, CalibratedModelParametersBase> { @@ -93,9 +96,79 @@ public sealed class LightGbmBinaryTrainer : LightGbmTrainerBase PredictionKind.BinaryClassification; + public sealed class Options : OptionsBase + { + + public enum EvaluateMetricType + { + None, + Default, + Logloss, + Error, + AreaUnderCurve, + }; + + /// + /// Whether training data is unbalanced. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Use for binary classification when training data is not balanced.", ShortName = "us")] + public bool UnbalancedSets = false; + + /// + /// Controls the balance of positive and negative weights in . + /// + /// + /// This is useful for training on unbalanced data. A typical value to consider is sum(negative cases) / sum(positive cases). + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Control the balance of positive and negative weights, useful for unbalanced classes." + + " A typical value to consider: sum(negative cases) / sum(positive cases).", + ShortName = "ScalePosWeight")] + public double WeightOfPositiveExamples = 1; + + /// + /// Parameter for the sigmoid function. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Parameter for the sigmoid function.", ShortName = "sigmoid")] + [TGUI(Label = "Sigmoid", SuggestedSweeps = "0.5,1")] + public double Sigmoid = 0.5; + + /// + /// Determines what evaluation metric to use. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Evaluation metrics.", + ShortName = "em")] + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Logloss; + + static Options() + { + NameMapping.Add(nameof(EvaluateMetricType), "metric"); + NameMapping.Add(nameof(EvaluateMetricType.None), ""); + NameMapping.Add(nameof(EvaluateMetricType.Logloss), "binary_logloss"); + NameMapping.Add(nameof(EvaluateMetricType.Error), "binary_error"); + NameMapping.Add(nameof(EvaluateMetricType.AreaUnderCurve), "auc"); + NameMapping.Add(nameof(WeightOfPositiveExamples), "scale_pos_weight"); + } + + internal override Dictionary ToDictionary(IHost host) + { + var res = base.ToDictionary(host); + res[GetOptionName(nameof(UnbalancedSets))] = UnbalancedSets; + res[GetOptionName(nameof(WeightOfPositiveExamples))] = WeightOfPositiveExamples; + res[GetOptionName(nameof(Sigmoid))] = Sigmoid; + if (EvaluationMetric != EvaluateMetricType.Default) + res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); + + return res; + } + } + internal LightGbmBinaryTrainer(IHostEnvironment env, Options options) : base(env, LoadNameValue, options, TrainerUtils.MakeBoolScalarLabel(options.LabelColumnName)) { + Contracts.CheckUserArg(options.Sigmoid > 0, nameof(Options.Sigmoid), "must be > 0."); + Contracts.CheckUserArg(options.WeightOfPositiveExamples > 0, nameof(Options.WeightOfPositiveExamples), "must be > 0."); } /// @@ -116,15 +189,25 @@ internal LightGbmBinaryTrainer(IHostEnvironment env, int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Trainers.LightGbm.Options.Defaults.NumberOfIterations) - : base(env, LoadNameValue, TrainerUtils.MakeBoolScalarLabel(labelColumnName), featureColumnName, exampleWeightColumnName, null, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) + : this(env, + new Options() + { + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = exampleWeightColumnName, + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations + }) { } private protected override CalibratedModelParametersBase CreatePredictor() { Host.Check(TrainedEnsemble != null, "The predictor cannot be created before training is complete"); - var innerArgs = LightGbmInterfaceUtils.JoinParameters(Options); + var innerArgs = LightGbmInterfaceUtils.JoinParameters(base.GbmOptions); var pred = new LightGbmBinaryModelParameters(Host, TrainedEnsemble, FeatureCount, innerArgs); var cali = new PlattCalibrator(Host, -0.5, 0); return new FeatureWeightsCalibratedModelParameters(Host, pred, cali); @@ -143,12 +226,7 @@ private protected override void CheckDataValid(IChannel ch, RoleMappedData data) } private protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, RoleMappedData data, float[] labels, int[] groups) - { - Options["objective"] = "binary"; - // Add default metric. - if (!Options.ContainsKey("metric")) - Options["metric"] = "binary_logloss"; - } + => GbmOptions["objective"] = "binary"; private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) { @@ -182,14 +260,14 @@ internal static partial class LightGbm Desc = LightGbmBinaryTrainer.Summary, UserName = LightGbmBinaryTrainer.UserName, ShortName = LightGbmBinaryTrainer.ShortName)] - public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, Options input) + public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, LightGbmBinaryTrainer.Options input) { Contracts.CheckValue(env, nameof(env)); var host = env.Register("TrainLightGBM"); host.CheckValue(input, nameof(input)); EntryPointUtils.CheckInputArgs(host, input); - return TrainerEntryPointsUtils.Train(host, input, + return TrainerEntryPointsUtils.Train(host, input, () => new LightGbmBinaryTrainer(host, input), getLabel: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumnName), getWeight: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.ExampleWeightColumnName)); diff --git a/src/Microsoft.ML.LightGbm/LightGbmCatalog.cs b/src/Microsoft.ML.LightGbm/LightGbmCatalog.cs index e740ee3836..40d5532cfb 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmCatalog.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmCatalog.cs @@ -38,7 +38,7 @@ public static LightGbmRegressionTrainer LightGbm(this RegressionCatalog.Regressi int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -58,7 +58,7 @@ public static LightGbmRegressionTrainer LightGbm(this RegressionCatalog.Regressi /// /// public static LightGbmRegressionTrainer LightGbm(this RegressionCatalog.RegressionTrainers catalog, - Options options) + LightGbmRegressionTrainer.Options options) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -90,7 +90,7 @@ public static LightGbmBinaryTrainer LightGbm(this BinaryClassificationCatalog.Bi int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -110,7 +110,7 @@ public static LightGbmBinaryTrainer LightGbm(this BinaryClassificationCatalog.Bi /// /// public static LightGbmBinaryTrainer LightGbm(this BinaryClassificationCatalog.BinaryClassificationTrainers catalog, - Options options) + LightGbmBinaryTrainer.Options options) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -144,11 +144,12 @@ public static LightGbmRankingTrainer LightGbm(this RankingCatalog.RankingTrainer int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new LightGbmRankingTrainer(env, labelColumnName, featureColumnName, rowGroupColumnName, exampleWeightColumnName, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations); + return new LightGbmRankingTrainer(env, labelColumnName, featureColumnName, rowGroupColumnName, exampleWeightColumnName, + numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations); } /// @@ -164,7 +165,7 @@ public static LightGbmRankingTrainer LightGbm(this RankingCatalog.RankingTrainer /// /// public static LightGbmRankingTrainer LightGbm(this RankingCatalog.RankingTrainers catalog, - Options options) + LightGbmRankingTrainer.Options options) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -196,7 +197,7 @@ public static LightGbmMulticlassTrainer LightGbm(this MulticlassClassificationCa int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Options.Defaults.NumberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); @@ -216,7 +217,7 @@ public static LightGbmMulticlassTrainer LightGbm(this MulticlassClassificationCa /// /// public static LightGbmMulticlassTrainer LightGbm(this MulticlassClassificationCatalog.MulticlassClassificationTrainers catalog, - Options options) + LightGbmMulticlassTrainer.Options options) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs index d4126d65d9..386a6112d7 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs @@ -3,16 +3,19 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Linq; using Microsoft.ML; using Microsoft.ML.Calibrators; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; -[assembly: LoadableClass(LightGbmMulticlassTrainer.Summary, typeof(LightGbmMulticlassTrainer), typeof(Options), +[assembly: LoadableClass(LightGbmMulticlassTrainer.Summary, typeof(LightGbmMulticlassTrainer), typeof(LightGbmMulticlassTrainer.Options), new[] { typeof(SignatureMulticlassClassifierTrainer), typeof(SignatureTrainer) }, "LightGBM Multi-class Classifier", LightGbmMulticlassTrainer.LoadNameValue, LightGbmMulticlassTrainer.ShortName, DocName = "trainer/LightGBM.md")] @@ -22,7 +25,10 @@ namespace Microsoft.ML.Trainers.LightGbm /// The for training a boosted decision tree multi-class classification model using LightGBM. /// /// - public sealed class LightGbmMulticlassTrainer : LightGbmTrainerBase, MulticlassPredictionTransformer, OneVersusAllModelParameters> + public sealed class LightGbmMulticlassTrainer : LightGbmTrainerBase, + MulticlassPredictionTransformer, + OneVersusAllModelParameters> { internal const string Summary = "LightGBM Multi Class Classifier"; internal const string LoadNameValue = "LightGBMMulticlass"; @@ -34,9 +40,61 @@ public sealed class LightGbmMulticlassTrainer : LightGbmTrainerBase PredictionKind.MulticlassClassification; + public sealed class Options : OptionsBase + { + public enum EvaluateMetricType + { + None, + Default, + Error, + LogLoss, + } + + /// + /// Whether to use softmax loss. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Use softmax loss for the multi classification.")] + [TlcModule.SweepableDiscreteParam("UseSoftmax", new object[] { true, false })] + public bool? UseSoftmax; + + /// + /// Parameter for the sigmoid function. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Parameter for the sigmoid function.", ShortName = "sigmoid")] + [TGUI(Label = "Sigmoid", SuggestedSweeps = "0.5,1")] + public double Sigmoid = 0.5; + + /// + /// Determines what evaluation metric to use. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Evaluation metrics.", + ShortName = "em")] + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Error; + + static Options() + { + NameMapping.Add(nameof(EvaluateMetricType), "metric"); + NameMapping.Add(nameof(EvaluateMetricType.Error), "multi_error"); + NameMapping.Add(nameof(EvaluateMetricType.LogLoss), "multi_logloss"); + } + + internal override Dictionary ToDictionary(IHost host) + { + var res = base.ToDictionary(host); + + res[GetOptionName(nameof(Sigmoid))] = Sigmoid; + if(EvaluationMetric != EvaluateMetricType.Default) + res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); + + return res; + } + } + internal LightGbmMulticlassTrainer(IHostEnvironment env, Options options) : base(env, LoadNameValue, options, TrainerUtils.MakeU4ScalarColumn(options.LabelColumnName)) { + Contracts.CheckUserArg(options.Sigmoid > 0, nameof(Options.Sigmoid), "must be > 0."); _numClass = -1; } @@ -58,10 +116,19 @@ internal LightGbmMulticlassTrainer(IHostEnvironment env, int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Trainers.LightGbm.Options.Defaults.NumberOfIterations) - : base(env, LoadNameValue, TrainerUtils.MakeU4ScalarColumn(labelColumnName), featureColumnName, exampleWeightColumnName, null, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) + : this(env, + new Options() + { + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = exampleWeightColumnName, + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations + }) { - _numClass = -1; } private InternalTreeEnsemble GetBinaryEnsemble(int classID) @@ -88,7 +155,7 @@ private protected override OneVersusAllModelParameters CreatePredictor() Host.Assert(_numClass > 1, "Must know the number of classes before creating a predictor."); Host.Assert(TrainedEnsemble.NumTrees % _numClass == 0, "Number of trees should be a multiple of number of classes."); - var innerArgs = LightGbmInterfaceUtils.JoinParameters(Options); + var innerArgs = LightGbmInterfaceUtils.JoinParameters(GbmOptions); IPredictorProducing[] predictors = new IPredictorProducing[_tlcNumClass]; for (int i = 0; i < _tlcNumClass; ++i) { @@ -163,13 +230,13 @@ private protected override void ConvertNaNLabels(IChannel ch, RoleMappedData dat private protected override void GetDefaultParameters(IChannel ch, int numRow, bool hasCategorical, int totalCats, bool hiddenMsg = false) { base.GetDefaultParameters(ch, numRow, hasCategorical, totalCats, true); - int numberOfLeaves = (int)Options["num_leaves"]; + int numberOfLeaves = (int)GbmOptions["num_leaves"]; int minimumExampleCountPerLeaf = LightGbmTrainerOptions.MinimumExampleCountPerLeaf ?? DefaultMinDataPerLeaf(numRow, numberOfLeaves, _numClass); - Options["min_data_per_leaf"] = minimumExampleCountPerLeaf; + GbmOptions["min_data_per_leaf"] = minimumExampleCountPerLeaf; if (!hiddenMsg) { if (!LightGbmTrainerOptions.LearningRate.HasValue) - ch.Info("Auto-tuning parameters: " + nameof(LightGbmTrainerOptions.LearningRate) + " = " + Options["learning_rate"]); + ch.Info("Auto-tuning parameters: " + nameof(LightGbmTrainerOptions.LearningRate) + " = " + GbmOptions["learning_rate"]); if (!LightGbmTrainerOptions.NumberOfLeaves.HasValue) ch.Info("Auto-tuning parameters: " + nameof(LightGbmTrainerOptions.NumberOfLeaves) + " = " + numberOfLeaves); if (!LightGbmTrainerOptions.MinimumExampleCountPerLeaf.HasValue) @@ -182,7 +249,7 @@ private protected override void CheckAndUpdateParametersBeforeTraining(IChannel Host.AssertValue(ch); ch.Assert(PredictionKind == PredictionKind.MulticlassClassification); ch.Assert(_numClass > 1); - Options["num_class"] = _numClass; + GbmOptions["num_class"] = _numClass; bool useSoftmax = false; if (LightGbmTrainerOptions.UseSoftmax.HasValue) @@ -196,13 +263,9 @@ private protected override void CheckAndUpdateParametersBeforeTraining(IChannel } if (useSoftmax) - Options["objective"] = "multiclass"; + GbmOptions["objective"] = "multiclass"; else - Options["objective"] = "multiclassova"; - - // Add default metric. - if (!Options.ContainsKey("metric")) - Options["metric"] = "multi_error"; + GbmOptions["objective"] = "multiclassova"; } private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) @@ -240,14 +303,14 @@ internal static partial class LightGbm Desc = "Train a LightGBM multi class model.", UserName = LightGbmMulticlassTrainer.Summary, ShortName = LightGbmMulticlassTrainer.ShortName)] - public static CommonOutputs.MulticlassClassificationOutput TrainMulticlass(IHostEnvironment env, Options input) + public static CommonOutputs.MulticlassClassificationOutput TrainMulticlass(IHostEnvironment env, LightGbmMulticlassTrainer.Options input) { Contracts.CheckValue(env, nameof(env)); var host = env.Register("TrainLightGBM"); host.CheckValue(input, nameof(input)); EntryPointUtils.CheckInputArgs(host, input); - return TrainerEntryPointsUtils.Train(host, input, + return TrainerEntryPointsUtils.Train(host, input, () => new LightGbmMulticlassTrainer(host, input), getLabel: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumnName), getWeight: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.ExampleWeightColumnName)); diff --git a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs index 6c6cc75dd5..0c8f183740 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs @@ -3,14 +3,17 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using Microsoft.ML; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; -[assembly: LoadableClass(LightGbmRankingTrainer.UserName, typeof(LightGbmRankingTrainer), typeof(Options), +[assembly: LoadableClass(LightGbmRankingTrainer.UserName, typeof(LightGbmRankingTrainer), typeof(LightGbmRankingTrainer.Options), new[] { typeof(SignatureRankerTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, "LightGBM Ranking", LightGbmRankingTrainer.LoadNameValue, LightGbmRankingTrainer.ShortName, DocName = "trainer/LightGBM.md")] @@ -45,7 +48,6 @@ private static VersionInfo GetVersionInfo() private protected override uint VerDefaultValueSerialized => 0x00010004; private protected override uint VerCategoricalSplitSerialized => 0x00010005; private protected override PredictionKind PredictionKind => PredictionKind.Ranking; - internal LightGbmRankingModelParameters(IHostEnvironment env, InternalTreeEnsemble trainedEnsemble, int featureCount, string innerArgs) : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs) { @@ -72,7 +74,10 @@ private static LightGbmRankingModelParameters Create(IHostEnvironment env, Model /// The for training a boosted decision tree ranking model using LightGBM. /// /// - public sealed class LightGbmRankingTrainer : LightGbmTrainerBase, LightGbmRankingModelParameters> + public sealed class LightGbmRankingTrainer : LightGbmTrainerBase, + LightGbmRankingModelParameters> { internal const string UserName = "LightGBM Ranking"; internal const string LoadNameValue = "LightGBMRanking"; @@ -80,9 +85,63 @@ public sealed class LightGbmRankingTrainer : LightGbmTrainerBase PredictionKind.Ranking; + public sealed class Options : OptionsBase + { + public enum EvaluateMetricType + { + None, + Default, + MeanAveragedPrecision, + NormalizedDiscountedCumulativeGain + }; + + /// + /// Comma-separated list of gains associated with each relevance label. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "An array of gains associated to each relevance label.", ShortName = "gains")] + [TGUI(Label = "Ranking Label Gain")] + public int[] CustomGains = { 0,3,7,15,31,63,127,255,511,1023,2047,4095 }; + + /// + /// Parameter for the sigmoid function. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Parameter for the sigmoid function.", ShortName = "sigmoid")] + [TGUI(Label = "Sigmoid", SuggestedSweeps = "0.5,1")] + public double Sigmoid = 0.5; + + /// + /// Determines what evaluation metric to use. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Evaluation metrics.", + ShortName = "em")] + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.NormalizedDiscountedCumulativeGain; + + static Options() + { + NameMapping.Add(nameof(CustomGains), "label_gain"); + NameMapping.Add(nameof(EvaluateMetricType), "metric"); + NameMapping.Add(nameof(EvaluateMetricType.None), ""); + NameMapping.Add(nameof(EvaluateMetricType.MeanAveragedPrecision), "map"); + NameMapping.Add(nameof(EvaluateMetricType.NormalizedDiscountedCumulativeGain), "ndcg"); + } + + internal override Dictionary ToDictionary(IHost host) + { + var res = base.ToDictionary(host); + res[GetOptionName(nameof(Sigmoid))] = Sigmoid; + res[GetOptionName(nameof(CustomGains))] = string.Join(",",CustomGains); + if(EvaluationMetric != EvaluateMetricType.Default) + res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); + + return res; + } + } + internal LightGbmRankingTrainer(IHostEnvironment env, Options options) : base(env, LoadNameValue, options, TrainerUtils.MakeR4ScalarColumn(options.LabelColumnName)) { + Contracts.CheckUserArg(options.Sigmoid > 0, nameof(Options.Sigmoid), "must be > 0."); } /// @@ -105,10 +164,19 @@ internal LightGbmRankingTrainer(IHostEnvironment env, int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Trainers.LightGbm.Options.Defaults.NumberOfIterations) - : base(env, LoadNameValue, TrainerUtils.MakeR4ScalarColumn(labelColumnName), - featureColumnName, weightsColumnName, rowGroupdColumnName, numberOfLeaves, - minimumExampleCountPerLeaf, learningRate, numberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) + : this(env, + new Options() + { + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = weightsColumnName, + RowGroupColumnName = rowGroupdColumnName, + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations + }) { Host.CheckNonEmpty(rowGroupdColumnName, nameof(rowGroupdColumnName)); } @@ -152,20 +220,18 @@ private protected override void CheckLabelCompatible(SchemaShape.Column labelCol private protected override LightGbmRankingModelParameters CreatePredictor() { Host.Check(TrainedEnsemble != null, "The predictor cannot be created before training is complete"); - var innerArgs = LightGbmInterfaceUtils.JoinParameters(Options); + var innerArgs = LightGbmInterfaceUtils.JoinParameters(GbmOptions); return new LightGbmRankingModelParameters(Host, TrainedEnsemble, FeatureCount, innerArgs); } private protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, RoleMappedData data, float[] labels, int[] groups) { Host.AssertValue(ch); - Options["objective"] = "lambdarank"; + GbmOptions["objective"] = "lambdarank"; ch.CheckValue(groups, nameof(groups)); - // Add default metric. - if (!Options.ContainsKey("metric")) - Options["metric"] = "ndcg"; + // Only output one ndcg score. - Options["eval_at"] = "5"; + GbmOptions["eval_at"] = "5"; } private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) @@ -196,14 +262,14 @@ internal static partial class LightGbm Desc = "Train a LightGBM ranking model.", UserName = LightGbmRankingTrainer.UserName, ShortName = LightGbmRankingTrainer.ShortName)] - public static CommonOutputs.RankingOutput TrainRanking(IHostEnvironment env, Options input) + public static CommonOutputs.RankingOutput TrainRanking(IHostEnvironment env, LightGbmRankingTrainer.Options input) { Contracts.CheckValue(env, nameof(env)); var host = env.Register("TrainLightGBM"); host.CheckValue(input, nameof(input)); EntryPointUtils.CheckInputArgs(host, input); - return TrainerEntryPointsUtils.Train(host, input, + return TrainerEntryPointsUtils.Train(host, input, () => new LightGbmRankingTrainer(host, input), getLabel: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumnName), getWeight: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.ExampleWeightColumnName), diff --git a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs index c9064f6cba..3a3843c106 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs @@ -2,14 +2,16 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using Microsoft.ML; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.EntryPoints; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Trainers.LightGbm; -[assembly: LoadableClass(LightGbmRegressionTrainer.Summary, typeof(LightGbmRegressionTrainer), typeof(Options), +[assembly: LoadableClass(LightGbmRegressionTrainer.Summary, typeof(LightGbmRegressionTrainer), typeof(LightGbmRegressionTrainer.Options), new[] { typeof(SignatureRegressorTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, LightGbmRegressionTrainer.UserNameValue, LightGbmRegressionTrainer.LoadNameValue, LightGbmRegressionTrainer.ShortName, DocName = "trainer/LightGBM.md")] @@ -74,7 +76,10 @@ private static LightGbmRegressionModelParameters Create(IHostEnvironment env, Mo /// The for training a boosted decision tree regression model using LightGBM. /// /// - public sealed class LightGbmRegressionTrainer : LightGbmTrainerBase, LightGbmRegressionModelParameters> + public sealed class LightGbmRegressionTrainer : LightGbmTrainerBase, + LightGbmRegressionModelParameters> { internal const string Summary = "LightGBM Regression"; internal const string LoadNameValue = "LightGBMRegression"; @@ -83,6 +88,44 @@ public sealed class LightGbmRegressionTrainer : LightGbmTrainerBase PredictionKind.Regression; + public sealed class Options : OptionsBase + { + public enum EvaluateMetricType + { + None, + Default, + MeanAbsoluteError, + RootMeanSquaredError, + MeanSquaredError + }; + + /// + /// Determines what evaluation metric to use. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Evaluation metrics.", + ShortName = "em")] + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError; + + static Options() + { + NameMapping.Add(nameof(EvaluateMetricType), "metric"); + NameMapping.Add(nameof(EvaluateMetricType.None), ""); + NameMapping.Add(nameof(EvaluateMetricType.MeanAbsoluteError), "mae"); + NameMapping.Add(nameof(EvaluateMetricType.RootMeanSquaredError), "rmse"); + NameMapping.Add(nameof(EvaluateMetricType.MeanSquaredError), "mse"); + } + + internal override Dictionary ToDictionary(IHost host) + { + var res = base.ToDictionary(host); + if (EvaluationMetric != EvaluateMetricType.Default) + res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); + + return res; + } + } + /// /// Initializes a new instance of /// @@ -101,8 +144,17 @@ internal LightGbmRegressionTrainer(IHostEnvironment env, int? numberOfLeaves = null, int? minimumExampleCountPerLeaf = null, double? learningRate = null, - int numberOfIterations = Trainers.LightGbm.Options.Defaults.NumberOfIterations) - : base(env, LoadNameValue, TrainerUtils.MakeR4ScalarColumn(labelColumnName), featureColumnName, exampleWeightColumnName, null, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations) + int numberOfIterations = Defaults.NumberOfIterations) + : this(env, new Options() + { + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = exampleWeightColumnName, + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations + }) { } @@ -115,7 +167,7 @@ private protected override LightGbmRegressionModelParameters CreatePredictor() { Host.Check(TrainedEnsemble != null, "The predictor cannot be created before training is complete"); - var innerArgs = LightGbmInterfaceUtils.JoinParameters(Options); + var innerArgs = LightGbmInterfaceUtils.JoinParameters(GbmOptions); return new LightGbmRegressionModelParameters(Host, TrainedEnsemble, FeatureCount, innerArgs); } @@ -133,10 +185,7 @@ private protected override void CheckDataValid(IChannel ch, RoleMappedData data) private protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, RoleMappedData data, float[] labels, int[] groups) { - Options["objective"] = "regression"; - // Add default metric. - if (!Options.ContainsKey("metric")) - Options["metric"] = "l2"; + GbmOptions["objective"] = "regression"; } private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) @@ -167,14 +216,14 @@ internal static partial class LightGbm Desc = LightGbmRegressionTrainer.Summary, UserName = LightGbmRegressionTrainer.UserNameValue, ShortName = LightGbmRegressionTrainer.ShortName)] - public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Options input) + public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, LightGbmRegressionTrainer.Options input) { Contracts.CheckValue(env, nameof(env)); var host = env.Register("TrainLightGBM"); host.CheckValue(input, nameof(input)); EntryPointUtils.CheckInputArgs(host, input); - return TrainerEntryPointsUtils.Train(host, input, + return TrainerEntryPointsUtils.Train(host, input, () => new LightGbmRegressionTrainer(host, input), getLabel: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumnName), getWeight: () => TrainerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.ExampleWeightColumnName)); diff --git a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs index e1992f1b63..bdaca7a1b7 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs @@ -4,7 +4,11 @@ using System; using System.Collections.Generic; +using System.Text; +using Microsoft.ML.CommandLine; using Microsoft.ML.Data; +using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; @@ -12,6 +16,12 @@ namespace Microsoft.ML.Trainers.LightGbm { + [BestFriend] + internal static class Defaults + { + public const int NumberOfIterations = 100; + } + /// /// Lock for LightGBM trainer. /// @@ -26,10 +36,239 @@ internal static class LightGbmShared /// /// Base class for all training with LightGBM. /// - public abstract class LightGbmTrainerBase : TrainerEstimatorBaseWithGroupId + public abstract class LightGbmTrainerBase : TrainerEstimatorBaseWithGroupId where TTransformer : ISingleFeaturePredictionTransformer where TModel : class + where TOptions : LightGbmTrainerBase.OptionsBase, new() { + public class OptionsBase : TrainerInputBaseWithGroupId + { + // Static override name map that maps friendly names to lightGBM arguments. + // If an argument is not here, then its name is identicaltto a lightGBM argument + // and does not require a mapping, for example, Subsample. + private protected static Dictionary NameMapping = new Dictionary() + { + {nameof(MinimumExampleCountPerLeaf), "min_data_per_leaf"}, + {nameof(NumberOfLeaves), "num_leaves"}, + {nameof(MaximumBinCountPerFeature), "max_bin" }, + {nameof(MinimumExampleCountPerGroup), "min_data_per_group" }, + {nameof(MaximumCategoricalSplitPointCount), "max_cat_threshold" }, + {nameof(CategoricalSmoothing), "cat_smooth" }, + {nameof(L2CategoricalRegularization), "cat_l2" }, + {nameof(HandleMissingValue), "use_missing" } + }; + + private protected string GetOptionName(string name) + { + if (NameMapping.ContainsKey(name)) + return NameMapping[name]; + return LightGbmInterfaceUtils.GetOptionName(name); + } + + private protected OptionsBase() { } + + /// + /// The number of boosting iterations. A new tree is created in each iteration, so this is equivalent to the number of trees. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Number of iterations.", SortOrder = 1, ShortName = "iter")] + [TGUI(Label = "Number of boosting iterations", SuggestedSweeps = "10,20,50,100,150,200")] + [TlcModule.SweepableDiscreteParam("NumBoostRound", new object[] { 10, 20, 50, 100, 150, 200 })] + public int NumberOfIterations = Defaults.NumberOfIterations; + + /// + /// The shrinkage rate for trees, used to prevent over-fitting. + /// + /// + /// Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Shrinkage rate for trees, used to prevent over-fitting. Range: (0,1].", + SortOrder = 2, ShortName = "lr", NullName = "")] + [TGUI(Label = "Learning Rate", SuggestedSweeps = "0.025-0.4;log")] + [TlcModule.SweepableFloatParamAttribute("LearningRate", 0.025f, 0.4f, isLogScale: true)] + public double? LearningRate; + + /// + /// The maximum number of leaves in one tree. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum leaves for trees.", + SortOrder = 2, ShortName = "nl", NullName = "")] + [TGUI(Description = "The maximum number of leaves per tree", SuggestedSweeps = "2-128;log;inc:4")] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, isLogScale: true, stepSize: 4)] + public int? NumberOfLeaves; + + /// + /// The minimal number of data points required to form a new tree leaf. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Minimum number of instances needed in a child.", + SortOrder = 2, ShortName = "mil", NullName = "")] + [TGUI(Label = "Min Documents In Leaves", SuggestedSweeps = "1,10,20,50 ")] + [TlcModule.SweepableDiscreteParamAttribute("MinDataPerLeaf", new object[] { 1, 10, 20, 50 })] + public int? MinimumExampleCountPerLeaf; + + /// + /// The maximum number of bins that feature values will be bucketed in. + /// + /// + /// The small number of bins may reduce training accuracy but may increase general power (deal with over-fitting). + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of bucket bin for features.", ShortName = "mb")] + public int MaximumBinCountPerFeature = 255; + + /// + /// Determines which booster to use. + /// + /// + /// Available boosters are , , and . + /// + [Argument(ArgumentType.Multiple, + HelpText = "Which booster to use, can be gbtree, gblinear or dart. gbtree and dart use tree based model while gblinear uses linear function.", + Name="Booster", + SortOrder = 3)] + internal IBoosterParameterFactory BoosterFactory = new GradientBooster.Options(); + + /// + /// Determines whether to output progress status during training and evaluation. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Verbose", ShortName = "v")] + public bool Verbose = false; + + /// + /// Controls the logging level in LighGBM. + /// + /// + /// means only output Fatal errors. means output Fatal, Warning, and Info level messages. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Printing running messages.")] + public bool Silent = true; + + /// + /// Determines the number of threads used to run LightGBM. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Number of parallel threads used to run LightGBM.", ShortName = "nt")] + public int? NumberOfThreads; + + /// + /// Determines the number of rounds, after which training will stop if validation metric doesn't improve. + /// + /// + /// 0 means disable early stopping. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Rounds of early stopping, 0 will disable it.", + ShortName = "es")] + public int EarlyStoppingRound = 0; + + /// + /// Number of data points per batch, when loading data. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Number of entries in a batch when loading data.", Hide = true)] + public int BatchSize = 1 << 20; + + /// + /// Whether to enable categorical split or not. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Enable categorical split or not.", ShortName = "cat")] + [TlcModule.SweepableDiscreteParam("UseCat", new object[] { true, false })] + public bool? UseCategoricalSplit; + + /// + /// Whether to enable special handling of missing value or not. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Enable special handling of missing value or not.")] + [TlcModule.SweepableDiscreteParam("UseMissing", new object[] { true, false })] + public bool HandleMissingValue = true; + + /// + /// The minimum number of data points per categorical group. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Minimum number of instances per categorical group.", ShortName = "mdpg")] + [TlcModule.Range(Inf = 0, Max = int.MaxValue)] + [TlcModule.SweepableDiscreteParam("MinDataPerGroup", new object[] { 10, 50, 100, 200 })] + public int MinimumExampleCountPerGroup = 100; + + /// + /// When the number of categories of one feature is smaller than or equal to , + /// one-vs-other split algorithm will be used. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Max number of categorical thresholds.", ShortName = "maxcat")] + [TlcModule.Range(Inf = 0, Max = int.MaxValue)] + [TlcModule.SweepableDiscreteParam("MaxCatThreshold", new object[] { 8, 16, 32, 64 })] + public int MaximumCategoricalSplitPointCount = 32; + + /// + /// Laplace smooth term in categorical feature split. + /// This can reduce the effect of noises in categorical features, especially for categories with few data. + /// + /// + /// Constraints: >= 0.0 + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Lapalace smooth term in categorical feature spilt. Avoid the bias of small categories.")] + [TlcModule.Range(Min = 0.0)] + [TlcModule.SweepableDiscreteParam("CatSmooth", new object[] { 1, 10, 20 })] + public double CategoricalSmoothing = 10; + + /// + /// L2 regularization for categorical split. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "L2 Regularization for categorical split.")] + [TlcModule.Range(Min = 0.0)] + [TlcModule.SweepableDiscreteParam("CatL2", new object[] { 0.1, 0.5, 1, 5, 10 })] + public double L2CategoricalRegularization = 10; + + /// + /// The random seed for LightGBM to use. + /// + /// + /// If not specified, will generate a random seed to be used. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Sets the random seed for LightGBM to use.")] + public int? Seed; + + [Argument(ArgumentType.Multiple, HelpText = "Parallel LightGBM Learning Algorithm", ShortName = "parag")] + internal ISupportParallel ParallelTrainer = new SingleTrainerFactory(); + + private BoosterParameterBase.OptionsBase _boosterParameter; + + /// + /// Booster parameter to use + /// + public BoosterParameterBase.OptionsBase Booster + { + get => _boosterParameter; + + set + { + _boosterParameter = value; + BoosterFactory = _boosterParameter; + } + } + + internal virtual Dictionary ToDictionary(IHost host) + { + Contracts.CheckValue(host, nameof(host)); + Dictionary res = new Dictionary(); + + var boosterParams = BoosterFactory.CreateComponent(host); + boosterParams.UpdateParameters(res); + res["boosting_type"] = boosterParams.BoosterName; + + res["verbose"] = Silent ? "-1" : "1"; + if (NumberOfThreads.HasValue) + res["nthread"] = NumberOfThreads.Value; + + res["seed"] = (Seed.HasValue) ? Seed : host.Rand.Next(); + + res[GetOptionName(nameof(MaximumBinCountPerFeature))] = MaximumBinCountPerFeature; + res[GetOptionName(nameof(HandleMissingValue))] = HandleMissingValue; + res[GetOptionName(nameof(MinimumExampleCountPerGroup))] = MinimumExampleCountPerGroup; + res[GetOptionName(nameof(MaximumCategoricalSplitPointCount))] = MaximumCategoricalSplitPointCount; + res[GetOptionName(nameof(CategoricalSmoothing))] = CategoricalSmoothing; + res[GetOptionName(nameof(L2CategoricalRegularization))] = L2CategoricalRegularization; + + return res; + } + } + private sealed class CategoricalMetaData { public int NumCol; @@ -40,14 +279,16 @@ private sealed class CategoricalMetaData public bool[] IsCategoricalFeature; } - private protected readonly Options LightGbmTrainerOptions; + // Contains the passed in options when the API is called + private protected readonly TOptions LightGbmTrainerOptions; /// /// Stores argumments as objects to convert them to invariant string type in the end so that /// the code is culture agnostic. When retrieving key value from this dictionary as string /// please convert to string invariant by string.Format(CultureInfo.InvariantCulture, "{0}", Option[key]). /// - private protected Dictionary Options; + private protected Dictionary GbmOptions; + private protected IParallel ParallelTraining; // Store _featureCount and _trainedEnsemble to construct predictor. @@ -67,29 +308,32 @@ private protected LightGbmTrainerBase(IHostEnvironment env, int? minimumExampleCountPerLeaf, double? learningRate, int numberOfIterations) - : base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(featureColumnName), - labelColumn, TrainerUtils.MakeR4ScalarWeightColumn(exampleWeightColumnName), TrainerUtils.MakeU4ScalarColumn(rowGroupColumnName)) + : this(env, name, new TOptions() + { + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations, + LabelColumnName = labelColumn.Name, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = exampleWeightColumnName, + RowGroupColumnName = rowGroupColumnName + }, + labelColumn) { - LightGbmTrainerOptions = new Options(); - - LightGbmTrainerOptions.NumberOfLeaves = numberOfLeaves; - LightGbmTrainerOptions.MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf; - LightGbmTrainerOptions.LearningRate = learningRate; - LightGbmTrainerOptions.NumberOfIterations = numberOfIterations; - - LightGbmTrainerOptions.LabelColumnName = labelColumn.Name; - LightGbmTrainerOptions.FeatureColumnName = featureColumnName; - LightGbmTrainerOptions.ExampleWeightColumnName = exampleWeightColumnName; - LightGbmTrainerOptions.RowGroupColumnName = rowGroupColumnName; - - InitParallelTraining(); } - private protected LightGbmTrainerBase(IHostEnvironment env, string name, Options options, SchemaShape.Column label) + private protected LightGbmTrainerBase(IHostEnvironment env, string name, TOptions options, SchemaShape.Column label) : base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(options.FeatureColumnName), label, TrainerUtils.MakeR4ScalarWeightColumn(options.ExampleWeightColumnName), TrainerUtils.MakeU4ScalarColumn(options.RowGroupColumnName)) { Host.CheckValue(options, nameof(options)); + Contracts.CheckUserArg(options.NumberOfIterations >= 0, nameof(options.NumberOfIterations), "must be >= 0."); + Contracts.CheckUserArg(options.MaximumBinCountPerFeature > 0, nameof(options.MaximumBinCountPerFeature), "must be > 0."); + Contracts.CheckUserArg(options.MinimumExampleCountPerGroup > 0, nameof(options.MinimumExampleCountPerGroup), "must be > 0."); + Contracts.CheckUserArg(options.MaximumCategoricalSplitPointCount > 0, nameof(options.MaximumCategoricalSplitPointCount), "must be > 0."); + Contracts.CheckUserArg(options.CategoricalSmoothing >= 0, nameof(options.CategoricalSmoothing), "must be >= 0."); + Contracts.CheckUserArg(options.L2CategoricalRegularization >= 0.0, nameof(options.L2CategoricalRegularization), "must be >= 0."); LightGbmTrainerOptions = options; InitParallelTraining(); @@ -130,17 +374,17 @@ private protected override TModel TrainModelCore(TrainContext context) private void InitParallelTraining() { - Options = LightGbmTrainerOptions.ToDictionary(Host); + GbmOptions = LightGbmTrainerOptions.ToDictionary(Host); ParallelTraining = LightGbmTrainerOptions.ParallelTrainer != null ? LightGbmTrainerOptions.ParallelTrainer.CreateComponent(Host) : new SingleTrainer(); if (ParallelTraining.ParallelType() != "serial" && ParallelTraining.NumMachines() > 1) { - Options["tree_learner"] = ParallelTraining.ParallelType(); + GbmOptions["tree_learner"] = ParallelTraining.ParallelType(); var otherParams = ParallelTraining.AdditionalParams(); if (otherParams != null) { foreach (var pair in otherParams) - Options[pair.Key] = pair.Value; + GbmOptions[pair.Key] = pair.Value; } Contracts.CheckValue(ParallelTraining.GetReduceScatterFunction(), nameof(ParallelTraining.GetReduceScatterFunction)); @@ -166,14 +410,14 @@ private protected virtual void CheckDataValid(IChannel ch, RoleMappedData data) ch.CheckParam(data.Schema.Label.HasValue, nameof(data), "Need a label column"); } - private protected virtual void GetDefaultParameters(IChannel ch, int numRow, bool hasCategarical, int totalCats, bool hiddenMsg = false) + private protected virtual void GetDefaultParameters(IChannel ch, int numRow, bool hasCategorical, int totalCats, bool hiddenMsg = false) { - double learningRate = LightGbmTrainerOptions.LearningRate ?? DefaultLearningRate(numRow, hasCategarical, totalCats); - int numberOfLeaves = LightGbmTrainerOptions.NumberOfLeaves ?? DefaultNumLeaves(numRow, hasCategarical, totalCats); + double learningRate = LightGbmTrainerOptions.LearningRate ?? DefaultLearningRate(numRow, hasCategorical, totalCats); + int numberOfLeaves = LightGbmTrainerOptions.NumberOfLeaves ?? DefaultNumLeaves(numRow, hasCategorical, totalCats); int minimumExampleCountPerLeaf = LightGbmTrainerOptions.MinimumExampleCountPerLeaf ?? DefaultMinDataPerLeaf(numRow, numberOfLeaves, 1); - Options["learning_rate"] = learningRate; - Options["num_leaves"] = numberOfLeaves; - Options["min_data_per_leaf"] = minimumExampleCountPerLeaf; + GbmOptions["learning_rate"] = learningRate; + GbmOptions["num_leaves"] = numberOfLeaves; + GbmOptions["min_data_per_leaf"] = minimumExampleCountPerLeaf; if (!hiddenMsg) { if (!LightGbmTrainerOptions.LearningRate.HasValue) @@ -186,7 +430,7 @@ private protected virtual void GetDefaultParameters(IChannel ch, int numRow, boo } [BestFriend] - internal Dictionary GetGbmParameters() => Options; + internal Dictionary GetGbmParameters() => GbmOptions; private FloatLabelCursor.Factory CreateCursorFactory(RoleMappedData data) { @@ -297,7 +541,7 @@ private CategoricalMetaData GetCategoricalMetaData(IChannel ch, RoleMappedData t { var catIndices = ConstructCategoricalFeatureMetaData(categoricalFeatures, rawNumCol, ref catMetaData); // Set categorical features - Options["categorical_feature"] = string.Join(",", catIndices); + GbmOptions["categorical_feature"] = string.Join(",", catIndices); } return catMetaData; } @@ -316,9 +560,10 @@ private Dataset LoadTrainingData(IChannel ch, RoleMappedData trainData, out Cate catMetaData = GetCategoricalMetaData(ch, trainData, numRow); GetDefaultParameters(ch, numRow, catMetaData.CategoricalBoudaries != null, catMetaData.TotalCats); - Dataset dtrain; - string param = LightGbmInterfaceUtils.JoinParameters(Options); + CheckAndUpdateParametersBeforeTraining(ch, trainData, labels, groups); + string param = LightGbmInterfaceUtils.JoinParameters(GbmOptions); + Dataset dtrain; // To reduce peak memory usage, only enable one sampling task at any given time. lock (LightGbmShared.SampleLock) { @@ -329,8 +574,6 @@ private Dataset LoadTrainingData(IChannel ch, RoleMappedData trainData, out Cate // Push rows into dataset. LoadDataset(ch, factory, dtrain, numRow, LightGbmTrainerOptions.BatchSize, catMetaData); - // Some checks. - CheckAndUpdateParametersBeforeTraining(ch, trainData, labels, groups); return dtrain; } @@ -362,15 +605,16 @@ private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, Catego Host.AssertValue(pch); Host.AssertValue(dtrain); Host.AssertValueOrNull(dvalid); + // For multi class, the number of labels is required. - ch.Assert(((ITrainer)this).PredictionKind != PredictionKind.MulticlassClassification || Options.ContainsKey("num_class"), + ch.Assert(((ITrainer)this).PredictionKind != PredictionKind.MulticlassClassification || GbmOptions.ContainsKey("num_class"), "LightGBM requires the number of classes to be specified in the parameters."); // Only enable one trainer to run at one time. lock (LightGbmShared.LockForMultiThreadingInside) { - ch.Info("LightGBM objective={0}", Options["objective"]); - using (Booster bst = WrappedLightGbmTraining.Train(ch, pch, Options, dtrain, + ch.Info("LightGBM objective={0}", GbmOptions["objective"]); + using (Booster bst = WrappedLightGbmTraining.Train(ch, pch, GbmOptions, dtrain, dvalid: dvalid, numIteration: LightGbmTrainerOptions.NumberOfIterations, verboseEval: LightGbmTrainerOptions.Verbose, earlyStoppingRound: LightGbmTrainerOptions.EarlyStoppingRound)) { diff --git a/src/Microsoft.ML.LightGbm/WrappedLightGbmInterface.cs b/src/Microsoft.ML.LightGbm/WrappedLightGbmInterface.cs index db72a689b6..aea7bd4162 100644 --- a/src/Microsoft.ML.LightGbm/WrappedLightGbmInterface.cs +++ b/src/Microsoft.ML.LightGbm/WrappedLightGbmInterface.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Globalization; using System.Runtime.InteropServices; +using System.Text; using Microsoft.ML.Runtime; namespace Microsoft.ML.Trainers.LightGbm @@ -225,6 +226,32 @@ public static string JoinParameters(Dictionary parameters) return string.Join(" ", res); } + /// + /// Helper function used for generating the LightGbm argument name. + /// When given a name, this will convert the name to lower-case with underscores. + /// The underscore will be placed when an upper-case letter is encountered. + /// + public static string GetOptionName(string name) + { + // Otherwise convert the name to the light gbm argument + StringBuilder strBuf = new StringBuilder(); + bool first = true; + foreach (char c in name) + { + if (char.IsUpper(c)) + { + if (first) + first = false; + else + strBuf.Append('_'); + strBuf.Append(char.ToLower(c)); + } + else + strBuf.Append(c); + } + return strBuf.ToString(); + } + /// /// Convert the pointer of c string to c# string. /// diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index af4afd7cfc..a23c9b29ae 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -53,10 +53,10 @@ Trainers.FieldAwareFactorizationMachineBinaryClassifier Train a field-aware fact Trainers.GeneralizedAdditiveModelBinaryClassifier Trains a gradient boosted stump per feature, on all features simultaneously, to fit target values using least-squares. It mantains no interactions between features. Microsoft.ML.Trainers.FastTree.Gam TrainBinary Microsoft.ML.Trainers.FastTree.GamBinaryTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.GeneralizedAdditiveModelRegressor Trains a gradient boosted stump per feature, on all features simultaneously, to fit target values using least-squares. It mantains no interactions between features. Microsoft.ML.Trainers.FastTree.Gam TrainRegression Microsoft.ML.Trainers.FastTree.GamRegressionTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput Trainers.KMeansPlusPlusClusterer K-means is a popular clustering algorithm. With K-means, the data is clustered into a specified number of clusters in order to minimize the within-cluster sum of squares. K-means++ improves upon K-means by using a better method for choosing the initial cluster centers. Microsoft.ML.Trainers.KMeansTrainer TrainKMeans Microsoft.ML.Trainers.KMeansTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+ClusteringOutput -Trainers.LightGbmBinaryClassifier Train a LightGBM binary classification model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainBinary Microsoft.ML.Trainers.LightGbm.Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput -Trainers.LightGbmClassifier Train a LightGBM multi class model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainMulticlass Microsoft.ML.Trainers.LightGbm.Options Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput -Trainers.LightGbmRanker Train a LightGBM ranking model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainRanking Microsoft.ML.Trainers.LightGbm.Options Microsoft.ML.EntryPoints.CommonOutputs+RankingOutput -Trainers.LightGbmRegressor LightGBM Regression Microsoft.ML.Trainers.LightGbm.LightGbm TrainRegression Microsoft.ML.Trainers.LightGbm.Options Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput +Trainers.LightGbmBinaryClassifier Train a LightGBM binary classification model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainBinary Microsoft.ML.Trainers.LightGbm.LightGbmBinaryTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput +Trainers.LightGbmClassifier Train a LightGBM multi class model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainMulticlass Microsoft.ML.Trainers.LightGbm.LightGbmMulticlassTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput +Trainers.LightGbmRanker Train a LightGBM ranking model. Microsoft.ML.Trainers.LightGbm.LightGbm TrainRanking Microsoft.ML.Trainers.LightGbm.LightGbmRankingTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+RankingOutput +Trainers.LightGbmRegressor LightGBM Regression Microsoft.ML.Trainers.LightGbm.LightGbm TrainRegression Microsoft.ML.Trainers.LightGbm.LightGbmRegressionTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput Trainers.LinearSvmBinaryClassifier Train a linear SVM. Microsoft.ML.Trainers.LinearSvmTrainer TrainLinearSvm Microsoft.ML.Trainers.LinearSvmTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Trainers.LogisticRegressionBinaryTrainer TrainBinary Microsoft.ML.Trainers.LogisticRegressionBinaryTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Trainers.LogisticRegressionBinaryTrainer TrainMulticlass Microsoft.ML.Trainers.LogisticRegressionMulticlassClassificationTrainer+Options Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index f0e18bf050..c9251c758f 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -11243,23 +11243,11 @@ "Default": "Auto" }, { - "Name": "MaximumBinCountPerFeature", - "Type": "Int", - "Desc": "Maximum number of bucket bin for features.", - "Aliases": [ - "mb" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 255 - }, - { - "Name": "Verbose", + "Name": "UnbalancedSets", "Type": "Bool", - "Desc": "Verbose", + "Desc": "Use for binary classification when training data is not balanced.", "Aliases": [ - "v" + "us" ], "Required": false, "SortOrder": 150.0, @@ -11267,41 +11255,39 @@ "Default": false }, { - "Name": "Silent", - "Type": "Bool", - "Desc": "Printing running messages.", + "Name": "WeightOfPositiveExamples", + "Type": "Float", + "Desc": "Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: sum(negative cases) / sum(positive cases).", + "Aliases": [ + "ScalePosWeight" + ], "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": true + "Default": 1.0 }, { - "Name": "NumberOfThreads", - "Type": "Int", - "Desc": "Number of parallel threads used to run LightGBM.", + "Name": "Sigmoid", + "Type": "Float", + "Desc": "Parameter for the sigmoid function.", "Aliases": [ - "nt" + "sigmoid" ], "Required": false, "SortOrder": 150.0, - "IsNullable": true, - "Default": null + "IsNullable": false, + "Default": 0.5 }, { "Name": "EvaluationMetric", "Type": { "Kind": "Enum", "Values": [ - "DefaultMetric", - "Rmse", - "Mae", + "None", + "Default", "Logloss", "Error", - "Merror", - "Mlogloss", - "Auc", - "Ndcg", - "Map" + "AreaUnderCurve" ] }, "Desc": "Evaluation metrics.", @@ -11311,59 +11297,64 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": "DefaultMetric" + "Default": "Logloss" }, { - "Name": "UseSoftmax", - "Type": "Bool", - "Desc": "Use softmax loss for the multi classification.", + "Name": "MaximumBinCountPerFeature", + "Type": "Int", + "Desc": "Maximum number of bucket bin for features.", + "Aliases": [ + "mb" + ], "Required": false, "SortOrder": 150.0, - "IsNullable": true, - "Default": null, - "SweepRange": { - "RangeType": "Discrete", - "Values": [ - true, - false - ] - } + "IsNullable": false, + "Default": 255 }, { - "Name": "EarlyStoppingRound", - "Type": "Int", - "Desc": "Rounds of early stopping, 0 will disable it.", + "Name": "Verbose", + "Type": "Bool", + "Desc": "Verbose", "Aliases": [ - "es" + "v" ], "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": 0 + "Default": false }, { - "Name": "CustomGains", - "Type": "String", - "Desc": "Comma separated list of gains associated to each relevance label.", + "Name": "Silent", + "Type": "Bool", + "Desc": "Printing running messages.", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "NumberOfThreads", + "Type": "Int", + "Desc": "Number of parallel threads used to run LightGBM.", "Aliases": [ - "gains" + "nt" ], "Required": false, "SortOrder": 150.0, - "IsNullable": false, - "Default": "0,3,7,15,31,63,127,255,511,1023,2047,4095" + "IsNullable": true, + "Default": null }, { - "Name": "Sigmoid", - "Type": "Float", - "Desc": "Parameter for the sigmoid function. Used only in LightGbmBinaryTrainer, LightGbmMulticlassTrainer and in LightGbmRankingTrainer.", + "Name": "EarlyStoppingRound", + "Type": "Int", + "Desc": "Rounds of early stopping, 0 will disable it.", "Aliases": [ - "sigmoid" + "es" ], "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": 0.5 + "Default": 0 }, { "Name": "BatchSize", @@ -11400,7 +11391,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": false, + "Default": true, "SweepRange": { "RangeType": "Discrete", "Values": [ @@ -11462,7 +11453,7 @@ { "Name": "CategoricalSmoothing", "Type": "Float", - "Desc": "Laplace smooth term in categorical feature split. Avoid the bias of small categories.", + "Desc": "Lapalace smooth term in categorical feature spilt. Avoid the bias of small categories.", "Required": false, "SortOrder": 150.0, "IsNullable": false, @@ -11745,6 +11736,54 @@ "IsNullable": false, "Default": "Auto" }, + { + "Name": "UseSoftmax", + "Type": "Bool", + "Desc": "Use softmax loss for the multi classification.", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null, + "SweepRange": { + "RangeType": "Discrete", + "Values": [ + true, + false + ] + } + }, + { + "Name": "Sigmoid", + "Type": "Float", + "Desc": "Parameter for the sigmoid function.", + "Aliases": [ + "sigmoid" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0.5 + }, + { + "Name": "EvaluationMetric", + "Type": { + "Kind": "Enum", + "Values": [ + "None", + "Default", + "Error", + "LogLoss" + ] + }, + "Desc": "Evaluation metrics.", + "Aliases": [ + "em" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "Error" + }, { "Name": "MaximumBinCountPerFeature", "Type": "Int", @@ -11790,48 +11829,6 @@ "IsNullable": true, "Default": null }, - { - "Name": "EvaluationMetric", - "Type": { - "Kind": "Enum", - "Values": [ - "DefaultMetric", - "Rmse", - "Mae", - "Logloss", - "Error", - "Merror", - "Mlogloss", - "Auc", - "Ndcg", - "Map" - ] - }, - "Desc": "Evaluation metrics.", - "Aliases": [ - "em" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "DefaultMetric" - }, - { - "Name": "UseSoftmax", - "Type": "Bool", - "Desc": "Use softmax loss for the multi classification.", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null, - "SweepRange": { - "RangeType": "Discrete", - "Values": [ - true, - false - ] - } - }, { "Name": "EarlyStoppingRound", "Type": "Int", @@ -11844,30 +11841,6 @@ "IsNullable": false, "Default": 0 }, - { - "Name": "CustomGains", - "Type": "String", - "Desc": "Comma separated list of gains associated to each relevance label.", - "Aliases": [ - "gains" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "0,3,7,15,31,63,127,255,511,1023,2047,4095" - }, - { - "Name": "Sigmoid", - "Type": "Float", - "Desc": "Parameter for the sigmoid function. Used only in LightGbmBinaryTrainer, LightGbmMulticlassTrainer and in LightGbmRankingTrainer.", - "Aliases": [ - "sigmoid" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0.5 - }, { "Name": "BatchSize", "Type": "Int", @@ -11903,7 +11876,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": false, + "Default": true, "SweepRange": { "RangeType": "Discrete", "Values": [ @@ -11965,7 +11938,7 @@ { "Name": "CategoricalSmoothing", "Type": "Float", - "Desc": "Laplace smooth term in categorical feature split. Avoid the bias of small categories.", + "Desc": "Lapalace smooth term in categorical feature spilt. Avoid the bias of small categories.", "Required": false, "SortOrder": 150.0, "IsNullable": false, @@ -12248,6 +12221,66 @@ "IsNullable": false, "Default": "Auto" }, + { + "Name": "CustomGains", + "Type": { + "Kind": "Array", + "ItemType": "Int" + }, + "Desc": "An array of gains associated to each relevance label.", + "Aliases": [ + "gains" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": [ + 0, + 3, + 7, + 15, + 31, + 63, + 127, + 255, + 511, + 1023, + 2047, + 4095 + ] + }, + { + "Name": "Sigmoid", + "Type": "Float", + "Desc": "Parameter for the sigmoid function.", + "Aliases": [ + "sigmoid" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0.5 + }, + { + "Name": "EvaluationMetric", + "Type": { + "Kind": "Enum", + "Values": [ + "None", + "Default", + "MeanAveragedPrecision", + "NormalizedDiscountedCumulativeGain" + ] + }, + "Desc": "Evaluation metrics.", + "Aliases": [ + "em" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "NormalizedDiscountedCumulativeGain" + }, { "Name": "MaximumBinCountPerFeature", "Type": "Int", @@ -12293,48 +12326,6 @@ "IsNullable": true, "Default": null }, - { - "Name": "EvaluationMetric", - "Type": { - "Kind": "Enum", - "Values": [ - "DefaultMetric", - "Rmse", - "Mae", - "Logloss", - "Error", - "Merror", - "Mlogloss", - "Auc", - "Ndcg", - "Map" - ] - }, - "Desc": "Evaluation metrics.", - "Aliases": [ - "em" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "DefaultMetric" - }, - { - "Name": "UseSoftmax", - "Type": "Bool", - "Desc": "Use softmax loss for the multi classification.", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null, - "SweepRange": { - "RangeType": "Discrete", - "Values": [ - true, - false - ] - } - }, { "Name": "EarlyStoppingRound", "Type": "Int", @@ -12347,30 +12338,6 @@ "IsNullable": false, "Default": 0 }, - { - "Name": "CustomGains", - "Type": "String", - "Desc": "Comma separated list of gains associated to each relevance label.", - "Aliases": [ - "gains" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "0,3,7,15,31,63,127,255,511,1023,2047,4095" - }, - { - "Name": "Sigmoid", - "Type": "Float", - "Desc": "Parameter for the sigmoid function. Used only in LightGbmBinaryTrainer, LightGbmMulticlassTrainer and in LightGbmRankingTrainer.", - "Aliases": [ - "sigmoid" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0.5 - }, { "Name": "BatchSize", "Type": "Int", @@ -12406,7 +12373,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": false, + "Default": true, "SweepRange": { "RangeType": "Discrete", "Values": [ @@ -12468,7 +12435,7 @@ { "Name": "CategoricalSmoothing", "Type": "Float", - "Desc": "Laplace smooth term in categorical feature split. Avoid the bias of small categories.", + "Desc": "Lapalace smooth term in categorical feature spilt. Avoid the bias of small categories.", "Required": false, "SortOrder": 150.0, "IsNullable": false, @@ -12751,6 +12718,27 @@ "IsNullable": false, "Default": "Auto" }, + { + "Name": "EvaluationMetric", + "Type": { + "Kind": "Enum", + "Values": [ + "None", + "Default", + "MeanAbsoluteError", + "RootMeanSquaredError", + "MeanSquaredError" + ] + }, + "Desc": "Evaluation metrics.", + "Aliases": [ + "em" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "RootMeanSquaredError" + }, { "Name": "MaximumBinCountPerFeature", "Type": "Int", @@ -12796,48 +12784,6 @@ "IsNullable": true, "Default": null }, - { - "Name": "EvaluationMetric", - "Type": { - "Kind": "Enum", - "Values": [ - "DefaultMetric", - "Rmse", - "Mae", - "Logloss", - "Error", - "Merror", - "Mlogloss", - "Auc", - "Ndcg", - "Map" - ] - }, - "Desc": "Evaluation metrics.", - "Aliases": [ - "em" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "DefaultMetric" - }, - { - "Name": "UseSoftmax", - "Type": "Bool", - "Desc": "Use softmax loss for the multi classification.", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null, - "SweepRange": { - "RangeType": "Discrete", - "Values": [ - true, - false - ] - } - }, { "Name": "EarlyStoppingRound", "Type": "Int", @@ -12850,30 +12796,6 @@ "IsNullable": false, "Default": 0 }, - { - "Name": "CustomGains", - "Type": "String", - "Desc": "Comma separated list of gains associated to each relevance label.", - "Aliases": [ - "gains" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": "0,3,7,15,31,63,127,255,511,1023,2047,4095" - }, - { - "Name": "Sigmoid", - "Type": "Float", - "Desc": "Parameter for the sigmoid function. Used only in LightGbmBinaryTrainer, LightGbmMulticlassTrainer and in LightGbmRankingTrainer.", - "Aliases": [ - "sigmoid" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0.5 - }, { "Name": "BatchSize", "Type": "Int", @@ -12909,7 +12831,7 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": false, + "Default": true, "SweepRange": { "RangeType": "Discrete", "Values": [ @@ -12971,7 +12893,7 @@ { "Name": "CategoricalSmoothing", "Type": "Float", - "Desc": "Laplace smooth term in categorical feature split. Avoid the bias of small categories.", + "Desc": "Lapalace smooth term in categorical feature spilt. Avoid the bias of small categories.", "Required": false, "SortOrder": 150.0, "IsNullable": false, @@ -23517,13 +23439,13 @@ "Components": [ { "Name": "dart", - "Desc": "Dropouts meet Multiple Additive Regression Trees. See https://arxiv.org/abs/1505.01866", + "Desc": "Dropouts meet Multiple Additive Regresion Trees. See https://arxiv.org/abs/1505.01866", "FriendlyName": "Tree Dropout Tree Booster", "Settings": [ { "Name": "TreeDropFraction", "Type": "Float", - "Desc": "The drop ratio for trees. Range:[0,1].", + "Desc": "The drop ratio for trees. Range:(0,1).", "Required": false, "SortOrder": 150.0, "IsNullable": false, @@ -23577,18 +23499,6 @@ "IsNullable": false, "Default": false }, - { - "Name": "UnbalancedSets", - "Type": "Bool", - "Desc": "Use for binary classification when training data is not balanced.", - "Aliases": [ - "us" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, { "Name": "MinimumSplitGain", "Type": "Float", @@ -23713,18 +23623,6 @@ 1.0 ] } - }, - { - "Name": "WeightOfPositiveExamples", - "Type": "Float", - "Desc": "Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: sum(negative cases) / sum(positive cases).", - "Aliases": [ - "ScalePosWeight" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 1.0 } ] }, @@ -23733,18 +23631,6 @@ "Desc": "Traditional Gradient Boosting Decision Tree.", "FriendlyName": "Tree Booster", "Settings": [ - { - "Name": "UnbalancedSets", - "Type": "Bool", - "Desc": "Use for binary classification when training data is not balanced.", - "Aliases": [ - "us" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, { "Name": "MinimumSplitGain", "Type": "Float", @@ -23869,18 +23755,6 @@ 1.0 ] } - }, - { - "Name": "WeightOfPositiveExamples", - "Type": "Float", - "Desc": "Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: sum(negative cases) / sum(positive cases).", - "Aliases": [ - "ScalePosWeight" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 1.0 } ] }, @@ -23915,18 +23789,6 @@ "Max": 1.0 } }, - { - "Name": "UnbalancedSets", - "Type": "Bool", - "Desc": "Use for binary classification when training data is not balanced.", - "Aliases": [ - "us" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, { "Name": "MinimumSplitGain", "Type": "Float", @@ -24051,18 +23913,6 @@ 1.0 ] } - }, - { - "Name": "WeightOfPositiveExamples", - "Type": "Float", - "Desc": "Control the balance of positive and negative weights, useful for unbalanced classes. A typical value to consider: sum(negative cases) / sum(positive cases).", - "Aliases": [ - "ScalePosWeight" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 1.0 } ] } diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt index 1fc6084997..c81802feb6 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt @@ -35,10 +35,10 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-l2: 37.107605006517 +[2] (%Time%) Iteration: 50 Training-rmse: 6.09160118577349 [2] 'Training with LightGBM' finished in %Time%. [3] 'Loading data for LightGBM #2' started. [3] 'Loading data for LightGBM #2' finished in %Time%. [4] 'Training with LightGBM #2' started. -[4] (%Time%) Iteration: 50 Training-l2: 27.7037679135951 +[4] (%Time%) Iteration: 50 Training-rmse: 5.26343689176522 [4] 'Training with LightGBM #2' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt index 909d9f0012..c88cad62b7 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt @@ -26,7 +26,7 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-l2: 26.0644295080124 +[2] (%Time%) Iteration: 50 Training-rmse: 5.10533343749577 [2] 'Training with LightGBM' finished in %Time%. [3] 'Saving model' started. [3] 'Saving model' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-out.txt index 4550a80d3c..62d41c06a5 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-out.txt @@ -1,4 +1,4 @@ -maml.exe CV tr=LightGBMR{nt=1 iter=50 em=mae v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 +maml.exe CV tr=LightGBMR{nt=1 iter=50 em=MeanAbsoluteError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 Not adding a normalizer. Auto-tuning parameters: UseCategoricalSplit = False LightGBM objective=regression @@ -35,10 +35,10 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-l1: 2.72125878362794 +[2] (%Time%) Iteration: 50 Training-mae: 2.72125878362794 [2] 'Training with LightGBM' finished in %Time%. [3] 'Loading data for LightGBM #2' started. [3] 'Loading data for LightGBM #2' finished in %Time%. [4] 'Training with LightGBM #2' started. -[4] (%Time%) Iteration: 50 Training-l1: 2.24116204430926 +[4] (%Time%) Iteration: 50 Training-mae: 2.24116204430926 [4] 'Training with LightGBM #2' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-rp.txt index d7359a5e25..8a563605cf 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-CV-generatedRegressionDataset.MAE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt /em Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -26.59978 1393.326 37.32081 1393.326 0.923402 50 0.2 20 10 + 1 Mae LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=mae v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1;/em:Mae +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +26.59978 1393.326 37.32081 1393.326 0.923402 MeanAbsoluteError 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=MeanAbsoluteError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /em:MeanAbsoluteError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-out.txt index 59d2ceaa05..11a1112241 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-out.txt @@ -1,4 +1,4 @@ -maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=mae v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 +maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=MeanAbsoluteError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 Not adding a normalizer. Auto-tuning parameters: UseCategoricalSplit = False LightGBM objective=regression @@ -26,7 +26,7 @@ Virtual memory usage(MB): %Number% [1] 'Loading data for LightGBM' started. [1] 'Loading data for LightGBM' finished in %Time%. [2] 'Training with LightGBM' started. -[2] (%Time%) Iteration: 50 Training-l1: 3.42889585604196 +[2] (%Time%) Iteration: 50 Training-mae: 3.42889585604196 [2] 'Training with LightGBM' finished in %Time%. [3] 'Saving model' started. [3] 'Saving model' finished in %Time%. diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-rp.txt index 59ac27dc61..d96adb798e 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegMae-TrainTest-generatedRegressionDataset.MAE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt /em Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -3.428896 25.23601 5.023546 25.23601 0.998616 50 0.2 20 10 + 1 Mae LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=mae v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1;/em:Mae +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /em /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +3.428896 25.23601 5.023546 25.23601 0.998616 MeanAbsoluteError 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=MeanAbsoluteError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /em:MeanAbsoluteError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-out.txt index 71d131bb5a..31c4280d50 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-out.txt @@ -1,4 +1,4 @@ -maml.exe CV tr=LightGBMR{nt=1 iter=50 em=rmse v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 +maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 Not adding a normalizer. Auto-tuning parameters: UseCategoricalSplit = False LightGBM objective=regression diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt index 1b893d09dc..d855ef6c79 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt /em Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -26.59978 1393.326 37.32081 1393.326 0.923402 50 0.2 20 10 + 1 Rmse LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=rmse v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1;/em:Rmse +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +26.59978 1393.326 37.32081 1393.326 0.923402 50 0.2 20 10 + 1 LightGBMR %Data% %Output% 99 0 0 maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-out.txt index c919475347..cd21c6e9d4 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-out.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-out.txt @@ -1,4 +1,4 @@ -maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=rmse v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 +maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 Not adding a normalizer. Auto-tuning parameters: UseCategoricalSplit = False LightGBM objective=regression diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt index b8f135e448..bf64ad5ed2 100644 --- a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt +++ b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt @@ -1,4 +1,4 @@ LightGBMR -L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt /em Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -3.428896 25.23601 5.023546 25.23601 0.998616 50 0.2 20 10 + 1 Rmse LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=rmse v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1;/em:Rmse +L1(avg) L2(avg) RMS(avg) Loss-fn(avg) R Squared /iter /lr /nl /mil /v /nt Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings +3.428896 25.23601 5.023546 25.23601 0.998616 50 0.2 20 10 + 1 LightGBMR %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1 /iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1 diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index ce141dbafd..70346af906 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -768,7 +768,7 @@ public void TestMulticlassEnsembleCombiner() var predictors = new PredictorModel[] { - LightGbm.TrainMulticlass(Env, new Options + LightGbm.TrainMulticlass(Env, new LightGbmMulticlassTrainer.Options { FeatureColumnName = "Features", NumberOfIterations = 5, diff --git a/test/Microsoft.ML.TestFramework/Learners.cs b/test/Microsoft.ML.TestFramework/Learners.cs index f369264db3..876710da6d 100644 --- a/test/Microsoft.ML.TestFramework/Learners.cs +++ b/test/Microsoft.ML.TestFramework/Learners.cs @@ -333,14 +333,14 @@ static TestLearnersBase() public static PredictorAndArgs LightGBMRegMae = new PredictorAndArgs { - Trainer = new SubComponent("LightGBMR", "nt=1 iter=50 em=mae v=+ lr=0.2 mil=10 nl=20"), + Trainer = new SubComponent("LightGBMR", "nt=1 iter=50 em=MeanAbsoluteError v=+ lr=0.2 mil=10 nl=20"), Tag = "LightGBMRegMae", BaselineProgress = true, }; public static PredictorAndArgs LightGBMRegRmse = new PredictorAndArgs { - Trainer = new SubComponent("LightGBMR", "nt=1 iter=50 em=rmse v=+ lr=0.2 mil=10 nl=20"), + Trainer = new SubComponent("LightGBMR", "nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20"), Tag = "LightGBMRegRmse", BaselineProgress = true, }; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index ff5f07379b..bc22299fb4 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -677,7 +677,7 @@ private void ExecuteTFTransformMNISTConvTrainingTest(bool shuffle, int? shuffleS batchSize: 20)) .Append(mlContext.Transforms.Concatenate("Features", "Prediction")) .AppendCacheCheckpoint(mlContext) - .Append(mlContext.MulticlassClassification.Trainers.LightGbm(new Trainers.LightGbm.Options() + .Append(mlContext.MulticlassClassification.Trainers.LightGbm(new Trainers.LightGbm.LightGbmMulticlassTrainer.Options() { LabelColumnName = "Label", FeatureColumnName = "Features", diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index d8ec78e585..beaec1a3a3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -49,7 +49,7 @@ public void LightGBMBinaryEstimator() { var (pipe, dataView) = GetBinaryClassificationPipeline(); - var trainer = ML.BinaryClassification.Trainers.LightGbm(new Options + var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options { NumberOfLeaves = 10, NumberOfThreads = 1, @@ -136,7 +136,7 @@ public void LightGBMRankerEstimator() { var (pipe, dataView) = GetRankingPipeline(); - var trainer = ML.Ranking.Trainers.LightGbm(new Options() { LabelColumnName = "Label0", FeatureColumnName = "NumericFeatures", RowGroupColumnName = "Group", LearningRate = 0.4 }); + var trainer = ML.Ranking.Trainers.LightGbm(new LightGbmRankingTrainer.Options() { LabelColumnName = "Label0", FeatureColumnName = "NumericFeatures", RowGroupColumnName = "Group", LearningRate = 0.4 }); var pipeWithTrainer = pipe.Append(trainer); TestEstimatorCore(pipeWithTrainer, dataView); @@ -162,13 +162,13 @@ public void FastTreeRegressorEstimator() } /// - /// LightGbmRegressorTrainer TrainerEstimator test + /// LightGbmRegressionTrainer TrainerEstimator test /// [LightGBMFact] public void LightGBMRegressorEstimator() { var dataView = GetRegressionPipeline(); - var trainer = ML.Regression.Trainers.LightGbm(new Options + var trainer = ML.Regression.Trainers.LightGbm(new LightGbmRegressionTrainer.Options { NumberOfThreads = 1, NormalizeFeatures = NormalizeOption.Warn, @@ -294,13 +294,14 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List