Skip to content

Enabling custom groupId column in the Ranking AutoML experiment #5246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kere-nel merged 3 commits into from
Jun 30, 2020
Merged

Enabling custom groupId column in the Ranking AutoML experiment #5246

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2519662
custom groupid
kere-nel Jun 17, 2020
63e4d51
groupid addition option 2
kere-nel Jun 29, 2020
a19bbca
removed prev code
kere-nel Jun 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Microsoft.ML.AutoML/API/ExperimentBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
validationData = preprocessorTransform.Transform(validationData);
}

var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, MetricsAgent,
preFeaturizer, preprocessorTransform, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
Expand All @@ -273,7 +273,7 @@ private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] tr
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, _logger);
preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);

// Execute experiment & get all pipelines run
Expand All @@ -300,7 +300,7 @@ private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatas
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, OptimizingMetricInfo, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
}
Expand Down
19 changes: 14 additions & 5 deletions src/Microsoft.ML.AutoML/API/RankingExperiment.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ public sealed class RankingExperimentSettings : ExperimentSettings
/// <value>The default value is <see cref="RankingMetric" />.</value>
public RankingMetric OptimizingMetric { get; set; }

/// <summary>
/// Name for the GroupId column.
/// </summary>
/// <value>The default value is GroupId.</value>
public string GroupIdColumnName { get; set; }

/// <summary>
/// Collection of trainers the AutoML experiment can leverage.
/// </summary>
Expand All @@ -28,6 +34,7 @@ public sealed class RankingExperimentSettings : ExperimentSettings
public ICollection<RankingTrainer> Trainers { get; }
public RankingExperimentSettings()
{
GroupIdColumnName = "GroupId";
OptimizingMetric = RankingMetric.Ndcg;
Trainers = Enum.GetValues(typeof(RankingTrainer)).OfType<RankingTrainer>().ToList();
}
Expand Down Expand Up @@ -68,10 +75,11 @@ public static class RankingExperimentResultExtensions
/// </summary>
/// <param name="results">Enumeration of AutoML experiment run results.</param>
/// <param name="metric">Metric to consider when selecting the best run.</param>
/// <param name="groupIdColumnName">Name for the GroupId column.</param>
/// <returns>The best experiment run.</returns>
public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg)
public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
{
var metricsAgent = new RankingMetricsAgent(null, metric);
var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
}
Expand All @@ -81,10 +89,11 @@ public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingM
/// </summary>
/// <param name="results">Enumeration of AutoML experiment cross validation run results.</param>
/// <param name="metric">Metric to consider when selecting the best run.</param>
/// <param name="groupIdColumnName">Name for the GroupId column.</param>
/// <returns>The best experiment run.</returns>
public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg)
public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
{
var metricsAgent = new RankingMetricsAgent(null, metric);
var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing);
}
Expand All @@ -103,7 +112,7 @@ public sealed class RankingExperiment : ExperimentBase<RankingMetrics, RankingEx
{
internal RankingExperiment(MLContext context, RankingExperimentSettings settings)
: base(context,
new RankingMetricsAgent(context, settings.OptimizingMetric),
new RankingMetricsAgent(context, settings.OptimizingMetric, settings.GroupIdColumnName),
new OptimizingMetricInfo(settings.OptimizingMetric),
settings,
TaskKind.Ranking,
Expand Down
6 changes: 4 additions & 2 deletions src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ internal class RankingMetricsAgent : IMetricsAgent<RankingMetrics>
{
private readonly MLContext _mlContext;
private readonly RankingMetric _optimizingMetric;
private readonly string _groupIdColumnName;

public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric)
public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric, string groupIdColumnName)
{
_mlContext = mlContext;
_optimizingMetric = optimizingMetric;
_groupIdColumnName = groupIdColumnName;
}

// Optimizing metric used: NDCG@10 and DCG@10
Expand Down Expand Up @@ -59,7 +61,7 @@ public bool IsModelPerfect(double score)

public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn)
{
return _mlContext.Ranking.Evaluate(data, labelColumn);
return _mlContext.Ranking.Evaluate(data, labelColumn, _groupIdColumnName);
}
}
}
3 changes: 3 additions & 0 deletions src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ internal class CrossValRunner<TMetrics> : IRunner<CrossValidationRunDetail<TMetr
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer[] _preprocessorTransforms;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly IChannel _logger;
private readonly DataViewSchema _modelInputSchema;

Expand All @@ -29,6 +30,7 @@ public CrossValRunner(MLContext context,
IMetricsAgent<TMetrics> metricsAgent,
IEstimator<ITransformer> preFeaturizer,
ITransformer[] preprocessorTransforms,
string groupIdColumn,
Comment thread
kere-nel marked this conversation as resolved.
Outdated
string labelColumn,
IChannel logger)
{
Expand All @@ -39,6 +41,7 @@ public CrossValRunner(MLContext context,
_preFeaturizer = preFeaturizer;
_preprocessorTransforms = preprocessorTransforms;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_logger = logger;
_modelInputSchema = trainDatasets[0].Schema;
}
Expand Down
3 changes: 3 additions & 0 deletions src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ internal class CrossValSummaryRunner<TMetrics> : IRunner<RunDetail<TMetrics>>
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer[] _preprocessorTransforms;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly OptimizingMetricInfo _optimizingMetricInfo;
private readonly IChannel _logger;
private readonly DataViewSchema _modelInputSchema;
Expand All @@ -32,6 +33,7 @@ public CrossValSummaryRunner(MLContext context,
IEstimator<ITransformer> preFeaturizer,
ITransformer[] preprocessorTransforms,
string labelColumn,
string groupIdColumn,
OptimizingMetricInfo optimizingMetricInfo,
IChannel logger)
{
Expand All @@ -42,6 +44,7 @@ public CrossValSummaryRunner(MLContext context,
_preFeaturizer = preFeaturizer;
_preprocessorTransforms = preprocessorTransforms;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_optimizingMetricInfo = optimizingMetricInfo;
_logger = logger;
_modelInputSchema = trainDatasets[0].Schema;
Expand Down
3 changes: 3 additions & 0 deletions src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ internal class TrainValidateRunner<TMetrics> : IRunner<RunDetail<TMetrics>>
private readonly IDataView _trainData;
private readonly IDataView _validData;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly IMetricsAgent<TMetrics> _metricsAgent;
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer _preprocessorTransform;
Expand All @@ -24,6 +25,7 @@ public TrainValidateRunner(MLContext context,
IDataView trainData,
IDataView validData,
string labelColumn,
string groupIdColumn,
IMetricsAgent<TMetrics> metricsAgent,
IEstimator<ITransformer> preFeaturizer,
ITransformer preprocessorTransform,
Expand All @@ -33,6 +35,7 @@ public TrainValidateRunner(MLContext context,
_trainData = trainData;
_validData = validData;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_metricsAgent = metricsAgent;
_preFeaturizer = preFeaturizer;
_preprocessorTransform = preprocessorTransform;
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ private static IDictionary<string, object> BuildBasePipelineNodeProps(IEnumerabl
}

private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnumerable<SweepableParam> sweepParams,
string labelColumn, string weightColumn, string groupColumn = null)
string labelColumn, string weightColumn, string groupColumn)
{
Dictionary<string, object> props = null;
if (sweepParams == null || !sweepParams.Any())
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ public static RunDetail<MulticlassClassificationMetrics> GetBestRun(IEnumerable<
}

public static RunDetail<RankingMetrics> GetBestRun(IEnumerable<RunDetail<RankingMetrics>> results,
RankingMetric metric)
RankingMetric metric, string groupIdColumnName)
{
var metricsAgent = new RankingMetricsAgent(null, metric);
var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
var metricInfo = new OptimizingMetricInfo(metric);
return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing);
}
Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public void AutoFitRankingTest()
{
string labelColumnName = "Label";
string scoreColumnName = "Score";
string groupIdColumnName = "GroupId";
string groupIdColumnName = "CustomGroupId";
string featuresColumnVectorNameA = "FeatureVectorA";
string featuresColumnVectorNameB = "FeatureVectorB";
var mlContext = new MLContext(1);
Expand All @@ -136,7 +136,7 @@ public void AutoFitRankingTest()
trainDataView = mlContext.Data.SkipRows(trainDataView, 500);
// STEP 2: Run AutoML experiment
ExperimentResult<RankingMetrics> experimentResult = mlContext.Auto()
.CreateRankingExperiment(5)
.CreateRankingExperiment(new RankingExperimentSettings() { GroupIdColumnName = "CustomGroupId", MaxExperimentTimeInSeconds = 5})
.Execute(trainDataView, testDataView,
new ColumnInformation()
{
Expand Down
24 changes: 12 additions & 12 deletions test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,14 @@ public void RankingMetricsGetScoreTest()
double[] ndcg = { 0.2, 0.3, 0.4 };
double[] dcg = { 0.2, 0.3, 0.4 };
var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg));
Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg));
Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg, "GroupId"));
Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg, "GroupId"));

double[] largeNdcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 };
double[] largeDcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 };
metrics = MetricsUtil.CreateRankingMetrics(largeDcg, largeNdcg);
Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg));
Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg));
Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg, "GroupId"));
Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg, "GroupId"));
}

[Fact]
Expand All @@ -143,8 +143,8 @@ public void RankingMetricsNonPerfectTest()
double[] ndcg = { 0.2, 0.3, 0.4 };
double[] dcg = { 0.2, 0.3, 0.4 };
var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg));
Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg));
Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId"));
Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId"));
}

[Fact]
Expand All @@ -153,8 +153,8 @@ public void RankingMetricsPerfectTest()
double[] ndcg = { 0.2, 0.3, 1 };
double[] dcg = { 0.2, 0.3, 1 };
var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg);
Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg)); //REVIEW: No true Perfect model
Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg));
Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId")); //REVIEW: No true Perfect model
Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId"));
}

[Fact]
Expand All @@ -179,9 +179,9 @@ private static double GetScore(RegressionMetrics metrics, RegressionMetric metri
return new RegressionMetricsAgent(null, metric).GetScore(metrics);
}

private static double GetScore(RankingMetrics metrics, RankingMetric metric)
private static double GetScore(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
{
return new RankingMetricsAgent(null, metric).GetScore(metrics);
return new RankingMetricsAgent(null, metric, groupIdColumnName).GetScore(metrics);
}

private static bool IsPerfectModel(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric)
Expand All @@ -202,9 +202,9 @@ private static bool IsPerfectModel(RegressionMetrics metrics, RegressionMetric m
return IsPerfectModel(metricsAgent, metrics);
}

private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric)
private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName)
{
var metricsAgent = new RankingMetricsAgent(null, metric);
var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
return IsPerfectModel(metricsAgent, metrics);
}

Expand Down