Skip to content

Enabling custom groupId column in the Ranking AutoML experiment #5246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kere-nel merged 3 commits into from
Jun 30, 2020
Merged

Enabling custom groupId column in the Ranking AutoML experiment #5246

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2519662
custom groupid
kere-nel Jun 17, 2020
63e4d51
groupid addition option 2
kere-nel Jun 29, 2020
a19bbca
removed prev code
kere-nel Jun 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Microsoft.ML.AutoML/API/ExperimentBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ private ExperimentResult<TMetrics> ExecuteTrainValidate(IDataView trainData,
validationData = preprocessorTransform.Transform(validationData);
}

var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent,
var runner = new TrainValidateRunner<TMetrics>(Context, trainData, validationData, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, MetricsAgent,
preFeaturizer, preprocessorTransform, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
Expand All @@ -273,7 +273,7 @@ private CrossValidationExperimentResult<TMetrics> ExecuteCrossVal(IDataView[] tr
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, _logger);
preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);

// Execute experiment & get all pipelines run
Expand All @@ -300,7 +300,7 @@ private ExperimentResult<TMetrics> ExecuteCrossValSummary(IDataView[] trainDatas
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

var runner = new CrossValSummaryRunner<TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, OptimizingMetricInfo, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner);
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public bool IsModelPerfect(double score)
}
}

public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn)
public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
Comment thread
kere-nel marked this conversation as resolved.
Outdated
{
return _mlContext.BinaryClassification.EvaluateNonCalibrated(data, labelColumn);
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ internal interface IMetricsAgent<T>

bool IsModelPerfect(double score);

T EvaluateMetrics(IDataView data, string labelColumn);
T EvaluateMetrics(IDataView data, string labelColumn, string groupId);
Comment thread
kere-nel marked this conversation as resolved.
Outdated
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public bool IsModelPerfect(double score)
}
}

public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn)
public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
{
return _mlContext.MulticlassClassification.Evaluate(data, labelColumn);
}
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@ public bool IsModelPerfect(double score)
}
}

public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn)
public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupIdColumn)

@LittleLittleCloud LittleLittleCloud Jun 26, 2020

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we just put groupId column as an option when creating RankingMetricsAgent? So that we don't have to add an extra groupIdColumn to IAgent.

@kere-nel kere-nel Jun 29, 2020

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we can do that. I think a possible concern is that GroupId is required by RankingTrainers, and passing it in as an option gives me the impression that it's optional. I think that if we make the default value of GroupId "GroupId" and document that, it might be okay. What you do think?

{
return _mlContext.Ranking.Evaluate(data, labelColumn);
return _mlContext.Ranking.Evaluate(data, labelColumn, groupIdColumn);
}
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public bool IsModelPerfect(double score)
}
}

public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn)
public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId)
{
return _mlContext.Regression.Evaluate(data, labelColumn);
}
Expand Down
5 changes: 4 additions & 1 deletion src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ internal class CrossValRunner<TMetrics> : IRunner<CrossValidationRunDetail<TMetr
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer[] _preprocessorTransforms;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly IChannel _logger;
private readonly DataViewSchema _modelInputSchema;

Expand All @@ -30,6 +31,7 @@ public CrossValRunner(MLContext context,
IEstimator<ITransformer> preFeaturizer,
ITransformer[] preprocessorTransforms,
string labelColumn,
string groupIdColumn,
Comment thread
kere-nel marked this conversation as resolved.
Outdated
IChannel logger)
{
_context = context;
Expand All @@ -39,6 +41,7 @@ public CrossValRunner(MLContext context,
_preFeaturizer = preFeaturizer;
_preprocessorTransforms = preprocessorTransforms;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_logger = logger;
_modelInputSchema = trainDatasets[0].Schema;
}
Expand All @@ -52,7 +55,7 @@ public CrossValRunner(MLContext context,
{
var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1);
var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i],
_labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger);
_labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger);
trainResults.Add(new SuggestedPipelineTrainResult<TMetrics>(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score));
}

Expand Down
5 changes: 4 additions & 1 deletion src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ internal class CrossValSummaryRunner<TMetrics> : IRunner<RunDetail<TMetrics>>
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer[] _preprocessorTransforms;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly OptimizingMetricInfo _optimizingMetricInfo;
private readonly IChannel _logger;
private readonly DataViewSchema _modelInputSchema;
Expand All @@ -32,6 +33,7 @@ public CrossValSummaryRunner(MLContext context,
IEstimator<ITransformer> preFeaturizer,
ITransformer[] preprocessorTransforms,
string labelColumn,
string groupIdColumn,
OptimizingMetricInfo optimizingMetricInfo,
IChannel logger)
{
Expand All @@ -42,6 +44,7 @@ public CrossValSummaryRunner(MLContext context,
_preFeaturizer = preFeaturizer;
_preprocessorTransforms = preprocessorTransforms;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_optimizingMetricInfo = optimizingMetricInfo;
_logger = logger;
_modelInputSchema = trainDatasets[0].Schema;
Expand All @@ -56,7 +59,7 @@ public CrossValSummaryRunner(MLContext context,
{
var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1);
var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i],
_labelColumn, _metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema,
_labelColumn, _groupIdColumn,_metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema,
_logger);
trainResults.Add(trainResult);
}
Expand Down
3 changes: 2 additions & 1 deletion src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
IDataView trainData,
IDataView validData,
string labelColumn,
string groupId,
IMetricsAgent<TMetrics> metricsAgent,
ITransformer preprocessorTransform,
FileInfo modelFileInfo,
Expand All @@ -28,7 +29,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
var model = estimator.Fit(trainData);

var scoredData = model.Transform(validData);
var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn);
var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn, groupId);
var score = metricsAgent.GetScore(metrics);

if (preprocessorTransform != null)
Expand Down
5 changes: 4 additions & 1 deletion src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ internal class TrainValidateRunner<TMetrics> : IRunner<RunDetail<TMetrics>>
private readonly IDataView _trainData;
private readonly IDataView _validData;
private readonly string _labelColumn;
private readonly string _groupIdColumn;
private readonly IMetricsAgent<TMetrics> _metricsAgent;
private readonly IEstimator<ITransformer> _preFeaturizer;
private readonly ITransformer _preprocessorTransform;
Expand All @@ -24,6 +25,7 @@ public TrainValidateRunner(MLContext context,
IDataView trainData,
IDataView validData,
string labelColumn,
string groupIdColumn,
IMetricsAgent<TMetrics> metricsAgent,
IEstimator<ITransformer> preFeaturizer,
ITransformer preprocessorTransform,
Expand All @@ -33,6 +35,7 @@ public TrainValidateRunner(MLContext context,
_trainData = trainData;
_validData = validData;
_labelColumn = labelColumn;
_groupIdColumn = groupIdColumn;
_metricsAgent = metricsAgent;
_preFeaturizer = preFeaturizer;
_preprocessorTransform = preprocessorTransform;
Expand All @@ -45,7 +48,7 @@ public TrainValidateRunner(MLContext context,
{
var modelFileInfo = GetModelFileInfo(modelDirectory, iterationNum);
var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainData, _validData,
_labelColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger);
_labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger);
var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail<TMetrics>(pipeline,
trainResult.score,
trainResult.exception == null,
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ private static IDictionary<string, object> BuildBasePipelineNodeProps(IEnumerabl
}

private static IDictionary<string, object> BuildLightGbmPipelineNodeProps(IEnumerable<SweepableParam> sweepParams,
string labelColumn, string weightColumn, string groupColumn = null)
string labelColumn, string weightColumn, string groupColumn)
{
Dictionary<string, object> props = null;
if (sweepParams == null || !sweepParams.Any())
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public void AutoFitRankingTest()
{
string labelColumnName = "Label";
string scoreColumnName = "Score";
string groupIdColumnName = "GroupId";
string groupIdColumnName = "CustomGroupId";
string featuresColumnVectorNameA = "FeatureVectorA";
string featuresColumnVectorNameB = "FeatureVectorB";
var mlContext = new MLContext(1);
Expand Down