PipelineSweeperMacro for Multi-Class Classification #539
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,67 +51,6 @@ public class LevelDependencyMap : Dictionary<ColumnInfo, List<TransformInference | |
| /// </summary> | ||
| public class DependencyMap : Dictionary<int, LevelDependencyMap> { } | ||
|
|
||
| /// <summary> | ||
| /// Class for encapsulating an entrypoint experiment graph | ||
| /// and keeping track of the input and output nodes. | ||
|
|
@@ -218,42 +157,8 @@ public sealed class AutoMlMlState : IMlState | |
| Desc = "State of an AutoML search and search space.")] | ||
| public sealed class Arguments : ISupportAutoMlStateFactory | ||
| { | ||
| [Argument(ArgumentType.Required, HelpText = "Supported metric for evaluator.", ShortName = "metric")] | ||
| public PipelineSweeperSupportedMetrics.Metrics Metric; | ||
|
|
||
| [Argument(ArgumentType.Required, HelpText = "AutoML engine (pipeline optimizer) that generates next candidates.", ShortName = "engine")] | ||
| public ISupportIPipelineOptimizerFactory Engine; | ||
|
|
@@ -271,7 +176,9 @@ public enum Metrics | |
| } | ||
|
|
||
| public AutoMlMlState(IHostEnvironment env, Arguments args) | ||
| : this(env, | ||
| PipelineSweeperSupportedMetrics.GetSupportedMetric(env, Enum.GetName(typeof(PipelineSweeperSupportedMetrics.Metrics), args.Metric)), | ||
| args.Engine.CreateComponent(env), | ||
| args.TerminatorArgs.CreateComponent(env), args.TrainerKind, requestedLearners: args.RequestedLearners) | ||
| { | ||
| } | ||
|
|
@@ -524,6 +431,18 @@ public void AddEvaluated(PipelinePattern pipeline) | |
| d += 1e-3; | ||
| _sortedSampledElements.Add(d, pipeline); | ||
| _history.Add(pipeline); | ||
|
|
||
| using (var ch = _host.Start("Suggested Pipeline")) | ||
| { | ||
| ch.Info($"PipelineSweeper Pipeline Id : {pipeline.UniqueId}"); | ||
|
Contributor
There was a problem hiding this comment. Can we also print an increasing pipeline number? This lets the user see the progress of the sweeping. #Resolved |
||
| foreach (var transform in pipeline.Transforms) | ||
| { | ||
| ch.Info($"PipelineSweeper Transform : {transform.Transform}"); | ||
| } | ||
| ch.Info($"PipelineSweeper Learner : {pipeline.Learner}"); | ||
| ch.Info($"PipelineSweeper Train Metric Value: {pipeline.PerformanceSummary.TrainingMetricValue}"); | ||
| ch.Info($"PipelineSweeper Test Metric Value: {pipeline.PerformanceSummary.MetricValue}"); | ||
|
Contributor
There was a problem hiding this comment. We're printing the primary sweeping metric here. How much work would it be to print all of the pipeline's metrics? Generally when hand sweeping over a dataset, I'm watching multiple metrics. This is important as one metric often increases at the detriment of others. For instance, micro & macro accuracy often trade-off as some models end-up better at smaller classes or the majority classes. The best models improve the metrics in unison. #Pending
Member
Author
There was a problem hiding this comment. Good idea. I need to explore how we can expose this functionality for the PipelineSweeper. One thing to note is that the other metrics would depend on the TrainerKind. I have created a separate task (269949) for this. Would not want to block this PR on this feature ask. In reply to: 203264334 [](ancestors = 203264334) |
||
| } | ||
| } | ||
|
|
||
| public void AddEvaluated(PipelinePattern[] pipelines) | ||
|
|
@@ -541,18 +460,7 @@ public PipelinePattern[] GetNextCandidates(int numberOfCandidates) | |
| currentBatchSize = Math.Min(itr.RemainingIterations(_history), numberOfCandidates); | ||
| BatchCandidates = AutoMlEngine.GetNextCandidates(_sortedSampledElements.Select(kvp => kvp.Value), currentBatchSize, _dataRoles); | ||
|
|
||
|
|
||
|
|
||
| return BatchCandidates; | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Linq; | ||
| using System.Reflection; | ||
| using Microsoft.ML.Runtime.CommandLine; | ||
| using Microsoft.ML.Runtime.EntryPoints; | ||
| using Microsoft.ML.Runtime.Data; | ||
| using Microsoft.ML.Runtime.PipelineInference; | ||
| using Microsoft.ML.Runtime.EntryPoints.JsonUtils; | ||
| using Newtonsoft.Json.Linq; | ||
|
|
||
| namespace Microsoft.ML.Runtime.PipelineInference | ||
| { | ||
| /// <summary> | ||
| /// PipelineSweeper will support metrics as they are added here. | ||
| /// </summary> | ||
| public sealed class PipelineSweeperSupportedMetrics | ||
| { | ||
| public enum Metrics | ||
| { | ||
| Auc, | ||
| AccuracyMicro, | ||
| AccuracyMacro, | ||
| L1, | ||
| L2, | ||
| F1, | ||
| AuPrc, | ||
| TopKAccuracy, | ||
| Rms, | ||
| LossFn, | ||
| RSquared, | ||
| LogLoss, | ||
| LogLossReduction, | ||
| Ndcg, | ||
| Dcg, | ||
| PositivePrecision, | ||
| PositiveRecall, | ||
| NegativePrecision, | ||
| NegativeRecall, | ||
| DrAtK, | ||
| DrAtPFpr, | ||
| DrAtNumPos, | ||
| NumAnomalies, | ||
| ThreshAtK, | ||
| ThreshAtP, | ||
| ThreshAtNumPos, | ||
| Nmi, | ||
| AvgMinScore, | ||
| Dbi | ||
| }; | ||
|
|
||
| /// <summary> | ||
| /// Map Enum Metrics to a SupportedMetric | ||
| /// </summary> | ||
| private static readonly Dictionary<string, SupportedMetric> _map = new Dictionary<string, SupportedMetric> | ||
|
Member
There was a problem hiding this comment.
if the only usage for this is to contain a list of known metrics, the enum might be enough.
Member
Author
There was a problem hiding this comment. To specify the metric argument in the Macro we are using an Enum. That's the list of metrics you see in he CSharpApi.cs file. However, for parsing the results from the train test macro we need to use the metric names used by the ML.NET evaluators. e.g. AcuracyMicro (Enum Argument) v/s 'Accuracy(micro-avg)' (Evaluator metric). This dictionary maps the two. Does that clarify why we need this mapping? In reply to: 203217600 [](ancestors = 203217600) There was a problem hiding this comment. I think Senja meant that the conversion from the enum to a string is not needed (see my comment below on the GetSupportedMetric method). In reply to: 203220227 [](ancestors = 203220227,203217600) |
||
| { | ||
| { Metrics.Auc.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Auc, true)}, | ||
| { Metrics.AccuracyMicro.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMicro, true)}, | ||
| { Metrics.AccuracyMacro.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMacro, true)}, | ||
| { Metrics.L1.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L1, false)}, | ||
| { Metrics.L2.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L2, false)}, | ||
| { Metrics.F1.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.F1, true)}, | ||
| { Metrics.AuPrc.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AuPrc, true)}, | ||
| { Metrics.TopKAccuracy.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.TopKAccuracy, true)}, | ||
| { Metrics.Rms.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Rms, false)}, | ||
| { Metrics.LossFn.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LossFn, false)}, | ||
| { Metrics.RSquared.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.RSquared, false)}, | ||
| { Metrics.LogLoss.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLoss, false)}, | ||
| { Metrics.LogLossReduction.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLossReduction, true)}, | ||
| { Metrics.Ndcg.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Ndcg, true)}, | ||
| { Metrics.Dcg.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dcg, true)}, | ||
| { Metrics.PositivePrecision.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositivePrecision, true)}, | ||
| { Metrics.PositiveRecall.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositiveRecall, true)}, | ||
| { Metrics.NegativePrecision.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativePrecision, true)}, | ||
| { Metrics.NegativeRecall.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativeRecall, true)}, | ||
| { Metrics.DrAtK.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtK, true)}, | ||
| { Metrics.DrAtPFpr.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtPFpr, true)}, | ||
| { Metrics.DrAtNumPos.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtNumPos, true)}, | ||
| { Metrics.NumAnomalies.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NumAnomalies, true)}, | ||
| { Metrics.ThreshAtK.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtK, false)}, | ||
| { Metrics.ThreshAtP.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtP, false)}, | ||
| { Metrics.ThreshAtNumPos.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtNumPos, false)}, | ||
| { Metrics.Nmi.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Nmi, false)}, | ||
| { Metrics.AvgMinScore.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AvgMinScore, false)}, | ||
| { Metrics.Dbi.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dbi, false)} | ||
| }; | ||
|
|
||
| public static SupportedMetric GetSupportedMetric(IHostEnvironment env, string metricName) | ||
|
There was a problem hiding this comment.
Is this argument needed? If it's just used for the one check below, we can either have an IExceptionContext instead, or simply use Contracts.CheckNonEmpty(). #Resolved
Member
Author
There was a problem hiding this comment. I removed the argument. Is there a suitable Contracts.Check* API that is used for an Enum argument? For now, I just removed the Checks since the default for the switch/case will throw a NotSupportedException anyway In reply to: 203461762 [](ancestors = 203461762) There was a problem hiding this comment.
You can pass a Metric here instead of a string, this way you won't need the dictionary, and instead just do a switch statement on the parameter. #Resolved
Member
Author
There was a problem hiding this comment. Thanks for the clarification. Modified to using switch case as suggested. In general, regarding usage of switch-case v.s. a dictionary mapping - is there some concrete benefit of using switch-case here or is it mostly a matter of personal preferance ? In reply to: 203462492 [](ancestors = 203462492)
Member
There was a problem hiding this comment. Maybe to avoid the dictionary & switch statement, you can just keep a whitelist of the maximizing metrics. In the GetSupportedMetric, you can retrieve the boolean from looking up the whitelist. In reply to: 203543398 [](ancestors = 203543398,203462492)
Member
Author
There was a problem hiding this comment. Hi Senja. I intend to re-factor this code (task 267452). I want to unblock the benchmarking efforts . Is it fine if i keep it as switch-case for now? In reply to: 203780367 [](ancestors = 203780367,203543398,203462492) |
||
| { | ||
| Contracts.CheckValue(env, nameof(env)); | ||
| env.CheckNonEmpty(metricName, nameof(metricName)); | ||
|
|
||
| if (_map.ContainsKey(metricName)) | ||
| { | ||
| return _map[metricName]; | ||
| } | ||
|
|
||
| throw new NotSupportedException($"Metric '{metricName}' not supported."); | ||
| } | ||
| } | ||
|
|
||
| public sealed class SupportedMetric | ||
| { | ||
| public string Name { get; } | ||
| public bool IsMaximizing { get; } | ||
|
|
||
| public SupportedMetric(string name, bool isMaximizing) | ||
| { | ||
| Name = name; | ||
| IsMaximizing = isMaximizing; | ||
| } | ||
|
|
||
| public override string ToString() => Name; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15626,36 +15626,37 @@ public sealed class UniformRandomAutoMlEngine : AutoMlEngine | |
|
|
||
| public abstract class AutoMlStateBase : ComponentKind {} | ||
|
|
||
| public enum PipelineSweeperSupportedMetricsMetrics | ||
|
Contributor
There was a problem hiding this comment. Isn't MetricsMetrics a bit odd sounding? #Resolved
Member
Author
There was a problem hiding this comment. This is the auto generated file (the CSharpAPI translation layer) . The name here results from concatenating the following from PipelineSweeperSupportedMetrics.cs public sealed class PipelineSweeperSupportedMetrics { public enum Metrics {The name used for the class / enum look fine to me. Any alternate suggestions ? In reply to: 203265071 [](ancestors = 203265071) |
||
| { | ||
| Auc = 0, | ||
| AccuracyMicro = 1, | ||
| AccuracyMacro = 2, | ||
| L1 = 3, | ||
| L2 = 4, | ||
| F1 = 5, | ||
| AuPrc = 6, | ||
| TopKAccuracy = 7, | ||
| Rms = 8, | ||
| LossFn = 9, | ||
| RSquared = 10, | ||
| LogLoss = 11, | ||
| LogLossReduction = 12, | ||
| Ndcg = 13, | ||
| Dcg = 14, | ||
| PositivePrecision = 15, | ||
| PositiveRecall = 16, | ||
| NegativePrecision = 17, | ||
| NegativeRecall = 18, | ||
| DrAtK = 19, | ||
| DrAtPFpr = 20, | ||
| DrAtNumPos = 21, | ||
| NumAnomalies = 22, | ||
| ThreshAtK = 23, | ||
| ThreshAtP = 24, | ||
| ThreshAtNumPos = 25, | ||
| Nmi = 26, | ||
| AvgMinScore = 27, | ||
| Dbi = 28 | ||
|
Member
There was a problem hiding this comment.
the numbers are implied, if they are just sequential. #Resolved
Member
Author
There was a problem hiding this comment. Looks like they are auto generated by the CSharp API generator. We did not specify the numbers in the Enum. In reply to: 203217022 [](ancestors = 203217022)
Contributor
There was a problem hiding this comment. If we serialize the enum, I think we'd want a fixed number. This could be serialized as a state object for the sweep so that the user can resume the sweep w/ the history of the sweep stored on disk. #Resolved
Member
Author
There was a problem hiding this comment. Not sure if this is a general comment , or something that needs to be addressed ? Btw, this is is auto generated file . The Enum you see here is defined in PipelineSweeperSupportedMetrics.cs In reply to: 203267499 [](ancestors = 203267499)
Contributor
There was a problem hiding this comment. It's a topic to consider for the future. |
||
| } | ||
|
|
||
|
|
||
|
|
@@ -15668,7 +15669,7 @@ public sealed class AutoMlStateAutoMlStateBase : AutoMlStateBase | |
| /// <summary> | ||
| /// Supported metric for evaluator. | ||
| /// </summary> | ||
| public PipelineSweeperSupportedMetricsMetrics Metric { get; set; } = PipelineSweeperSupportedMetricsMetrics.Auc; | ||
|
|
||
| /// <summary> | ||
| /// AutoML engine (pipeline optimizer) that generates next candidates. | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If you change the GetSupportedMetric method to take the Metrics value, then you won't need this, you can just pass args.Metric. #Resolved