Skip to content

PipelineSweeperMacro for Multi-Class Classification #539

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abgoswam merged 5 commits into from
Jul 19, 2018
Merged

PipelineSweeperMacro for Multi-Class Classification #539

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
41f8598
failing test case for multiclass
abgoswam Jul 16, 2018
b7653de
Refactored PipelineSweeperSupportedMetrics Class; added unit test for…
abgoswam Jul 17, 2018
0ccfbab
take care of review comments; display transforms/learners + metrics i…
abgoswam Jul 18, 2018
46af33f
taking care of PR comments + refactor PipelineSweeperRunSummary
abgoswam Jul 18, 2018
d7e8402
taking care of review comments
abgoswam Jul 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 4 additions & 97 deletions src/Microsoft.ML.PipelineInference/AutoInference.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,67 +51,6 @@ public class LevelDependencyMap : Dictionary<ColumnInfo, List<TransformInference
/// </summary>
public class DependencyMap : Dictionary<int, LevelDependencyMap> { }

/// <summary>
/// AutoInference will support metrics as they are added here.
/// </summary>
public sealed class SupportedMetric
{
public static readonly SupportedMetric Auc = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Auc, true);
public static readonly SupportedMetric AccuracyMicro = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMicro, true);
public static readonly SupportedMetric AccuracyMacro = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMacro, true);
public static readonly SupportedMetric L1 = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L1, false);
public static readonly SupportedMetric L2 = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L2, false);
public static readonly SupportedMetric F1 = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.F1, true);
public static readonly SupportedMetric AuPrc = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AuPrc, true);
public static readonly SupportedMetric TopKAccuracy = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.TopKAccuracy, true);
public static readonly SupportedMetric Rms = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Rms, false);
public static readonly SupportedMetric LossFn = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LossFn, false);
public static readonly SupportedMetric RSquared = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.RSquared, false);
public static readonly SupportedMetric LogLoss = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLoss, false);
public static readonly SupportedMetric LogLossReduction = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLossReduction, true);
public static readonly SupportedMetric Ndcg = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Ndcg, true);
public static readonly SupportedMetric Dcg = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dcg, true);
public static readonly SupportedMetric PositivePrecision = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositivePrecision, true);
public static readonly SupportedMetric PositiveRecall = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositiveRecall, true);
public static readonly SupportedMetric NegativePrecision = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativePrecision, true);
public static readonly SupportedMetric NegativeRecall = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativeRecall, true);
public static readonly SupportedMetric DrAtK = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtK, true);
public static readonly SupportedMetric DrAtPFpr = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtPFpr, true);
public static readonly SupportedMetric DrAtNumPos = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtNumPos, true);
public static readonly SupportedMetric NumAnomalies = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NumAnomalies, true);
public static readonly SupportedMetric ThreshAtK = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtK, false);
public static readonly SupportedMetric ThreshAtP = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtP, false);
public static readonly SupportedMetric ThreshAtNumPos = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtNumPos, false);
public static readonly SupportedMetric Nmi = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Nmi, true);
public static readonly SupportedMetric AvgMinScore = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AvgMinScore, false);
public static readonly SupportedMetric Dbi = new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dbi, false);

public string Name { get; }
public bool IsMaximizing { get; }

private SupportedMetric(string name, bool isMaximizing)
{
Name = name;
IsMaximizing = isMaximizing;
}

public static SupportedMetric ByName(string name)
{
var fields =
typeof(SupportedMetric).GetFields(BindingFlags.Static | BindingFlags.Public);

foreach (var field in fields)
{
var metric = (SupportedMetric)field.GetValue(Auc);
if (name.Equals(metric.Name, StringComparison.OrdinalIgnoreCase))
return metric;
}
throw new NotSupportedException($"Metric '{name}' not supported.");
}

public override string ToString() => Name;
}

/// <summary>
/// Class for encapsulating an entrypoint experiment graph
/// and keeping track of the input and output nodes.
Expand Down Expand Up @@ -218,42 +157,8 @@ public sealed class AutoMlMlState : IMlState
Desc = "State of an AutoML search and search space.")]
public sealed class Arguments : ISupportAutoMlStateFactory
{
// REVIEW: These should be the same as SupportedMetrics above. Not sure how to reference that class,
// without the C# API generator trying to create a version of that class in the API as well.
public enum Metrics
{
Auc,
AccuracyMicro,
AccuracyMacro,
L2,
F1,
AuPrc,
TopKAccuracy,
Rms,
LossFn,
RSquared,
LogLoss,
LogLossReduction,
Ndcg,
Dcg,
PositivePrecision,
PositiveRecall,
NegativePrecision,
NegativeRecall,
DrAtK,
DrAtPFpr,
DrAtNumPos,
NumAnomalies,
ThreshAtK,
ThreshAtP,
ThreshAtNumPos,
Nmi,
AvgMinScore,
Dbi
};

[Argument(ArgumentType.Required, HelpText = "Supported metric for evaluator.", ShortName = "metric")]
public Metrics Metric;
public PipelineSweeperSupportedMetrics.Metrics Metric;

[Argument(ArgumentType.Required, HelpText = "AutoML engine (pipeline optimizer) that generates next candidates.", ShortName = "engine")]
public ISupportIPipelineOptimizerFactory Engine;
Expand All @@ -271,7 +176,9 @@ public enum Metrics
}

public AutoMlMlState(IHostEnvironment env, Arguments args)
: this(env, SupportedMetric.ByName(Enum.GetName(typeof(Arguments.Metrics), args.Metric)), args.Engine.CreateComponent(env),
: this(env,
PipelineSweeperSupportedMetrics.GetSupportedMetric(env, Enum.GetName(typeof(PipelineSweeperSupportedMetrics.Metrics), args.Metric)),

@yaeldekel yaeldekel Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Enum.GetName [](start = 78, length = 12)

If you change the GetSupportedMetric method to take the Metrics value, then you won't need this, you can just pass args.Metric. #Resolved

args.Engine.CreateComponent(env),
args.TerminatorArgs.CreateComponent(env), args.TrainerKind, requestedLearners: args.RequestedLearners)
{
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ML.PipelineInference/PipelinePattern.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ public Models.TrainTestEvaluator.Output AddAsTrainTest(Var<IDataView> trainData,
/// Runs a train-test experiment on the current pipeline, through entrypoints.
/// </summary>
public void RunTrainTestExperiment(IDataView trainData, IDataView testData,
AutoInference.SupportedMetric metric, MacroUtils.TrainerKinds trainerKind, out double testMetricValue,
SupportedMetric metric, MacroUtils.TrainerKinds trainerKind, out double testMetricValue,
out double trainMetricValue)
{
var experiment = CreateTrainTestExperiment(trainData, testData, trainerKind, true, out var trainTestOutput);
Expand Down
120 changes: 120 additions & 0 deletions src/Microsoft.ML.PipelineInference/PipelineSweeperSupportedMetrics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using Microsoft.ML.Runtime.CommandLine;
using Microsoft.ML.Runtime.EntryPoints;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.PipelineInference;
using Microsoft.ML.Runtime.EntryPoints.JsonUtils;
using Newtonsoft.Json.Linq;

namespace Microsoft.ML.Runtime.PipelineInference
{
/// <summary>
/// PipelineSweeper will support metrics as they are added here.
/// </summary>
public sealed class PipelineSweeperSupportedMetrics
{
public enum Metrics
{
Auc,
AccuracyMicro,
AccuracyMacro,
L1,
L2,
F1,
AuPrc,
TopKAccuracy,
Rms,
LossFn,
RSquared,
LogLoss,
LogLossReduction,
Ndcg,
Dcg,
PositivePrecision,
PositiveRecall,
NegativePrecision,
NegativeRecall,
DrAtK,
DrAtPFpr,
DrAtNumPos,
NumAnomalies,
ThreshAtK,
ThreshAtP,
ThreshAtNumPos,
Nmi,
AvgMinScore,
Dbi
};

/// <summary>
/// Mapp Enum Metrics to a SupportedMetric
/// </summary>
private static readonly Dictionary<string, SupportedMetric> _map = new Dictionary<string, SupportedMetric>

@sfilipi sfilipi Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_map [](start = 67, length = 5)

if the only usage for this is to contain a list of known metrics, the enum might be enough.
Take a look at Enum.Parse. You can move the construction of new SupportedMetric in the GetSupportedMetric, if the conversion succeeds. #Resolved

@abgoswam abgoswam Jul 18, 2018 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To specify the metric argument in the Macro we are using an Enum. That's the list of metrics you see in he CSharpApi.cs file. However, for parsing the results from the train test macro we need to use the metric names used by the ML.NET evaluators.

e.g. AcuracyMicro (Enum Argument) v/s 'Accuracy(micro-avg)' (Evaluator metric). This dictionary maps the two. Does that clarify why we need this mapping?

In reply to: 203217600 [](ancestors = 203217600)

@yaeldekel yaeldekel Jul 18, 2018

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think Senja meant that the conversion from the enum to a string is not needed (see my comment below on the GetSupportedMetric method).

In reply to: 203220227 [](ancestors = 203220227,203217600)

{
{ Metrics.Auc.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Auc, true)},
{ Metrics.AccuracyMicro.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMicro, true)},
{ Metrics.AccuracyMacro.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AccuracyMacro, true)},
{ Metrics.L1.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L1, false)},
{ Metrics.L2.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.L2, false)},
{ Metrics.F1.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.F1, true)},
{ Metrics.AuPrc.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AuPrc, true)},
{ Metrics.TopKAccuracy.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.TopKAccuracy, true)},
{ Metrics.Rms.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Rms, false)},
{ Metrics.LossFn.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LossFn, false)},
{ Metrics.RSquared.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.RSquared, false)},
{ Metrics.LogLoss.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLoss, false)},
{ Metrics.LogLossReduction.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.LogLossReduction, true)},
{ Metrics.Ndcg.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Ndcg, true)},
{ Metrics.Dcg.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dcg, true)},
{ Metrics.PositivePrecision.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositivePrecision, true)},
{ Metrics.PositiveRecall.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.PositiveRecall, true)},
{ Metrics.NegativePrecision.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativePrecision, true)},
{ Metrics.NegativeRecall.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NegativeRecall, true)},
{ Metrics.DrAtK.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtK, true)},
{ Metrics.DrAtPFpr.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtPFpr, true)},
{ Metrics.DrAtNumPos.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.DrAtNumPos, true)},
{ Metrics.NumAnomalies.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.NumAnomalies, true)},
{ Metrics.ThreshAtK.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtK, false)},
{ Metrics.ThreshAtP.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtP, false)},
{ Metrics.ThreshAtNumPos.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.ThreshAtNumPos, false)},
{ Metrics.Nmi.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Nmi, false)},
{ Metrics.AvgMinScore.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.AvgMinScore, false)},
{ Metrics.Dbi.ToString(), new SupportedMetric(FieldNames.PipelineSweeperSupportedMetrics.Dbi, false)}
};

public static SupportedMetric GetSupportedMetric(IHostEnvironment env, string metricName)

@yaeldekel yaeldekel Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

env [](start = 74, length = 3)

Is this argument needed? If it's just used for the one check below, we can either have an IExceptionContext instead, or simply use Contracts.CheckNonEmpty(). #Resolved

@abgoswam abgoswam Jul 18, 2018 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed the argument. Is there a suitable Contracts.Check* API that is used for an Enum argument?

For now, I just removed the Checks since the default for the switch/case will throw a NotSupportedException anyway

In reply to: 203461762 [](ancestors = 203461762)

@yaeldekel yaeldekel Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

string [](start = 79, length = 6)

You can pass a Metric here instead of a string, this way you won't need the dictionary, and instead just do a switch statement on the parameter. #Resolved

@abgoswam abgoswam Jul 18, 2018 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the clarification. Modified to using switch case as suggested.

In general, regarding usage of switch-case v.s. a dictionary mapping - is there some concrete benefit of using switch-case here or is it mostly a matter of personal preferance ?

In reply to: 203462492 [](ancestors = 203462492)

@sfilipi sfilipi Jul 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe to avoid the dictionary & switch statement, you can just keep a whitelist of the maximizing metrics. In the GetSupportedMetric, you can retrieve the boolean from looking up the whitelist.

In reply to: 203543398 [](ancestors = 203543398,203462492)

@abgoswam abgoswam Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi Senja. I intend to re-factor this code (task 267452).
I have added the metrics issue as part of the same task.

I want to unblock the benchmarking efforts . Is it fine if i keep it as switch-case for now?

In reply to: 203780367 [](ancestors = 203780367,203543398,203462492)

{
Contracts.CheckValue(env, nameof(env));
env.CheckNonEmpty(metricName, nameof(metricName));

if (_map.ContainsKey(metricName))
{
return _map[metricName];
}

throw new NotSupportedException($"Metric '{metricName}' not supported.");
}
}

public sealed class SupportedMetric
{
public string Name { get; }
public bool IsMaximizing { get; }

public SupportedMetric(string name, bool isMaximizing)
{
Name = name;
IsMaximizing = isMaximizing;
}

public override string ToString() => Name;
}
}
55 changes: 28 additions & 27 deletions src/Microsoft.ML/CSharpApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15626,36 +15626,37 @@ public sealed class UniformRandomAutoMlEngine : AutoMlEngine

public abstract class AutoMlStateBase : ComponentKind {}

public enum AutoInferenceAutoMlMlStateArgumentsMetrics
public enum PipelineSweeperSupportedMetricsMetrics

@justinormont justinormont Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't MetricsMetrics a bit odd sounding? #Resolved

@abgoswam abgoswam Jul 18, 2018

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the auto generated file (the CSharpAPI translation layer) .

The name here results from concatenating the following from PipelineSweeperSupportedMetrics.cs

public sealed class PipelineSweeperSupportedMetrics { public enum Metrics {

The name used for the class / enum look fine to me. Any alternate suggestions ?

In reply to: 203265071 [](ancestors = 203265071)

{
Auc = 0,
AccuracyMicro = 1,
AccuracyMacro = 2,
L2 = 3,
F1 = 4,
AuPrc = 5,
TopKAccuracy = 6,
Rms = 7,
LossFn = 8,
RSquared = 9,
LogLoss = 10,
LogLossReduction = 11,
Ndcg = 12,
Dcg = 13,
PositivePrecision = 14,
PositiveRecall = 15,
NegativePrecision = 16,
NegativeRecall = 17,
DrAtK = 18,
DrAtPFpr = 19,
DrAtNumPos = 20,
NumAnomalies = 21,
ThreshAtK = 22,
ThreshAtP = 23,
ThreshAtNumPos = 24,
Nmi = 25,
AvgMinScore = 26,
Dbi = 27
L1 = 3,
L2 = 4,
F1 = 5,
AuPrc = 6,
TopKAccuracy = 7,
Rms = 8,
LossFn = 9,
RSquared = 10,
LogLoss = 11,
LogLossReduction = 12,
Ndcg = 13,
Dcg = 14,
PositivePrecision = 15,
PositiveRecall = 16,
NegativePrecision = 17,
NegativeRecall = 18,
DrAtK = 19,
DrAtPFpr = 20,
DrAtNumPos = 21,
NumAnomalies = 22,
ThreshAtK = 23,
ThreshAtP = 24,
ThreshAtNumPos = 25,
Nmi = 26,
AvgMinScore = 27,
Dbi = 28

@sfilipi sfilipi Jul 17, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

= 28 [](start = 16, length = 4)

the numbers are implied, if they are just sequential. #Resolved

@abgoswam abgoswam Jul 18, 2018 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like they are auto generated by the CSharp API generator. We did not specify the numbers in the Enum.

In reply to: 203217022 [](ancestors = 203217022)

@justinormont justinormont Jul 18, 2018 •
edited by abgoswam
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we serialize the enum, I think we'd want a fixed number. This could be serialized as a state object for the sweep so that the user can resume the sweep w/ the history of the sweep stored on disk. #Resolved

@abgoswam abgoswam Jul 18, 2018

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this is a general comment , or something that needs to be addressed ?

Btw, this is is auto generated file . The Enum you see here is defined in PipelineSweeperSupportedMetrics.cs

In reply to: 203267499 [](ancestors = 203267499)

@justinormont justinormont Jul 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a topic to consider for the future.
When we serialize the enum, we'll either have to serialize the numeric form or the text form. A numeric form will be more stable, and for that we'd have to assign the numbers in the primary definition of the enum.

}


Expand All @@ -15668,7 +15669,7 @@ public sealed class AutoMlStateAutoMlStateBase : AutoMlStateBase
/// <summary>
/// Supported metric for evaluator.
/// </summary>
public AutoInferenceAutoMlMlStateArgumentsMetrics Metric { get; set; } = AutoInferenceAutoMlMlStateArgumentsMetrics.Auc;
public PipelineSweeperSupportedMetricsMetrics Metric { get; set; } = PipelineSweeperSupportedMetricsMetrics.Auc;

/// <summary>
/// AutoML engine (pipeline optimizer) that generates next candidates.
Expand Down
1 change: 1 addition & 0 deletions test/BaselineOutput/Common/EntryPoints/core_manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21644,6 +21644,7 @@
"Auc",
"AccuracyMicro",
"AccuracyMacro",
"L1",
"L2",
"F1",
"AuPrc",
Expand Down
Loading