From 8e2bbda245a9bc4b232b81cf8668af9512d27259 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 15 Apr 2020 17:26:18 -0700 Subject: [PATCH 01/11] Handle all folds returning NaN optimization metric in CrossValSummaryRunner --- .../Experiment/Runners/CrossValSummaryRunner.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index e640174fd8..814fd704cf 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML @@ -70,6 +71,9 @@ public CrossValSummaryRunner(MLContext context, // Get the model from the best fold var bestFoldIndex = BestResultUtil.GetIndexOfBestScore(trainResults.Select(r => r.score), _optimizingMetricInfo.IsMaximizing); + // bestFoldIndex will be -1 if the optimization metric for all folds is NaN. + // In this case, return model from the first fold. + bestFoldIndex = bestFoldIndex != -1 ? bestFoldIndex : 0; var bestModel = trainResults.ElementAt(bestFoldIndex).model; // Get the metrics from the fold whose score is closest to avg of all fold scores From 307d51da5231c30aa9620f3c8cc08a7a80eea76c Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 16 Apr 2020 16:32:33 -0700 Subject: [PATCH 02/11] Handle NaN in calculation of average scores and index of closest fold --- .../Runners/CrossValSummaryRunner.cs | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 814fd704cf..8dead71318 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.ML.Data; +using Google.Protobuf.WellKnownTypes; using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML @@ -77,7 +77,7 @@ public CrossValSummaryRunner(MLContext context, var bestModel = trainResults.ElementAt(bestFoldIndex).model; // Get the metrics from the fold whose score is closest to avg of all fold scores - var avgScore = trainResults.Average(r => r.score); + var avgScore = GetAverageOfNonNaNScores(trainResults); var indexClosestToAvg = GetIndexClosestToAverage(trainResults.Select(r => r.score), avgScore); var metricsClosestToAvg = trainResults[indexClosestToAvg].metrics; @@ -87,14 +87,32 @@ public CrossValSummaryRunner(MLContext context, return (suggestedPipelineRunDetail, runDetail); } + private static double GetAverageOfNonNaNScores(List<(ModelContainer model, TMetrics metrics, Exception exception, double score)> results) + { + var newResults = results.Where(r => !double.IsNaN(r.score)); + // Return NaN iff all scores are NaN + if (newResults.Count() ==0) + return double.NaN; + // Return average of non-NaN scores otherwise + return newResults.Average(r => r.score); + } + private static int GetIndexClosestToAverage(IEnumerable values, double average) { + // Average will be NaN iff all values are NaN. + // Return the first index in this case. + if (double.IsNaN(average)) + return 0; + int avgFoldIndex = -1; var smallestDistFromAvg = double.PositiveInfinity; for (var i = 0; i < values.Count(); i++) { - var distFromAvg = Math.Abs(values.ElementAt(i) - average); - if (distFromAvg < smallestDistFromAvg || smallestDistFromAvg == double.PositiveInfinity) + var value = values.ElementAt(i); + if (double.IsNaN(value)) + continue; + var distFromAvg = Math.Abs(value - average); + if (distFromAvg < smallestDistFromAvg) { smallestDistFromAvg = distFromAvg; avgFoldIndex = i; From 1a5a85289908abf0f1a537b21851b6d034b8a2c3 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 16 Apr 2020 17:25:58 -0700 Subject: [PATCH 03/11] Handle all metrics being NaN in finding Best Run --- src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 05cba5e8a7..2f848b39ed 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.ML.Data; +using Microsoft.ML.Internal.CpuMath.Core; namespace Microsoft.ML.AutoML { @@ -41,6 +42,8 @@ public static RunDetail GetBestRun(IEnumerable metricsAgent.GetScore(r.ValidationMetrics)); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); + Contracts.Check(indexOfBestScore != -1, + "The optimization metric in all runs was undefined. A best run could not be found."); return results.ElementAt(indexOfBestScore); } @@ -51,6 +54,8 @@ public static CrossValidationRunDetail GetBestRun(IEnumerabl if (!results.Any()) { return null; } var scores = results.Select(r => r.Results.Average(x => metricsAgent.GetScore(x.ValidationMetrics))); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); + Contracts.Check(indexOfBestScore != -1, + "The average optimization metric in all runs was undefined. A best run could not be found."); return results.ElementAt(indexOfBestScore); } From 2ec8cb73c0e0c2f40c4b1367a6636334985d07cb Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 16 Apr 2020 17:33:22 -0700 Subject: [PATCH 04/11] nit --- .../Experiment/Runners/CrossValSummaryRunner.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 8dead71318..1e6e2c8b61 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Google.Protobuf.WellKnownTypes; using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML From dd38e5996a136c0c8d35fc611f3240879bc6325b Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 16 Apr 2020 19:38:19 -0700 Subject: [PATCH 05/11] nit --- .../Experiment/Runners/CrossValSummaryRunner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 1e6e2c8b61..552fe86ceb 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,7 +6,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.FastTree; namespace Microsoft.ML.AutoML { @@ -90,7 +92,7 @@ private static double GetAverageOfNonNaNScores(List<(ModelContainer model, TMetr { var newResults = results.Where(r => !double.IsNaN(r.score)); // Return NaN iff all scores are NaN - if (newResults.Count() ==0) + if (newResults.Count() == 0) return double.NaN; // Return average of non-NaN scores otherwise return newResults.Average(r => r.score); From 0782e58d438c2af532e2b1544a08481bbe234000 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 16 Apr 2020 20:09:31 -0700 Subject: [PATCH 06/11] nit --- .../Experiment/Runners/CrossValSummaryRunner.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 552fe86ceb..74163646c5 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,9 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.ML.Data; using Microsoft.ML.Runtime; -using Microsoft.ML.Trainers.FastTree; namespace Microsoft.ML.AutoML { From d800b5338eefea83e0dbb6d42ddf2d7eca983b6a Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Fri, 17 Apr 2020 14:48:01 -0700 Subject: [PATCH 07/11] Handle all NaNs in best model selection --- src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 2f848b39ed..a82782a17b 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using Microsoft.ML.Data; using Microsoft.ML.Internal.CpuMath.Core; @@ -42,8 +43,9 @@ public static RunDetail GetBestRun(IEnumerable metricsAgent.GetScore(r.ValidationMetrics)); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); - Contracts.Check(indexOfBestScore != -1, - "The optimization metric in all runs was undefined. A best run could not be found."); + // indexOfBestScore will be -1 if the optimization metric for all models is NaN. + // In this case, return the first model. + indexOfBestScore = indexOfBestScore != -1 ? indexOfBestScore : 0; return results.ElementAt(indexOfBestScore); } @@ -54,8 +56,9 @@ public static CrossValidationRunDetail GetBestRun(IEnumerabl if (!results.Any()) { return null; } var scores = results.Select(r => r.Results.Average(x => metricsAgent.GetScore(x.ValidationMetrics))); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); - Contracts.Check(indexOfBestScore != -1, - "The average optimization metric in all runs was undefined. A best run could not be found."); + // indexOfBestScore will be -1 if the optimization metric for all models is NaN. + // In this case, return the first model. + indexOfBestScore = indexOfBestScore != -1 ? indexOfBestScore : 0; return results.ElementAt(indexOfBestScore); } From a8112820ffa394adfd9c0fd15d27ba0bc2613ffa Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Fri, 17 Apr 2020 15:48:12 -0700 Subject: [PATCH 08/11] Return average metrics instead of metrics form the fold with optimizing metric closest to average --- .../Runners/CrossValSummaryRunner.cs | 71 ++++++++++++++++--- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 74163646c5..70a1c85026 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML @@ -75,25 +76,79 @@ public CrossValSummaryRunner(MLContext context, bestFoldIndex = bestFoldIndex != -1 ? bestFoldIndex : 0; var bestModel = trainResults.ElementAt(bestFoldIndex).model; - // Get the metrics from the fold whose score is closest to avg of all fold scores - var avgScore = GetAverageOfNonNaNScores(trainResults); - var indexClosestToAvg = GetIndexClosestToAverage(trainResults.Select(r => r.score), avgScore); - var metricsClosestToAvg = trainResults[indexClosestToAvg].metrics; + // Get the average metrics across all folds + var avgScore = GetAverageOfNonNaNScores(trainResults.Select(x => x.score)); + var avgMetrics = GetAverageMetrics(trainResults.Select(x => x.metrics)); // Build result objects - var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, avgScore, allRunsSucceeded, metricsClosestToAvg, bestModel, null); + var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, avgScore, allRunsSucceeded, avgMetrics, bestModel, null); var runDetail = suggestedPipelineRunDetail.ToIterationResult(_preFeaturizer); return (suggestedPipelineRunDetail, runDetail); } - private static double GetAverageOfNonNaNScores(List<(ModelContainer model, TMetrics metrics, Exception exception, double score)> results) + private static TMetrics GetAverageMetrics(IEnumerable metrics) { - var newResults = results.Where(r => !double.IsNaN(r.score)); + if (typeof(TMetrics) == typeof(BinaryClassificationMetrics)) + { + var newMetrics = metrics.Select(x => x as BinaryClassificationMetrics); + Contracts.Assert(newMetrics != null); + + var result = new BinaryClassificationMetrics( + auc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderRocCurve)), + accuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.Accuracy)), + positivePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositivePrecision)), + positiveRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositiveRecall)), + negativePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativePrecision)), + negativeRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativeRecall)), + f1Score: GetAverageOfNonNaNScores(newMetrics.Select(x => x.F1Score)), + auprc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderPrecisionRecallCurve))); + return result as TMetrics; + } + + if (typeof(TMetrics) == typeof(MulticlassClassificationMetrics)) + { + var newMetrics = metrics.Select(x => x as MulticlassClassificationMetrics); + Contracts.Assert(newMetrics != null); + + var result = new MulticlassClassificationMetrics( + accuracyMicro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MicroAccuracy)), + accuracyMacro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MacroAccuracy)), + logLoss: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLoss)), + logLossReduction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLossReduction)), + topKPredictionCount: newMetrics.ElementAt(0).TopKPredictionCount, + topKAccuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.TopKAccuracy)), + // TODO: + // Figure out whether class label ordering can be different across different folds. + // If yes, whether it is possible to get the information from the objects available. + perClassLogLoss: null); + return result as TMetrics; + } + + if (typeof(TMetrics) == typeof(RegressionMetrics)) + { + var newMetrics = metrics.Select(x => x as RegressionMetrics); + Contracts.Assert(newMetrics != null); + + var result = new RegressionMetrics( + l1: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanAbsoluteError)), + l2: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanSquaredError)), + rms: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RootMeanSquaredError)), + lossFunction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LossFunction)), + rSquared: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RSquared))); + return result as TMetrics; + } + + return null; + } + + private static double GetAverageOfNonNaNScores(IEnumerable results) + { + var newResults = results.Where(r => !double.IsNaN(r)); // Return NaN iff all scores are NaN if (newResults.Count() == 0) return double.NaN; // Return average of non-NaN scores otherwise - return newResults.Average(r => r.score); + return newResults.Average(r => r); } private static int GetIndexClosestToAverage(IEnumerable values, double average) From 9d937cd3ecee17e4eab97c53cccfc065618b63f5 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Fri, 17 Apr 2020 15:49:44 -0700 Subject: [PATCH 09/11] nit --- src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index a82782a17b..a40eb6bfe6 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -4,9 +4,7 @@ using System.Collections.Generic; using System.Linq; -using System.Runtime.CompilerServices; using Microsoft.ML.Data; -using Microsoft.ML.Internal.CpuMath.Core; namespace Microsoft.ML.AutoML { From 401cb4ada927aafb5192400bbbee969dfc123512 Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Wed, 22 Apr 2020 20:46:33 -0700 Subject: [PATCH 10/11] Add PerClassLogLoss and ConfusionMatrix from the fold closest to average score --- .../Experiment/Runners/CrossValSummaryRunner.cs | 17 ++++++++++------- .../Metrics/BinaryClassificationMetrics.cs | 7 +++++++ .../Metrics/MulticlassClassificationMetrics.cs | 7 +++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 70a1c85026..fc012c1310 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -78,7 +78,9 @@ public CrossValSummaryRunner(MLContext context, // Get the average metrics across all folds var avgScore = GetAverageOfNonNaNScores(trainResults.Select(x => x.score)); - var avgMetrics = GetAverageMetrics(trainResults.Select(x => x.metrics)); + var indexClosestToAvg = GetIndexClosestToAverage(trainResults.Select(r => r.score), avgScore); + var metricsClosestToAvg = trainResults[indexClosestToAvg].metrics; + var avgMetrics = GetAverageMetrics(trainResults.Select(x => x.metrics), metricsClosestToAvg); // Build result objects var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, avgScore, allRunsSucceeded, avgMetrics, bestModel, null); @@ -86,7 +88,7 @@ public CrossValSummaryRunner(MLContext context, return (suggestedPipelineRunDetail, runDetail); } - private static TMetrics GetAverageMetrics(IEnumerable metrics) + private static TMetrics GetAverageMetrics(IEnumerable metrics, TMetrics metricsClosestToAvg) { if (typeof(TMetrics) == typeof(BinaryClassificationMetrics)) { @@ -101,7 +103,9 @@ private static TMetrics GetAverageMetrics(IEnumerable metrics) negativePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativePrecision)), negativeRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativeRecall)), f1Score: GetAverageOfNonNaNScores(newMetrics.Select(x => x.F1Score)), - auprc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderPrecisionRecallCurve))); + auprc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderPrecisionRecallCurve)), + // Return ConfusionMatrix from the fold closest to average score + confusionMatrix: (metricsClosestToAvg as BinaryClassificationMetrics).ConfusionMatrix); return result as TMetrics; } @@ -117,10 +121,9 @@ private static TMetrics GetAverageMetrics(IEnumerable metrics) logLossReduction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLossReduction)), topKPredictionCount: newMetrics.ElementAt(0).TopKPredictionCount, topKAccuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.TopKAccuracy)), - // TODO: - // Figure out whether class label ordering can be different across different folds. - // If yes, whether it is possible to get the information from the objects available. - perClassLogLoss: null); + // Return PerClassLogLoss and ConfusionMatrix from the fold closest to average score + perClassLogLoss: (metricsClosestToAvg as MulticlassClassificationMetrics).PerClassLogLoss.ToArray(), + confusionMatrix: (metricsClosestToAvg as MulticlassClassificationMetrics).ConfusionMatrix); return result as TMetrics; } diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs index 8a621b9de5..25bf956527 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs @@ -122,5 +122,12 @@ internal BinaryClassificationMetrics(double auc, double accuracy, double positiv F1Score = f1Score; AreaUnderPrecisionRecallCurve = auprc; } + + internal BinaryClassificationMetrics(double auc, double accuracy, double positivePrecision, double positiveRecall, + double negativePrecision, double negativeRecall, double f1Score, double auprc, ConfusionMatrix confusionMatrix) + : this(auc, accuracy, positivePrecision, positiveRecall, negativePrecision, negativeRecall, f1Score, auprc) + { + ConfusionMatrix = confusionMatrix; + } } } \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs index 68fed130d2..05d8f050d0 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs @@ -134,5 +134,12 @@ internal MulticlassClassificationMetrics(double accuracyMicro, double accuracyMa TopKAccuracy = topKAccuracy; PerClassLogLoss = perClassLogLoss.ToImmutableArray(); } + + internal MulticlassClassificationMetrics(double accuracyMicro, double accuracyMacro, double logLoss, double logLossReduction, + int topKPredictionCount, double topKAccuracy, double[] perClassLogLoss, ConfusionMatrix confusionMatrix) + : this(accuracyMicro, accuracyMacro, logLoss, logLossReduction, topKPredictionCount, topKAccuracy, perClassLogLoss) + { + ConfusionMatrix = confusionMatrix; + } } } From 7141539f672db7a80dd43490864539b2387302fc Mon Sep 17 00:00:00 2001 From: "REDMOND\\nakazmi" Date: Thu, 23 Apr 2020 14:13:29 -0700 Subject: [PATCH 11/11] feedback --- .../Experiment/Runners/CrossValRunner.cs | 8 ++++---- .../Experiment/Runners/CrossValSummaryRunner.cs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index fa12e10ada..98e54d4b46 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -66,11 +66,11 @@ public CrossValRunner(MLContext context, private static double CalcAverageScore(IEnumerable scores) { - if (scores.Any(s => double.IsNaN(s))) - { + var newScores = scores.Where(r => !double.IsNaN(r)); + // Return NaN iff all scores are NaN + if (newScores.Count() == 0) return double.NaN; - } - return scores.Average(); + return newScores.Average(); } } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index fc012c1310..eb5c02f89a 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -141,7 +141,7 @@ private static TMetrics GetAverageMetrics(IEnumerable metrics, TMetric return result as TMetrics; } - return null; + throw new NotImplementedException($"Metric {typeof(TMetrics)} not implemented"); } private static double GetAverageOfNonNaNScores(IEnumerable results)