diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index fa12e10ada..98e54d4b46 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -66,11 +66,11 @@ public CrossValRunner(MLContext context, private static double CalcAverageScore(IEnumerable scores) { - if (scores.Any(s => double.IsNaN(s))) - { + var newScores = scores.Where(r => !double.IsNaN(r)); + // Return NaN iff all scores are NaN + if (newScores.Count() == 0) return double.NaN; - } - return scores.Average(); + return newScores.Average(); } } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index e640174fd8..eb5c02f89a 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML @@ -70,27 +71,105 @@ public CrossValSummaryRunner(MLContext context, // Get the model from the best fold var bestFoldIndex = BestResultUtil.GetIndexOfBestScore(trainResults.Select(r => r.score), _optimizingMetricInfo.IsMaximizing); + // bestFoldIndex will be -1 if the optimization metric for all folds is NaN. + // In this case, return model from the first fold. + bestFoldIndex = bestFoldIndex != -1 ? bestFoldIndex : 0; var bestModel = trainResults.ElementAt(bestFoldIndex).model; - // Get the metrics from the fold whose score is closest to avg of all fold scores - var avgScore = trainResults.Average(r => r.score); + // Get the average metrics across all folds + var avgScore = GetAverageOfNonNaNScores(trainResults.Select(x => x.score)); var indexClosestToAvg = GetIndexClosestToAverage(trainResults.Select(r => r.score), avgScore); var metricsClosestToAvg = trainResults[indexClosestToAvg].metrics; + var avgMetrics = GetAverageMetrics(trainResults.Select(x => x.metrics), metricsClosestToAvg); // Build result objects - var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, avgScore, allRunsSucceeded, metricsClosestToAvg, bestModel, null); + var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, avgScore, allRunsSucceeded, avgMetrics, bestModel, null); var runDetail = suggestedPipelineRunDetail.ToIterationResult(_preFeaturizer); return (suggestedPipelineRunDetail, runDetail); } + private static TMetrics GetAverageMetrics(IEnumerable metrics, TMetrics metricsClosestToAvg) + { + if (typeof(TMetrics) == typeof(BinaryClassificationMetrics)) + { + var newMetrics = metrics.Select(x => x as BinaryClassificationMetrics); + Contracts.Assert(newMetrics != null); + + var result = new BinaryClassificationMetrics( + auc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderRocCurve)), + accuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.Accuracy)), + positivePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositivePrecision)), + positiveRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.PositiveRecall)), + negativePrecision: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativePrecision)), + negativeRecall: GetAverageOfNonNaNScores(newMetrics.Select(x => x.NegativeRecall)), + f1Score: GetAverageOfNonNaNScores(newMetrics.Select(x => x.F1Score)), + auprc: GetAverageOfNonNaNScores(newMetrics.Select(x => x.AreaUnderPrecisionRecallCurve)), + // Return ConfusionMatrix from the fold closest to average score + confusionMatrix: (metricsClosestToAvg as BinaryClassificationMetrics).ConfusionMatrix); + return result as TMetrics; + } + + if (typeof(TMetrics) == typeof(MulticlassClassificationMetrics)) + { + var newMetrics = metrics.Select(x => x as MulticlassClassificationMetrics); + Contracts.Assert(newMetrics != null); + + var result = new MulticlassClassificationMetrics( + accuracyMicro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MicroAccuracy)), + accuracyMacro: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MacroAccuracy)), + logLoss: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLoss)), + logLossReduction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LogLossReduction)), + topKPredictionCount: newMetrics.ElementAt(0).TopKPredictionCount, + topKAccuracy: GetAverageOfNonNaNScores(newMetrics.Select(x => x.TopKAccuracy)), + // Return PerClassLogLoss and ConfusionMatrix from the fold closest to average score + perClassLogLoss: (metricsClosestToAvg as MulticlassClassificationMetrics).PerClassLogLoss.ToArray(), + confusionMatrix: (metricsClosestToAvg as MulticlassClassificationMetrics).ConfusionMatrix); + return result as TMetrics; + } + + if (typeof(TMetrics) == typeof(RegressionMetrics)) + { + var newMetrics = metrics.Select(x => x as RegressionMetrics); + Contracts.Assert(newMetrics != null); + + var result = new RegressionMetrics( + l1: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanAbsoluteError)), + l2: GetAverageOfNonNaNScores(newMetrics.Select(x => x.MeanSquaredError)), + rms: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RootMeanSquaredError)), + lossFunction: GetAverageOfNonNaNScores(newMetrics.Select(x => x.LossFunction)), + rSquared: GetAverageOfNonNaNScores(newMetrics.Select(x => x.RSquared))); + return result as TMetrics; + } + + throw new NotImplementedException($"Metric {typeof(TMetrics)} not implemented"); + } + + private static double GetAverageOfNonNaNScores(IEnumerable results) + { + var newResults = results.Where(r => !double.IsNaN(r)); + // Return NaN iff all scores are NaN + if (newResults.Count() == 0) + return double.NaN; + // Return average of non-NaN scores otherwise + return newResults.Average(r => r); + } + private static int GetIndexClosestToAverage(IEnumerable values, double average) { + // Average will be NaN iff all values are NaN. + // Return the first index in this case. + if (double.IsNaN(average)) + return 0; + int avgFoldIndex = -1; var smallestDistFromAvg = double.PositiveInfinity; for (var i = 0; i < values.Count(); i++) { - var distFromAvg = Math.Abs(values.ElementAt(i) - average); - if (distFromAvg < smallestDistFromAvg || smallestDistFromAvg == double.PositiveInfinity) + var value = values.ElementAt(i); + if (double.IsNaN(value)) + continue; + var distFromAvg = Math.Abs(value - average); + if (distFromAvg < smallestDistFromAvg) { smallestDistFromAvg = distFromAvg; avgFoldIndex = i; diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 05cba5e8a7..a40eb6bfe6 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -41,6 +41,9 @@ public static RunDetail GetBestRun(IEnumerable metricsAgent.GetScore(r.ValidationMetrics)); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); + // indexOfBestScore will be -1 if the optimization metric for all models is NaN. + // In this case, return the first model. + indexOfBestScore = indexOfBestScore != -1 ? indexOfBestScore : 0; return results.ElementAt(indexOfBestScore); } @@ -51,6 +54,9 @@ public static CrossValidationRunDetail GetBestRun(IEnumerabl if (!results.Any()) { return null; } var scores = results.Select(r => r.Results.Average(x => metricsAgent.GetScore(x.ValidationMetrics))); var indexOfBestScore = GetIndexOfBestScore(scores, isMetricMaximizing); + // indexOfBestScore will be -1 if the optimization metric for all models is NaN. + // In this case, return the first model. + indexOfBestScore = indexOfBestScore != -1 ? indexOfBestScore : 0; return results.ElementAt(indexOfBestScore); } diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs index 8a621b9de5..25bf956527 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs @@ -122,5 +122,12 @@ internal BinaryClassificationMetrics(double auc, double accuracy, double positiv F1Score = f1Score; AreaUnderPrecisionRecallCurve = auprc; } + + internal BinaryClassificationMetrics(double auc, double accuracy, double positivePrecision, double positiveRecall, + double negativePrecision, double negativeRecall, double f1Score, double auprc, ConfusionMatrix confusionMatrix) + : this(auc, accuracy, positivePrecision, positiveRecall, negativePrecision, negativeRecall, f1Score, auprc) + { + ConfusionMatrix = confusionMatrix; + } } } \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs index 68fed130d2..05d8f050d0 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs @@ -134,5 +134,12 @@ internal MulticlassClassificationMetrics(double accuracyMicro, double accuracyMa TopKAccuracy = topKAccuracy; PerClassLogLoss = perClassLogLoss.ToImmutableArray(); } + + internal MulticlassClassificationMetrics(double accuracyMicro, double accuracyMacro, double logLoss, double logLossReduction, + int topKPredictionCount, double topKAccuracy, double[] perClassLogLoss, ConfusionMatrix confusionMatrix) + : this(accuracyMicro, accuracyMacro, logLoss, logLossReduction, topKPredictionCount, topKAccuracy, perClassLogLoss) + { + ConfusionMatrix = confusionMatrix; + } } }