From c14a3678d763a90b6733737ea3405552ea6f608d Mon Sep 17 00:00:00 2001 From: Daniel Holstein Date: Thu, 25 Apr 2019 15:08:57 -0700 Subject: [PATCH 1/3] If first three iterations all fail, short-circuit AutoML experiment --- src/Microsoft.ML.Auto/Experiment/Experiment.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Microsoft.ML.Auto/Experiment/Experiment.cs b/src/Microsoft.ML.Auto/Experiment/Experiment.cs index ec167e9b17..ae5e28d588 100644 --- a/src/Microsoft.ML.Auto/Experiment/Experiment.cs +++ b/src/Microsoft.ML.Auto/Experiment/Experiment.cs @@ -89,6 +89,12 @@ public IList Execute() break; } + // If after third run, all runs have failed so far, break + if (_history.Count() == 3 && _history.All(r => !r.RunSucceded)) + { + break; + } + } while (_history.Count < _experimentSettings.MaxModels && !_experimentSettings.CancellationToken.IsCancellationRequested && stopwatch.Elapsed.TotalSeconds < _experimentSettings.MaxExperimentTimeInSeconds); From 931631ae20df672671eb04958af91dc09daca74c Mon Sep 17 00:00:00 2001 From: Daniel Holstein Date: Thu, 25 Apr 2019 19:15:48 -0700 Subject: [PATCH 2/3] throw exception if first 3 runs failed --- src/Microsoft.ML.Auto/Experiment/Experiment.cs | 4 ++-- .../SuggestedPipelineCrossValRunDetail.cs | 1 + .../SuggestedPipelineRunDetails/SuggestedPipelineRunDetail.cs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.Auto/Experiment/Experiment.cs b/src/Microsoft.ML.Auto/Experiment/Experiment.cs index ae5e28d588..da1321b02f 100644 --- a/src/Microsoft.ML.Auto/Experiment/Experiment.cs +++ b/src/Microsoft.ML.Auto/Experiment/Experiment.cs @@ -89,10 +89,10 @@ public IList Execute() break; } - // If after third run, all runs have failed so far, break + // If after third run, all runs have failed so far, throw exception if (_history.Count() == 3 && _history.All(r => !r.RunSucceded)) { - break; + throw new InvalidOperationException($"Training failed on the data, with exception: {_history.Last().Exception}"); } } while (_history.Count < _experimentSettings.MaxModels && diff --git a/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineCrossValRunDetail.cs b/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineCrossValRunDetail.cs index e21b55428a..835292800a 100644 --- a/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineCrossValRunDetail.cs +++ b/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineCrossValRunDetail.cs @@ -42,6 +42,7 @@ internal SuggestedPipelineCrossValRunDetail(SuggestedPipeline pipeline, IEnumerable> results) : base(pipeline, score, runSucceeded) { Results = results; + Exception = Results.Select(r => r.Exception).FirstOrDefault(e => e != null); } public CrossValidationRunDetail ToIterationResult(IEstimator preFeaturizer) diff --git a/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineRunDetail.cs b/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineRunDetail.cs index 7cb76e1ab3..7af2f84be9 100644 --- a/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineRunDetail.cs +++ b/src/Microsoft.ML.Auto/Experiment/SuggestedPipelineRunDetails/SuggestedPipelineRunDetail.cs @@ -12,6 +12,8 @@ internal class SuggestedPipelineRunDetail public readonly bool RunSucceded; public readonly double Score; + public Exception Exception { get; set; } + public SuggestedPipelineRunDetail(SuggestedPipeline pipeline, double score, bool runSucceeded) { Pipeline = pipeline; @@ -34,7 +36,6 @@ internal class SuggestedPipelineRunDetail : SuggestedPipelineRunDetail { public readonly TMetrics ValidationMetrics; public readonly ModelContainer ModelContainer; - public readonly Exception Exception; internal SuggestedPipelineRunDetail(SuggestedPipeline pipeline, double score, From 1530f083fc3cbc5eef2bc796160dc1d6c909f339 Mon Sep 17 00:00:00 2001 From: Daniel Holstein Date: Thu, 25 Apr 2019 19:34:26 -0700 Subject: [PATCH 3/3] rev error message --- src/Microsoft.ML.Auto/Experiment/Experiment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Auto/Experiment/Experiment.cs b/src/Microsoft.ML.Auto/Experiment/Experiment.cs index da1321b02f..f6dda403a3 100644 --- a/src/Microsoft.ML.Auto/Experiment/Experiment.cs +++ b/src/Microsoft.ML.Auto/Experiment/Experiment.cs @@ -92,7 +92,7 @@ public IList Execute() // If after third run, all runs have failed so far, throw exception if (_history.Count() == 3 && _history.All(r => !r.RunSucceded)) { - throw new InvalidOperationException($"Training failed on the data, with exception: {_history.Last().Exception}"); + throw new InvalidOperationException($"Training failed with the exception: {_history.Last().Exception}"); } } while (_history.Count < _experimentSettings.MaxModels &&