From f4a910ee75bb41165d19514b8c763760f38e2266 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 27 Mar 2020 13:28:06 -0700 Subject: [PATCH 01/22] Free Tensor objects in finally statement --- .../Experiment/Runners/RunnerUtil.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index c6aeef49a5..cc2fb099f9 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -10,6 +10,8 @@ namespace Microsoft.ML.AutoML { internal static class RunnerUtil { + private static ITransformer _estimatorModel; + public static (ModelContainer model, TMetrics metrics, Exception exception, double score) TrainAndScorePipeline(MLContext context, SuggestedPipeline pipeline, @@ -25,21 +27,21 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub try { var estimator = pipeline.ToEstimator(trainData, validData); - var model = estimator.Fit(trainData); + _estimatorModel = estimator.Fit(trainData); - var scoredData = model.Transform(validData); + var scoredData = _estimatorModel.Transform(validData); var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn); var score = metricsAgent.GetScore(metrics); if (preprocessorTransform != null) { - model = preprocessorTransform.Append(model); + _estimatorModel = preprocessorTransform.Append(_estimatorModel); } // Build container for model var modelContainer = modelFileInfo == null ? - new ModelContainer(context, model) : - new ModelContainer(context, modelFileInfo, model, modelInputSchema); + new ModelContainer(context, _estimatorModel) : + new ModelContainer(context, modelFileInfo, _estimatorModel, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -48,6 +50,12 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub logger.Error($"Pipeline crashed: {pipeline.ToString()} . Exception: {ex}"); return (null, null, ex, double.NaN); } + finally + { + // Free Tensor objects in model. Tensor objects made in TensorFlow's C + // libraries are not automatically cleaned up by C#'s Garbage Collector. + (_estimatorModel as IDisposable)?.Dispose(); + } } public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum) From 4f8475cbf8df5cef679810ce03b2cb186f3c543d Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 27 Mar 2020 16:13:40 -0700 Subject: [PATCH 02/22] Update RunnerUtil.cs --- .../Experiment/Runners/RunnerUtil.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index cc2fb099f9..f9fce06b68 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -10,8 +10,6 @@ namespace Microsoft.ML.AutoML { internal static class RunnerUtil { - private static ITransformer _estimatorModel; - public static (ModelContainer model, TMetrics metrics, Exception exception, double score) TrainAndScorePipeline(MLContext context, SuggestedPipeline pipeline, @@ -24,24 +22,25 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub DataViewSchema modelInputSchema, IChannel logger) where TMetrics : class { + ITransformer estimatorModel = null; try { var estimator = pipeline.ToEstimator(trainData, validData); - _estimatorModel = estimator.Fit(trainData); + estimatorModel = estimator.Fit(trainData); - var scoredData = _estimatorModel.Transform(validData); + var scoredData = estimatorModel.Transform(validData); var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn); var score = metricsAgent.GetScore(metrics); if (preprocessorTransform != null) { - _estimatorModel = preprocessorTransform.Append(_estimatorModel); + estimatorModel = preprocessorTransform.Append(estimatorModel); } // Build container for model var modelContainer = modelFileInfo == null ? - new ModelContainer(context, _estimatorModel) : - new ModelContainer(context, modelFileInfo, _estimatorModel, modelInputSchema); + new ModelContainer(context, estimatorModel) : + new ModelContainer(context, modelFileInfo, estimatorModel, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -54,7 +53,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub { // Free Tensor objects in model. Tensor objects made in TensorFlow's C // libraries are not automatically cleaned up by C#'s Garbage Collector. - (_estimatorModel as IDisposable)?.Dispose(); + (estimatorModel as IDisposable)?.Dispose(); } } From f7e13376e87a81e54955c073ac346a32b0136238 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Mon, 30 Mar 2020 10:13:04 -0700 Subject: [PATCH 03/22] Re-enable AutoFitImageClassificationTrainTest after fix --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6523f89f0d..30b1251980 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -54,8 +54,6 @@ public void AutoFitMultiTest() } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void AutoFitImageClassificationTrainTest() { var context = new MLContext(seed: 1); From 83ad312f86d6e22217c3ade6898ba12aa1bf13b2 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Mon, 6 Apr 2020 23:23:39 -0700 Subject: [PATCH 04/22] Added IDisposable support to ModelContainer & corrected name of modelContainer in used cases --- .../Experiment/ModelContainer.cs | 17 ++++++++++++++++- .../Experiment/Runners/CrossValRunner.cs | 2 +- .../Experiment/Runners/RunnerUtil.cs | 2 +- .../Experiment/Runners/TrainValidateRunner.cs | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index 8b829da7ab..bcd63d93ce 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -2,11 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.IO; namespace Microsoft.ML.AutoML { - internal class ModelContainer + internal class ModelContainer : IDisposable { private readonly MLContext _mlContext; private readonly FileInfo _fileInfo; @@ -16,12 +17,14 @@ internal ModelContainer(MLContext mlContext, ITransformer model) { _mlContext = mlContext; _model = model; + _disposed = false; } internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema) { _mlContext = mlContext; _fileInfo = fileInfo; + _disposed = false; // Write model to disk using (var fs = File.Create(fileInfo.FullName)) @@ -46,5 +49,17 @@ public ITransformer GetModel() } return model; } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + (_model as IDisposable)?.Dispose(); + _disposed = true; + } + #endregion } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index fa12e10ada..7312bd1725 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -53,7 +53,7 @@ public CrossValRunner(MLContext context, var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i], _labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger); - trainResults.Add(new SuggestedPipelineTrainResult(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score)); + trainResults.Add(new SuggestedPipelineTrainResult(trainResult.modelContainer, trainResult.metrics, trainResult.exception, trainResult.score)); } var avgScore = CalcAverageScore(trainResults.Select(r => r.Score)); diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index f9fce06b68..97111d7189 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -10,7 +10,7 @@ namespace Microsoft.ML.AutoML { internal static class RunnerUtil { - public static (ModelContainer model, TMetrics metrics, Exception exception, double score) + public static (ModelContainer modelContainer, TMetrics metrics, Exception exception, double score) TrainAndScorePipeline(MLContext context, SuggestedPipeline pipeline, IDataView trainData, diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index d608f7dd2f..4289f45a0c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -50,7 +50,7 @@ public TrainValidateRunner(MLContext context, trainResult.score, trainResult.exception == null, trainResult.metrics, - trainResult.model, + trainResult.modelContainer, trainResult.exception); var runDetail = suggestedPipelineRunDetail.ToIterationResult(_preFeaturizer); return (suggestedPipelineRunDetail, runDetail); From b366fbe361c52a3cf382df4bc44cadd020ab3fa1 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Mon, 6 Apr 2020 23:28:18 -0700 Subject: [PATCH 05/22] Corrected name of modelContainer in used cases --- .../Experiment/Runners/RunnerUtil.cs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index 97111d7189..a1625a56a0 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -22,25 +22,24 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except DataViewSchema modelInputSchema, IChannel logger) where TMetrics : class { - ITransformer estimatorModel = null; try { var estimator = pipeline.ToEstimator(trainData, validData); - estimatorModel = estimator.Fit(trainData); + var model = estimator.Fit(trainData); - var scoredData = estimatorModel.Transform(validData); + var scoredData = model.Transform(validData); var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn); var score = metricsAgent.GetScore(metrics); if (preprocessorTransform != null) { - estimatorModel = preprocessorTransform.Append(estimatorModel); + model = preprocessorTransform.Append(model); } // Build container for model var modelContainer = modelFileInfo == null ? - new ModelContainer(context, estimatorModel) : - new ModelContainer(context, modelFileInfo, estimatorModel, modelInputSchema); + new ModelContainer(context, model) : + new ModelContainer(context, modelFileInfo, model, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -49,12 +48,6 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except logger.Error($"Pipeline crashed: {pipeline.ToString()} . Exception: {ex}"); return (null, null, ex, double.NaN); } - finally - { - // Free Tensor objects in model. Tensor objects made in TensorFlow's C - // libraries are not automatically cleaned up by C#'s Garbage Collector. - (estimatorModel as IDisposable)?.Dispose(); - } } public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum) From cb22b21d8f661cd90c2dc04b387b6de5b8345e5b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 8 Apr 2020 21:05:09 -0700 Subject: [PATCH 06/22] Clean up Tensor objects through finalizer/destructor of ImageClassificationModelParameters --- .../Experiment/ModelContainer.cs | 17 +---------------- .../ImageClassificationTrainer.cs | 4 ++++ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index bcd63d93ce..8b829da7ab 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -2,12 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.IO; namespace Microsoft.ML.AutoML { - internal class ModelContainer : IDisposable + internal class ModelContainer { private readonly MLContext _mlContext; private readonly FileInfo _fileInfo; @@ -17,14 +16,12 @@ internal ModelContainer(MLContext mlContext, ITransformer model) { _mlContext = mlContext; _model = model; - _disposed = false; } internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema) { _mlContext = mlContext; _fileInfo = fileInfo; - _disposed = false; // Write model to disk using (var fs = File.Create(fileInfo.FullName)) @@ -49,17 +46,5 @@ public ITransformer GetModel() } return model; } - - #region IDisposable Support - private bool _disposed; - - public void Dispose() - { - if (_disposed) - return; - (_model as IDisposable)?.Dispose(); - _disposed = true; - } - #endregion } } diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index f3ebfe0dd7..cfaeba1d2d 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1509,5 +1509,9 @@ public void Dispose() _isDisposed = true; } + + // Finalizer to clean-up remaining Tensor objects generated in TensorFlow C libraries + // not cleaned up by GC + ~ImageClassificationModelParameters() => Dispose(); } } From eefa76f449c64c2040452e02ab1d0a303b56f2c0 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Apr 2020 16:31:04 -0700 Subject: [PATCH 07/22] Dispose ExperimentResult objects at the end --- .../CrossValidationExperimentResult.cs | 23 ++++++++++++++++++- .../API/ExperimentResults/ExperimentResult.cs | 23 ++++++++++++++++++- .../ImageClassificationTrainer.cs | 4 ---- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 6 +++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 5c1db48a59..975fe5ddbe 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using Microsoft.ML.Data; @@ -11,7 +12,7 @@ namespace Microsoft.ML.AutoML /// Result of an AutoML experiment that includes cross validation details. /// /// Metrics type for the experiment (like ). - public class CrossValidationExperimentResult + public class CrossValidationExperimentResult : IDisposable { /// /// Details of the cross validation runs in this experiment. @@ -36,5 +37,25 @@ internal CrossValidationExperimentResult(IEnumerable + /// Releases unmanaged resources in CrossValidationExperimentResult instances + /// + /// + /// Invocation of Dispoe() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void Dispose() + { + if (_disposed) + return; + (BestRun as IDisposable)?.Dispose(); + (RunDetails as IDisposable)?.Dispose(); + _disposed = true; + } + #endregion } } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index 85eecfdb7f..86f7d90907 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using Microsoft.ML.Data; @@ -11,7 +12,7 @@ namespace Microsoft.ML.AutoML /// Result of an AutoML experiment. /// /// Metrics type for the experiment (like ). - public class ExperimentResult + public class ExperimentResult : IDisposable { /// /// Details of the runs in this experiment. @@ -36,5 +37,25 @@ internal ExperimentResult(IEnumerable> runDetails, RunDetails = runDetails; BestRun = bestRun; } + + #region IDisposable Support + private bool _disposed; + + /// + /// Releases unmanaged resources in ExperimentResult instances + /// + /// + /// Invocation of Dispoe() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void Dispose() + { + if (_disposed) + return; + (BestRun as IDisposable)?.Dispose(); + (RunDetails as IDisposable)?.Dispose(); + _disposed = true; + } + #endregion } } \ No newline at end of file diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index cfaeba1d2d..f3ebfe0dd7 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1509,9 +1509,5 @@ public void Dispose() _isDisposed = true; } - - // Finalizer to clean-up remaining Tensor objects generated in TensorFlow C libraries - // not cleaned up by GC - ~ImageClassificationModelParameters() => Dispose(); } } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 30b1251980..6eca51bd52 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -36,6 +36,7 @@ public void AutoFitBinaryTest() Assert.NotNull(result.BestRun.Estimator); Assert.NotNull(result.BestRun.Model); Assert.NotNull(result.BestRun.TrainerName); + result.Dispose(); } [Fact] @@ -51,6 +52,7 @@ public void AutoFitMultiTest() Assert.True(result.BestRun.Results.First().ValidationMetrics.MicroAccuracy >= 0.7); var scoredData = result.BestRun.Results.First().Model.Transform(trainData); Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); + result.Dispose(); } [TensorFlowFact] @@ -73,6 +75,7 @@ public void AutoFitImageClassificationTrainTest() var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); + result.Dispose(); } [Fact(Skip ="Takes too much time, ~10 minutes.")] @@ -94,6 +97,7 @@ public void AutoFitImageClassification() Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.80, 0.9); var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); + result.Dispose(); } private void Context_Log(object sender, LoggingEventArgs e) @@ -117,6 +121,7 @@ public void AutoFitRegressionTest() new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); + result.Dispose(); } [Fact] @@ -163,6 +168,7 @@ public void AutoFitRecommendationTest() var metrices = mlContext.Recommendation().Evaluate(testDataViewWithBestScore, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); Assert.NotEqual(0, metrices.MeanSquaredError); + experimentResult.Dispose(); } private TextLoader.Options GetLoaderArgs(string labelColumnName, string userIdColumnName, string itemIdColumnName) From 45681b424d09c03b2ed5c9764c8ee54eee8ce580 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 9 Apr 2020 17:16:34 -0700 Subject: [PATCH 08/22] Dispose only Tensor objects in models --- .../CrossValidationExperimentResult.cs | 11 +++++++---- .../API/ExperimentResults/ExperimentResult.cs | 9 +++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 975fe5ddbe..dc815f4288 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -42,18 +42,21 @@ internal CrossValidationExperimentResult(IEnumerable - /// Releases unmanaged resources in CrossValidationExperimentResult instances + /// Releases unmanaged Tensor objects in models stored in CrossValidationRunDetail instances /// /// - /// Invocation of Dispoe() is necessary to clean up remaining C library Tensor objects and + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and /// avoid a memory leak /// public void Dispose() { if (_disposed) return; - (BestRun as IDisposable)?.Dispose(); - (RunDetails as IDisposable)?.Dispose(); + foreach (var runDetail in RunDetails) + foreach(var result in runDetail.Results) + (result.Model as IDisposable)?.Dispose(); + foreach (var result in BestRun.Results) + (result.Model as IDisposable)?.Dispose(); _disposed = true; } #endregion diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index 86f7d90907..865a2d7b80 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -42,18 +42,19 @@ internal ExperimentResult(IEnumerable> runDetails, private bool _disposed; /// - /// Releases unmanaged resources in ExperimentResult instances + /// Releases unmanaged Tensor objects in models stored in RunDetail instances /// /// - /// Invocation of Dispoe() is necessary to clean up remaining C library Tensor objects and + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and /// avoid a memory leak /// public void Dispose() { if (_disposed) return; - (BestRun as IDisposable)?.Dispose(); - (RunDetails as IDisposable)?.Dispose(); + foreach(var runDetail in RunDetails) + (runDetail.Model as IDisposable)?.Dispose(); + (BestRun.Model as IDisposable)?.Dispose(); _disposed = true; } #endregion From fbd3fd906b441bcdf71ec36a22e1bc2d06931c72 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Sat, 11 Apr 2020 21:11:23 -0700 Subject: [PATCH 09/22] Don't free BestModel models --- .../API/ExperimentResults/CrossValidationExperimentResult.cs | 2 -- .../API/ExperimentResults/ExperimentResult.cs | 1 - 2 files changed, 3 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index dc815f4288..b3a60583bb 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -55,8 +55,6 @@ public void Dispose() foreach (var runDetail in RunDetails) foreach(var result in runDetail.Results) (result.Model as IDisposable)?.Dispose(); - foreach (var result in BestRun.Results) - (result.Model as IDisposable)?.Dispose(); _disposed = true; } #endregion diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index 865a2d7b80..ebe7b10bde 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -54,7 +54,6 @@ public void Dispose() return; foreach(var runDetail in RunDetails) (runDetail.Model as IDisposable)?.Dispose(); - (BestRun.Model as IDisposable)?.Dispose(); _disposed = true; } #endregion From 7dad2424689f4a4b2c0f15c8e25eed95cd87483c Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Mon, 13 Apr 2020 17:29:53 -0700 Subject: [PATCH 10/22] Throw Exception if model is trying to be accessed after disposal --- .../ExperimentResults/CrossValidationExperimentResult.cs | 7 ++++++- .../API/ExperimentResults/ExperimentResult.cs | 5 ++++- .../API/RunDetails/CrossValidationRunDetail.cs | 4 +++- src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs | 4 +++- src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs | 5 ++++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index b3a60583bb..75e39c2073 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -53,8 +53,13 @@ public void Dispose() if (_disposed) return; foreach (var runDetail in RunDetails) - foreach(var result in runDetail.Results) + { + foreach (var result in runDetail.Results) + { (result.Model as IDisposable)?.Dispose(); + result.IsModelDisposed = true; + } + } _disposed = true; } #endregion diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index ebe7b10bde..f6f5aeb230 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -52,8 +52,11 @@ public void Dispose() { if (_disposed) return; - foreach(var runDetail in RunDetails) + foreach (var runDetail in RunDetails) + { (runDetail.Model as IDisposable)?.Dispose(); + runDetail.IsModelDisposed = true; + } _disposed = true; } #endregion diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs index b4ec8f4817..45c718bdaf 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs @@ -51,7 +51,7 @@ public sealed class TrainResult /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer.GetModel(); } } + public ITransformer Model { get { return _modelContainer.GetModel(IsModelDisposed); } } /// /// Exception encountered while training the fold. This property is @@ -65,6 +65,8 @@ public sealed class TrainResult private readonly ModelContainer _modelContainer; + public bool IsModelDisposed = false; + internal TrainResult(ModelContainer modelContainer, TMetrics metrics, Exception exception) diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs index 45ebe696bd..ab5a9a6ae1 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs @@ -35,7 +35,7 @@ public sealed class RunDetail : RunDetail /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer?.GetModel(); } } + public ITransformer Model { get { return _modelContainer?.GetModel(IsModelDisposed); } } /// /// Exception encountered during the run. This property is if @@ -49,6 +49,8 @@ public sealed class RunDetail : RunDetail private readonly ModelContainer _modelContainer; + public bool IsModelDisposed = false; + internal RunDetail(string trainerName, IEstimator estimator, Pipeline pipeline, diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index 8b829da7ab..738ac19022 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.IO; namespace Microsoft.ML.AutoML @@ -30,8 +31,10 @@ internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer mod } } - public ITransformer GetModel() + public ITransformer GetModel(bool isModelDisposed) { + if (isModelDisposed) + throw new ObjectDisposedException("Trained model was disposed before"); // If model stored in memory, return it if (_model != null) { From 1488d0ce2450883f802e4eb36419742c79a3cd2b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Mon, 13 Apr 2020 21:32:14 -0700 Subject: [PATCH 11/22] Initialize IsModelDisposed inside constructors --- .../API/RunDetails/CrossValidationRunDetail.cs | 6 +++++- src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs index 45c718bdaf..10aaf82ac5 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs @@ -65,7 +65,10 @@ public sealed class TrainResult private readonly ModelContainer _modelContainer; - public bool IsModelDisposed = false; + /// + /// Flag to show whether or not Model has been disposed using IDisposable's .Dispose() + /// + public bool IsModelDisposed; internal TrainResult(ModelContainer modelContainer, TMetrics metrics, @@ -74,6 +77,7 @@ internal TrainResult(ModelContainer modelContainer, _modelContainer = modelContainer; ValidationMetrics = metrics; Exception = exception; + IsModelDisposed = false; } } diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs index ab5a9a6ae1..e997005276 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs @@ -49,7 +49,10 @@ public sealed class RunDetail : RunDetail private readonly ModelContainer _modelContainer; - public bool IsModelDisposed = false; + /// + /// Flag to show whether or not Model has been disposed using IDisposable's .Dispose() + /// + public bool IsModelDisposed; internal RunDetail(string trainerName, IEstimator estimator, @@ -61,6 +64,7 @@ internal RunDetail(string trainerName, _modelContainer = modelContainer; ValidationMetrics = metrics; Exception = exception; + IsModelDisposed = false; } } From 78bba9cbf3b747876f0be8d90ec1edc469552719 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 15 Apr 2020 22:00:56 -0700 Subject: [PATCH 12/22] Model always written to disk, no longer stored in memory, simplify model disposal --- .../CrossValidationExperimentResult.cs | 32 +++++++++++++------ .../API/ExperimentResults/ExperimentResult.cs | 29 +++++++++++++---- .../RunDetails/CrossValidationRunDetail.cs | 2 +- .../API/RunDetails/RunDetail.cs | 2 +- .../Experiment/ModelContainer.cs | 17 +--------- .../Experiment/Runners/RunnerUtil.cs | 6 ++-- .../Experiment/Runners/TrainValidateRunner.cs | 2 +- 7 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 75e39c2073..3fe213a204 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -36,13 +36,16 @@ internal CrossValidationExperimentResult(IEnumerable - /// Releases unmanaged Tensor objects in models stored in CrossValidationRunDetail instances + /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances /// /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and @@ -52,15 +55,26 @@ public void Dispose() { if (_disposed) return; - foreach (var runDetail in RunDetails) - { - foreach (var result in runDetail.Results) - { - (result.Model as IDisposable)?.Dispose(); - result.IsModelDisposed = true; - } - } + if (!_disposedRunDetails) + (RunDetails as IDisposable)?.Dispose(); + (BestRun as IDisposable)?.Dispose(); _disposed = true; + _disposedRunDetails = true; + } + + /// + /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// + /// + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void DisposeRunDetails() + { + if (_disposedRunDetails || _disposed) + return; + (RunDetails as IDisposable)?.Dispose(); + _disposedRunDetails = true; } #endregion } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index f6f5aeb230..1843d2798b 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -36,13 +36,16 @@ internal ExperimentResult(IEnumerable> runDetails, { RunDetails = runDetails; BestRun = bestRun; + _disposed = false; + _disposedRunDetails = false; } #region IDisposable Support private bool _disposed; + private bool _disposedRunDetails; /// - /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances /// /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and @@ -52,12 +55,26 @@ public void Dispose() { if (_disposed) return; - foreach (var runDetail in RunDetails) - { - (runDetail.Model as IDisposable)?.Dispose(); - runDetail.IsModelDisposed = true; - } + if (!_disposedRunDetails) + (RunDetails as IDisposable)?.Dispose(); + (BestRun as IDisposable)?.Dispose(); _disposed = true; + _disposedRunDetails = true; + } + + /// + /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// + /// + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void DisposeRunDetails() + { + if (_disposedRunDetails || _disposed) + return; + (RunDetails as IDisposable)?.Dispose(); + _disposedRunDetails = true; } #endregion } diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs index 10aaf82ac5..deaf61d430 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs @@ -51,7 +51,7 @@ public sealed class TrainResult /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer.GetModel(IsModelDisposed); } } + public ITransformer Model { get { return _modelContainer.GetModel(); } } /// /// Exception encountered while training the fold. This property is diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs index e997005276..307248708a 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs @@ -35,7 +35,7 @@ public sealed class RunDetail : RunDetail /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer?.GetModel(IsModelDisposed); } } + public ITransformer Model { get { return _modelContainer?.GetModel(); } } /// /// Exception encountered during the run. This property is if diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index 738ac19022..6c6729ea6c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -11,13 +11,6 @@ internal class ModelContainer { private readonly MLContext _mlContext; private readonly FileInfo _fileInfo; - private readonly ITransformer _model; - - internal ModelContainer(MLContext mlContext, ITransformer model) - { - _mlContext = mlContext; - _model = model; - } internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema) { @@ -31,16 +24,8 @@ internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer mod } } - public ITransformer GetModel(bool isModelDisposed) + public ITransformer GetModel() { - if (isModelDisposed) - throw new ObjectDisposedException("Trained model was disposed before"); - // If model stored in memory, return it - if (_model != null) - { - return _model; - } - // Load model from disk ITransformer model; using (var stream = new FileStream(_fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Read)) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index a1625a56a0..93ce9ebe67 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -37,9 +37,7 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except } // Build container for model - var modelContainer = modelFileInfo == null ? - new ModelContainer(context, model) : - new ModelContainer(context, modelFileInfo, model, modelInputSchema); + var modelContainer = new ModelContainer(context, modelFileInfo, model, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -53,7 +51,7 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum) { return modelDirectory == null ? - null : + new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}_{foldNum}.zip")) : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}_{foldNum}.zip")); } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index 4289f45a0c..335bf4507c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -59,7 +59,7 @@ public TrainValidateRunner(MLContext context, private static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum) { return modelDirectory == null ? - null : + new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}.zip")) : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}.zip")); } } From bf848236f2f23cacd9b708a26698ecaf1c8e2896 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 15 Apr 2020 22:04:47 -0700 Subject: [PATCH 13/22] Model always written to disk, no longer stored in memory, simplify model disposal --- .../CrossValidationExperimentResult.cs | 32 +++++++++++++------ .../API/ExperimentResults/ExperimentResult.cs | 29 +++++++++++++---- .../RunDetails/CrossValidationRunDetail.cs | 8 +---- .../API/RunDetails/RunDetail.cs | 8 +---- .../Experiment/ModelContainer.cs | 17 +--------- .../Experiment/Runners/RunnerUtil.cs | 6 ++-- .../Experiment/Runners/TrainValidateRunner.cs | 2 +- 7 files changed, 52 insertions(+), 50 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 75e39c2073..3fe213a204 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -36,13 +36,16 @@ internal CrossValidationExperimentResult(IEnumerable - /// Releases unmanaged Tensor objects in models stored in CrossValidationRunDetail instances + /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances /// /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and @@ -52,15 +55,26 @@ public void Dispose() { if (_disposed) return; - foreach (var runDetail in RunDetails) - { - foreach (var result in runDetail.Results) - { - (result.Model as IDisposable)?.Dispose(); - result.IsModelDisposed = true; - } - } + if (!_disposedRunDetails) + (RunDetails as IDisposable)?.Dispose(); + (BestRun as IDisposable)?.Dispose(); _disposed = true; + _disposedRunDetails = true; + } + + /// + /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// + /// + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void DisposeRunDetails() + { + if (_disposedRunDetails || _disposed) + return; + (RunDetails as IDisposable)?.Dispose(); + _disposedRunDetails = true; } #endregion } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index f6f5aeb230..1843d2798b 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -36,13 +36,16 @@ internal ExperimentResult(IEnumerable> runDetails, { RunDetails = runDetails; BestRun = bestRun; + _disposed = false; + _disposedRunDetails = false; } #region IDisposable Support private bool _disposed; + private bool _disposedRunDetails; /// - /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances /// /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and @@ -52,12 +55,26 @@ public void Dispose() { if (_disposed) return; - foreach (var runDetail in RunDetails) - { - (runDetail.Model as IDisposable)?.Dispose(); - runDetail.IsModelDisposed = true; - } + if (!_disposedRunDetails) + (RunDetails as IDisposable)?.Dispose(); + (BestRun as IDisposable)?.Dispose(); _disposed = true; + _disposedRunDetails = true; + } + + /// + /// Releases unmanaged Tensor objects in models stored in RunDetail instances + /// + /// + /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and + /// avoid a memory leak + /// + public void DisposeRunDetails() + { + if (_disposedRunDetails || _disposed) + return; + (RunDetails as IDisposable)?.Dispose(); + _disposedRunDetails = true; } #endregion } diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs index 10aaf82ac5..b4ec8f4817 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/CrossValidationRunDetail.cs @@ -51,7 +51,7 @@ public sealed class TrainResult /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer.GetModel(IsModelDisposed); } } + public ITransformer Model { get { return _modelContainer.GetModel(); } } /// /// Exception encountered while training the fold. This property is @@ -65,11 +65,6 @@ public sealed class TrainResult private readonly ModelContainer _modelContainer; - /// - /// Flag to show whether or not Model has been disposed using IDisposable's .Dispose() - /// - public bool IsModelDisposed; - internal TrainResult(ModelContainer modelContainer, TMetrics metrics, Exception exception) @@ -77,7 +72,6 @@ internal TrainResult(ModelContainer modelContainer, _modelContainer = modelContainer; ValidationMetrics = metrics; Exception = exception; - IsModelDisposed = false; } } diff --git a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs index e997005276..45ebe696bd 100644 --- a/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs +++ b/src/Microsoft.ML.AutoML/API/RunDetails/RunDetail.cs @@ -35,7 +35,7 @@ public sealed class RunDetail : RunDetail /// /// You can use the trained model to obtain predictions on input data. /// - public ITransformer Model { get { return _modelContainer?.GetModel(IsModelDisposed); } } + public ITransformer Model { get { return _modelContainer?.GetModel(); } } /// /// Exception encountered during the run. This property is if @@ -49,11 +49,6 @@ public sealed class RunDetail : RunDetail private readonly ModelContainer _modelContainer; - /// - /// Flag to show whether or not Model has been disposed using IDisposable's .Dispose() - /// - public bool IsModelDisposed; - internal RunDetail(string trainerName, IEstimator estimator, Pipeline pipeline, @@ -64,7 +59,6 @@ internal RunDetail(string trainerName, _modelContainer = modelContainer; ValidationMetrics = metrics; Exception = exception; - IsModelDisposed = false; } } diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index 738ac19022..6c6729ea6c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -11,13 +11,6 @@ internal class ModelContainer { private readonly MLContext _mlContext; private readonly FileInfo _fileInfo; - private readonly ITransformer _model; - - internal ModelContainer(MLContext mlContext, ITransformer model) - { - _mlContext = mlContext; - _model = model; - } internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema) { @@ -31,16 +24,8 @@ internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer mod } } - public ITransformer GetModel(bool isModelDisposed) + public ITransformer GetModel() { - if (isModelDisposed) - throw new ObjectDisposedException("Trained model was disposed before"); - // If model stored in memory, return it - if (_model != null) - { - return _model; - } - // Load model from disk ITransformer model; using (var stream = new FileStream(_fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Read)) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index a1625a56a0..93ce9ebe67 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -37,9 +37,7 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except } // Build container for model - var modelContainer = modelFileInfo == null ? - new ModelContainer(context, model) : - new ModelContainer(context, modelFileInfo, model, modelInputSchema); + var modelContainer = new ModelContainer(context, modelFileInfo, model, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -53,7 +51,7 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum) { return modelDirectory == null ? - null : + new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}_{foldNum}.zip")) : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}_{foldNum}.zip")); } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index 4289f45a0c..335bf4507c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -59,7 +59,7 @@ public TrainValidateRunner(MLContext context, private static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum) { return modelDirectory == null ? - null : + new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}.zip")) : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}.zip")); } } From 087c0d5e386c4c2d0c321ce90ac94f0df29f115b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 15 Apr 2020 22:06:27 -0700 Subject: [PATCH 14/22] Update ModelContainer.cs --- src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index 6c6729ea6c..a1d828f00e 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.IO; namespace Microsoft.ML.AutoML From 476bc0e30590c06e473abaa1b6904942852e283a Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 16 Apr 2020 02:18:25 -0700 Subject: [PATCH 15/22] Added descriptions for Dispose functions and "using" instead of direct .Dispose() calls --- .../CrossValidationExperimentResult.cs | 11 ++++++----- .../API/ExperimentResults/ExperimentResult.cs | 11 ++++++----- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 18 ++++++------------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 3fe213a204..229e15bc47 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -36,8 +36,6 @@ internal CrossValidationExperimentResult(IEnumerable /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak + /// avoid a memory leak. Dispose() guarantees that models in BestRun and models in RunDetails + /// have the same lifetime. /// public void Dispose() { @@ -66,8 +65,10 @@ public void Dispose() /// Releases unmanaged Tensor objects in models stored in RunDetail instances /// /// - /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak + /// Invocation of DisposeRunDetails() is necessary to clean up remaining C library + /// Tensor objects and avoid a memory leak. Compared to Dispose(), DisposeRunDetails() + /// only disposes of RunDetails so that the best determined model in BestRun can have + /// a different lifetime than models in other experimental runs. /// public void DisposeRunDetails() { diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index 1843d2798b..45905082fa 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -36,8 +36,6 @@ internal ExperimentResult(IEnumerable> runDetails, { RunDetails = runDetails; BestRun = bestRun; - _disposed = false; - _disposedRunDetails = false; } #region IDisposable Support @@ -49,7 +47,8 @@ internal ExperimentResult(IEnumerable> runDetails, /// /// /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak + /// avoid a memory leak. Dispose() guarantees that models in BestRun and models in RunDetails + /// have the same lifetime. /// public void Dispose() { @@ -66,8 +65,10 @@ public void Dispose() /// Releases unmanaged Tensor objects in models stored in RunDetail instances /// /// - /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak + /// Invocation of DisposeRunDetails() is necessary to clean up remaining C library + /// Tensor objects and avoid a memory leak. Compared to Dispose(), DisposeRunDetails() + /// only disposes of RunDetails so that the best determined model in BestRun can have + /// a different lifetime than models in other experimental runs. /// public void DisposeRunDetails() { diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6eca51bd52..6007dd5714 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -29,14 +29,13 @@ public void AutoFitBinaryTest() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); - var result = context.Auto() + using var result = context.Auto() .CreateBinaryClassificationExperiment(0) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); Assert.NotNull(result.BestRun.Estimator); Assert.NotNull(result.BestRun.Model); Assert.NotNull(result.BestRun.TrainerName); - result.Dispose(); } [Fact] @@ -46,13 +45,12 @@ public void AutoFitMultiTest() var columnInference = context.Auto().InferColumns(DatasetUtil.TrivialMulticlassDatasetPath, DatasetUtil.TrivialMulticlassDatasetLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(DatasetUtil.TrivialMulticlassDatasetPath); - var result = context.Auto() + using var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainData, 5, DatasetUtil.TrivialMulticlassDatasetLabel); Assert.True(result.BestRun.Results.First().ValidationMetrics.MicroAccuracy >= 0.7); var scoredData = result.BestRun.Results.First().Model.Transform(trainData); Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); - result.Dispose(); } [TensorFlowFact] @@ -67,7 +65,7 @@ public void AutoFitImageClassificationTrainTest() TrainTestData trainTestData = context.Data.TrainTestSplit(trainData, testFraction: 0.2, seed: 1); IDataView trainDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TrainSet, originalColumnNames); IDataView testDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TestSet, originalColumnNames); - var result = context.Auto() + using var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainDataset, testDataset, columnInference.ColumnInformation); @@ -75,7 +73,6 @@ public void AutoFitImageClassificationTrainTest() var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); - result.Dispose(); } [Fact(Skip ="Takes too much time, ~10 minutes.")] @@ -90,14 +87,13 @@ public void AutoFitImageClassification() var columnInference = context.Auto().InferColumns(datasetPath, "Label"); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(datasetPath); - var result = context.Auto() + using var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainData, columnInference.ColumnInformation); Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.80, 0.9); var scoredData = result.BestRun.Model.Transform(trainData); Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); - result.Dispose(); } private void Context_Log(object sender, LoggingEventArgs e) @@ -115,13 +111,12 @@ public void AutoFitRegressionTest() var trainData = textLoader.Load(dataPath); var validationData = context.Data.TakeRows(trainData, 20); trainData = context.Data.SkipRows(trainData, 20); - var result = context.Auto() + using var result = context.Auto() .CreateRegressionExperiment(0) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9)); - result.Dispose(); } [Fact] @@ -140,7 +135,7 @@ public void AutoFitRecommendationTest() var testDataView = reader.Load(new MultiFileSource(GetDataPath(TestDatasets.trivialMatrixFactorization.testFilename))); // STEP 2: Run AutoML experiment - ExperimentResult experimentResult = mlContext.Auto() + using ExperimentResult experimentResult = mlContext.Auto() .CreateRecommendationExperiment(5) .Execute(trainDataView, testDataView, new ColumnInformation() @@ -168,7 +163,6 @@ public void AutoFitRecommendationTest() var metrices = mlContext.Recommendation().Evaluate(testDataViewWithBestScore, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); Assert.NotEqual(0, metrices.MeanSquaredError); - experimentResult.Dispose(); } private TextLoader.Options GetLoaderArgs(string labelColumnName, string userIdColumnName, string itemIdColumnName) From ef170739084ae058708a67f6298fcf9d5f4bc3e5 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 29 May 2020 17:05:03 -0500 Subject: [PATCH 16/22] Added disposable/finalizer --- .../CrossValidationExperimentResult.cs | 44 +------------------ .../API/ExperimentResults/ExperimentResult.cs | 44 +------------------ .../Experiment/ModelContainer.cs | 13 ++++++ .../Experiment/Runners/RunnerUtil.cs | 6 ++- .../Experiment/Runners/TrainValidateRunner.cs | 2 +- .../ImageClassificationTrainer.cs | 29 ++++++++++++ 6 files changed, 49 insertions(+), 89 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs index 229e15bc47..5c1db48a59 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using Microsoft.ML.Data; @@ -12,7 +11,7 @@ namespace Microsoft.ML.AutoML /// Result of an AutoML experiment that includes cross validation details. /// /// Metrics type for the experiment (like ). - public class CrossValidationExperimentResult : IDisposable + public class CrossValidationExperimentResult { /// /// Details of the cross validation runs in this experiment. @@ -37,46 +36,5 @@ internal CrossValidationExperimentResult(IEnumerable - /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances - /// - /// - /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak. Dispose() guarantees that models in BestRun and models in RunDetails - /// have the same lifetime. - /// - public void Dispose() - { - if (_disposed) - return; - if (!_disposedRunDetails) - (RunDetails as IDisposable)?.Dispose(); - (BestRun as IDisposable)?.Dispose(); - _disposed = true; - _disposedRunDetails = true; - } - - /// - /// Releases unmanaged Tensor objects in models stored in RunDetail instances - /// - /// - /// Invocation of DisposeRunDetails() is necessary to clean up remaining C library - /// Tensor objects and avoid a memory leak. Compared to Dispose(), DisposeRunDetails() - /// only disposes of RunDetails so that the best determined model in BestRun can have - /// a different lifetime than models in other experimental runs. - /// - public void DisposeRunDetails() - { - if (_disposedRunDetails || _disposed) - return; - (RunDetails as IDisposable)?.Dispose(); - _disposedRunDetails = true; - } - #endregion } } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs index 45905082fa..85eecfdb7f 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using Microsoft.ML.Data; @@ -12,7 +11,7 @@ namespace Microsoft.ML.AutoML /// Result of an AutoML experiment. /// /// Metrics type for the experiment (like ). - public class ExperimentResult : IDisposable + public class ExperimentResult { /// /// Details of the runs in this experiment. @@ -37,46 +36,5 @@ internal ExperimentResult(IEnumerable> runDetails, RunDetails = runDetails; BestRun = bestRun; } - - #region IDisposable Support - private bool _disposed; - private bool _disposedRunDetails; - - /// - /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances - /// - /// - /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and - /// avoid a memory leak. Dispose() guarantees that models in BestRun and models in RunDetails - /// have the same lifetime. - /// - public void Dispose() - { - if (_disposed) - return; - if (!_disposedRunDetails) - (RunDetails as IDisposable)?.Dispose(); - (BestRun as IDisposable)?.Dispose(); - _disposed = true; - _disposedRunDetails = true; - } - - /// - /// Releases unmanaged Tensor objects in models stored in RunDetail instances - /// - /// - /// Invocation of DisposeRunDetails() is necessary to clean up remaining C library - /// Tensor objects and avoid a memory leak. Compared to Dispose(), DisposeRunDetails() - /// only disposes of RunDetails so that the best determined model in BestRun can have - /// a different lifetime than models in other experimental runs. - /// - public void DisposeRunDetails() - { - if (_disposedRunDetails || _disposed) - return; - (RunDetails as IDisposable)?.Dispose(); - _disposedRunDetails = true; - } - #endregion } } \ No newline at end of file diff --git a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs index a1d828f00e..8b829da7ab 100644 --- a/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs +++ b/src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs @@ -10,6 +10,13 @@ internal class ModelContainer { private readonly MLContext _mlContext; private readonly FileInfo _fileInfo; + private readonly ITransformer _model; + + internal ModelContainer(MLContext mlContext, ITransformer model) + { + _mlContext = mlContext; + _model = model; + } internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema) { @@ -25,6 +32,12 @@ internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer mod public ITransformer GetModel() { + // If model stored in memory, return it + if (_model != null) + { + return _model; + } + // Load model from disk ITransformer model; using (var stream = new FileStream(_fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Read)) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index 93ce9ebe67..a1625a56a0 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -37,7 +37,9 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except } // Build container for model - var modelContainer = new ModelContainer(context, modelFileInfo, model, modelInputSchema); + var modelContainer = modelFileInfo == null ? + new ModelContainer(context, model) : + new ModelContainer(context, modelFileInfo, model, modelInputSchema); return (modelContainer, metrics, null, score); } @@ -51,7 +53,7 @@ public static (ModelContainer modelContainer, TMetrics metrics, Exception except public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum) { return modelDirectory == null ? - new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}_{foldNum}.zip")) : + null : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}_{foldNum}.zip")); } } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index 335bf4507c..4289f45a0c 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -59,7 +59,7 @@ public TrainValidateRunner(MLContext context, private static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum) { return modelDirectory == null ? - new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}.zip")) : + null : new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}.zip")); } } diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 79973cd23b..b5f446bd47 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1493,6 +1493,12 @@ ValueMapper IValueMapper.GetMapper() } public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + public void Dispose(bool disposing) { if (_isDisposed) return; @@ -1509,5 +1515,28 @@ public void Dispose() _isDisposed = true; } + + // Finalizer to clean-up remaining Tensor objects generated in TensorFlow C libraries + // not cleaned up by GC + ~ImageClassificationModelParameters() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private unsafe void Dispose(bool disposing) + { + // Free unmanaged resources. + if (_pMFModel != null) + { + MFDestroyModel(ref _pMFModel); + _host.Assert(_pMFModel == null); + } + } } } From a16fccf273dd41ccf4c417fe3d9e960721fee2ad Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Sun, 31 May 2020 12:06:27 -0500 Subject: [PATCH 17/22] Run specific test 100 iterations --- .vsts-dotnet-ci.yml | 21 ++++++++++++------- .../ImageClassificationTrainer.cs | 16 -------------- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 5 +++-- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 5c2dcf30cb..6932184c01 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,7 +27,8 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -36,7 +37,8 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -44,7 +46,8 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted macOS @@ -63,7 +66,8 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -72,7 +76,8 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -92,7 +97,8 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -102,7 +108,8 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index b5f446bd47..cbaabc4337 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -1522,21 +1522,5 @@ public void Dispose(bool disposing) { Dispose(false); } - - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - private unsafe void Dispose(bool disposing) - { - // Free unmanaged resources. - if (_pMFModel != null) - { - MFDestroyModel(ref _pMFModel); - _host.Assert(_pMFModel == null); - } - } } } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 6007dd5714..0b80026a56 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -53,9 +53,10 @@ public void AutoFitMultiTest() Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } - [TensorFlowFact] - public void AutoFitImageClassificationTrainTest() + [Theory, TestCategory("RunSpecificTest"), TestFrameworkCommon.Attributes.IterationData(100)] + public void AutoFitImageClassificationTrainTest(int iterations) { + Console.WriteLine(String.Format("AutoFitImageClassificationTrainTest iteration: {0}", iterations)); var context = new MLContext(seed: 1); var datasetPath = DatasetUtil.GetFlowersDataset(); var columnInference = context.Auto().InferColumns(datasetPath, "Label"); From 3c6045f17d59ec574a097927c1f89f440c767c4b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Sun, 31 May 2020 23:10:04 -0500 Subject: [PATCH 18/22] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 0b80026a56..f43b597fc2 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -29,7 +29,7 @@ public void AutoFitBinaryTest() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); - using var result = context.Auto() + var result = context.Auto() .CreateBinaryClassificationExperiment(0) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); Assert.True(result.BestRun.ValidationMetrics.Accuracy > 0.70); @@ -45,7 +45,7 @@ public void AutoFitMultiTest() var columnInference = context.Auto().InferColumns(DatasetUtil.TrivialMulticlassDatasetPath, DatasetUtil.TrivialMulticlassDatasetLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(DatasetUtil.TrivialMulticlassDatasetPath); - using var result = context.Auto() + var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainData, 5, DatasetUtil.TrivialMulticlassDatasetLabel); Assert.True(result.BestRun.Results.First().ValidationMetrics.MicroAccuracy >= 0.7); @@ -66,7 +66,7 @@ public void AutoFitImageClassificationTrainTest(int iterations) TrainTestData trainTestData = context.Data.TrainTestSplit(trainData, testFraction: 0.2, seed: 1); IDataView trainDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TrainSet, originalColumnNames); IDataView testDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TestSet, originalColumnNames); - using var result = context.Auto() + var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainDataset, testDataset, columnInference.ColumnInformation); @@ -88,7 +88,7 @@ public void AutoFitImageClassification() var columnInference = context.Auto().InferColumns(datasetPath, "Label"); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(datasetPath); - using var result = context.Auto() + var result = context.Auto() .CreateMulticlassClassificationExperiment(0) .Execute(trainData, columnInference.ColumnInformation); @@ -112,7 +112,7 @@ public void AutoFitRegressionTest() var trainData = textLoader.Load(dataPath); var validationData = context.Data.TakeRows(trainData, 20); trainData = context.Data.SkipRows(trainData, 20); - using var result = context.Auto() + var result = context.Auto() .CreateRegressionExperiment(0) .Execute(trainData, validationData, new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel }); @@ -136,7 +136,7 @@ public void AutoFitRecommendationTest() var testDataView = reader.Load(new MultiFileSource(GetDataPath(TestDatasets.trivialMatrixFactorization.testFilename))); // STEP 2: Run AutoML experiment - using ExperimentResult experimentResult = mlContext.Auto() + ExperimentResult experimentResult = mlContext.Auto() .CreateRecommendationExperiment(5) .Execute(trainDataView, testDataView, new ColumnInformation() From 289bc12e841237d56030a568229a5ff0a6530dbb Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 3 Jun 2020 21:55:02 -0700 Subject: [PATCH 19/22] Update .vsts-dotnet-ci.yml --- .vsts-dotnet-ci.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 6932184c01..5c2dcf30cb 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,8 +27,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -37,8 +36,7 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -46,8 +44,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted macOS @@ -66,8 +63,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -76,8 +72,7 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -97,8 +92,7 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -108,8 +102,7 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 From 5fb9450f3fad23e20525781fd11afb5de238419c Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 3 Jun 2020 21:56:48 -0700 Subject: [PATCH 20/22] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index f43b597fc2..fb8ba599cc 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -53,7 +53,7 @@ public void AutoFitMultiTest() Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } - [Theory, TestCategory("RunSpecificTest"), TestFrameworkCommon.Attributes.IterationData(100)] + [TensorFlowFact] public void AutoFitImageClassificationTrainTest(int iterations) { Console.WriteLine(String.Format("AutoFitImageClassificationTrainTest iteration: {0}", iterations)); From 7668f047d272523d10e868a8c0069ff688064c5b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 3 Jun 2020 21:57:24 -0700 Subject: [PATCH 21/22] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index fb8ba599cc..81bb23adc6 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -56,7 +56,6 @@ public void AutoFitMultiTest() [TensorFlowFact] public void AutoFitImageClassificationTrainTest(int iterations) { - Console.WriteLine(String.Format("AutoFitImageClassificationTrainTest iteration: {0}", iterations)); var context = new MLContext(seed: 1); var datasetPath = DatasetUtil.GetFlowersDataset(); var columnInference = context.Auto().InferColumns(datasetPath, "Label"); From cb90b8f2a1d28030caa7d00423a7ff2709d3eeda Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 3 Jun 2020 21:57:51 -0700 Subject: [PATCH 22/22] Update AutoFitTests.cs --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 81bb23adc6..30b1251980 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -54,7 +54,7 @@ public void AutoFitMultiTest() } [TensorFlowFact] - public void AutoFitImageClassificationTrainTest(int iterations) + public void AutoFitImageClassificationTrainTest() { var context = new MLContext(seed: 1); var datasetPath = DatasetUtil.GetFlowersDataset();