diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs index 611eb501c1..f3201f3389 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs @@ -34,8 +34,8 @@ public static void Example() var idv = mlContext.Data.LoadFromEnumerable(data); // Create a ML pipeline. - var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation) - .ScoreTensorFlowModel( + using var model = mlContext.Model.LoadTensorFlowModel(modelLocation); + var pipeline = model.ScoreTensorFlowModel( new[] { nameof(OutputScores.output) }, new[] { nameof(TensorData.input) }, addBatchDimensionInput: true); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs index 131ed744b2..2dbadbc3f2 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs @@ -60,7 +60,7 @@ public static void Example() // Unfrozen (SavedModel format) models are loaded by providing the // path to the directory containing the model file and other model // artifacts like pre-trained weights. - var tensorFlowModel = mlContext.Model.LoadTensorFlowModel( + using var tensorFlowModel = mlContext.Model.LoadTensorFlowModel( modelLocation); var schema = tensorFlowModel.GetModelSchema(); var featuresType = (VectorDataViewType)schema["Features"].Type; diff --git a/src/Microsoft.ML.Data/DataLoadSave/CompositeDataLoader.cs b/src/Microsoft.ML.Data/DataLoadSave/CompositeDataLoader.cs index 8e849802ca..3fccbebd20 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/CompositeDataLoader.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/CompositeDataLoader.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using Microsoft.ML; using Microsoft.ML.Data; using Microsoft.ML.Runtime; @@ -15,7 +16,7 @@ namespace Microsoft.ML.Data /// This class represents a data loader that applies a transformer chain after loading. /// It also has methods to save itself to a repository. /// - public sealed class CompositeDataLoader : IDataLoader + public sealed class CompositeDataLoader : IDataLoader, IDisposable where TLastTransformer : class, ITransformer { internal const string TransformerDirectory = TransformerChain.LoaderSignature; @@ -110,5 +111,19 @@ private static VersionInfo GetVersionInfo() loaderSignature: LoaderSignature, loaderAssemblyName: typeof(CompositeDataLoader<,>).Assembly.FullName); } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + Transformer.Dispose(); + + _disposed = true; + } + #endregion } } diff --git a/src/Microsoft.ML.Data/DataLoadSave/TransformerChain.cs b/src/Microsoft.ML.Data/DataLoadSave/TransformerChain.cs index f9f7a1f413..b43d457326 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/TransformerChain.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/TransformerChain.cs @@ -50,7 +50,7 @@ internal interface ITransformerChainAccessor /// A chain of transformers (possibly empty) that end with a . /// For an empty chain, is always . /// - public sealed class TransformerChain : ITransformer, IEnumerable, ITransformerChainAccessor + public sealed class TransformerChain : ITransformer, IEnumerable, ITransformerChainAccessor, IDisposable where TLastTransformer : class, ITransformer { private readonly ITransformer[] _transformers; @@ -232,6 +232,21 @@ IRowToRowMapper ITransformer.GetRowToRowMapper(DataViewSchema inputSchema) } return new CompositeRowToRowMapper(inputSchema, mappers); } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + foreach (var transformer in _transformers) + (transformer as IDisposable)?.Dispose(); + + _disposed = true; + } + #endregion } /// diff --git a/src/Microsoft.ML.Data/DataView/CompositeRowToRowMapper.cs b/src/Microsoft.ML.Data/DataView/CompositeRowToRowMapper.cs index 980d12da51..d634ac9e85 100644 --- a/src/Microsoft.ML.Data/DataView/CompositeRowToRowMapper.cs +++ b/src/Microsoft.ML.Data/DataView/CompositeRowToRowMapper.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.Data /// A row-to-row mapper that is the result of a chained application of multiple mappers. /// [BestFriend] - internal sealed class CompositeRowToRowMapper : IRowToRowMapper + internal sealed class CompositeRowToRowMapper : IRowToRowMapper, IDisposable { [BestFriend] internal IRowToRowMapper[] InnerMappers { get; } @@ -118,5 +118,20 @@ public SubsetActive(DataViewRow row, Func pred) /// public override bool IsColumnActive(DataViewSchema.Column column) => _pred(column.Index); } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + foreach (var mapper in InnerMappers) + (mapper as IDisposable)?.Dispose(); + + _disposed = true; + } + #endregion } } diff --git a/src/Microsoft.ML.Data/Prediction/PredictionEngine.cs b/src/Microsoft.ML.Data/Prediction/PredictionEngine.cs index 3b919292ba..43d71555ba 100644 --- a/src/Microsoft.ML.Data/Prediction/PredictionEngine.cs +++ b/src/Microsoft.ML.Data/Prediction/PredictionEngine.cs @@ -157,18 +157,13 @@ private protected virtual Func TransformerCheck } public void Dispose() - { - Disposing(true); - GC.SuppressFinalize(this); - } - - [BestFriend] - private protected void Disposing(bool disposing) { if (_disposed) return; - if (disposing) - _disposer?.Invoke(); + + _disposer?.Invoke(); + (Transformer as IDisposable)?.Dispose(); + _disposed = true; } diff --git a/src/Microsoft.ML.Data/Scorers/MulticlassClassificationScorer.cs b/src/Microsoft.ML.Data/Scorers/MulticlassClassificationScorer.cs index 8563e1ffe4..a82092f430 100644 --- a/src/Microsoft.ML.Data/Scorers/MulticlassClassificationScorer.cs +++ b/src/Microsoft.ML.Data/Scorers/MulticlassClassificationScorer.cs @@ -64,7 +64,8 @@ private static VersionInfo GetVersionInfo() /// // REVIEW: It seems like the attachment of metadata should be solvable in a manner // less ridiculously verbose than this. - public sealed class LabelNameBindableMapper : ISchemaBindableMapper, ICanSaveModel, IBindableCanSavePfa, IBindableCanSaveOnnx + public sealed class LabelNameBindableMapper : ISchemaBindableMapper, ICanSaveModel, IBindableCanSavePfa, + IBindableCanSaveOnnx, IDisposable { private static readonly FuncInstanceMethodInfo1 _decodeInitMethodInfo = FuncInstanceMethodInfo1.Create(target => target.DecodeInit); @@ -379,6 +380,21 @@ public RowImpl(DataViewRow row, DataViewSchema schema) public override ValueGetter GetGetter(DataViewSchema.Column column) => Input.GetGetter(column); } } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + // TODO: Is it necessary to call the base class Dispose()? + if (_disposed) + return; + + (_bindable as IDisposable)?.Dispose(); + + _disposed = true; + } + #endregion } /// diff --git a/src/Microsoft.ML.Data/Scorers/PredictedLabelScorerBase.cs b/src/Microsoft.ML.Data/Scorers/PredictedLabelScorerBase.cs index 3f2e517d31..106f2718a5 100644 --- a/src/Microsoft.ML.Data/Scorers/PredictedLabelScorerBase.cs +++ b/src/Microsoft.ML.Data/Scorers/PredictedLabelScorerBase.cs @@ -17,7 +17,7 @@ namespace Microsoft.ML.Data /// Class for scorers that compute on additional "PredictedLabel" column from the score column. /// Currently, this scorer is used for binary classification, multi-class classification, and clustering. /// - internal abstract class PredictedLabelScorerBase : RowToRowScorerBase, ITransformCanSavePfa, ITransformCanSaveOnnx + internal abstract class PredictedLabelScorerBase : RowToRowScorerBase, ITransformCanSavePfa, ITransformCanSaveOnnx, IDisposable { public abstract class ThresholdArgumentsBase : ScorerArgumentsBase { @@ -435,5 +435,20 @@ protected void EnsureCachedPosition(ref long cachedPosition, ref TScore cachedPosition = boundRow.Position; } } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + (Bindings.RowMapper as IDisposable)?.Dispose(); + (Bindable as IDisposable)?.Dispose(); + + _disposed = true; + } + #endregion } } \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs index d86aaa320e..8c0b822dc7 100644 --- a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs @@ -40,7 +40,7 @@ internal static class PredictionTransformerBase /// Base class for transformers with no feature column, or more than one feature columns. /// /// The type of the model parameters used by this prediction transformer. - public abstract class PredictionTransformerBase : IPredictionTransformer + public abstract class PredictionTransformerBase : IPredictionTransformer, IDisposable where TModel : class { /// @@ -181,6 +181,22 @@ private protected void SaveModelCore(ModelSaveContext ctx) } }); } + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + (Model as IDisposable)?.Dispose(); + (BindableMapper as IDisposable)?.Dispose(); + (Scorer as IDisposable)?.Dispose(); + + _disposed = true; + } + #endregion } /// diff --git a/src/Microsoft.ML.Data/Scorers/SchemaBindablePredictorWrapper.cs b/src/Microsoft.ML.Data/Scorers/SchemaBindablePredictorWrapper.cs index 77ae754110..7450f90de0 100644 --- a/src/Microsoft.ML.Data/Scorers/SchemaBindablePredictorWrapper.cs +++ b/src/Microsoft.ML.Data/Scorers/SchemaBindablePredictorWrapper.cs @@ -32,7 +32,7 @@ namespace Microsoft.ML.Data /// This is a base class for wrapping s in an . /// internal abstract class SchemaBindablePredictorWrapperBase : ISchemaBindableMapper, ICanSaveModel, ICanSaveSummary, - IBindableCanSavePfa, IBindableCanSaveOnnx + IBindableCanSavePfa, IBindableCanSaveOnnx, IDisposable { // The ctor guarantees that Predictor is non-null. It also ensures that either // ValueMapper or FloatPredictor is non-null (or both). With these guarantees, @@ -193,7 +193,7 @@ void ICanSaveSummary.SaveSummary(TextWriter writer, RoleMappedSchema schema) /// This class doesn't care. It DOES care that the role mapped schema specifies a unique Feature column. /// It also requires that the output schema has ColumnCount == 1. /// - protected sealed class SingleValueRowMapper : ISchemaBoundRowMapper + protected sealed class SingleValueRowMapper : ISchemaBoundRowMapper, IDisposable { private readonly SchemaBindablePredictorWrapperBase _parent; @@ -241,7 +241,35 @@ DataViewRow ISchemaBoundRowMapper.GetRow(DataViewRow input, IEnumerable diff --git a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs index d035a5c308..f8fd1b476e 100644 --- a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs +++ b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using Microsoft.ML.Runtime; using Microsoft.ML.TensorFlow; using Tensorflow; @@ -13,7 +14,7 @@ namespace Microsoft.ML.Transforms /// It provides some convenient methods to query model schema as well as /// creation of object. /// - public sealed class TensorFlowModel + public sealed class TensorFlowModel : IDisposable { internal Session Session { get; } internal string ModelPath { get; } @@ -31,6 +32,7 @@ internal TensorFlowModel(IHostEnvironment env, Session session, string modelLoca Session = session; ModelPath = modelLocation; _env = env; + _disposed = false; } /// @@ -83,5 +85,19 @@ public TensorFlowEstimator ScoreTensorFlowModel(string outputColumnName, string /// public TensorFlowEstimator ScoreTensorFlowModel(string[] outputColumnNames, string[] inputColumnNames, bool addBatchDimensionInput = false) => new TensorFlowEstimator(_env, outputColumnNames, inputColumnNames, this, addBatchDimensionInput); + + #region IDisposable Support + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + + Session.Dispose(); + + _disposed = true; + } + #endregion } } \ No newline at end of file diff --git a/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs b/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs index 04873c1f46..2ad63321d4 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs @@ -15,6 +15,9 @@ public static class TensorflowCatalog /// Load TensorFlow model into memory. This is the convenience method that allows the model to be loaded once and subsequently use it for querying schema and creation of /// using . /// usage of this API requires additional NuGet dependencies on TensorFlow redist, see linked document for more information. + /// also holds references to unmanaged resources that need to be freed either with an explicit + /// call to Dispose() or implicitly by declaring the variable with the "using" syntax/> + /// /// /// Model to load. internal static DataViewSchema GetModelSchema(IHostEnvironment env, string modelPath) { - var model = LoadTensorFlowModel(env, modelPath); + using var model = LoadTensorFlowModel(env, modelPath); return GetModelSchema(env, model.Session.graph); } diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index 23f343d6df..2388ba8e19 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -738,6 +738,8 @@ public void Dispose() { if (_session != null && _session != IntPtr.Zero) { + if (_session.graph != null) + _session.graph.Dispose(); _session.close(); } } diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 0f174b1e53..f3ebfe0dd7 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -77,10 +77,8 @@ namespace Microsoft.ML.Vision /// public sealed class ImageClassificationTrainer : TrainerEstimatorBase, - ImageClassificationModelParameters>, IDisposable + ImageClassificationModelParameters> { - private bool _isDisposed; - internal const string LoadName = "ImageClassificationTrainer"; internal const string UserName = "Image Classification Trainer"; internal const string ShortName = "IMGCLSS"; @@ -1206,6 +1204,9 @@ private void UpdateTransferLearningModelOnDisk(int classCount) _session.graph.Dispose(); _session.Dispose(); _session = LoadTFSessionByModelFilePath(Host, frozenModelPath, false); + + sess.graph.Dispose(); + sess.Dispose(); } private void VariableSummaries(RefVariable var) @@ -1322,24 +1323,6 @@ private static TensorFlowSessionWrapper LoadTensorFlowSessionFromMetaGraph(IHost return new TensorFlowSessionWrapper(GetSession(env, modelFilePath, true), modelFilePath); } - public void Dispose() - { - if (_isDisposed) - return; - - if (_session?.graph != IntPtr.Zero) - { - _session.graph.Dispose(); - } - - if (_session != null && _session != IntPtr.Zero) - { - _session.close(); - } - - _isDisposed = true; - } - /// /// Trains a using both training and validation data, /// returns a . diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index ab1070fe55..b11624ebff 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -54,8 +54,6 @@ public void AutoFitMultiTest() } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void AutoFitImageClassificationTrainTest() { var context = new MLContext(seed: 1); diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index d7eb9bec93..6af3bbeefc 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -6365,8 +6365,6 @@ public void TestOvaMacroWithUncalibratedLearner() } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void EntryPointTensorFlowTransform() { Env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly); @@ -6382,8 +6380,6 @@ public void EntryPointTensorFlowTransform() } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TestTensorFlowEntryPoint() { var dataPath = GetDataPath("Train-Tiny-28x28.txt"); diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 6a1b0cba6c..ea17c4aae0 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -132,7 +132,7 @@ public void TensorFlowTransforCifarEndToEndTest2() .Append(mlContext.MulticlassClassification.Trainers.SdcaMaximumEntropy()); - var transformer = pipeEstimator.Fit(data); + using var transformer = pipeEstimator.Fit(data); var predictions = transformer.Transform(data); var metrics = mlContext.MulticlassClassification.Evaluate(predictions); @@ -172,7 +172,9 @@ public void TensorFlowTransformMatrixMultiplicationTest() 2.0f, 2.0f }, b = new[] { 3.0f, 3.0f, 3.0f, 3.0f } } })); - var trans = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }).Fit(loader).Transform(loader); + + using var tfModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var trans = tfModel.ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }).Fit(loader).Transform(loader); using (var cursor = trans.GetRowCursorForAllColumns()) { @@ -262,7 +264,8 @@ public void TensorFlowTransformInputShapeTest() var inputs = new string[] { "OneDim", "TwoDim", "ThreeDim", "FourDim", "FourDimKnown" }; var outputs = new string[] { "o_OneDim", "o_TwoDim", "o_ThreeDim", "o_FourDim", "o_FourDimKnown" }; - var trans = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(outputs, inputs).Fit(loader).Transform(loader); + using var tfModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var trans = tfModel.ScoreTensorFlowModel(outputs, inputs).Fit(loader).Transform(loader); using (var cursor = trans.GetRowCursorForAllColumns()) { @@ -382,7 +385,8 @@ public void TensorFlowTransformInputOutputTypesTest() var inputs = new string[] { "f64", "f32", "i64", "i32", "i16", "i8", "u64", "u32", "u16", "u8", "b" }; var outputs = new string[] { "o_f64", "o_f32", "o_i64", "o_i32", "o_i16", "o_i8", "o_u64", "o_u32", "o_u16", "o_u8", "o_b" }; - var trans = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(outputs, inputs).Fit(loader).Transform(loader); ; + using var tfModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var trans = tfModel.ScoreTensorFlowModel(outputs, inputs).Fit(loader).Transform(loader); using (var cursor = trans.GetRowCursorForAllColumns()) { @@ -477,8 +481,8 @@ public void TensorFlowTransformObjectDetectionTest() var cropped = new ImageResizingTransformer(mlContext, "ImageCropped", 32, 32, "ImageReal").Transform(images); var pixels = mlContext.Transforms.ExtractPixels("image_tensor", "ImageCropped", outputAsFloatArray: false).Fit(cropped).Transform(cropped); - var tf = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel( - new[] { "detection_boxes", "detection_scores", "num_detections", "detection_classes" }, new[] { "image_tensor" }).Fit(pixels).Transform(pixels); + using var tfModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var tf = tfModel.ScoreTensorFlowModel(new[] { "detection_boxes", "detection_scores", "num_detections", "detection_classes" }, new[] { "image_tensor" }).Fit(pixels).Transform(pixels); using (var curs = tf.GetRowCursor(tf.Schema["image_tensor"], tf.Schema["detection_boxes"], tf.Schema["detection_scores"], tf.Schema["detection_classes"], tf.Schema["num_detections"])) { @@ -524,7 +528,8 @@ public void TensorFlowTransformInceptionTest() var images = mlContext.Transforms.LoadImages("ImageReal", "ImagePath", imageFolder).Fit(data).Transform(data); var cropped = mlContext.Transforms.ResizeImages("ImageCropped", 224, 224, "ImageReal").Fit(images).Transform(images); var pixels = mlContext.Transforms.ExtractPixels(inputName, "ImageCropped", interleavePixelColors: true).Fit(cropped).Transform(cropped); - var tf = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(outputName, inputName, true).Fit(pixels).Transform(pixels); + using var tfModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + var tf = tfModel.ScoreTensorFlowModel(outputName, inputName, true).Fit(pixels).Transform(pixels); tf.Schema.TryGetColumnIndex(inputName, out int input); tf.Schema.TryGetColumnIndex(outputName, out int b); @@ -642,7 +647,7 @@ public void TensorFlowTransformMNISTConvTest() .Append(mlContext.Transforms.Concatenate("Features", "Softmax", "dense/Relu")) .Append(mlContext.MulticlassClassification.Trainers.LightGbm("Label", "Features")); - var trainedModel = pipe.Fit(trainData); + using var trainedModel = pipe.Fit(trainData); var predicted = trainedModel.Transform(testData); var metrics = mlContext.MulticlassClassification.Evaluate(predicted); @@ -696,7 +701,7 @@ public void TensorFlowTransformMNISTLRTrainingTest() .Append(mlContext.Transforms.Conversion.MapValueToKey("KeyLabel", "Label", maximumNumberOfKeys: 10)) .Append(mlContext.MulticlassClassification.Trainers.LightGbm("KeyLabel", "Features")); - var trainedModel = pipe.Fit(trainData); + using var trainedModel = pipe.Fit(trainData); var predicted = trainedModel.Transform(testData); var metrics = mlContext.MulticlassClassification.Evaluate(predicted, labelColumnName: "KeyLabel"); Assert.InRange(metrics.MicroAccuracy, expectedMicroAccuracy, 1); @@ -741,8 +746,6 @@ private void CleanUp(string modelLocation) } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowTransformMNISTConvTrainingTest() { double expectedMicro = 0.73304347826086956; @@ -821,7 +824,7 @@ private void ExecuteTFTransformMNISTConvTrainingTest(bool shuffle, int? shuffleS NumberOfIterations = 1 })); - var trainedModel = pipe.Fit(preprocessedTrainData); + using var trainedModel = pipe.Fit(preprocessedTrainData); var predicted = trainedModel.Transform(preprocessedTestData); var metrics = mlContext.MulticlassClassification.Evaluate(predicted); Assert.InRange(metrics.MicroAccuracy, expectedMicroAccuracy - 0.1, expectedMicroAccuracy + 0.1); @@ -869,7 +872,7 @@ public void TensorFlowTransformMNISTConvSavedModelTest() .Append(mlContext.Transforms.Concatenate("Features", new[] { "Softmax", "dense/Relu" })) .Append(mlContext.MulticlassClassification.Trainers.LightGbm("Label", "Features")); - var trainedModel = pipe.Fit(trainData); + using var trainedModel = pipe.Fit(trainData); var predicted = trainedModel.Transform(testData); var metrics = mlContext.MulticlassClassification.Evaluate(predicted); @@ -974,7 +977,7 @@ public void TensorFlowTransformCifar() var mlContext = new MLContext(seed: 1); List logMessages = new List(); mlContext.Log += (sender, e) => logMessages.Add(e.Message); - var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + using var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation); var schema = tensorFlowModel.GetInputSchema(); Assert.True(schema.TryGetColumnIndex("Input", out int column)); var type = (VectorDataViewType)schema[column].Type; @@ -1061,7 +1064,7 @@ public void TensorFlowTransformCifarSavedModel() { var modelLocation = "cifar_saved_model"; var mlContext = new MLContext(seed: 1); - var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation); + using var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation); var schema = tensorFlowModel.GetInputSchema(); Assert.True(schema.TryGetColumnIndex("Input", out int column)); var type = (VectorDataViewType)schema[column].Type; @@ -1131,10 +1134,11 @@ public void TensorFlowTransformCifarInvalidShape() var cropped = new ImageResizingTransformer(mlContext, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(mlContext, "Input", "ImageCropped").Transform(cropped); + using TensorFlowModel model = mlContext.Model.LoadTensorFlowModel(modelLocation); var thrown = false; try { - IDataView trans = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel("Output", "Input").Fit(pixels).Transform(pixels); + IDataView trans = model.ScoreTensorFlowModel("Output", "Input").Fit(pixels).Transform(pixels); } catch { @@ -1185,10 +1189,10 @@ public void TensorFlowSentimentClassificationTest() // For explanation on how was the `sentiment_model` created // c.f. https://github.com/dotnet/machinelearning-testdata/blob/master/Microsoft.ML.TensorFlow.TestModels/sentiment_model/README.md string modelLocation = @"sentiment_model"; - var pipelineModel = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(new[] { "Prediction/Softmax" }, new[] { "Features" }) + using var pipelineModel = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(new[] { "Prediction/Softmax" }, new[] { "Features" }) .Append(mlContext.Transforms.CopyColumns("Prediction", "Prediction/Softmax")) .Fit(dataView); - var tfEnginePipe = mlContext.Model.CreatePredictionEngine(pipelineModel); + using var tfEnginePipe = mlContext.Model.CreatePredictionEngine(pipelineModel); var processedData = dataPipe.Predict(data[0]); Array.Resize(ref processedData.Features, 600); @@ -1222,7 +1226,7 @@ class TextOutput public void TensorFlowStringTest() { var mlContext = new MLContext(seed: 1); - var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(@"model_string_test"); + using var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(@"model_string_test"); var schema = tensorFlowModel.GetModelSchema(); Assert.True(schema.TryGetColumnIndex("A", out var colIndex)); Assert.True(schema.TryGetColumnIndex("B", out colIndex)); @@ -1246,10 +1250,14 @@ public void TensorFlowStringTest() } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationDefault() { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + Output.WriteLine("TODO TEST_STABILITY: TensorFlowImageClassificationDefault hangs on Linux."); + return; + } + string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", "images"); @@ -1285,7 +1293,7 @@ public void TensorFlowImageClassificationDefault() .Append(mlContext.MulticlassClassification.Trainers.ImageClassification("Label", "Image") .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); ; - var trainedModel = pipeline.Fit(trainDataset); + using var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, "model.zip"); @@ -1302,6 +1310,7 @@ public void TensorFlowImageClassificationDefault() Assert.InRange(metrics.MicroAccuracy, 0.8, 1); Assert.InRange(metrics.MacroAccuracy, 0.8, 1); + (loadedModel as IDisposable)?.Dispose(); } internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedValuesFileName, string validationSetBottleneckCachedValuesFileName) @@ -1333,10 +1342,14 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV [InlineData(ImageClassificationTrainer.Architecture.MobilenetV2)] [InlineData(ImageClassificationTrainer.Architecture.ResnetV250)] [InlineData(ImageClassificationTrainer.Architecture.InceptionV3)] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassification(ImageClassificationTrainer.Architecture arch) { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + Output.WriteLine("TODO TEST_STABILITY: TensorFlowImageClassification hangs on Linux."); + return; + } + string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", "images"); @@ -1400,7 +1413,7 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur .Append(mlContext.MulticlassClassification.Trainers.ImageClassification(options) .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); - var trainedModel = pipeline.Fit(trainDataset); + using var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, "model.zip"); @@ -1420,7 +1433,7 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur // Testing TrySinglePrediction: Utilizing PredictionEngine for single // predictions. Here, two pre-selected images are utilized in testing // the Prediction engine. - var predictionEngine = mlContext.Model + using var predictionEngine = mlContext.Model .CreatePredictionEngine(loadedModel); IEnumerable testImages = LoadImagesFromDirectory( @@ -1462,19 +1475,17 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur Assert.Equal("roses", predictionSecond.PredictedLabel); Assert.True(Array.IndexOf(labels, predictionFirst.PredictedLabel) > -1); Assert.True(Array.IndexOf(labels, predictionSecond.PredictedLabel) > -1); + + (loadedModel as IDisposable)?.Dispose(); } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationWithExponentialLRScheduling() { TensorFlowImageClassificationWithLRScheduling(new ExponentialLRDecay(), 50); } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationWithPolynomialLRScheduling() { @@ -1548,7 +1559,7 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule .Append(mlContext.Transforms.Conversion.MapKeyToValue( outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel")); - var trainedModel = pipeline.Fit(trainDataset); + using var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, "model.zip"); @@ -1568,7 +1579,7 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule // Testing TrySinglePrediction: Utilizing PredictionEngine for single // predictions. Here, two pre-selected images are utilized in testing // the Prediction engine. - var predictionEngine = mlContext.Model + using var predictionEngine = mlContext.Model .CreatePredictionEngine(loadedModel); IEnumerable testImages = LoadImagesFromDirectory( @@ -1614,13 +1625,13 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule Assert.True(File.Exists(Path.Combine(options.WorkspacePath, options.TrainSetBottleneckCachedValuesFileName))); Assert.True(File.Exists(Path.Combine(options.WorkspacePath, options.ValidationSetBottleneckCachedValuesFileName))); Assert.True(File.Exists(Path.Combine(Path.GetTempPath(), "MLNET", ImageClassificationTrainer.ModelFileName[options.Arch]))); + + (loadedModel as IDisposable)?.Dispose(); } [TensorFlowTheory] [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Accuracy)] [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Loss)] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTrainer.EarlyStoppingMetric earlyStoppingMetric) { string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", @@ -1690,7 +1701,7 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") .Append(mlContext.MulticlassClassification.Trainers.ImageClassification(options)); - var trainedModel = pipeline.Fit(trainDataset); + using var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, "model.zip"); @@ -1707,11 +1718,11 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine //Assert that the training ran and stopped within half epochs due to EarlyStopping Assert.InRange(lastEpoch, 1, 49); + + (loadedModel as IDisposable)?.Dispose(); } [TensorFlowFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationBadImages() { string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", @@ -1763,7 +1774,7 @@ public void TensorFlowImageClassificationBadImages() var pipeline = mlContext.MulticlassClassification.Trainers.ImageClassification(options); - var trainedModel = pipeline.Fit(trainDataset); + using var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, "model.zip"); @@ -1779,6 +1790,8 @@ public void TensorFlowImageClassificationBadImages() // by skipping bad images. Assert.InRange(metrics.MicroAccuracy, 0.3, 1); Assert.InRange(metrics.MacroAccuracy, 0.3, 1); + + (loadedModel as IDisposable)?.Dispose(); } public static IEnumerable LoadImagesFromDirectory(string folder, diff --git a/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs b/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs index b3228017c1..de1d8427f1 100644 --- a/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs +++ b/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs @@ -78,7 +78,8 @@ public void TestSimpleCase() var xyData = new List { new TestDataXY() { A = new float[4], B = new float[4] } }; var stringData = new List { new TestDataDifferntType() { a = new string[4], b = new string[4] } }; var sizeData = new List { new TestDataSize() { a = new float[2], b = new float[2] } }; - var pipe = ML.Model.LoadTensorFlowModel(modelFile).ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }); + using var model = ML.Model.LoadTensorFlowModel(modelFile); + var pipe = model.ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }); var invalidDataWrongNames = ML.Data.LoadFromEnumerable(xyData); var invalidDataWrongTypes = ML.Data.LoadFromEnumerable( stringData); @@ -119,7 +120,8 @@ public void TestOldSavingAndLoading() b = new[] { 10.0f, 8.0f, 6.0f, 6.0f } } })); - var est = ML.Model.LoadTensorFlowModel(modelFile).ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }); + using var model = ML.Model.LoadTensorFlowModel(modelFile); + var est = model.ScoreTensorFlowModel(new[] { "c" }, new[] { "a", "b" }); var transformer = est.Fit(dataView); var result = transformer.Transform(dataView); var resultRoles = new RoleMappedData(result); @@ -164,7 +166,8 @@ public void TestTensorFlow() TestEstimatorCore(pipe, data); - var result = pipe.Fit(data).Transform(data); + using var model = pipe.Fit(data); + var result = model.Transform(data); result.Schema.TryGetColumnIndex("Output", out int output); using (var cursor = result.GetRowCursor(result.Schema["Output"])) { @@ -187,7 +190,7 @@ public void TestTensorFlowWithSchema() const string modelLocation = "cifar_model/frozen_model.pb"; var mlContext = new MLContext(seed: 1); - var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation); + using var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation); var schema = tensorFlowModel.GetInputSchema(); Assert.True(schema.TryGetColumnIndex("Input", out int column)); var type = (VectorDataViewType)schema[column].Type; @@ -210,7 +213,8 @@ public void TestTensorFlowWithSchema() TestEstimatorCore(pipe, data); - var result = pipe.Fit(data).Transform(data); + using var model = pipe.Fit(data); + var result = model.Transform(data); result.Schema.TryGetColumnIndex("Output", out int output); using (var cursor = result.GetRowCursor(result.Schema["Output"])) {