From 130137884e46e9e924f0a07811149617d064673d Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Thu, 2 Apr 2020 18:08:36 -0700 Subject: [PATCH 1/4] fix tensorflow test hanging issue --- .../Utilities/ResourceManagerUtils.cs | 17 ++- .../TensorflowTests.cs | 102 ++++++------------ 2 files changed, 47 insertions(+), 72 deletions(-) diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs index 319b39b9c2..246c1c0a7f 100644 --- a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs @@ -257,6 +257,8 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient string tempPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path), "temp-resource-" + guid.ToString())); try { + int blockSize = 4096; + using (var s = webClient.OpenRead(uri)) using (var fh = env.CreateOutputFile(tempPath)) using (var ws = fh.CreateWriteStream()) @@ -268,21 +270,28 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient size = 10000000; long printFreq = (long)(size / 10.0); - var buffer = new byte[4096]; + var buffer = new byte[blockSize]; long total = 0; - int count; + + var task = s.ReadAsync(buffer, 0, blockSize); + task.Wait(ct); + int count = task.Result; // REVIEW: use a progress channel instead. - while ((count = s.Read(buffer, 0, 4096)) > 0) + while (count > 0) { ws.Write(buffer, 0, count); total += count; - if ((total - (total / printFreq) * printFreq) <= 4096) + if ((total - (total / printFreq) * printFreq) <= blockSize) ch.Info($"{fileName}: Downloaded {total} bytes out of {size}"); if (ct.IsCancellationRequested) { ch.Error($"{fileName}: Download timed out"); return ch.Except("Download timed out"); } + + task = s.ReadAsync(buffer, 0, blockSize); + task.Wait(ct); + count = task.Result; } } File.Move(tempPath, path); diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 97f473dee6..8a013a5aa0 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -61,8 +61,20 @@ public void Dispose() [Collection("NoParallelization")] public sealed class TensorFlowScenariosTests : BaseTestClass, IClassFixture { + private readonly string _fullImagesetFolderPath; + private readonly string _finalImagesFolderName; + public TensorFlowScenariosTests(ITestOutputHelper output) : base(output) { + string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", + "images"); + + //Download the image set and unzip + _finalImagesFolderName = DownloadImageSet( + imagesDownloadFolderPath); + + _fullImagesetFolderPath = Path.Combine( + imagesDownloadFolderPath, _finalImagesFolderName); } private class TestData @@ -1250,25 +1262,13 @@ public void TensorFlowStringTest() } [TensorFlowFact] - // This test hangs occasionally - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationDefault() { - string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", - "images"); - - //Download the image set and unzip - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + folder: _fullImagesetFolderPath, useFolderNameAsLabel: true); IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( mlContext.Data.LoadFromEnumerable(images), seed: 1); @@ -1285,7 +1285,7 @@ public void TensorFlowImageClassificationDefault() IDataView trainDataset = trainTestData.TrainSet; IDataView testDataset = trainTestData.TestSet; - var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Append(mlContext.MulticlassClassification.Trainers.ImageClassification("Label", "Image") .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); ; @@ -1338,25 +1338,13 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV [InlineData(ImageClassificationTrainer.Architecture.MobilenetV2)] [InlineData(ImageClassificationTrainer.Architecture.ResnetV250)] [InlineData(ImageClassificationTrainer.Architecture.InceptionV3)] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassification(ImageClassificationTrainer.Architecture arch) { - string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", - "images"); - - //Download the image set and unzip - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + folder: _fullImagesetFolderPath, useFolderNameAsLabel: true); IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( mlContext.Data.LoadFromEnumerable(images), seed: 1); @@ -1372,13 +1360,13 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur IDataView trainDataset = trainTestData.TrainSet; IDataView testDataset = trainTestData.TestSet; - var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Fit(testDataset) .Transform(testDataset); // Check if the bottleneck cached values already exist var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, - workspacePath, isReuse) = getInitialParameters(arch, finalImagesFolderName); + workspacePath, isReuse) = getInitialParameters(arch, _finalImagesFolderName); var options = new ImageClassificationTrainer.Options() { @@ -1401,7 +1389,7 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur ValidationSet = validationSet }; - var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Append(mlContext.MulticlassClassification.Trainers.ImageClassification(options) .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); @@ -1429,9 +1417,9 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur .CreatePredictionEngine(loadedModel); IEnumerable testImages = LoadImagesFromDirectory( - fullImagesetFolderPath, true); + _fullImagesetFolderPath, true); - string[] directories = Directory.GetDirectories(fullImagesetFolderPath); + string[] directories = Directory.GetDirectories(_fullImagesetFolderPath); string[] labels = new string[directories.Length]; for (int j = 0; j < labels.Length; j++) { @@ -1442,13 +1430,13 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur // Test daisy image ImageData firstImageToPredict = new ImageData { - ImagePath = Path.Combine(fullImagesetFolderPath, "daisy", "5794835_d15905c7c8_n.jpg") + ImagePath = Path.Combine(_fullImagesetFolderPath, "daisy", "5794835_d15905c7c8_n.jpg") }; // Test rose image ImageData secondImageToPredict = new ImageData { - ImagePath = Path.Combine(fullImagesetFolderPath, "roses", "12240303_80d87f77a3_n.jpg") + ImagePath = Path.Combine(_fullImagesetFolderPath, "roses", "12240303_80d87f77a3_n.jpg") }; var predictionFirst = predictionEngine.Predict(firstImageToPredict); @@ -1486,21 +1474,11 @@ public void TensorFlowImageClassificationWithPolynomialLRScheduling() internal void TensorFlowImageClassificationWithLRScheduling(LearningRateScheduler learningRateScheduler, int epoch) { - string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", - "images"); - - //Download the image set and unzip - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + folder: _fullImagesetFolderPath, useFolderNameAsLabel: true); IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( mlContext.Data.LoadFromEnumerable(images), seed: 1); @@ -1516,13 +1494,13 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule IDataView trainDataset = trainTestData.TrainSet; IDataView testDataset = trainTestData.TestSet; - var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Fit(testDataset) .Transform(testDataset); // Check if the bottleneck cached values already exist var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, - workspacePath, isReuse) = getInitialParameters(ImageClassificationTrainer.Architecture.ResnetV2101, finalImagesFolderName); + workspacePath, isReuse) = getInitialParameters(ImageClassificationTrainer.Architecture.ResnetV2101, _finalImagesFolderName); var options = new ImageClassificationTrainer.Options() { @@ -1546,7 +1524,7 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule LearningRateScheduler = learningRateScheduler }; - var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Append(mlContext.MulticlassClassification.Trainers.ImageClassification(options)) .Append(mlContext.Transforms.Conversion.MapKeyToValue( outputColumnName: "PredictedLabel", @@ -1575,9 +1553,9 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule .CreatePredictionEngine(loadedModel); IEnumerable testImages = LoadImagesFromDirectory( - fullImagesetFolderPath, true); + _fullImagesetFolderPath, true); - string[] directories = Directory.GetDirectories(fullImagesetFolderPath); + string[] directories = Directory.GetDirectories(_fullImagesetFolderPath); string[] labels = new string[directories.Length]; for (int j = 0; j < labels.Length; j++) { @@ -1588,13 +1566,13 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule // Test daisy image ImageData firstImageToPredict = new ImageData { - ImagePath = Path.Combine(fullImagesetFolderPath, "daisy", "5794835_d15905c7c8_n.jpg") + ImagePath = Path.Combine(_fullImagesetFolderPath, "daisy", "5794835_d15905c7c8_n.jpg") }; // Test rose image ImageData secondImageToPredict = new ImageData { - ImagePath = Path.Combine(fullImagesetFolderPath, "roses", "12240303_80d87f77a3_n.jpg") + ImagePath = Path.Combine(_fullImagesetFolderPath, "roses", "12240303_80d87f77a3_n.jpg") }; var predictionFirst = predictionEngine.Predict(firstImageToPredict); @@ -1624,25 +1602,13 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule [TensorFlowTheory] [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Accuracy)] [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Loss)] - // This test hangs ocassionally - [Trait("Category", "SkipInCI")] public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTrainer.EarlyStoppingMetric earlyStoppingMetric) { - string imagesDownloadFolderPath = Path.Combine(TensorFlowScenariosTestsFixture.assetsPath, "inputs", - "images"); - - //Download the image set and unzip - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + folder: _fullImagesetFolderPath, useFolderNameAsLabel: true); IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( mlContext.Data.LoadFromEnumerable(images), seed: 1); @@ -1660,13 +1626,13 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine IDataView testDataset = trainTestData.TestSet; int lastEpoch = 0; - var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var validationSet = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Fit(testDataset) .Transform(testDataset); // Check if the bottleneck cached values already exist var (trainSetBottleneckCachedValuesFileName, validationSetBottleneckCachedValuesFileName, - workspacePath, isReuse) = getInitialParameters(ImageClassificationTrainer.Architecture.ResnetV2101, finalImagesFolderName); + workspacePath, isReuse) = getInitialParameters(ImageClassificationTrainer.Architecture.ResnetV2101, _finalImagesFolderName); @@ -1692,7 +1658,7 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine ValidationSet = validationSet }; - var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", fullImagesetFolderPath, "ImagePath") + var pipeline = mlContext.Transforms.LoadRawImageBytes("Image", _fullImagesetFolderPath, "ImagePath") .Append(mlContext.MulticlassClassification.Trainers.ImageClassification(options)); using var trainedModel = pipeline.Fit(trainDataset); From 3f2d6e8dbae21ef0218e355a5366965784e40ad8 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Thu, 2 Apr 2020 20:53:01 -0700 Subject: [PATCH 2/4] set smaller timeout for download resource and ingore exception within retry --- .../Utilities/ResourceManagerUtils.cs | 20 ++++++++++++----- .../TensorflowTests.cs | 22 +++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs index 246c1c0a7f..d4583d1b28 100644 --- a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs @@ -127,14 +127,22 @@ private async Task DownloadFromUrlWithRetryAsync(IHostEnvironment env, I for (int i = 0; i < retryTimes; ++i) { - var thisDownloadResult = await DownloadFromUrlAsync(env, ch, url, fileName, timeout, filePath); + try + { + var thisDownloadResult = await DownloadFromUrlAsync(env, ch, url, fileName, timeout, filePath); - if (string.IsNullOrEmpty(thisDownloadResult)) - return thisDownloadResult; - else - downloadResult += thisDownloadResult + @"\n"; + if (string.IsNullOrEmpty(thisDownloadResult)) + return thisDownloadResult; + else + downloadResult += thisDownloadResult + @"\n"; - await Task.Delay(10 * 1000); + await Task.Delay(10 * 1000); + } + catch (Exception ex) + { + // ignore any Exception and retrying download + ch.Warning($"{i+1} - th try: Dowload {fileName} from {url} fail with exception {ex.Message}"); + } } return downloadResult; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 8a013a5aa0..7acafd6e4e 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -23,6 +23,7 @@ using static Microsoft.ML.DataOperationsCatalog; using Microsoft.ML.Trainers; using Microsoft.ML.TestFrameworkCommon.Attributes; +using Microsoft.ML.Internal.Utilities; namespace Microsoft.ML.Scenarios { @@ -1264,6 +1265,10 @@ public void TensorFlowStringTest() [TensorFlowFact] public void TensorFlowImageClassificationDefault() { + // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download + string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); + MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1307,6 +1312,9 @@ public void TensorFlowImageClassificationDefault() Assert.InRange(metrics.MacroAccuracy, 0.8, 1); (loadedModel as IDisposable)?.Dispose(); + + // set back timeout value + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedValuesFileName, string validationSetBottleneckCachedValuesFileName) @@ -1340,6 +1348,10 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV [InlineData(ImageClassificationTrainer.Architecture.InceptionV3)] public void TensorFlowImageClassification(ImageClassificationTrainer.Architecture arch) { + // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download + string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); + MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1457,6 +1469,9 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur Assert.True(Array.IndexOf(labels, predictionSecond.PredictedLabel) > -1); (loadedModel as IDisposable)?.Dispose(); + + // set back timeout value + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } [TensorFlowFact] @@ -1604,6 +1619,10 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Loss)] public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTrainer.EarlyStoppingMetric earlyStoppingMetric) { + // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download + string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); + MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1680,6 +1699,9 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine Assert.InRange(lastEpoch, 1, 49); (loadedModel as IDisposable)?.Dispose(); + + // set back timeout value + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } [TensorFlowFact] From 0f8668f37837cb945ce3dfcfcdc65db9647a0f1c Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Fri, 3 Apr 2020 13:39:09 -0700 Subject: [PATCH 3/4] take comments --- .../Utilities/ResourceManagerUtils.cs | 18 +++++++++------- .../UnitTests/TestResourceDownload.cs | 2 -- .../BaseTestClass.cs | 8 +++++++ .../TensorflowTests.cs | 21 ------------------- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs index d4583d1b28..2f2af12bc7 100644 --- a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs @@ -281,12 +281,18 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient var buffer = new byte[blockSize]; long total = 0; - var task = s.ReadAsync(buffer, 0, blockSize); - task.Wait(ct); - int count = task.Result; // REVIEW: use a progress channel instead. - while (count > 0) + while (true) { + var task = s.ReadAsync(buffer, 0, blockSize, ct); + task.Wait(); + int count = task.Result; + + if(count <= 0) + { + break; + } + ws.Write(buffer, 0, count); total += count; if ((total - (total / printFreq) * printFreq) <= blockSize) @@ -296,10 +302,6 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient ch.Error($"{fileName}: Download timed out"); return ch.Except("Download timed out"); } - - task = s.ReadAsync(buffer, 0, blockSize); - task.Wait(ct); - count = task.Result; } } File.Move(tempPath, path); diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestResourceDownload.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestResourceDownload.cs index ee7c102fcf..0663235b28 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestResourceDownload.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestResourceDownload.cs @@ -26,7 +26,6 @@ public TestResourceDownload(ITestOutputHelper helper) public async Task TestDownloadError() { var envVarOld = Environment.GetEnvironmentVariable(ResourceManagerUtils.CustomResourcesUrlEnvVariable); - var timeoutVarOld = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); var resourcePathVarOld = Environment.GetEnvironmentVariable(Utils.CustomSearchDirEnvVariable); Environment.SetEnvironmentVariable(Utils.CustomSearchDirEnvVariable, null); @@ -134,7 +133,6 @@ public async Task TestDownloadError() { // Set environment variable back to its old value. Environment.SetEnvironmentVariable(ResourceManagerUtils.CustomResourcesUrlEnvVariable, envVarOld); - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timeoutVarOld); Environment.SetEnvironmentVariable(Utils.CustomSearchDirEnvVariable, resourcePathVarOld); } } diff --git a/test/Microsoft.ML.TestFramework/BaseTestClass.cs b/test/Microsoft.ML.TestFramework/BaseTestClass.cs index 40a1a01120..a1b5f60ad9 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestClass.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestClass.cs @@ -8,6 +8,7 @@ using System.Reflection; using System.Threading; using Microsoft.ML.Internal.Internallearn.Test; +using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TestFrameworkCommon.Attributes; @@ -79,16 +80,23 @@ void IDisposable.Dispose() protected virtual void Initialize() { + // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to fail fast and retry download + _timeOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); } protected virtual void Cleanup() { + // set back timeout value + Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, _timeOutOldValue); } protected static string RootDir { get; } protected string OutDir { get; } protected static string DataDir { get; } + private string _timeOutOldValue; + protected ITestOutputHelper Output { get; } public static string GetDataPath(string name) diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 7acafd6e4e..afc8693176 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -1265,10 +1265,6 @@ public void TensorFlowStringTest() [TensorFlowFact] public void TensorFlowImageClassificationDefault() { - // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download - string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1312,9 +1308,6 @@ public void TensorFlowImageClassificationDefault() Assert.InRange(metrics.MacroAccuracy, 0.8, 1); (loadedModel as IDisposable)?.Dispose(); - - // set back timeout value - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedValuesFileName, string validationSetBottleneckCachedValuesFileName) @@ -1348,10 +1341,6 @@ internal bool ShouldReuse(string workspacePath, string trainSetBottleneckCachedV [InlineData(ImageClassificationTrainer.Architecture.InceptionV3)] public void TensorFlowImageClassification(ImageClassificationTrainer.Architecture arch) { - // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download - string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1469,9 +1458,6 @@ public void TensorFlowImageClassification(ImageClassificationTrainer.Architectur Assert.True(Array.IndexOf(labels, predictionSecond.PredictedLabel) > -1); (loadedModel as IDisposable)?.Dispose(); - - // set back timeout value - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } [TensorFlowFact] @@ -1619,10 +1605,6 @@ internal void TensorFlowImageClassificationWithLRScheduling(LearningRateSchedule [InlineData(ImageClassificationTrainer.EarlyStoppingMetric.Loss)] public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTrainer.EarlyStoppingMetric earlyStoppingMetric) { - // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to retry download - string timoOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); - MLContext mlContext = new MLContext(seed: 1); //Load all the original images info @@ -1699,9 +1681,6 @@ public void TensorFlowImageClassificationEarlyStopping(ImageClassificationTraine Assert.InRange(lastEpoch, 1, 49); (loadedModel as IDisposable)?.Dispose(); - - // set back timeout value - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, timoOutOldValue); } [TensorFlowFact] From 62f73bb17e9d9c840af7d18be4c39c58d3df1bdc Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Fri, 3 Apr 2020 14:53:15 -0700 Subject: [PATCH 4/4] only override timeout variable for tensorflow tests --- test/Microsoft.ML.TestFramework/BaseTestClass.cs | 8 -------- .../TensorflowTests.cs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.ML.TestFramework/BaseTestClass.cs b/test/Microsoft.ML.TestFramework/BaseTestClass.cs index a1b5f60ad9..40a1a01120 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestClass.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestClass.cs @@ -8,7 +8,6 @@ using System.Reflection; using System.Threading; using Microsoft.ML.Internal.Internallearn.Test; -using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Runtime; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.TestFrameworkCommon.Attributes; @@ -80,23 +79,16 @@ void IDisposable.Dispose() protected virtual void Initialize() { - // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to fail fast and retry download - _timeOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable); - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString()); } protected virtual void Cleanup() { - // set back timeout value - Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, _timeOutOldValue); } protected static string RootDir { get; } protected string OutDir { get; } protected static string DataDir { get; } - private string _timeOutOldValue; - protected ITestOutputHelper Output { get; } public static string GetDataPath(string name) diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index afc8693176..d2357d138a 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -64,6 +64,7 @@ public sealed class TensorFlowScenariosTests : BaseTestClass, IClassFixture