dotnet · mstfbl · Mar 25, 2020 · Mar 13, 2020 · Mar 16, 2020 · Mar 16, 2020
diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs
@@ -113,8 +113,7 @@ public async Task<ResourceDownloadResults> EnsureResourceAsync(IHostEnvironment
                 await DownloadFromUrlWithRetryAsync(env, ch, absoluteUrl.AbsoluteUri, fileName, timeout, filePath), absoluteUrl.AbsoluteUri);
         }
 
-        private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, IChannel ch, string url, string fileName,
-            int timeout, string filePath, int retryTimes = 5)
+        private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, IChannel ch, string url, string fileName, int timeout, string filePath, int retryTimes = 5)
         {
             var downloadResult = "";
 
@@ -125,8 +124,12 @@ private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, I
                 if (string.IsNullOrEmpty(thisDownloadResult))
                     return thisDownloadResult;
                 else
+                {
                     downloadResult += thisDownloadResult + @"\n";
-
+                    // Do not retry if the URL does not exist
+                    if (thisDownloadResult.Contains("does not exist"))
+                        return downloadResult;
+                }
                 await Task.Delay(10 * 1000);
             }
 
@@ -136,7 +139,7 @@ private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, I
         /// <returns>Returns the error message if an error occurred, null if download was successful.</returns>
         private async Task<string> DownloadFromUrlAsync(IHostEnvironment env, IChannel ch, string url, string fileName, int timeout, string filePath)
         {
-            using (var webClient = new WebClient())
+            using (var webClient = new WebClientResponseUri())
             using (var downloadCancel = new CancellationTokenSource())
             {
                 bool deleteNeeded = false;
@@ -160,27 +163,8 @@ private async Task<string> DownloadFromUrlAsync(IHostEnvironment env, IChannel c
                     deleteNeeded = true;
                     return (await t).Message;
                 }
-
-                return CheckValidDownload(ch, filePath, url, ref deleteNeeded);
-            }
-        }
-
-        private static string CheckValidDownload(IChannel ch, string filePath, string url, ref bool deleteNeeded)
-        {
-            // If the relative url does not exist, aka.ms redirects to www.microsoft.com. Make sure this did not happen.
-            // If the file is big then it is definitely not the redirect.
-            var info = new FileInfo(filePath);
-            if (info.Length > 4096)
                 return null;
-            string error = null;
-            using (var r = new StreamReader(filePath))
-            {
-                var text = r.ReadToEnd();
-                if (text.Contains("<head>") && text.Contains("<body>") && text.Contains("microsoft.com"))
-                    error = $"The url '{url}' does not exist. Url was redirected to www.microsoft.com.";
             }
-            deleteNeeded = error != null;
-            return error;
         }
 
         private static void TryDelete(IChannel ch, string filePath, bool warn = true)
@@ -252,7 +236,7 @@ private static string GetFilePath(IChannel ch, string fileName, string dir, out
             return filePath;
         }
 
-        private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient webClient, Uri uri, string path, string fileName, CancellationToken ct)
+        private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClientResponseUri webClient, Uri uri, string path, string fileName, CancellationToken ct)
         {
             if (File.Exists(path))
                 return null;
@@ -269,12 +253,19 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient
             string tempPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path), "temp-resource-" + guid.ToString()));
             try
             {
+                Console.WriteLine(String.Format("Trying to view file through the url: {0}", uri.ToString()));
                 using (var s = webClient.OpenRead(uri))
                 using (var fh = env.CreateOutputFile(tempPath))
                 using (var ws = fh.CreateWriteStream())
                 {
                     var headers = webClient.ResponseHeaders.GetValues("Content-Length");
-                    if (Utils.Size(headers) == 0 || !long.TryParse(headers[0], out var size))
+                    var requestUri = webClient.ResponseUri;
+                    if (requestUri.Host == "www.microsoft.com" && requestUri.ToString().Length < 60)
+                    {
+                        return new ArgumentException($"The url '{uri}' does not exist. Url was redirected to '{requestUri}'.");
+                    }
+
+                    if (!long.TryParse(headers[0], out var size))
                         size = 10000000;
 
                     long printFreq = (long)(size / 10.0);
@@ -328,4 +319,17 @@ public static ResourceDownloadResults GetErrorMessage(out string errorMessage, p
         private static extern int chmod(string pathname, int mode);
 #pragma warning restore IDE1006
     }
+
+    public class WebClientResponseUri : WebClient
+    {
+        public Uri ResponseUri { get; private set; }
+
+        protected override WebResponse GetWebResponse(WebRequest request)
+        {
+            WebResponse response = null;
+            response = base.GetWebResponse(request);
+            ResponseUri = response.ResponseUri;
+            return response;
+        }
+    }
 }
diff --git a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs
@@ -181,7 +181,7 @@ internal static void DownloadIfNeeded(IHostEnvironment env, string url, string d
             using (var ch = env.Start("Ensuring meta files are present."))
             {
                 var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(env, ch, url, fileName, dir, timeout);
-                ensureModel.Wait();
+                ensureModel.Wait(5 * 60 * 1000); // 5 minute timeout period
                 var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
                 if (errorResult != null)
                 {

diff --git a/src/Microsoft.ML.Transforms/Text/WordEmbeddingsExtractor.cs b/src/Microsoft.ML.Transforms/Text/WordEmbeddingsExtractor.cs
@@ -628,8 +628,8 @@ private string EnsureModelFile(IHostEnvironment env, out int linesToSkip, WordEm
                 {
                     string dir = kind == WordEmbeddingEstimator.PretrainedModelKind.SentimentSpecificWordEmbedding ? Path.Combine("Text", "Sswe") : "WordVectors";
                     var url = $"{dir}/{modelFileName}";
-                    var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(Host, ch, url, modelFileName, dir, Timeout);
-                    ensureModel.Wait();
+                    var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(env, ch, url, modelFileName, dir, Timeout);
+                    ensureModel.Wait(5 * 60 * 1000); // 5 minute timeout period
                     var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
                     if (errorResult != null)
                     {