Merge pull request #512 from lorengordon/retry-urlopen

Retries urlopen up to 5 times on URLerror

setup.cfg

@@ -29,6 +29,7 @@ classifiers =
 
 [options]
 install_requires =
+    backoff
     click
     defusedxml;platform_system=="Windows"
     futures;python_version<"3"

src/watchmaker/__init__.py

@@ -263,7 +263,7 @@ def _get_config(self):
         # Get the raw config data
         data = ''
         try:
-            data = urllib.request.urlopen(self.config_path).read()
+            data = watchmaker.utils.urlopen_retry(self.config_path).read()
         except (ValueError, urllib.error.URLError):
             msg = (
                 'Could not read the config from {0}! Please make sure your '

src/watchmaker/managers/base.py

@@ -82,7 +82,7 @@ def retrieve_file(self, url, filename):
 
         try:
             self.log.debug('Establishing connection to the host, %s', url)
-            response = urllib.request.urlopen(url)
+            response = watchmaker.utils.urlopen_retry(url)
             self.log.debug('Opening the file handle, %s', filename)
             with open(filename, 'wb') as outfile:
                 self.log.debug('Saving file to local filesystem...')

src/watchmaker/utils/__init__.py

@@ -5,7 +5,9 @@
 
 import os
 
-from watchmaker.utils.urllib import parse, request
+import backoff
+
+from watchmaker.utils import urllib
 
 
 def scheme_from_uri(uri):
@@ -14,27 +16,33 @@ def scheme_from_uri(uri):
     # i.e. '/abspath/foo' or 'relpath/foo'
     # Do not test `if parts.scheme` because of how urlparse handles Windows
     # file paths -- i.e. 'C:\\foo' => scheme = 'c' :(
-    return uri.scheme if '://' in parse.urlunparse(uri) else 'file'
+    return uri.scheme if '://' in urllib.parse.urlunparse(uri) else 'file'
 
 
 def uri_from_filepath(filepath):
     """Return a URI compatible with urllib, handling URIs and file paths."""
-    parts = parse.urlparse(filepath)
+    parts = urllib.parse.urlparse(filepath)
     scheme = scheme_from_uri(parts)
 
     if scheme != 'file':
         # Return non-file paths unchanged
         return filepath
 
     # Expand relative file paths and convert them to uri-style
-    path = request.pathname2url(os.path.abspath(os.path.expanduser(
+    path = urllib.request.pathname2url(os.path.abspath(os.path.expanduser(
         ''.join([x for x in [parts.netloc, parts.path] if x]))))
 
-    return parse.urlunparse((scheme, '', path, '', '', ''))
+    return urllib.parse.urlunparse((scheme, '', path, '', '', ''))
 
 
 def basename_from_uri(uri):
     """Return the basename/filename/leaf part of a URI."""
     # Do not split on '/' and return the last part because that will also
     # include any query in the uri. Instead, parse the uri.
-    return os.path.basename(parse.urlparse(uri).path)
+    return os.path.basename(urllib.parse.urlparse(uri).path)
+
+
+@backoff.on_exception(backoff.expo, urllib.error.URLError, max_tries=5)
+def urlopen_retry(uri):
+    """Retry urlopen on exception."""
+    return urllib.request.urlopen(uri)