Applying Backoff for rate-limited Maven Downloads in setup.py

setup.py

@@ -3,20 +3,23 @@
 from __future__ import print_function
 
 import glob
-import sys
-
 import os
 import shutil
+import sys
+import time
+
 from setuptools import Command
 from setuptools import setup
 from setuptools.command.install import install
 
 if sys.version_info[0] >= 3:
     # Python 3
     from urllib.request import urlopen
+    from urllib.error import HTTPError
 else:
     # Python 2
     from urllib2 import urlopen
+    from urllib2 import HTTPError
 
 #
 # This script modifies the basic setuptools by adding some functionality to the standard
@@ -40,7 +43,7 @@
 
 PACKAGE_NAME = 'amazon_kclpy'
 JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars')
-PACKAGE_VERSION = '2.0.2'
+PACKAGE_VERSION = '2.0.3'
 PYTHON_REQUIREMENTS = [
     'boto',
     # argparse is part of python2.7 but must be declared for python2.6
@@ -115,6 +118,7 @@
     ('commons-collections', 'commons-collections', '3.2.2'),
     ('software.amazon.glue', 'schema-registry-serde', '1.0.2')
 ]
+MAX_URL_DOWNLOAD_ATTEMPTS = 5
 
 
 class MavenJarDownloader:
@@ -175,7 +179,8 @@ def download_file(self, url, dest):
         """
         print('Attempting to retrieve remote jar {url}'.format(url=url))
         try:
-            response = urlopen(url)
+            response = self.make_request_with_backoff(url)
+
             with open(dest, 'wb') as dest_file:
                 shutil.copyfileobj(response, dest_file)
             print('Saving {url} -> {dest}'.format(url=url, dest=dest))
@@ -192,6 +197,21 @@ def download_files(self):
                 url = self.package_url(package[0], package[1], package[2])
                 self.download_file(url, dest)
 
+    def make_request_with_backoff(self, url):
+        for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS):
+            print('Attempting to retrieve remote jar {url}'.format(url=url))
+            try:
+                return urlopen(url)
+            except HTTPError as e:
+                if e.code == 429:
+                    sleep_time = 2 ** attempt_number
+                    print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time))
+                    time.sleep(sleep_time)
+                else:
+                    raise
+
+        raise Exception('"429 Too Many Requests" responses received.')
+
 
 class DownloadJarsCommand(Command):
     description = "Download the jar files needed to run the sample application"