Fixes #913 - Fix limitrate to recover of sudden changes in bandwidth …

…and to not slow down too much the transfers

A "maximum" throttle was also added to avoid excessive
throttling/limitrate.
Transfer performances should be improved when using the limitrate
feature.

S3/Config.py

@@ -164,6 +164,9 @@ class Config(object):
     # expected for every send file requests.
     use_http_expect = False
     signurl_use_https = False
+    # Maximum sleep duration for throtte / limitrate.
+    # s3 will timeout if a request/transfer is stuck for more than a short time
+    throttle_max = 100
 
     ## Creating a singleton
     def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None):

S3/S3.py

@@ -1427,6 +1427,7 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
                     # CONTINUE case. Reset the response
                     http_response.read()
                     conn.c._HTTPConnection__state = ConnMan._CS_REQ_SENT
+
                 while (size_left > 0):
                     #debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, filename, size_left))
                     l = min(self.config.send_chunk, size_left)
@@ -1439,20 +1440,22 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
                         start_time = time.time()
 
                     md5_hash.update(data)
+
                     conn.c.wrapper_send_body(data)
                     if self.config.progress_meter:
                         progress.update(delta_position = len(data))
                     size_left -= len(data)
 
                     #throttle
+                    limitrate_throttle = throttle
                     if self.config.limitrate > 0:
                         real_duration = time.time() - start_time
-                        expected_duration = float(l)/self.config.limitrate
-                        throttle = max(expected_duration - real_duration, throttle)
-                    if throttle:
-                        time.sleep(throttle)
-                md5_computed = md5_hash.hexdigest()
+                        expected_duration = float(l) / self.config.limitrate
+                        limitrate_throttle = max(expected_duration - real_duration, limitrate_throttle)
+                    if limitrate_throttle:
+                        time.sleep(min(limitrate_throttle, self.config.throttle_max))
 
+                md5_computed = md5_hash.hexdigest()
                 http_response = conn.c.getresponse()
 
             response = {}