Fixes #908 - Fixes unicode in req headers for py2 and py3

S3/CloudFront.py

@@ -26,6 +26,7 @@
 from .Crypto import sign_string_v2
 from .S3Uri import S3Uri, S3UriS3
 from .ConnMan import ConnMan
+from .SortedDict import SortedDict
 
 cloudfront_api_version = "2010-11-01"
 cloudfront_resource = "/%(api_ver)s/distribution" % { 'api_ver' : cloudfront_api_version }
@@ -397,7 +398,7 @@ def DeleteDistribution(self, cfuri):
                     break
                 warning("Still waiting...")
                 time.sleep(10)
-        headers = {}
+        headers = SortedDict(ignore_case = True)
         headers['if-match'] = response['headers']['etag']
         response = self.send_request("DeleteDist", dist_id = cfuri.dist_id(),
                                      headers = headers)
@@ -424,7 +425,7 @@ def SetDistConfig(self, cfuri, dist_config, etag = None):
         debug("SetDistConfig(): Etag = %s" % etag)
         request_body = str(dist_config)
         debug("SetDistConfig(): request_body: %s" % request_body)
-        headers = {}
+        headers = SortedDict(ignore_case = True)
         headers['if-match'] = etag
         response = self.send_request("SetDistConfig", dist_id = cfuri.dist_id(),
                                      body = request_body, headers = headers)
@@ -498,7 +499,7 @@ def GetInvalInfo(self, cfuri):
 
     def send_request(self, op_name, dist_id = None, request_id = None, body = None, headers = None, retries = _max_retries):
         if headers is None:
-            headers = {}
+            headers = SortedDict(ignore_case = True)
         operation = self.operations[op_name]
         if body:
             headers['content-type'] = 'text/plain'
@@ -537,7 +538,7 @@ def create_request(self, operation, dist_id = None, request_id = None, headers =
                    operation['resource'] % { 'dist_id' : dist_id, 'request_id' : request_id })
 
         if not headers:
-            headers = {}
+            headers = SortedDict(ignore_case = True)
 
         if "date" in headers:
             if "x-amz-date" not in headers:

S3/Crypto.py

@@ -69,7 +69,7 @@ def sign_string_v2(string_to_sign):
     return signature
 __all__.append("sign_string_v2")
 
-def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={}):
+def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers=None):
     """Sign a string with the secret key, returning base64 encoded results.
     By default the configured secret key is used, but may be overridden as
     an argument.
@@ -86,6 +86,9 @@ def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={}
                                # Missing of aws s3 doc but needed
                                'delete', 'cors']
 
+    if cur_headers is None:
+        cur_headers = SortedDict(ignore_case = True)
+
     access_key = Config.Config().access_key
 
     string_to_sign  = method + "\n"
@@ -111,7 +114,7 @@ def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={}
     debug("SignHeaders: " + repr(string_to_sign))
     signature = decode_from_s3(sign_string_v2(encode_to_s3(string_to_sign)))
 
-    new_headers = dict(list(cur_headers.items()))
+    new_headers = SortedDict(list(cur_headers.items()), ignore_case=True)
     new_headers["Authorization"] = "AWS " + access_key + ":" + signature
 
     return new_headers
@@ -174,8 +177,10 @@ def getSignatureKey(key, dateStamp, regionName, serviceName):
     return kSigning
 
 def sign_request_v4(method='GET', host='', canonical_uri='/', params=None,
-                    region='us-east-1', cur_headers={}, body=b''):
+                    region='us-east-1', cur_headers=None, body=b''):
     service = 's3'
+    if cur_headers is None:
+        cur_headers = SortedDict(ignore_case = True)
 
     cfg = Config.Config()
     access_key = cfg.access_key
@@ -207,7 +212,7 @@ def sign_request_v4(method='GET', host='', canonical_uri='/', params=None,
         # avoid duplicate headers and previous Authorization
         if header == 'Authorization' or header in signed_headers.split(';'):
             continue
-        canonical_headers[header.strip()] = str(cur_headers[header]).strip()
+        canonical_headers[header.strip()] = cur_headers[header].strip()
         signed_headers += ';' + header.strip()
 
     # sort headers into a string

S3/Custom_httplib27.py

@@ -138,12 +138,13 @@ def httpconnection_patched_send_request(self, method, url, body, headers):
         if 'expect' == hdr.lower() and '100-continue' in value.lower():
             expect_continue = True
 
+    url = encode_to_s3(url)
     self.putrequest(method, url, **skips)
 
     if 'content-length' not in header_names:
         self._set_content_length(body, method)
     for hdr, value in headers.iteritems():
-        self.putheader(hdr, value)
+        self.putheader(encode_to_s3(hdr), encode_to_s3(value))
 
     # If an Expect: 100-continue was sent, we need to check for a 417
     # Expectation Failed to avoid unecessarily sending the body

S3/Custom_httplib3x.py

@@ -11,6 +11,8 @@
 
 from io import StringIO
 
+from .Utils import encode_to_s3
+
 
 _METHODS_EXPECTING_BODY = ['PATCH', 'POST', 'PUT']
 
@@ -162,7 +164,6 @@ def httpconnection_patched_send_request(self, method, url, body, headers,
     # 1. content-length has not been explicitly set
     # 2. the body is a file or iterable, but not a str or bytes-like
     # 3. Transfer-Encoding has NOT been explicitly set by the caller
-
     if 'content-length' not in header_names:
         # only chunk body if not explicitly set for backwards
         # compatibility, assuming the client code is already handling the
@@ -184,7 +185,7 @@ def httpconnection_patched_send_request(self, method, url, body, headers,
         encode_chunked = False
 
     for hdr, value in headers.items():
-        self.putheader(hdr, value)
+        self.putheader(encode_to_s3(hdr), encode_to_s3(value))
 
     if isinstance(body, str):
         # RFC 2616 Section 3.7.1 says that text default has a

S3/S3.py

@@ -740,8 +740,8 @@ def compose_batch_del_xml(bucket, key_list):
             raise ValueError("Key list is empty")
         bucket = S3Uri(batch[0]).bucket()
         request_body = compose_batch_del_xml(bucket, batch)
-        headers = {'content-md5': compute_content_md5(request_body),
-                   'content-type': 'application/xml'}
+        headers = SortedDict({'content-md5': compute_content_md5(request_body),
+                   'content-type': 'application/xml'}, ignore_case=True)
         request = self.create_request("BATCH_DELETE", bucket = bucket,
                                       headers = headers, body = request_body,
                                       uri_params = {'delete': None})
@@ -944,7 +944,7 @@ def set_acl(self, uri, acl):
         body = u"%s"% acl
         debug(u"set_acl(%s): acl-xml: %s" % (uri, body))
 
-        headers = {'content-type': 'application/xml'}
+        headers = SortedDict({'content-type': 'application/xml'}, ignore_case = True)
         if uri.has_object():
             request = self.create_request("OBJECT_PUT", uri = uri,
                                           headers = headers, body = body,
@@ -964,7 +964,7 @@ def get_policy(self, uri):
         return response['data']
 
     def set_policy(self, uri, policy):
-        headers = {}
+        headers = SortedDict(ignore_case = True)
         # TODO check policy is proper json string
         headers['content-type'] = 'application/json'
         request = self.create_request("BUCKET_CREATE", uri = uri,
@@ -987,7 +987,7 @@ def get_cors(self, uri):
         return response['data']
 
     def set_cors(self, uri, cors):
-        headers = {}
+        headers = SortedDict(ignore_case = True)
         # TODO check cors is proper json string
         headers['content-type'] = 'application/xml'
         headers['content-md5'] = compute_content_md5(cors)
@@ -1015,7 +1015,7 @@ def set_lifecycle_policy(self, uri, policy):
         return response
 
     def set_payer(self, uri):
-        headers = {}
+        headers = SortedDict(ignore_case = True)
         headers['content-type'] = 'application/xml'
         body = '<RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">\n'
         if self.config.requester_pays:
@@ -1367,7 +1367,7 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
             conn = ConnMan.get(self.get_hostname(resource['bucket']))
             conn.c.putrequest(method_string, self.format_uri(resource, conn.path))
             for header in headers.keys():
-                conn.c.putheader(header, str(headers[header]))
+                conn.c.putheader(encode_to_s3(header), encode_to_s3(headers[header]))
             conn.c.endheaders()
         except ParameterError as e:
             raise
@@ -1600,7 +1600,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
         try:
             conn.c.putrequest(method_string, self.format_uri(resource, conn.path))
             for header in headers.keys():
-                conn.c.putheader(header, str(headers[header]))
+                conn.c.putheader(encode_to_s3(header), encode_to_s3(headers[header]))
             if start_position > 0:
                 debug("Requesting Range: %d .. end" % start_position)
                 conn.c.putheader("Range", "bytes=%d-" % start_position)

s3cmd

@@ -2801,16 +2801,16 @@ def main():
     if options.add_header:
         for hdr in options.add_header:
             try:
-                key, val = hdr.split(":", 1)
+                key, val = unicodise_s(hdr).split(":", 1)
             except ValueError:
-                raise ParameterError("Invalid header format: %s" % hdr)
+                raise ParameterError("Invalid header format: %s" % unicodise_s(hdr))
             key_inval = re.sub("[a-zA-Z0-9-.]", "", key)
             if key_inval:
                 key_inval = key_inval.replace(" ", "<space>")
                 key_inval = key_inval.replace("\t", "<tab>")
                 raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval))
-            debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip().lower(), val.strip()))
-            cfg.extra_headers[key.strip().lower()] = val.strip()
+            debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.replace('_', '-').strip().lower(), val.strip()))
+            cfg.extra_headers[key.replace('_', '-').strip().lower()] = val.strip()
 
     # Process --remove-header
     if options.remove_headers:
@@ -2841,9 +2841,12 @@ def main():
     ## Update Config with other parameters
     for option in cfg.option_list():
         try:
-            if getattr(options, option) != None:
-                debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option)))
-                cfg.update_option(option, getattr(options, option))
+            value = getattr(options, option)
+            if value != None:
+                if type(value) == type(b''):
+                    value = unicodise_s(value)
+                debug(u"Updating Config.Config %s -> %s" % (option, value))
+                cfg.update_option(option, value)
         except AttributeError:
             ## Some Config() options are not settable from command line
             pass
@@ -2876,9 +2879,13 @@ def main():
     ## Update CloudFront options if some were set
     for option in CfCmd.options.option_list():
         try:
-            if getattr(options, option) != None:
-                debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option)))
-                CfCmd.options.update_option(option, getattr(options, option))
+            value = getattr(options, option)
+            if value != None:
+                if type(value) == type(b''):
+                    value = unicodise_s(value)
+            if value != None:
+                debug(u"Updating CloudFront.Cmd %s -> %s" % (option, value))
+                CfCmd.options.update_option(option, value)
         except AttributeError:
             ## Some CloudFront.Cmd.Options() options are not settable from command line
             pass