From db37cf75ff372e81f4c8a7d42418135de37fdb2f Mon Sep 17 00:00:00 2001 From: Florent Viard Date: Sat, 19 Aug 2017 16:36:15 +0200 Subject: [PATCH] Fixes #908 - Fixes unicode in req headers for py2 and py3 --- S3/CloudFront.py | 9 +++++---- S3/Crypto.py | 13 +++++++++---- S3/Custom_httplib27.py | 3 ++- S3/Custom_httplib3x.py | 5 +++-- S3/S3.py | 16 ++++++++-------- s3cmd | 27 +++++++++++++++++---------- 6 files changed, 44 insertions(+), 29 deletions(-) diff --git a/S3/CloudFront.py b/S3/CloudFront.py index 119f2aadb..e2e08ef4d 100644 --- a/S3/CloudFront.py +++ b/S3/CloudFront.py @@ -26,6 +26,7 @@ from .Crypto import sign_string_v2 from .S3Uri import S3Uri, S3UriS3 from .ConnMan import ConnMan +from .SortedDict import SortedDict cloudfront_api_version = "2010-11-01" cloudfront_resource = "/%(api_ver)s/distribution" % { 'api_ver' : cloudfront_api_version } @@ -397,7 +398,7 @@ def DeleteDistribution(self, cfuri): break warning("Still waiting...") time.sleep(10) - headers = {} + headers = SortedDict(ignore_case = True) headers['if-match'] = response['headers']['etag'] response = self.send_request("DeleteDist", dist_id = cfuri.dist_id(), headers = headers) @@ -424,7 +425,7 @@ def SetDistConfig(self, cfuri, dist_config, etag = None): debug("SetDistConfig(): Etag = %s" % etag) request_body = str(dist_config) debug("SetDistConfig(): request_body: %s" % request_body) - headers = {} + headers = SortedDict(ignore_case = True) headers['if-match'] = etag response = self.send_request("SetDistConfig", dist_id = cfuri.dist_id(), body = request_body, headers = headers) @@ -498,7 +499,7 @@ def GetInvalInfo(self, cfuri): def send_request(self, op_name, dist_id = None, request_id = None, body = None, headers = None, retries = _max_retries): if headers is None: - headers = {} + headers = SortedDict(ignore_case = True) operation = self.operations[op_name] if body: headers['content-type'] = 'text/plain' @@ -537,7 +538,7 @@ def create_request(self, operation, dist_id = None, request_id = None, headers = operation['resource'] % { 'dist_id' : dist_id, 'request_id' : request_id }) if not headers: - headers = {} + headers = SortedDict(ignore_case = True) if "date" in headers: if "x-amz-date" not in headers: diff --git a/S3/Crypto.py b/S3/Crypto.py index 529517226..d3771e3ca 100644 --- a/S3/Crypto.py +++ b/S3/Crypto.py @@ -69,7 +69,7 @@ def sign_string_v2(string_to_sign): return signature __all__.append("sign_string_v2") -def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={}): +def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers=None): """Sign a string with the secret key, returning base64 encoded results. By default the configured secret key is used, but may be overridden as an argument. @@ -86,6 +86,9 @@ def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={} # Missing of aws s3 doc but needed 'delete', 'cors'] + if cur_headers is None: + cur_headers = SortedDict(ignore_case = True) + access_key = Config.Config().access_key string_to_sign = method + "\n" @@ -111,7 +114,7 @@ def sign_request_v2(method='GET', canonical_uri='/', params=None, cur_headers={} debug("SignHeaders: " + repr(string_to_sign)) signature = decode_from_s3(sign_string_v2(encode_to_s3(string_to_sign))) - new_headers = dict(list(cur_headers.items())) + new_headers = SortedDict(list(cur_headers.items()), ignore_case=True) new_headers["Authorization"] = "AWS " + access_key + ":" + signature return new_headers @@ -174,8 +177,10 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): return kSigning def sign_request_v4(method='GET', host='', canonical_uri='/', params=None, - region='us-east-1', cur_headers={}, body=b''): + region='us-east-1', cur_headers=None, body=b''): service = 's3' + if cur_headers is None: + cur_headers = SortedDict(ignore_case = True) cfg = Config.Config() access_key = cfg.access_key @@ -207,7 +212,7 @@ def sign_request_v4(method='GET', host='', canonical_uri='/', params=None, # avoid duplicate headers and previous Authorization if header == 'Authorization' or header in signed_headers.split(';'): continue - canonical_headers[header.strip()] = str(cur_headers[header]).strip() + canonical_headers[header.strip()] = cur_headers[header].strip() signed_headers += ';' + header.strip() # sort headers into a string diff --git a/S3/Custom_httplib27.py b/S3/Custom_httplib27.py index 1f4234ec7..4a201a52e 100644 --- a/S3/Custom_httplib27.py +++ b/S3/Custom_httplib27.py @@ -138,12 +138,13 @@ def httpconnection_patched_send_request(self, method, url, body, headers): if 'expect' == hdr.lower() and '100-continue' in value.lower(): expect_continue = True + url = encode_to_s3(url) self.putrequest(method, url, **skips) if 'content-length' not in header_names: self._set_content_length(body, method) for hdr, value in headers.iteritems(): - self.putheader(hdr, value) + self.putheader(encode_to_s3(hdr), encode_to_s3(value)) # If an Expect: 100-continue was sent, we need to check for a 417 # Expectation Failed to avoid unecessarily sending the body diff --git a/S3/Custom_httplib3x.py b/S3/Custom_httplib3x.py index 42d84228b..c7b15dfb9 100644 --- a/S3/Custom_httplib3x.py +++ b/S3/Custom_httplib3x.py @@ -11,6 +11,8 @@ from io import StringIO +from .Utils import encode_to_s3 + _METHODS_EXPECTING_BODY = ['PATCH', 'POST', 'PUT'] @@ -162,7 +164,6 @@ def httpconnection_patched_send_request(self, method, url, body, headers, # 1. content-length has not been explicitly set # 2. the body is a file or iterable, but not a str or bytes-like # 3. Transfer-Encoding has NOT been explicitly set by the caller - if 'content-length' not in header_names: # only chunk body if not explicitly set for backwards # compatibility, assuming the client code is already handling the @@ -184,7 +185,7 @@ def httpconnection_patched_send_request(self, method, url, body, headers, encode_chunked = False for hdr, value in headers.items(): - self.putheader(hdr, value) + self.putheader(encode_to_s3(hdr), encode_to_s3(value)) if isinstance(body, str): # RFC 2616 Section 3.7.1 says that text default has a diff --git a/S3/S3.py b/S3/S3.py index f8eb39079..0ca935e7f 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -740,8 +740,8 @@ def compose_batch_del_xml(bucket, key_list): raise ValueError("Key list is empty") bucket = S3Uri(batch[0]).bucket() request_body = compose_batch_del_xml(bucket, batch) - headers = {'content-md5': compute_content_md5(request_body), - 'content-type': 'application/xml'} + headers = SortedDict({'content-md5': compute_content_md5(request_body), + 'content-type': 'application/xml'}, ignore_case=True) request = self.create_request("BATCH_DELETE", bucket = bucket, headers = headers, body = request_body, uri_params = {'delete': None}) @@ -944,7 +944,7 @@ def set_acl(self, uri, acl): body = u"%s"% acl debug(u"set_acl(%s): acl-xml: %s" % (uri, body)) - headers = {'content-type': 'application/xml'} + headers = SortedDict({'content-type': 'application/xml'}, ignore_case = True) if uri.has_object(): request = self.create_request("OBJECT_PUT", uri = uri, headers = headers, body = body, @@ -964,7 +964,7 @@ def get_policy(self, uri): return response['data'] def set_policy(self, uri, policy): - headers = {} + headers = SortedDict(ignore_case = True) # TODO check policy is proper json string headers['content-type'] = 'application/json' request = self.create_request("BUCKET_CREATE", uri = uri, @@ -987,7 +987,7 @@ def get_cors(self, uri): return response['data'] def set_cors(self, uri, cors): - headers = {} + headers = SortedDict(ignore_case = True) # TODO check cors is proper json string headers['content-type'] = 'application/xml' headers['content-md5'] = compute_content_md5(cors) @@ -1015,7 +1015,7 @@ def set_lifecycle_policy(self, uri, policy): return response def set_payer(self, uri): - headers = {} + headers = SortedDict(ignore_case = True) headers['content-type'] = 'application/xml' body = '\n' if self.config.requester_pays: @@ -1367,7 +1367,7 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0, conn = ConnMan.get(self.get_hostname(resource['bucket'])) conn.c.putrequest(method_string, self.format_uri(resource, conn.path)) for header in headers.keys(): - conn.c.putheader(header, str(headers[header])) + conn.c.putheader(encode_to_s3(header), encode_to_s3(headers[header])) conn.c.endheaders() except ParameterError as e: raise @@ -1600,7 +1600,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_ try: conn.c.putrequest(method_string, self.format_uri(resource, conn.path)) for header in headers.keys(): - conn.c.putheader(header, str(headers[header])) + conn.c.putheader(encode_to_s3(header), encode_to_s3(headers[header])) if start_position > 0: debug("Requesting Range: %d .. end" % start_position) conn.c.putheader("Range", "bytes=%d-" % start_position) diff --git a/s3cmd b/s3cmd index 3fbd93e3f..763a77105 100755 --- a/s3cmd +++ b/s3cmd @@ -2801,16 +2801,16 @@ def main(): if options.add_header: for hdr in options.add_header: try: - key, val = hdr.split(":", 1) + key, val = unicodise_s(hdr).split(":", 1) except ValueError: - raise ParameterError("Invalid header format: %s" % hdr) + raise ParameterError("Invalid header format: %s" % unicodise_s(hdr)) key_inval = re.sub("[a-zA-Z0-9-.]", "", key) if key_inval: key_inval = key_inval.replace(" ", "") key_inval = key_inval.replace("\t", "") raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval)) - debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip().lower(), val.strip())) - cfg.extra_headers[key.strip().lower()] = val.strip() + debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.replace('_', '-').strip().lower(), val.strip())) + cfg.extra_headers[key.replace('_', '-').strip().lower()] = val.strip() # Process --remove-header if options.remove_headers: @@ -2841,9 +2841,12 @@ def main(): ## Update Config with other parameters for option in cfg.option_list(): try: - if getattr(options, option) != None: - debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option))) - cfg.update_option(option, getattr(options, option)) + value = getattr(options, option) + if value != None: + if type(value) == type(b''): + value = unicodise_s(value) + debug(u"Updating Config.Config %s -> %s" % (option, value)) + cfg.update_option(option, value) except AttributeError: ## Some Config() options are not settable from command line pass @@ -2876,9 +2879,13 @@ def main(): ## Update CloudFront options if some were set for option in CfCmd.options.option_list(): try: - if getattr(options, option) != None: - debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option))) - CfCmd.options.update_option(option, getattr(options, option)) + value = getattr(options, option) + if value != None: + if type(value) == type(b''): + value = unicodise_s(value) + if value != None: + debug(u"Updating CloudFront.Cmd %s -> %s" % (option, value)) + CfCmd.options.update_option(option, value) except AttributeError: ## Some CloudFront.Cmd.Options() options are not settable from command line pass