diff --git a/.coveragerc b/.coveragerc index 047ce53d2bd5..490564b5cd74 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,3 +2,8 @@ omit = */demo/* */demo.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ diff --git a/gcloud/datastore/entity.py b/gcloud/datastore/entity.py index 5101a637222a..59948084fa7a 100644 --- a/gcloud/datastore/entity.py +++ b/gcloud/datastore/entity.py @@ -230,7 +230,7 @@ def delete(self): dataset_id=self.dataset().id(), key_pb=self.key().to_protobuf()) # pylint: enable=maybe-no-member - def __repr__(self): # pragma NO COVER + def __repr__(self): # An entity should have a key all the time (even if it's partial). if self.key(): # pylint: disable=maybe-no-member diff --git a/gcloud/datastore/key.py b/gcloud/datastore/key.py index 0ba282735cf0..9950b1cd9b75 100644 --- a/gcloud/datastore/key.py +++ b/gcloud/datastore/key.py @@ -274,7 +274,7 @@ def id_or_name(self): """ return self.id() or self.name() - def parent(self): # pragma NO COVER + def parent(self): """Getter: return a new key for the next highest element in path. :rtype: :class:`gcloud.datastore.key.Key` @@ -286,5 +286,5 @@ def parent(self): # pragma NO COVER return None return self.path(self.path()[:-1]) - def __repr__(self): # pragma NO COVER + def __repr__(self): return '' % self.path() diff --git a/gcloud/storage/acl.py b/gcloud/storage/acl.py index b0546e212bc9..2d40e2733318 100644 --- a/gcloud/storage/acl.py +++ b/gcloud/storage/acl.py @@ -110,7 +110,7 @@ def __str__(self): else: return '{self.type}-{self.identifier}'.format(self=self) - def __repr__(self): # pragma NO COVER + def __repr__(self): return ''.format( self=self, roles=', '.join(self.roles)) @@ -353,7 +353,7 @@ def get_entities(self): return self.entities.values() - def save(self): # pragma NO COVER + def save(self): """A method to be overridden by subclasses. :raises: NotImplementedError diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index d6ab28f88bdd..6fbf4883fa9b 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -42,7 +42,7 @@ def from_dict(cls, bucket_dict, connection=None): return cls(connection=connection, name=bucket_dict['name'], metadata=bucket_dict) - def __repr__(self): # pragma NO COVER + def __repr__(self): return '' % self.name def __iter__(self): @@ -126,7 +126,7 @@ def new_key(self, key): # Support Python 2 and 3. try: string_type = basestring - except NameError: # pragma NO COVER PY3k + except NameError: # pragma: NO COVER PY3k string_type = str if isinstance(key, string_type): diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index b51058f82edd..72a93c6fabd7 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -1,9 +1,9 @@ """Create / interact with gcloud storage connections.""" import base64 +import calendar import datetime import json -import time import urllib from Crypto.Hash import SHA256 @@ -18,6 +18,14 @@ from gcloud.storage.iterator import BucketIterator +def _utcnow(): # pragma: NO COVER testing replaces + """Returns current time as UTC datetime. + + NOTE: on the module namespace so tests can replace it. + """ + return datetime.datetime.utcnow() + + class Connection(connection.Connection): """A connection to Google Cloud Storage via the JSON REST API. @@ -419,7 +427,7 @@ def new_bucket(self, bucket): # Support Python 2 and 3. try: string_type = basestring - except NameError: # pragma NO COVER PY3k + except NameError: # pragma: NO COVER PY3k string_type = str if isinstance(bucket, string_type): @@ -429,7 +437,7 @@ def new_bucket(self, bucket): def generate_signed_url(self, resource, expiration, method='GET', content_md5=None, - content_type=None): # pragma NO COVER + content_type=None): """Generate signed URL to provide query-string auth'n to a resource. :type resource: string @@ -455,31 +463,7 @@ def generate_signed_url(self, resource, expiration, until expiration. """ - # expiration can be an absolute timestamp (int, long), - # an absolute time (datetime.datetime), - # or a relative time (datetime.timedelta). - # We should convert all of these into an absolute timestamp. - - # If it's a timedelta, add it to `now` in UTC. - if isinstance(expiration, datetime.timedelta): - now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - expiration = now + expiration - - # If it's a datetime, convert to a timestamp. - if isinstance(expiration, datetime.datetime): - # Make sure the timezone on the value is UTC - # (either by converting or replacing the value). - if expiration.tzinfo: - expiration = expiration.astimezone(pytz.utc) - else: - expiration = expiration.replace(tzinfo=pytz.utc) - - # Turn the datetime into a timestamp (seconds, not microseconds). - expiration = int(time.mktime(expiration.timetuple())) - - if not isinstance(expiration, (int, long)): - raise ValueError('Expected an integer timestamp, datetime, or ' - 'timedelta. Got %s' % type(expiration)) + expiration = _get_expiration_seconds(expiration) # Generate the string to sign. signature_string = '\n'.join([ @@ -514,3 +498,36 @@ def generate_signed_url(self, resource, expiration, return '{endpoint}{resource}?{querystring}'.format( endpoint=self.API_ACCESS_ENDPOINT, resource=resource, querystring=urllib.urlencode(query_params)) + + +def _get_expiration_seconds(expiration): + """Convert 'expiration' to a number of seconds in the future. + + :type expiration: int, long, datetime.datetime, datetime.timedelta + :param expiration: When the signed URL should expire. + + :rtype: int + :returns: a timestamp as an absolute number of seconds. + """ + + # If it's a timedelta, add it to `now` in UTC. + if isinstance(expiration, datetime.timedelta): + now = _utcnow().replace(tzinfo=pytz.utc) + expiration = now + expiration + + # If it's a datetime, convert to a timestamp. + if isinstance(expiration, datetime.datetime): + # Make sure the timezone on the value is UTC + # (either by converting or replacing the value). + if expiration.tzinfo: + expiration = expiration.astimezone(pytz.utc) + else: + expiration = expiration.replace(tzinfo=pytz.utc) + + # Turn the datetime into a timestamp (seconds, not microseconds). + expiration = int(calendar.timegm(expiration.timetuple())) + + if not isinstance(expiration, (int, long)): + raise TypeError('Expected an integer timestamp, datetime, or ' + 'timedelta. Got %s' % type(expiration)) + return expiration diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py index b69c6d70a768..cd134a50ade2 100644 --- a/gcloud/storage/iterator.py +++ b/gcloud/storage/iterator.py @@ -112,7 +112,7 @@ def reset(self): self.page_number = 0 self.next_page_token = None - def get_items_from_response(self, response): # pragma NO COVER + def get_items_from_response(self, response): """Factory method called while iterating. This should be overriden. This method should be overridden by a subclass. diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 73f4f799da40..9bf76f820643 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -58,7 +58,7 @@ def from_dict(cls, key_dict, bucket=None): return cls(bucket=bucket, name=key_dict['name'], metadata=key_dict) - def __repr__(self): # pragma NO COVER + def __repr__(self): if self.bucket: bucket_name = self.bucket.name else: @@ -102,8 +102,7 @@ def public_url(self): storage_base_url='http://commondatastorage.googleapis.com', self=self) - def generate_signed_url(self, expiration, - method='GET'): # pragma NO COVER + def generate_signed_url(self, expiration, method='GET'): """Generates a signed URL for this key. If you have a key that you want to allow access to @@ -181,11 +180,7 @@ def get_contents_to_file(self, fh): """ for chunk in KeyDataIterator(self): - try: - fh.write(chunk) - except IOError, e: # pragma NO COVER - if e.errno == errno.ENOSPC: - raise Exception('No space left on device.') + fh.write(chunk) def get_contents_to_filename(self, filename): """Get the contents of this key to a file by name. diff --git a/gcloud/storage/test_acl.py b/gcloud/storage/test_acl.py index b33bf42696f9..d72ec0fb782e 100644 --- a/gcloud/storage/test_acl.py +++ b/gcloud/storage/test_acl.py @@ -376,6 +376,21 @@ def test_all_authenticated(self): self.assertEqual(list(acl), [{'entity': 'allAuthenticatedUsers', 'role': ROLE}]) + def test_get_entities_empty(self): + acl = self._makeOne() + self.assertEqual(acl.get_entities(), []) + + def test_get_entities_nonempty(self): + TYPE = 'type' + ID = 'id' + acl = self._makeOne() + entity = acl.entity(TYPE, ID) + self.assertEqual(acl.get_entities(), [entity]) + + def test_save_raises_NotImplementedError(self): + acl = self._makeOne() + self.assertRaises(NotImplementedError, acl.save) + class Test_BucketACL(unittest2.TestCase): diff --git a/gcloud/storage/test_connection.py b/gcloud/storage/test_connection.py index 207a221d4a6e..c3787cbe2c82 100644 --- a/gcloud/storage/test_connection.py +++ b/gcloud/storage/test_connection.py @@ -557,6 +557,121 @@ def test_new_bucket_w_invalid(self): conn = self._makeOne(PROJECT) self.assertRaises(TypeError, conn.new_bucket, object()) + def test_generate_signed_url_w_expiration_int(self): + import base64 + import urlparse + from gcloud._testing import _Monkey + from gcloud.storage import connection as MUT + + ENDPOINT = 'http://api.example.com' + RESOURCE = '/name/key' + PROJECT = 'project' + KEY = 'key' + SIGNED = base64.b64encode('DEADBEEF') + crypto = _Crypto() + rsa = _RSA() + pkcs_v1_5 = _PKCS1_v1_5() + sha256 = _SHA256() + conn = self._makeOne(PROJECT, _Credentials()) + conn.API_ACCESS_ENDPOINT = ENDPOINT + + with _Monkey(MUT, crypto=crypto, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, + SHA256=sha256): + url = conn.generate_signed_url(RESOURCE, 1000) + + scheme, netloc, path, qs, frag = urlparse.urlsplit(url) + self.assertEqual(scheme, 'http') + self.assertEqual(netloc, 'api.example.com') + self.assertEqual(path, RESOURCE) + params = urlparse.parse_qs(qs) + self.assertEqual(len(params), 3) + self.assertEqual(params['Signature'], [SIGNED]) + self.assertEqual(params['Expires'], ['1000']) + self.assertEqual(params['GoogleAccessId'], + [_Credentials.service_account_name]) + self.assertEqual(frag, '') + + +class Test__get_expiration_seconds(unittest2.TestCase): + + def _callFUT(self, expiration): + from gcloud.storage.connection import _get_expiration_seconds + + return _get_expiration_seconds(expiration) + + def _utc_seconds(self, when): + import calendar + + return int(calendar.timegm(when.timetuple())) + + def test__get_expiration_seconds_w_invalid(self): + self.assertRaises(TypeError, self._callFUT, object()) + self.assertRaises(TypeError, self._callFUT, None) + + def test__get_expiration_seconds_w_int(self): + self.assertEqual(self._callFUT(123), 123) + + def test__get_expiration_seconds_w_long(self): + try: + long + except NameError: # pragma: NO COVER Py3K + pass + else: + self.assertEqual(self._callFUT(long(123)), 123) + + def test__get_expiration_w_naive_datetime(self): + import datetime + + expiration_no_tz = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + utc_seconds = self._utc_seconds(expiration_no_tz) + self.assertEqual(self._callFUT(expiration_no_tz), utc_seconds) + + def test__get_expiration_w_utc_datetime(self): + import datetime + import pytz + + expiration_utc = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, pytz.utc) + utc_seconds = self._utc_seconds(expiration_utc) + self.assertEqual(self._callFUT(expiration_utc), utc_seconds) + + def test__get_expiration_w_other_zone_datetime(self): + import datetime + import pytz + + zone = pytz.timezone('CET') + expiration_other = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, zone) + utc_seconds = self._utc_seconds(expiration_other) + cet_seconds = utc_seconds - (60 * 60) # CET one hour earlier than UTC + self.assertEqual(self._callFUT(expiration_other), cet_seconds) + + def test__get_expiration_seconds_w_timedelta_seconds(self): + import datetime + from gcloud.storage import connection + from gcloud._testing import _Monkey + + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + utc_seconds = self._utc_seconds(dummy_utcnow) + expiration_as_delta = datetime.timedelta(seconds=10) + + with _Monkey(connection, _utcnow=lambda: dummy_utcnow): + result = self._callFUT(expiration_as_delta) + + self.assertEqual(result, utc_seconds + 10) + + def test__get_expiration_seconds_w_timedelta_days(self): + import datetime + from gcloud.storage import connection + from gcloud._testing import _Monkey + + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + utc_seconds = self._utc_seconds(dummy_utcnow) + expiration_as_delta = datetime.timedelta(days=1) + + with _Monkey(connection, _utcnow=lambda: dummy_utcnow): + result = self._callFUT(expiration_as_delta) + + self.assertEqual(result, utc_seconds + 86400) + class Http(object): @@ -570,3 +685,61 @@ def __init__(self, headers, content): def request(self, **kw): self._called_with = kw return self._response, self._content + + +class _Credentials(object): + + service_account_name = 'testing@example.com' + + @property + def private_key(self): + import base64 + return base64.b64encode('SEEKRIT') + + +class _Crypto(object): + + FILETYPE_PEM = 'pem' + _loaded = _dumped = None + + def load_pkcs12(self, buffer, passphrase): + self._loaded = (buffer, passphrase) + return self + + def get_privatekey(self): + return '__PKCS12__' + + def dump_privatekey(self, type, pkey, cipher=None, passphrase=None): + self._dumped = (type, pkey, cipher, passphrase) + return '__PEM__' + + +class _RSA(object): + + _imported = None + + def importKey(self, pem): + self._imported = pem + return 'imported:%s' % pem + + +class _PKCS1_v1_5(object): + + _pem_key = _signature_hash = None + + def new(self, pem_key): + self._pem_key = pem_key + return self + + def sign(self, signature_hash): + self._signature_hash = signature_hash + return 'DEADBEEF' + + +class _SHA256(object): + + _signature_string = None + + def new(self, signature_string): + self._signature_string = signature_string + return self diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index ba197d3b727c..07155d00b077 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -111,6 +111,13 @@ def test_reset(self): self.assertEqual(iterator.page_number, 0) self.assertEqual(iterator.next_page_token, None) + def test_get_items_from_response_raises_NotImplementedError(self): + PATH = '/foo' + connection = _Connection() + iterator = self._makeOne(connection, PATH) + self.assertRaises(NotImplementedError, + iterator.get_items_from_response, object()) + class TestBucketIterator(unittest2.TestCase): diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index 538dd056ede6..8b41912ecdfc 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -80,6 +80,30 @@ def test_public_url(self): 'http://commondatastorage.googleapis.com/name/%s' % KEY) + def test_generate_signed_url_w_default_method(self): + KEY = 'key' + EXPIRATION = '2014-10-16T20:34:37Z' + connection = _Connection() + bucket = _Bucket(connection) + key = self._makeOne(bucket, KEY) + self.assertEqual(key.generate_signed_url(EXPIRATION), + 'http://example.com/abucket/akey?Signature=DEADBEEF' + '&Expiration=2014-10-16T20:34:37Z') + self.assertEqual(connection._signed, + [('/name/key', EXPIRATION, {'method': 'GET'})]) + + def test_generate_signed_url_w_explicit_method(self): + KEY = 'key' + EXPIRATION = '2014-10-16T20:34:37Z' + connection = _Connection() + bucket = _Bucket(connection) + key = self._makeOne(bucket, KEY) + self.assertEqual(key.generate_signed_url(EXPIRATION, method='POST'), + 'http://example.com/abucket/akey?Signature=DEADBEEF' + '&Expiration=2014-10-16T20:34:37Z') + self.assertEqual(connection._signed, + [('/name/key', EXPIRATION, {'method': 'POST'})]) + def test_exists_miss(self): NONESUCH = 'nonesuch' connection = _Connection() @@ -571,6 +595,7 @@ class _Connection(object): def __init__(self, *responses): self._responses = responses self._requested = [] + self._signed = [] def make_request(self, **kw): self._requested.append(kw) @@ -591,6 +616,11 @@ def build_api_url(self, path, query_params=None, scheme, netloc, _, _, _ = urlsplit(api_base_url) return urlunsplit((scheme, netloc, path, qs, '')) + def generate_signed_url(self, resource, expiration, **kw): + self._signed.append((resource, expiration, kw)) + return ('http://example.com/abucket/akey?Signature=DEADBEEF' + '&Expiration=%s' % expiration) + class _Bucket(object): path = '/b/name'