Skip to content

Making _BlobIterator accumulate prefixes. #932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dhermes merged 1 commit into from
Jun 23, 2015
Merged

Making _BlobIterator accumulate prefixes. #932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions gcloud/storage/bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ class _BlobIterator(Iterator):
def __init__(self, bucket, extra_params=None, connection=None):
connection = _require_connection(connection)
self.bucket = bucket
self.prefixes = ()
self.prefixes = set()
self._current_prefixes = None
super(_BlobIterator, self).__init__(
connection=connection, path=bucket.path + '/o',
extra_params=extra_params)
Expand All @@ -63,7 +64,8 @@ def get_items_from_response(self, response):
:type response: dict
:param response: The JSON API response for a page of blobs.
"""
self.prefixes = tuple(response.get('prefixes', ()))
self._current_prefixes = tuple(response.get('prefixes', ()))
self.prefixes.update(self._current_prefixes)
for item in response.get('items', []):
name = item.get('name')
blob = Blob(name, bucket=self.bucket)
Expand Down
72 changes: 44 additions & 28 deletions gcloud/storage/test_bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def test_ctor_w_implicit_connection(self):
self.assertEqual(iterator.path, '%s/o' % bucket.path)
self.assertEqual(iterator.page_number, 0)
self.assertEqual(iterator.next_page_token, None)
self.assertEqual(iterator.prefixes, ())
self.assertEqual(iterator.prefixes, set())

def test_ctor_w_explicit_connection(self):
connection = _Connection()
Expand All @@ -48,33 +48,52 @@ def test_ctor_w_explicit_connection(self):
self.assertEqual(iterator.path, '%s/o' % bucket.path)
self.assertEqual(iterator.page_number, 0)
self.assertEqual(iterator.next_page_token, None)
self.assertEqual(iterator.prefixes, ())
self.assertEqual(iterator.prefixes, set())

def test_get_items_from_response_empty(self):
from gcloud.storage._testing import _monkey_defaults
connection = _Connection()
bucket = _Bucket()
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response({}))
iterator = self._makeOne(bucket, connection=connection)
blobs = list(iterator.get_items_from_response({}))
self.assertEqual(blobs, [])
self.assertEqual(iterator.prefixes, ())
self.assertEqual(iterator.prefixes, set())

def test_get_items_from_response_non_empty(self):
from gcloud.storage.blob import Blob
from gcloud.storage._testing import _monkey_defaults
BLOB_NAME = 'blob-name'
response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
connection = _Connection()
bucket = _Bucket()
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response(response))
iterator = self._makeOne(bucket, connection=connection)
blobs = list(iterator.get_items_from_response(response))
self.assertEqual(len(blobs), 1)
blob = blobs[0]
self.assertTrue(isinstance(blob, Blob))
self.assertEqual(blob.name, BLOB_NAME)
self.assertEqual(iterator.prefixes, set(['foo']))

def test_get_items_from_response_cumulative_prefixes(self):
from gcloud.storage.blob import Blob
BLOB_NAME = 'blob-name1'
response1 = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
response2 = {
'items': [],
'prefixes': ['foo', 'bar'],
}
connection = _Connection()
bucket = _Bucket()
iterator = self._makeOne(bucket, connection=connection)
# Parse first response.
blobs = list(iterator.get_items_from_response(response1))
self.assertEqual(len(blobs), 1)
blob = blobs[0]
self.assertTrue(isinstance(blob, Blob))
self.assertEqual(blob.name, BLOB_NAME)
self.assertEqual(iterator.prefixes, ('foo',))
self.assertEqual(iterator.prefixes, set(['foo']))
# Parse second response.
blobs = list(iterator.get_items_from_response(response2))
self.assertEqual(len(blobs), 0)
self.assertEqual(iterator.prefixes, set(['foo', 'bar']))


class Test_Bucket(unittest2.TestCase):
Expand Down Expand Up @@ -243,21 +262,18 @@ def test_get_blob_hit(self):
self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME))

def test_list_blobs_defaults(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs()
blobs = list(iterator)
iterator = bucket.list_blobs(connection=connection)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test_list_blobs_explicit(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
MAX_RESULTS = 10
PAGE_TOKEN = 'ABCD'
Expand All @@ -277,17 +293,17 @@ def test_list_blobs_explicit(self):
}
connection = _Connection({'items': []})
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
)
blobs = list(iterator)
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
connection=connection,
)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
Expand Down
8 changes: 4 additions & 4 deletions regression/storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ def test_root_level_w_delimiter(self):
self.assertEqual([blob.name for blob in blobs], ['file01.txt'])
self.assertEqual(iterator.page_number, 1)
self.assertTrue(iterator.next_page_token is None)
self.assertEqual(iterator.prefixes, ('parent/',))
self.assertEqual(iterator.prefixes, set(['parent/']))

def test_first_level(self):
iterator = self.bucket.list_blobs(delimiter='/', prefix='parent/')
Expand All @@ -268,7 +268,7 @@ def test_first_level(self):
self.assertEqual([blob.name for blob in blobs], ['parent/file11.txt'])
self.assertEqual(iterator.page_number, 1)
self.assertTrue(iterator.next_page_token is None)
self.assertEqual(iterator.prefixes, ('parent/child/',))
self.assertEqual(iterator.prefixes, set(['parent/child/']))

def test_second_level(self):
iterator = self.bucket.list_blobs(delimiter='/',
Expand All @@ -281,7 +281,7 @@ def test_second_level(self):
self.assertEqual(iterator.page_number, 1)
self.assertTrue(iterator.next_page_token is None)
self.assertEqual(iterator.prefixes,
('parent/child/grand/', 'parent/child/other/'))
set(['parent/child/grand/', 'parent/child/other/']))

def test_third_level(self):
# Pseudo-hierarchy can be arbitrarily deep, subject to the limit
Expand All @@ -296,7 +296,7 @@ def test_third_level(self):
['parent/child/grand/file31.txt'])
self.assertEqual(iterator.page_number, 1)
self.assertTrue(iterator.next_page_token is None)
self.assertEqual(iterator.prefixes, ())
self.assertEqual(iterator.prefixes, set())


class TestStorageSignURLs(TestStorageFiles):
Expand Down