diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index c32832a926ce..9960b560624d 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -19,7 +19,6 @@ import six -import google.auth.transport.requests from google import resumable_media from google.resumable_media.requests import MultipartUpload from google.resumable_media.requests import ResumableUpload @@ -823,8 +822,8 @@ def insert_data(self, return errors - def _make_transport(self, client): - """Make an authenticated transport with a client's credentials. + def _get_transport(self, client): + """Return the client's transport. :type client: :class:`~google.cloud.bigquery.client.Client` :param client: The client to use. @@ -834,10 +833,7 @@ def _make_transport(self, client): :returns: The transport (with credentials) that will make authenticated requests. """ - # Create a ``requests`` transport with the client's credentials. - transport = google.auth.transport.requests.AuthorizedSession( - client._credentials) - return transport + return client._http def _initiate_resumable_upload(self, client, stream, metadata, num_retries): @@ -865,7 +861,7 @@ def _initiate_resumable_upload(self, client, stream, * The ``transport`` used to initiate the upload. """ chunk_size = _DEFAULT_CHUNKSIZE - transport = self._make_transport(client) + transport = self._get_transport(client) headers = _get_upload_headers(client._connection.USER_AGENT) upload_url = _RESUMABLE_URL_TEMPLATE.format(project=self.project) upload = ResumableUpload(upload_url, chunk_size, headers=headers) @@ -941,7 +937,7 @@ def _do_multipart_upload(self, client, stream, metadata, msg = _READ_LESS_THAN_SIZE.format(size, len(data)) raise ValueError(msg) - transport = self._make_transport(client) + transport = self._get_transport(client) headers = _get_upload_headers(client._connection.USER_AGENT) upload_url = _MULTIPART_URL_TEMPLATE.format(project=self.project) diff --git a/bigquery/tests/unit/test_table.py b/bigquery/tests/unit/test_table.py index eebb40a2e736..3bab58b6c8f8 100644 --- a/bigquery/tests/unit/test_table.py +++ b/bigquery/tests/unit/test_table.py @@ -1561,14 +1561,14 @@ def _row_data(row): self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['data'], SENT) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test__make_transport(self, session_factory): - client = mock.Mock(spec=[u'_credentials']) + def test__get_transport(self): + client = mock.Mock(spec=[u'_credentials', '_http']) + client._http = mock.sentinel.http table = self._make_one(self.TABLE_NAME, None) - transport = table._make_transport(client) - self.assertIs(transport, session_factory.return_value) - session_factory.assert_called_once_with(client._credentials) + transport = table._get_transport(client) + + self.assertIs(transport, mock.sentinel.http) @staticmethod def _mock_requests_response(status_code, headers, content=b''): @@ -1600,8 +1600,7 @@ def _initiate_resumable_upload_helper(self, num_retries=None): response_headers = {'location': resumable_url} fake_transport = self._mock_transport( http_client.OK, response_headers) - table._make_transport = mock.Mock( - return_value=fake_transport, spec=[]) + client._http = fake_transport # Create some mock arguments and call the method under test. data = b'goodbye gudbi gootbee' @@ -1640,7 +1639,6 @@ def _initiate_resumable_upload_helper(self, num_retries=None): self.assertEqual(stream.tell(), 0) # Check the mocks. - table._make_transport.assert_called_once_with(client) request_headers = expected_headers.copy() request_headers['x-upload-content-type'] = _GENERIC_CONTENT_TYPE fake_transport.request.assert_called_once_with( @@ -1668,7 +1666,7 @@ def _do_multipart_upload_success_helper( # Create mocks to be checked for doing transport. fake_transport = self._mock_transport(http_client.OK, {}) - table._make_transport = mock.Mock(return_value=fake_transport, spec=[]) + client._http = fake_transport # Create some mock arguments. data = b'Bzzzz-zap \x00\x01\xf4' @@ -1682,7 +1680,6 @@ def _do_multipart_upload_success_helper( # Check the mocks and the returned value. self.assertIs(response, fake_transport.request.return_value) self.assertEqual(stream.tell(), size) - table._make_transport.assert_called_once_with(client) get_boundary.assert_called_once_with() upload_url = ( @@ -1723,7 +1720,7 @@ class TestTableUpload(object): # rather than `unittest`-style. @staticmethod - def _make_table(): + def _make_table(transport=None): from google.cloud.bigquery import _http from google.cloud.bigquery import client from google.cloud.bigquery import dataset @@ -1733,6 +1730,7 @@ def _make_table(): client = mock.create_autospec(client.Client, instance=True) client._connection = connection client._credentials = mock.sentinel.credentials + client._http = transport client.project = 'project_id' dataset = dataset.Dataset('test_dataset', client) @@ -1955,57 +1953,54 @@ def _make_resumable_upload_responses(cls, size): return [initial_response, data_response, final_response] @staticmethod - def _make_transport_patch(table, responses=None): - """Patch a table's _make_transport method to return given responses.""" + def _make_transport(responses=None): import google.auth.transport.requests transport = mock.create_autospec( google.auth.transport.requests.AuthorizedSession, instance=True) transport.request.side_effect = responses - return mock.patch.object( - table, '_make_transport', return_value=transport, autospec=True) + return transport def test__do_resumable_upload(self): - table = self._make_table() file_obj = self._make_file_obj() file_obj_len = len(file_obj.getvalue()) - responses = self._make_resumable_upload_responses(file_obj_len) + transport = self._make_transport( + self._make_resumable_upload_responses(file_obj_len)) + table = self._make_table(transport) - with self._make_transport_patch(table, responses) as transport: - result = table._do_resumable_upload( - table._dataset._client, - file_obj, - self.EXPECTED_CONFIGURATION, - None) + result = table._do_resumable_upload( + table._dataset._client, + file_obj, + self.EXPECTED_CONFIGURATION, + None) content = result.content.decode('utf-8') assert json.loads(content) == {'size': file_obj_len} # Verify that configuration data was passed in with the initial # request. - transport.return_value.request.assert_any_call( + transport.request.assert_any_call( 'POST', mock.ANY, data=json.dumps(self.EXPECTED_CONFIGURATION).encode('utf-8'), headers=mock.ANY) def test__do_multipart_upload(self): - table = self._make_table() + transport = self._make_transport([self._make_response(http_client.OK)]) + table = self._make_table(transport) file_obj = self._make_file_obj() file_obj_len = len(file_obj.getvalue()) - responses = [self._make_response(http_client.OK)] - with self._make_transport_patch(table, responses) as transport: - table._do_multipart_upload( - table._dataset._client, - file_obj, - self.EXPECTED_CONFIGURATION, - file_obj_len, - None) + table._do_multipart_upload( + table._dataset._client, + file_obj, + self.EXPECTED_CONFIGURATION, + file_obj_len, + None) # Verify that configuration data was passed in with the initial # request. - request_args = transport.return_value.request.mock_calls[0][2] + request_args = transport.request.mock_calls[0][2] request_data = request_args['data'].decode('utf-8') request_headers = request_args['headers'] diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index b515d1e2c8c2..836cfb645f42 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -36,7 +36,6 @@ from six.moves.urllib.parse import quote -import google.auth.transport.requests from google import resumable_media from google.resumable_media.requests import ChunkedDownload from google.resumable_media.requests import Download @@ -361,8 +360,8 @@ def delete(self, client=None): """ return self.bucket.delete_blob(self.name, client=client) - def _make_transport(self, client): - """Make an authenticated transport with a client's credentials. + def _get_transport(self, client): + """Return the client's transport. :type client: :class:`~google.cloud.storage.client.Client` :param client: (Optional) The client to use. If not passed, falls back @@ -374,10 +373,7 @@ def _make_transport(self, client): make authenticated requests. """ client = self._require_client(client) - # Create a ``requests`` transport with the client's credentials. - transport = google.auth.transport.requests.AuthorizedSession( - client._credentials) - return transport + return client._http def _get_download_url(self): """Get the download URL for the current blob. @@ -463,7 +459,7 @@ def download_to_file(self, file_obj, client=None): """ download_url = self._get_download_url() headers = _get_encryption_headers(self._encryption_key) - transport = self._make_transport(client) + transport = self._get_transport(client) try: self._do_download(transport, file_obj, download_url, headers) @@ -638,7 +634,7 @@ def _do_multipart_upload(self, client, stream, content_type, msg = _READ_LESS_THAN_SIZE.format(size, len(data)) raise ValueError(msg) - transport = self._make_transport(client) + transport = self._get_transport(client) info = self._get_upload_arguments(content_type) headers, object_metadata, content_type = info @@ -708,7 +704,7 @@ def _initiate_resumable_upload(self, client, stream, content_type, if chunk_size is None: chunk_size = self.chunk_size - transport = self._make_transport(client) + transport = self._get_transport(client) info = self._get_upload_arguments(content_type) headers, object_metadata, content_type = info if extra_headers is not None: diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index 7904ce86e89b..7cc0dadc2691 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -327,14 +327,14 @@ def test_delete(self): self.assertFalse(blob.exists()) self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test__make_transport(self, fake_session_factory): - client = mock.Mock(spec=[u'_credentials']) + def test__get_transport(self): + client = mock.Mock(spec=[u'_credentials', '_http']) + client._http = mock.sentinel.transport blob = self._make_one(u'blob-name', bucket=None) - transport = blob._make_transport(client) - self.assertIs(transport, fake_session_factory.return_value) - fake_session_factory.assert_called_once_with(client._credentials) + transport = blob._get_transport(client) + + self.assertIs(transport, mock.sentinel.transport) def test__get_download_url_with_media_link(self): blob_name = 'something.txt' @@ -400,13 +400,10 @@ def _mock_download_transport(self): fake_transport.request.side_effect = [chunk1_response, chunk2_response] return fake_transport - def _check_session_mocks(self, client, fake_session_factory, + def _check_session_mocks(self, client, transport, expected_url, headers=None): - # Check that exactly one transport was created. - fake_session_factory.assert_called_once_with(client._credentials) - fake_transport = fake_session_factory.return_value # Check that the transport was called exactly twice. - self.assertEqual(fake_transport.request.call_count, 2) + self.assertEqual(transport.request.call_count, 2) if headers is None: headers = {} # NOTE: bytes=0-2 never shows up because the mock was called with @@ -415,7 +412,7 @@ def _check_session_mocks(self, client, fake_session_factory, headers['range'] = 'bytes=3-5' call = mock.call( 'GET', expected_url, data=None, headers=headers) - self.assertEqual(fake_transport.request.mock_calls, [call, call]) + self.assertEqual(transport.request.mock_calls, [call, call]) def test__do_download_simple(self): blob_name = 'blob-name' @@ -471,8 +468,7 @@ def test__do_download_chunked(self): 'GET', download_url, data=None, headers=headers) self.assertEqual(transport.request.mock_calls, [call, call]) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_to_file_with_failure(self, fake_session_factory): + def test_download_to_file_with_failure(self): from google.cloud import exceptions blob_name = 'blob-name' @@ -483,10 +479,8 @@ def test_download_to_file_with_failure(self, fake_session_factory): } transport.request.return_value = self._mock_requests_response( http_client.NOT_FOUND, bad_response_headers, content=b'Not found') - fake_session_factory.return_value = transport # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=[u'_http']) bucket = _Bucket(client) blob = self._make_one(blob_name, bucket=bucket) # Set the media link on the blob @@ -497,19 +491,15 @@ def test_download_to_file_with_failure(self, fake_session_factory): blob.download_to_file(file_obj) self.assertEqual(file_obj.tell(), 0) - # Check that exactly one transport was created. - fake_session_factory.assert_called_once_with(client._credentials) # Check that the transport was called once. transport.request.assert_called_once_with( 'GET', blob.media_link, data=None, headers={}) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_to_file_wo_media_link(self, fake_session_factory): + def test_download_to_file_wo_media_link(self): blob_name = 'blob-name' - fake_session_factory.return_value = self._mock_download_transport() + transport = self._mock_download_transport() # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=[u'_http']) bucket = _Bucket(client) blob = self._make_one(blob_name, bucket=bucket) # Modify the blob so there there will be 2 chunks of size 3. @@ -525,16 +515,13 @@ def test_download_to_file_wo_media_link(self, fake_session_factory): expected_url = ( 'https://www.googleapis.com/download/storage/v1/b/' 'name/o/blob-name?alt=media') - self._check_session_mocks(client, fake_session_factory, expected_url) + self._check_session_mocks(client, transport, expected_url) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def _download_to_file_helper(self, fake_session_factory, use_chunks=False): + def _download_to_file_helper(self, use_chunks=False): blob_name = 'blob-name' - fake_transport = self._mock_download_transport() - fake_session_factory.return_value = fake_transport + transport = self._mock_download_transport() # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=[u'_http']) bucket = _Bucket(client) media_link = 'http://example.com/media/' properties = {'mediaLink': media_link} @@ -549,18 +536,16 @@ def _download_to_file_helper(self, fake_session_factory, use_chunks=False): http_client.OK, {'content-length': '6', 'content-range': 'bytes 0-5/6'}, content=b'abcdef') - fake_transport.request.side_effect = [single_chunk_response] + transport.request.side_effect = [single_chunk_response] file_obj = io.BytesIO() blob.download_to_file(file_obj) self.assertEqual(file_obj.getvalue(), b'abcdef') if use_chunks: - self._check_session_mocks(client, fake_session_factory, media_link) + self._check_session_mocks(client, transport, media_link) else: - # Check that exactly one transport was created. - fake_session_factory.assert_called_once_with(client._credentials) - fake_transport.request.assert_called_once_with( + transport.request.assert_called_once_with( 'GET', media_link, data=None, headers={}) def test_download_to_file_default(self): @@ -569,16 +554,15 @@ def test_download_to_file_default(self): def test_download_to_file_with_chunk_size(self): self._download_to_file_helper(use_chunks=True) - def _download_to_filename_helper(self, fake_session_factory, updated=None): + def _download_to_filename_helper(self, updated=None): import os import time from google.cloud._testing import _NamedTemporaryFile blob_name = 'blob-name' - fake_session_factory.return_value = self._mock_download_transport() + transport = self._mock_download_transport() # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=['_http']) bucket = _Bucket(client) media_link = 'http://example.com/media/' properties = {'mediaLink': media_link} @@ -603,29 +587,24 @@ def _download_to_filename_helper(self, fake_session_factory, updated=None): self.assertEqual(wrote, b'abcdef') - self._check_session_mocks(client, fake_session_factory, media_link) + self._check_session_mocks(client, transport, media_link) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_to_filename(self, fake_session_factory): + def test_download_to_filename(self): updated = '2014-12-06T13:13:50.690Z' - self._download_to_filename_helper( - fake_session_factory, updated=updated) + self._download_to_filename_helper(updated=updated) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_to_filename_wo_updated(self, fake_session_factory): - self._download_to_filename_helper(fake_session_factory) + def test_download_to_filename_wo_updated(self): + self._download_to_filename_helper() - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_to_filename_w_key(self, fake_session_factory): + def test_download_to_filename_w_key(self): import os import time from google.cloud._testing import _NamedTemporaryFile blob_name = 'blob-name' - fake_session_factory.return_value = self._mock_download_transport() + transport = self._mock_download_transport() # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=['_http']) bucket = _Bucket(client) media_link = 'http://example.com/media/' properties = {'mediaLink': media_link, @@ -655,15 +634,13 @@ def test_download_to_filename_w_key(self, fake_session_factory): 'X-Goog-Encryption-Key': header_key_value, } self._check_session_mocks( - client, fake_session_factory, media_link, headers=key_headers) + client, transport, media_link, headers=key_headers) - @mock.patch('google.auth.transport.requests.AuthorizedSession') - def test_download_as_string(self, fake_session_factory): + def test_download_as_string(self): blob_name = 'blob-name' - fake_session_factory.return_value = self._mock_download_transport() + transport = self._mock_download_transport() # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock( - _credentials=_make_credentials(), spec=['_credentials']) + client = mock.Mock(_http=transport, spec=['_http']) bucket = _Bucket(client) media_link = 'http://example.com/media/' properties = {'mediaLink': media_link} @@ -675,7 +652,7 @@ def test_download_as_string(self, fake_session_factory): fetched = blob.download_as_string() self.assertEqual(fetched, b'abcdef') - self._check_session_mocks(client, fake_session_factory, media_link) + self._check_session_mocks(client, transport, media_link) def test__get_content_type_explicit(self): blob = self._make_one(u'blob-name', bucket=None) @@ -777,11 +754,10 @@ def _do_multipart_success(self, mock_get_boundary, size=None, self.assertIsNone(blob.chunk_size) # Create mocks to be checked for doing transport. - fake_transport = self._mock_transport(http_client.OK, {}) - blob._make_transport = mock.Mock(return_value=fake_transport, spec=[]) + transport = self._mock_transport(http_client.OK, {}) # Create some mock arguments. - client = mock.sentinel.client + client = mock.Mock(_http=transport, spec=['_http']) data = b'data here hear hier' stream = io.BytesIO(data) content_type = u'application/xml' @@ -789,7 +765,7 @@ def _do_multipart_success(self, mock_get_boundary, size=None, client, stream, content_type, size, num_retries) # Check the mocks and the returned value. - self.assertIs(response, fake_transport.request.return_value) + self.assertIs(response, transport.request.return_value) if size is None: data_read = data self.assertEqual(stream.tell(), len(data)) @@ -797,7 +773,6 @@ def _do_multipart_success(self, mock_get_boundary, size=None, data_read = data[:size] self.assertEqual(stream.tell(), size) - blob._make_transport.assert_called_once_with(client) mock_get_boundary.assert_called_once_with() upload_url = ( @@ -813,7 +788,7 @@ def _do_multipart_success(self, mock_get_boundary, size=None, data_read + b'\r\n--==0==--') headers = {'content-type': b'multipart/related; boundary="==0=="'} - fake_transport.request.assert_called_once_with( + transport.request.assert_called_once_with( 'POST', upload_url, data=payload, headers=headers) @mock.patch(u'google.resumable_media._upload.get_boundary', @@ -866,12 +841,10 @@ def _initiate_resumable_helper(self, size=None, extra_headers=None, # Create mocks to be checked for doing transport. resumable_url = 'http://test.invalid?upload_id=hey-you' response_headers = {'location': resumable_url} - fake_transport = self._mock_transport( - http_client.OK, response_headers) - blob._make_transport = mock.Mock(return_value=fake_transport, spec=[]) + transport = self._mock_transport(http_client.OK, response_headers) # Create some mock arguments and call the method under test. - client = mock.sentinel.client + client = mock.Mock(_http=transport, spec=[u'_http']) data = b'hello hallo halo hi-low' stream = io.BytesIO(data) content_type = u'text/plain' @@ -912,13 +885,12 @@ def _initiate_resumable_helper(self, size=None, extra_headers=None, else: self.assertIsNone(retry_strategy.max_cumulative_retry) self.assertEqual(retry_strategy.max_retries, num_retries) - self.assertIs(transport, fake_transport) + self.assertIs(transport, transport) # Make sure we never read from the stream. self.assertEqual(stream.tell(), 0) # Check the mocks. blob._get_writable_metadata.assert_called_once_with() - blob._make_transport.assert_called_once_with(client) payload = json.dumps(object_metadata).encode('utf-8') expected_headers = { 'content-type': 'application/json; charset=UTF-8', @@ -928,7 +900,7 @@ def _initiate_resumable_helper(self, size=None, extra_headers=None, expected_headers['x-upload-content-length'] = str(size) if extra_headers is not None: expected_headers.update(extra_headers) - fake_transport.request.assert_called_once_with( + transport.request.assert_called_once_with( 'POST', upload_url, data=payload, headers=expected_headers) def test__initiate_resumable_upload_no_size(self): @@ -1034,12 +1006,11 @@ def _do_resumable_helper(self, use_size=False, num_retries=None): resumable_url = 'http://test.invalid?upload_id=and-then-there-was-1' headers1 = {'location': resumable_url} headers2 = {'range': 'bytes=0-{:d}'.format(blob.chunk_size - 1)} - fake_transport, responses = self._make_resumable_transport( + transport, responses = self._make_resumable_transport( headers1, headers2, {}, total_bytes) - blob._make_transport = mock.Mock(return_value=fake_transport, spec=[]) # Create some mock arguments and call the method under test. - client = mock.sentinel.client + client = mock.Mock(_http=transport, spec=['_http']) stream = io.BytesIO(data) content_type = u'text/html' response = blob._do_resumable_upload( @@ -1050,14 +1021,13 @@ def _do_resumable_helper(self, use_size=False, num_retries=None): self.assertEqual(stream.tell(), total_bytes) # Check the mocks. - blob._make_transport.assert_called_once_with(client) call0 = self._do_resumable_upload_call0(blob, content_type, size=size) call1 = self._do_resumable_upload_call1( blob, content_type, data, resumable_url, size=size) call2 = self._do_resumable_upload_call2( blob, content_type, data, resumable_url, total_bytes) self.assertEqual( - fake_transport.request.mock_calls, [call0, call1, call2]) + transport.request.mock_calls, [call0, call1, call2]) def test__do_resumable_upload_no_size(self): self._do_resumable_helper() @@ -1272,16 +1242,15 @@ def _create_resumable_upload_session_helper(self, origin=None, # Create mocks to be checked for doing transport. resumable_url = 'http://test.invalid?upload_id=clean-up-everybody' response_headers = {'location': resumable_url} - fake_transport = self._mock_transport( + transport = self._mock_transport( http_client.OK, response_headers) - blob._make_transport = mock.Mock(return_value=fake_transport, spec=[]) if side_effect is not None: - fake_transport.request.side_effect = side_effect + transport.request.side_effect = side_effect # Create some mock arguments and call the method under test. content_type = u'text/plain' size = 10000 - client = mock.sentinel.client + client = mock.Mock(_http=transport, spec=[u'_http']) new_url = blob.create_resumable_upload_session( content_type=content_type, size=size, origin=origin, client=client) @@ -1291,7 +1260,6 @@ def _create_resumable_upload_session_helper(self, origin=None, self.assertEqual(blob.chunk_size, chunk_size) # Check the mocks. - blob._make_transport.assert_called_once_with(client) upload_url = ( 'https://www.googleapis.com/upload/storage/v1' + bucket.path + @@ -1304,7 +1272,7 @@ def _create_resumable_upload_session_helper(self, origin=None, } if origin is not None: expected_headers['Origin'] = origin - fake_transport.request.assert_called_once_with( + transport.request.assert_called_once_with( 'POST', upload_url, data=payload, headers=expected_headers) def test_create_resumable_upload_session(self):