From 53591aecd23e3c20e4b5b89f3d921e494a2a6bd6 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 9 Oct 2019 13:06:21 -0700 Subject: [PATCH 1/4] Fix requests asyncio if input data is async gen --- .../azure/core/pipeline/transport/requests_asyncio.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py index bc3d50d4f04b..583ce0351e6a 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py @@ -102,6 +102,14 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse: loop = kwargs.get("loop", _get_running_loop()) response = None error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] + if hasattr(request.data, '__aiter__'): + # Need to consume that async generator, since requests can't do anything with it + # That's not ideal, but a list is our only choice. Memory not optimal here, + # but providing an async generator to a requests based transport is not optimal too + new_data = [] + async for part in request.data: + new_data.append(part) + request.data = iter(new_data) try: response = await loop.run_in_executor( None, From aad66ffbbc0dcef850b1f9f342301a82ac85c00c Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 9 Oct 2019 15:09:32 -0700 Subject: [PATCH 2/4] ChangeLog --- sdk/core/azure-core/HISTORY.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/HISTORY.md b/sdk/core/azure-core/HISTORY.md index 9ab8fb5ed6ee..6c8dd378f79d 100644 --- a/sdk/core/azure-core/HISTORY.md +++ b/sdk/core/azure-core/HISTORY.md @@ -3,6 +3,12 @@ ------------------- +## 2019-XX-XX Versuib 1.0.0 + +### Bug fixes + +- Fix AsyncioRequestsTransport is input stream is an async generator #7743 + ## 2019-10-07 Version 1.0.0b4 ### Features @@ -32,7 +38,7 @@ - Tracing: `link` renamed `link_from_headers` and `link` takes now a string - Tracing: opencensus implementation has been moved to the package `azure-core-tracing-opencensus` - Some modules and classes that were importables from several differente places have been removed: - + - `azure.core.HttpResponseError` is now only `azure.core.exceptions.HttpResponseError` - `azure.core.Configuration` is now only `azure.core.configuration.Configuration` - `azure.core.HttpRequest` is now only `azure.core.pipeline.transport.HttpRequest` From 0f349e9543b4a82ac32e4accb5ca16ace9d5f226 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 9 Oct 2019 15:26:02 -0700 Subject: [PATCH 3/4] Feedback --- .../pipeline/transport/requests_asyncio.py | 6 ++- .../test_request_asyncio.py | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 sdk/core/azure-core/tests/azure_core_asynctests/test_request_asyncio.py diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py index 583ce0351e6a..f63bfed3ead6 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py @@ -109,7 +109,9 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse: new_data = [] async for part in request.data: new_data.append(part) - request.data = iter(new_data) + data_to_send = iter(new_data) + else: + data_to_send = request.data try: response = await loop.run_in_executor( None, @@ -118,7 +120,7 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse: request.method, request.url, headers=request.headers, - data=request.data, + data=data_to_send, files=request.files, verify=kwargs.pop('connection_verify', self.connection_config.verify), timeout=kwargs.pop('connection_timeout', self.connection_config.timeout), diff --git a/sdk/core/azure-core/tests/azure_core_asynctests/test_request_asyncio.py b/sdk/core/azure-core/tests/azure_core_asynctests/test_request_asyncio.py new file mode 100644 index 000000000000..666bdd39276e --- /dev/null +++ b/sdk/core/azure-core/tests/azure_core_asynctests/test_request_asyncio.py @@ -0,0 +1,40 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import json + +from azure.core.pipeline.transport import AsyncioRequestsTransport, HttpRequest + +import pytest + + +@pytest.mark.asyncio +async def test_async_gen_data(): + transport = AsyncioRequestsTransport() + + class AsyncGen: + def __init__(self): + self._range = iter([b"azerty"]) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._range) + except StopIteration: + raise StopAsyncIteration + + req = HttpRequest('GET', 'http://httpbin.org/post', data=AsyncGen()) + + await transport.send(req) + +@pytest.mark.asyncio +async def test_send_data(): + transport = AsyncioRequestsTransport() + req = HttpRequest('PUT', 'http://httpbin.org/anything', data=b"azerty") + response = await transport.send(req) + + assert json.loads(response.text())['data'] == "azerty" \ No newline at end of file From 573b2cf2a3021c5c7bf942298ee3c75da2e6e986 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 9 Oct 2019 15:26:53 -0700 Subject: [PATCH 4/4] Update HISTORY.md --- sdk/core/azure-core/HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/HISTORY.md b/sdk/core/azure-core/HISTORY.md index 6c8dd378f79d..c5346004e1e2 100644 --- a/sdk/core/azure-core/HISTORY.md +++ b/sdk/core/azure-core/HISTORY.md @@ -3,7 +3,7 @@ ------------------- -## 2019-XX-XX Versuib 1.0.0 +## 2019-XX-XX Version 1.0.0 ### Bug fixes