Get latest master

sdk/core/azure-core/HISTORY.md

@@ -3,6 +3,13 @@
 
 -------------------
 
+## 2019-XX-XX Version 1.0.0
+
+### Bug fixes
+
+- Fix AsyncioRequestsTransport if input stream is an async generator  #7743
+- Fix form-data with aiohttp transport  #7749
+
 ## 2019-10-07 Version 1.0.0b4
 
 ### Features
@@ -32,7 +39,7 @@
 - Tracing: `link` renamed `link_from_headers`  and `link` takes now a string
 - Tracing: opencensus implementation has been moved to the package `azure-core-tracing-opencensus`
 - Some modules and classes that were importables from several differente places have been removed:
-   
+
    - `azure.core.HttpResponseError` is now only `azure.core.exceptions.HttpResponseError`
    - `azure.core.Configuration` is now only `azure.core.configuration.Configuration`
    - `azure.core.HttpRequest` is now only `azure.core.pipeline.transport.HttpRequest`

sdk/core/azure-core/azure/core/pipeline/policies/universal.py

@@ -131,10 +131,10 @@ def __init__(self, base_user_agent=None, **kwargs):  # pylint: disable=super-ini
         self.use_env = kwargs.pop('user_agent_use_env', True)
 
         if base_user_agent is None:
-            self._user_agent = "python/{} ({}) azure-core/{}".format(
+            self._user_agent = "azsdk-python-core/{} Python/{} {}".format(
+                azcore_version,
                 platform.python_version(),
-                platform.platform(),
-                azcore_version
+                platform.platform()
             )
         else:
             self._user_agent = base_user_agent

sdk/core/azure-core/azure/core/pipeline/transport/aiohttp.py

@@ -92,7 +92,6 @@ async def open(self):
             self.session = aiohttp.ClientSession(
                 loop=self._loop,
                 trust_env=self._use_env_settings,
-                skip_auto_headers=['Content-Type']
             )
         if self.session is not None:
             await self.session.__aenter__()
@@ -169,6 +168,11 @@ async def send(self, request: HttpRequest, **config: Any) -> Optional[AsyncHttpR
             cert=config.pop('connection_cert', self.connection_config.cert),
             verify=config.pop('connection_verify', self.connection_config.verify)
         )
+        # If we know for sure there is not body, disable "auto content type"
+        # Otherwise, aiohttp will send "application/octect-stream" even for empty POST request
+        # and that break services like storage signature
+        if not request.data and not request.files:
+            config['skip_auto_headers'] = ['Content-Type']
         try:
             stream_response = config.pop("stream", False)
             result = await self.session.request(
@@ -231,14 +235,14 @@ async def __anext__(self):
                     retry_active = False
                 else:
                     await asyncio.sleep(retry_interval)
-                    headers = {'range': 'bytes=' + self.downloaded + '-'}
+                    headers = {'range': 'bytes=' + str(self.downloaded) + '-'}
                     resp = self.pipeline.run(self.request, stream=True, headers=headers)
                     if resp.status_code == 416:
                         raise
                     chunk = await self.response.internal_response.content.read(self.block_size)
                     if not chunk:
                         raise StopIteration()
-                    self.downloaded += chunk
+                    self.downloaded += len(chunk)
                     return chunk
                 continue
             except StreamConsumedError:

sdk/core/azure-core/azure/core/pipeline/transport/requests_asyncio.py

@@ -102,6 +102,16 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse:
         loop = kwargs.get("loop", _get_running_loop())
         response = None
         error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]]
+        if hasattr(request.data, '__aiter__'):
+            # Need to consume that async generator, since requests can't do anything with it
+            # That's not ideal, but a list is our only choice. Memory not optimal here,
+            # but providing an async generator to a requests based transport is not optimal too
+            new_data = []
+            async for part in request.data:
+                new_data.append(part)
+            data_to_send = iter(new_data)
+        else:
+            data_to_send = request.data
         try:
             response = await loop.run_in_executor(
                 None,
@@ -110,7 +120,7 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AsyncHttpResponse:
                     request.method,
                     request.url,
                     headers=request.headers,
-                    data=request.data,
+                    data=data_to_send,
                     files=request.files,
                     verify=kwargs.pop('connection_verify', self.connection_config.verify),
                     timeout=kwargs.pop('connection_timeout', self.connection_config.timeout),
@@ -182,7 +192,7 @@ async def __anext__(self):
                     retry_active = False
                 else:
                     await asyncio.sleep(retry_interval)
-                    headers = {'range': 'bytes=' + self.downloaded + '-'}
+                    headers = {'range': 'bytes=' + str(self.downloaded) + '-'}
                     resp = self.pipeline.run(self.request, stream=True, headers=headers)
                     if resp.status_code == 416:
                         raise
@@ -193,7 +203,7 @@ async def __anext__(self):
                     )
                     if not chunk:
                         raise StopIteration()
-                    self.downloaded += chunk
+                    self.downloaded += len(chunk)
                     return chunk
                 continue
             except requests.exceptions.StreamConsumedError:

sdk/core/azure-core/azure/core/pipeline/transport/requests_basic.py

@@ -120,14 +120,14 @@ def __next__(self):
                     retry_active = False
                 else:
                     time.sleep(retry_interval)
-                    headers = {'range': 'bytes=' + self.downloaded + '-'}
+                    headers = {'range': 'bytes=' + str(self.downloaded) + '-'}
                     resp = self.pipeline.run(self.request, stream=True, headers=headers)
                     if resp.status_code == 416:
                         raise
                     chunk = next(self.iter_content_func)
                     if not chunk:
                         raise StopIteration()
-                    self.downloaded += chunk
+                    self.downloaded += len(chunk)
                     return chunk
                 continue
             except requests.exceptions.StreamConsumedError:

sdk/core/azure-core/azure/core/pipeline/transport/requests_trio.py

@@ -98,7 +98,7 @@ async def __anext__(self):
                     retry_active = False
                 else:
                     await trio.sleep(1000)
-                    headers = {'range': 'bytes=' + self.downloaded + '-'}
+                    headers = {'range': 'bytes=' + str(self.downloaded) + '-'}
                     resp = self.pipeline.run(self.request, stream=True, headers=headers)
                     if resp.status_code == 416:
                         raise
@@ -114,7 +114,7 @@ async def __anext__(self):
                         )
                     if not chunk:
                         raise StopIteration()
-                    self.downloaded += chunk
+                    self.downloaded += len(chunk)
                     return chunk
                 continue
             except requests.exceptions.StreamConsumedError:

sdk/core/azure-core/tests/azure_core_asynctests/test_request_asyncio.py

@@ -0,0 +1,40 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE.txt in the project root for
+# license information.
+# -------------------------------------------------------------------------
+import json
+
+from azure.core.pipeline.transport import AsyncioRequestsTransport, HttpRequest
+
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_async_gen_data():
+    transport = AsyncioRequestsTransport()
+
+    class AsyncGen:
+        def __init__(self):
+            self._range = iter([b"azerty"])
+
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self):
+            try:
+                return next(self._range)
+            except StopIteration:
+                raise StopAsyncIteration
+
+    req = HttpRequest('GET', 'http://httpbin.org/post', data=AsyncGen())
+
+    await transport.send(req)
+
+@pytest.mark.asyncio
+async def test_send_data():
+    transport = AsyncioRequestsTransport()
+    req = HttpRequest('PUT', 'http://httpbin.org/anything', data=b"azerty")
+    response = await transport.send(req)
+
+    assert json.loads(response.text())['data'] == "azerty"

sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/tests/test_storage_blob_partition_manager.py

@@ -28,7 +28,7 @@ def get_live_storage_blob_client():
     container_str = str(uuid.uuid4())
     blob_service_client = BlobServiceClient.from_connection_string(storage_connection_str)
     blob_service_client.create_container(container_str)
-    container_client = ContainerClient.from_connection_string(storage_connection_str, container=container_str)
+    container_client = ContainerClient.from_connection_string(storage_connection_str, container_str)
     return container_str, container_client
 
 

sdk/keyvault/azure-keyvault-certificates/HISTORY.md

@@ -1,5 +1,9 @@
 # Release History
 
+## 4.0.0b5
+### Breaking changes
+- Removed redundant method `get_pending_certificate_signing_request()`. A pending CSR can be retrieved via `get_certificate_operation()`.
+
 ## 4.0.0b4 (2019-10-08)
 ### Breaking changes
 - Enums `JsonWebKeyCurveName` and `JsonWebKeyType` have been renamed to `KeyCurveName` and `KeyType`, respectively.
@@ -14,6 +18,7 @@ for details.
 - The `vault_url` parameter of `CertificateClient` has been renamed to `vault_endpoint`
 - The property `vault_url` has been renamed to `vault_endpoint` in all models
 
+
 ## 4.0.0b3 (2019-09-11)
 Version 4.0.0b3 is the first preview of our efforts to create a user-friendly and Pythonic client library for Azure Key Vault's certificates.
 

sdk/keyvault/azure-keyvault-certificates/azure/keyvault/certificates/_version.py

@@ -3,4 +3,4 @@
 # Licensed under the MIT License.
 # ------------------------------------
 
-VERSION = "4.0.0b4"
+VERSION = "4.0.0b5"

sdk/keyvault/azure-keyvault-certificates/azure/keyvault/certificates/aio/client.py

@@ -4,7 +4,6 @@
 # ------------------------------------
 # pylint:disable=too-many-lines,too-many-public-methods
 import base64
-import uuid
 from typing import Any, AsyncIterable, Optional, Iterable, List, Dict, Coroutine
 from functools import partial
 
@@ -705,58 +704,6 @@ async def cancel_certificate_operation(self, name: str, **kwargs: "**Any") -> Ce
         )
         return CertificateOperation._from_certificate_operation_bundle(certificate_operation_bundle=bundle)
 
-    @distributed_trace_async
-    async def get_pending_certificate_signing_request(
-        self, name: str, custom_headers: Optional[Dict[str, str]] = None, **kwargs: "**Any"
-    ) -> str:
-        """Gets the Base64 pending certificate signing request (PKCS-10).
-
-        :param str name: The name of the certificate
-        :param custom_headers: headers that will be added to the request
-        :type custom_headers: dict[str, str]
-        :return: Base64 encoded pending certificate signing request (PKCS-10).
-        :rtype: str
-        :raises: :class:`~azure.core.exceptions.HttpResponseError`
-        """
-        vault_base_url = self.vault_endpoint
-        # Construct URL
-        url = "/certificates/{certificate-name}/pending"
-        path_format_arguments = {
-            "vaultBaseUrl": self._client._serialize.url("vault_base_url", vault_base_url, "str", skip_quote=True),
-            "certificate-name": self._client._serialize.url("certificate_name", name, "str"),
-        }
-        url = self._client._client.format_url(url, **path_format_arguments)
-
-        # Construct parameters
-        query_parameters = {}
-        query_parameters["api-version"] = self._client._serialize.query(
-            name="self.api_version", data=self._client.api_version, data_type="str"
-        )
-
-        # Construct headers
-        header_parameters = {}
-        header_parameters["Accept"] = "application/pkcs10"
-        if self._client._config.generate_client_request_id:
-            header_parameters["x-ms-client-request-id"] = str(uuid.uuid1())
-        if custom_headers:
-            header_parameters.update(custom_headers)
-
-        # Construct and send request
-        request = self._client._client.get(url=url, params=query_parameters, headers=header_parameters)
-        pipeline_response = await self._client._client._pipeline.run(request, stream=False, **kwargs)
-        response = pipeline_response.http_response
-
-        if response.status_code not in [200]:
-            self._client.map_error(status_code=response.status_code, response=response, error_map=_error_map)
-            raise self._client.models.KeyVaultErrorException(response, self._client._deserialize)
-
-        deserialized = None
-
-        if response.status_code == 200:
-            deserialized = response.body() if hasattr(response, "body") else response.content
-
-        return deserialized
-
     @distributed_trace_async
     async def merge_certificate(self, name: str, x509_certificates: List[bytearray], **kwargs: "**Any") -> Certificate:
         """Merges a certificate or a certificate chain with a key pair existing on the server.

sdk/keyvault/azure-keyvault-certificates/azure/keyvault/certificates/client.py

@@ -4,7 +4,6 @@
 # ------------------------------------
 # pylint:disable=too-many-lines,too-many-public-methods
 import base64
-import uuid
 from functools import partial
 
 from azure.core.polling import LROPoller
@@ -765,59 +764,6 @@ def merge_certificate(
         )
         return Certificate._from_certificate_bundle(certificate_bundle=bundle)
 
-    @distributed_trace
-    def get_pending_certificate_signing_request(
-        self,
-        name,  # type: str
-        **kwargs  # type: **Any
-    ):
-        # type: (...) -> str
-        """Gets the Base64 pending certificate signing request (PKCS-10).
-
-        :param str name: The name of the certificate
-        :param custom_headers: headers that will be added to the request
-        :type custom_headers: dict[str, str]
-        :return: Base64 encoded pending certificate signing request (PKCS-10).
-        :rtype: str
-        :raises: :class:`~azure.core.exceptions.HttpResponseError`
-        """
-        vault_base_url = self.vault_endpoint
-        # Construct URL
-        url = "/certificates/{certificate-name}/pending"
-        path_format_arguments = {
-            "vaultBaseUrl": self._client._serialize.url("vault_base_url", vault_base_url, "str", skip_quote=True),
-            "certificate-name": self._client._serialize.url("certificate_name", name, "str"),
-        }
-        url = self._client._client.format_url(url, **path_format_arguments)
-
-        # Construct parameters
-        query_parameters = {}
-        query_parameters["api-version"] = self._client._serialize.query(
-            name="self.api_version", data=self._client.api_version, data_type="str"
-        )
-
-        # Construct headers
-        header_parameters = {}
-        header_parameters["Accept"] = "application/pkcs10"
-        if self._client._config.generate_client_request_id:
-            header_parameters["x-ms-client-request-id"] = str(uuid.uuid1())
-
-        # Construct and send request
-        request = self._client._client.get(url=url, params=query_parameters, headers=header_parameters)
-        pipeline_response = self._client._client._pipeline.run(request, stream=False, **kwargs)
-        response = pipeline_response.http_response
-
-        if response.status_code not in [200]:
-            self._client.map_error(status_code=response.status_code, response=response, error_map=_error_map)
-            raise self._client.models.KeyVaultErrorException(response, self._client._deserialize)
-
-        deserialized = None
-
-        if response.status_code == 200:
-            deserialized = response.body() if hasattr(response, "body") else response.content
-
-        return deserialized
-
     @distributed_trace
     def get_issuer(self, name, **kwargs):
         # type: (str, **Any) -> Issuer