diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index 9766b6816ab8..4de864f577ce 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -3,31 +3,33 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "1.3.1" +__version__ = "2.0.0-preview.1" from azure.eventhub.common import EventData, EventPosition -from azure.eventhub.error import EventHubError, EventDataError, ConnectError, AuthenticationError +from azure.eventhub.error import EventHubError, EventDataError, ConnectError, \ + AuthenticationError, EventDataSendError, ConnectionLostError from azure.eventhub.client import EventHubClient -from azure.eventhub.sender import Sender -from azure.eventhub.receiver import Receiver +from azure.eventhub.sender import EventSender +from azure.eventhub.receiver import EventReceiver from .constants import MessageSendResult from .constants import TransportType -from .common import FIRST_AVAILABLE, NEW_EVENTS_ONLY, SharedKeyCredentials, SASTokenCredentials +from .common import EventHubSharedKeyCredential, EventHubSASTokenCredential __all__ = [ "__version__", "EventData", "EventHubError", "ConnectError", + "ConnectionLostError", "EventDataError", + "EventDataSendError", "AuthenticationError", "EventPosition", "EventHubClient", - "Sender", - "Receiver", + "EventSender", + "EventReceiver", "MessageSendResult", "TransportType", - "FIRST_AVAILABLE", "NEW_EVENTS_ONLY", - "SharedKeyCredentials", - "SASTokenCredentials", + "EventHubSharedKeyCredential", + "EventHubSASTokenCredential", ] diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py index 020392000d1f..88fd0673f4df 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/__init__.py @@ -1,9 +1,9 @@ from .event_hubs_client_async import EventHubClient -from .receiver_async import Receiver -from .sender_async import Sender +from .receiver_async import EventReceiver +from .sender_async import EventSender __all__ = [ "EventHubClient", - "Receiver", - "Sender" + "EventReceiver", + "EventSender" ] diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index d88461c98d0c..2a006373fccd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -15,13 +15,13 @@ AMQPClientAsync, ) -from azure.eventhub.common import parse_sas_token, SharedKeyCredentials, SASTokenCredentials +from azure.eventhub.common import parse_sas_token, EventPosition, EventHubSharedKeyCredential, EventHubSASTokenCredential from azure.eventhub import ( EventHubError) from ..client_abstract import EventHubClientAbstract -from .sender_async import Sender -from .receiver_async import Receiver +from .sender_async import EventSender +from .receiver_async import EventReceiver log = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def _create_auth(self, username=None, password=None): transport_type = self.config.transport_type auth_timeout = self.config.auth_timeout - if isinstance(self.credential, SharedKeyCredentials): + if isinstance(self.credential, EventHubSharedKeyCredential): username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: @@ -66,7 +66,7 @@ def _create_auth(self, username=None, password=None): self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, SASTokenCredentials): + elif isinstance(self.credential, EventHubSASTokenCredential): token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) @@ -85,10 +85,14 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - async def get_properties(self): """ - Get details on the specified EventHub async. + Get properties of the specified EventHub async. + Keys in the details dictionary include: + + -'path' + -'created_at' + -'partition_ids' :rtype: dict """ @@ -117,21 +121,25 @@ async def get_properties(self): await mgmt_client.close_async() async def get_partition_ids(self): + """ + Get partition ids of the specified EventHub async. + + :rtype: list[str] + """ return (await self.get_properties())['partition_ids'] async def get_partition_properties(self, partition): """ - Get information on the specified partition async. + Get properties of the specified partition async. Keys in the details dictionary include: - -'name' - -'type' - -'partition' - -'begin_sequence_number' + -'event_hub_path' + -'id' + -'beginning_sequence_number' -'last_enqueued_sequence_number' -'last_enqueued_offset' -'last_enqueued_time_utc' - -'is_partition_empty' + -'is_empty' :param partition: The target partition id. :type partition: str @@ -168,23 +176,27 @@ async def get_partition_properties(self, partition): await mgmt_client.close_async() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, - prefetch=None, loop=None): + self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), exclusive_receiver_priority=None, + operation=None, prefetch=None, loop=None): """ - Add an async receiver to the client for a particular consumer group and partition. + Create an async receiver to the client for a particular consumer group and partition. - :param consumer_group: The name of the consumer group. + :param partition_id: The ID of the partition. + :type partition_id: str + :param consumer_group: The name of the consumer group. Default value is `$Default`. :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str :param event_position: The position from which to start receiving. :type event_position: ~azure.eventhub.common.EventPosition - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :operation: An optional operation to be appended to the hostname in the source URL. + :param exclusive_receiver_priority: The priority of the exclusive receiver. The client will create an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int + :param operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str - :rtype: ~azure.eventhub.aio.receiver_async.ReceiverAsync + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param loop: An event loop. If not specified the default event loop will be used. + :rtype: ~azure.eventhub.aio.receiver_async.EventReceiver Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py @@ -200,35 +212,30 @@ def create_receiver( path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) - handler = Receiver( - self, source_url, offset=event_position, exclusive_receiver_priority=exclusive_receiver_priority, + handler = EventReceiver( + self, source_url, event_position=event_position, exclusive_receiver_priority=exclusive_receiver_priority, prefetch=prefetch, loop=loop) return handler def create_sender( self, partition_id=None, operation=None, send_timeout=None, loop=None): """ - Add an async sender to the client to send ~azure.eventhub.common.EventData object + Create an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. - :param partition: Optionally specify a particular partition to send to. + :param partition_id: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. - :type partition: str - :operation: An optional operation to be appended to the hostname in the target URL. + :type partition_id: str + :param operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int - :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during - periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not - be pinged. - :type keep_alive: int - :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. - Default value is `True`. - :type auto_reconnect: bool - :rtype: ~azure.eventhub.aio.sender_async.SenderAsync + :type send_timeout: float + :param loop: An event loop. If not specified the default event loop will be used. + :rtype ~azure.eventhub.aio.sender_async.EventSender + Example: .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py @@ -245,6 +252,6 @@ def create_sender( target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - handler = Sender( + handler = EventSender( self, target, partition=partition_id, send_timeout=send_timeout, loop=loop) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index 6614001dc93d..6ee82725d2e2 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -7,33 +7,25 @@ import uuid import logging -from uamqp import errors, types +from uamqp import errors, types, compat from uamqp import ReceiveClientAsync, Source from azure.eventhub import EventHubError, EventData -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) -class Receiver(object): +class EventReceiver(object): """ - Implements the async API of a Receiver. - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START create_eventhub_client_async_receiver_instance] - :end-before: [END create_eventhub_client_async_receiver_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Async Receiver. + Implements the async API of a EventReceiver. """ timeout = 0 _epoch = b'com.microsoft:epoch' def __init__( # pylint: disable=super-init-not-called - self, client, source, offset=None, prefetch=300, exclusive_receiver_priority=None, + self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an async receiver. @@ -42,18 +34,21 @@ def __init__( # pylint: disable=super-init-not-called :type client: ~azure.eventhub.aio.EventHubClientAsync :param source: The source EventHub from which to receive events. :type source: ~uamqp.address.Source + :param event_position: The position from which to start receiving. + :type event_position: ~azure.eventhub.common.EventPosition :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int + :param exclusive_receiver_priority: The priority of the exclusive receiver. It will an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int :param loop: An event loop. """ self.loop = loop or asyncio.get_event_loop() self.running = False self.client = client self.source = source - self.offset = offset + self.offset = event_position self.messages_iter = None self.prefetch = prefetch self.exclusive_receiver_priority = exclusive_receiver_priority @@ -68,7 +63,7 @@ def __init__( # pylint: disable=super-init-not-called self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access if exclusive_receiver_priority: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClientAsync( @@ -81,7 +76,7 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) async def __aenter__(self): @@ -95,7 +90,10 @@ def __aiter__(self): async def __anext__(self): await self._open() + max_retries = self.client.config.max_retries + connecting_count = 0 while True: + connecting_count += 1 try: if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter_async() @@ -103,62 +101,76 @@ async def __anext__(self): event_data = EventData(message=message) self.offset = event_data.offset return event_data - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - await self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() + log.info("EventReceiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() + if connecting_count < max_retries: + log.info("EventReceiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver timed out. Shutting down.") + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except StopAsyncIteration: raise - except asyncio.CancelledError: - # TODO: stop self.message_iter - raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close(exception=error) raise error + def _check_closed(self): + if self.error: + raise EventHubError("This receiver has been closed. Please create a new receiver to receive event data.", + self.error) async def _open(self): """ - Open the Receiver using the supplied conneciton. + Open the EventReceiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START eventhub_client_async_receiver_open] - :end-before: [END eventhub_client_async_receiver_open] - :language: python - :dedent: 4 - :caption: Open the Receiver using the supplied conneciton. - """ # pylint: disable=protected-access + self._check_closed() if self.redirected: self.source = self.redirected.address source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -172,107 +184,93 @@ async def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) - if not self.running: - try: - await self._handler.open_async() - self.running = True - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - except errors.AuthenticationException: - log.info("Receiver failed authentication. Retrying...") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Receiver detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate (%r).", shutdown) - error = AuthenticationError(str(shutdown)) - raise error - else: - log.info("Receiver connection error (%r).", shutdown) - error = ConnectError(str(shutdown)) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Receiver connect failed: {}".format(e)) - raise error + await self._connect() + self.running = True - async def _reconnect(self): # pylint: disable=too-many-statements + async def _connect(self): + connected = await self._build_connection() + if not connected: + await asyncio.sleep(self.reconnect_backoff) + while not await self._build_connection(is_reconnect=True): + await asyncio.sleep(self.reconnect_backoff) + + async def _build_connection(self, is_reconnect=False): # pylint: disable=too-many-statements # pylint: disable=protected-access - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password":self.client._auth_config.get("iot_password")} - await self._handler.close_async() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset.selector()) - self._handler = ReceiveClientAsync( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), - loop=self.loop) - self.messages_iter = None + if is_reconnect: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password":self.client._auth_config.get("iot_password")} + await self._handler.close_async() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) # pylint: disable=protected-access + self._handler = ReceiveClientAsync( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access + loop=self.loop) + self.messages_iter = None try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) return True except errors.AuthenticationException as shutdown: - log.info("AsyncReceiver disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + if is_reconnect: + log.info("EventReceiver couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("EventReceiver detached. Attempting reconnect.") return False - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("EventReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") + if is_reconnect: + log.info("EventReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventReceiver detached. Attempting reconnect.") return False - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("AsyncReceiver couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("EventReceiver connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False - log.info("AsyncReceiver connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown)) - await self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receiver reconnect failed: {}".format(e)) + error = EventHubError("EventReceiver reconnect failed: {}".format(e)) await self.close(exception=error) raise error - async def reconnect(self): - """If the Receiver was disconnected from the service with + async def _reconnect(self): + """If the EventReceiver was disconnected from the service with a retryable error - attempt to reconnect.""" - while not await self._reconnect(): - await asyncio.sleep(self.reconnect_backoff) + return await self._build_connection(is_reconnect=True) async def close(self, exception=None): """ @@ -330,6 +328,11 @@ async def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int + :param timeout: The timeout time in seconds to receive a batch of events + from an Event Hub. Results will be returned after timeout. If combined + with max_batch_size, it will return after either the count of received events + reaches the max_batch_size or the operation has timed out. + :type timeout: int :rtype: list[~azure.eventhub.common.EventData] Example: @@ -338,16 +341,19 @@ async def receive(self, max_batch_size=None, timeout=None): :end-before: [END eventhub_client_async_receive] :language: python :dedent: 4 - :caption: Sends an event data and asynchronously waits - until acknowledgement is received or operation times out. + :caption: Receives events asynchronously """ - if self.error: - raise self.error await self._open() + max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size + timeout = self.client.config.receive_timeout if timeout is None else timeout + data_batch = [] + max_retries = self.client.config.max_retries + connecting_count = 0 while True: + connecting_count += 1 try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = await self._handler.receive_message_batch_async( @@ -358,29 +364,52 @@ async def receive(self, max_batch_size=None, timeout=None): self.offset = event_data.offset data_batch.append(event_data) return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("AsyncReceiver disconnected due to token error. Attempting reconnect.") - await self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() + log.info("EventReceiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() + if connecting_count < max_retries: + log.info("EventReceiver detached. Attempting reconnect.") + await self._reconnect() else: - log.info("AsyncReceiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventReceiver timed out. Shutting down.") + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close(exception=error) - raise error + raise error \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index e263131ff859..f2d09a2df457 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -7,29 +7,21 @@ import asyncio import logging -from uamqp import constants, errors +from uamqp import constants, errors, compat from uamqp import SendClientAsync from azure.eventhub import MessageSendResult from azure.eventhub import EventHubError from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, _error_handler + AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) -class Sender(object): +class EventSender(object): """ - Implements the async API of a Sender. - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START create_eventhub_client_async_sender_instance] - :end-before: [END create_eventhub_client_async_sender_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Async Sender. + Implements the async API of a EventSender. """ @@ -48,10 +40,10 @@ def __init__( # pylint: disable=super-init-not-called :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is `None`, i.e. no keep alive pings. - :type keep_alive: int + :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool @@ -68,6 +60,7 @@ def __init__( # pylint: disable=super-init-not-called self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHSender-{}".format(uuid.uuid4()) + self.unsent_events = None self.redirected = None self.error = None if partition: @@ -81,7 +74,7 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) self._outcome = None self._condition = None @@ -94,21 +87,10 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def _open(self): """ - Open the Sender using the supplied conneciton. + Open the EventSender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.async_ops.connection_async.ConnectionAsync - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START eventhub_client_async_sender_open] - :end-before: [END eventhub_client_async_sender_open] - :language: python - :dedent: 4 - :caption: Open the Sender using the supplied conneciton. - """ if self.redirected: self.target = self.redirected.address @@ -120,99 +102,87 @@ async def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), + properties=self.client._create_properties(self.client.config.user_agent), # pylint: disable=protected-access loop=self.loop) if not self.running: - try: - await self._handler.open_async() - self.running = True - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) - except errors.AuthenticationException: - log.info("Sender failed authentication. Retrying...") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Sender couldn't authenticate.", shutdown) - error = AuthenticationError(str(shutdown)) - raise error - else: - log.info("Sender connection error (%r).", shutdown) - error = ConnectError(str(shutdown)) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Sender connect failed: {}".format(e)) - raise error + await self._connect() + self.running = True - async def _reconnect(self): - await self._handler.close_async() - unsent_events = self._handler.pending_messages - self._handler = SendClientAsync( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent), - loop=self.loop) + async def _connect(self): + connected = await self._build_connection() + if not connected: + await asyncio.sleep(self.reconnect_backoff) + while not await self._build_connection(is_reconnect=True): + await asyncio.sleep(self.reconnect_backoff) + + async def _build_connection(self, is_reconnect=False): + """ + + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ + # pylint: disable=protected-access + if is_reconnect: + await self._handler.close_async() + self._handler = SendClientAsync( + self.target, + auth=self.client.get_auth(), + debug=self.client.config.network_tracing, + msg_timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent)) try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) - self._handler.queue_message(*unsent_events) - await self._handler.wait_async() return True except errors.AuthenticationException as shutdown: - log.info("AsyncSender disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + if is_reconnect: + log.info("EventSender couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventSender couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("EventSender detached. Attempting reconnect.") return False - log.info("AsyncSender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("EventSender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") + if is_reconnect: + log.info("EventSender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventSender detached. Attempting reconnect.") return False - log.info("AsyncSender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - await self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("AsyncSender couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("EventSender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + else: + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False - log.info("AsyncSender connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown)) - await self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Sender reconnect failed: {}".format(e)) + error = EventHubError("EventSender Reconnect failed: {}".format(e)) await self.close(exception=error) raise error - async def reconnect(self): - """If the Receiver was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not await self._reconnect(): - await asyncio.sleep(self.reconnect_backoff) + async def _reconnect(self): + return await self._build_connection(is_reconnect=True) async def close(self, exception=None): """ @@ -248,85 +218,130 @@ async def close(self, exception=None): self.error = EventHubError("This send handler is now closed.") await self._handler.close_async() - async def _send_event_data(self, event_data): + async def _send_event_data(self): await self._open() - try: - self._handler.send_message(event_data.message) - if self._outcome != MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except errors.MessageException as failed: - error = EventDataError(str(failed), failed) - await self.close(exception=error) - raise error - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Sender disconnected due to token error. Attempting reconnect.") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + max_retries = self.client.config.max_retries + connecting_count = 0 + while True: + connecting_count += 1 + try: + if self.unsent_events: + self._handler.queue_message(*self.unsent_events) + await self._handler.wait_async() + self.unsent_events = self._handler.pending_messages + if self._outcome != constants.MessageSendResult.Ok: + EventSender._error(self._outcome, self._condition) + return + except (errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) as msg_error: + raise EventDataError(str(msg_error), msg_error) + except errors.MessageException as failed: + log.info("Send event data error (%r)", failed) + error = EventDataSendError(str(failed), failed) await self.close(exception=error) raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventSender disconnected due to token error. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventSender authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + await self.close(auth_error) + raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + log.info("EventSender detached. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventSender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if connecting_count < max_retries: + log.info("EventSender detached. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventSender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventSender connection lost. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventSender connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + await self.close(error) + raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventSender timed out sending event data. Attempting reconnect.") + await self._reconnect() + else: + log.info("EventSender timed out. Shutting down.") + await self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Send failed: {}".format(e)) await self.close(exception=error) raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e)) - await self.close(exception=error) - raise error - else: - return self._outcome + + def _check_closed(self): + if self.error: + raise EventHubError("This sender has been closed. Please create a new sender to send event data.", + self.error) @staticmethod - def _set_batching_label(event_datas, batching_label): + def _set_partition_key(event_datas, partition_key): ed_iter = iter(event_datas) for ed in ed_iter: - ed._batching_label = batching_label + ed._set_partition_key(partition_key) yield ed - async def send(self, event_data, batching_label=None): + async def send(self, event_data, partition_key=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData + :param partition_key: With the given partition_key, event data will land to + a particular partition of the Event Hub decided by the service. + :type partition_key: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult + :return: None + :rtype: None Example: .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_sync_send] - :end-before: [END eventhub_client_sync_send] + :start-after: [START eventhub_client_async_send] + :end-before: [END eventhub_client_async_send] :language: python :dedent: 4 :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ - if self.error: - raise self.error + self._check_closed() if isinstance(event_data, EventData): - if batching_label: - event_data._batching_label = batching_label + if partition_key: + event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: wrapper_event_data = _BatchSendEventData( - self._set_batching_label(event_data, batching_label), - batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) + self._set_partition_key(event_data, partition_key), + partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - await self._send_event_data(wrapper_event_data) + self.unsent_events = [wrapper_event_data.message] + await self._send_event_data() def _on_outcome(self, outcome, condition): """ @@ -334,10 +349,13 @@ def _on_outcome(self, outcome, condition): :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult + :param condition: Detail information of the outcome. + """ self._outcome = outcome self._condition = condition @staticmethod def _error(outcome, condition): - return None if outcome == MessageSendResult.Ok else EventHubError(outcome, condition) + if outcome != MessageSendResult.Ok: + raise condition diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 8fb7940850e9..f77dccf19203 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -15,6 +15,7 @@ from urllib import unquote_plus, urlencode, quote_plus except ImportError: from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus +from typing import Any, List, Dict import uamqp from uamqp import Message, AMQPClient @@ -22,12 +23,12 @@ from uamqp import constants from azure.eventhub import __version__ -from azure.eventhub.sender import Sender -from azure.eventhub.receiver import Receiver -from azure.eventhub.common import parse_sas_token +from azure.eventhub.sender import EventSender +from azure.eventhub.receiver import EventReceiver +from azure.eventhub.common import parse_sas_token, EventPosition from azure.eventhub.error import EventHubError from .client_abstract import EventHubClientAbstract -from .common import SASTokenCredentials, SharedKeyCredentials +from .common import EventHubSASTokenCredential, EventHubSharedKeyCredential log = logging.getLogger(__name__) @@ -63,7 +64,7 @@ def _create_auth(self, username=None, password=None): auth_timeout = self.config.auth_timeout # TODO: the following code can be refactored to create auth from classes directly instead of using if-else - if isinstance(self.credential, SharedKeyCredentials): + if isinstance(self.credential, EventHubSharedKeyCredential): username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: @@ -73,7 +74,7 @@ def _create_auth(self, username=None, password=None): self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, SASTokenCredentials): + elif isinstance(self.credential, EventHubSASTokenCredential): token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) @@ -93,16 +94,14 @@ def _create_auth(self, username=None, password=None): get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - def get_properties(self): + # type:() -> Dict[str, Any] """ - Get details on the specified EventHub. + Get properties of the specified EventHub. Keys in the details dictionary include: - -'name' - -'type' + -'path' -'created_at' - -'partition_count' -'partition_ids' :rtype: dict @@ -132,21 +131,27 @@ def get_properties(self): mgmt_client.close() def get_partition_ids(self): + # type:() -> List[str] + """ + Get partition ids of the specified EventHub. + + :rtype: list[str] + """ return self.get_properties()['partition_ids'] def get_partition_properties(self, partition): + # type:(str) -> Dict[str, str] """ - Get information on the specified partition async. + Get properties of the specified partition. Keys in the details dictionary include: - -'name' - -'type' - -'partition' - -'begin_sequence_number' + -'event_hub_path' + -'id' + -'beginning_sequence_number' -'last_enqueued_sequence_number' -'last_enqueued_offset' -'last_enqueued_time_utc' - -'is_partition_empty' + -'is_empty' :param partition: The target partition id. :type partition: str @@ -171,7 +176,6 @@ def get_partition_properties(self, partition): output = {} if partition_info: output['event_hub_path'] = partition_info[b'name'].decode('utf-8') - # output['type'] = partition_info[b'type'].decode('utf-8') output['id'] = partition_info[b'partition'].decode('utf-8') output['beginning_sequence_number'] = partition_info[b'begin_sequence_number'] output['last_enqueued_sequence_number'] = partition_info[b'last_enqueued_sequence_number'] @@ -184,24 +188,28 @@ def get_partition_properties(self, partition): mgmt_client.close() def create_receiver( - self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, - prefetch=None, + self, partition_id, consumer_group="$Default", event_position=EventPosition.first_available_event(), + exclusive_receiver_priority=None, operation=None, prefetch=None, ): + # type: (str, str, EventPosition, int, str, int) -> EventReceiver """ - Add a receiver to the client for a particular consumer group and partition. + Create a receiver to the client for a particular consumer group and partition. - :param consumer_group: The name of the consumer group. + :param partition_id: The ID of the partition. + :type partition_id: str + :param consumer_group: The name of the consumer group. Default value is `$Default`. :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str :param event_position: The position from which to start receiving. :type event_position: ~azure.eventhub.common.EventPosition - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :operation: An optional operation to be appended to the hostname in the source URL. + :param exclusive_receiver_priority: The priority of the exclusive receiver. The client will create an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int + :param operation: An optional operation to be appended to the hostname in the source URL. The value must start with `/` character. :type operation: str - :rtype: ~azure.eventhub.receiver.Receiver + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :rtype: ~azure.eventhub.receiver.EventReceiver Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -217,32 +225,27 @@ def create_receiver( path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( self.address.hostname, path, consumer_group, partition_id) - handler = Receiver( + handler = EventReceiver( self, source_url, event_position=event_position, exclusive_receiver_priority=exclusive_receiver_priority, prefetch=prefetch) return handler def create_sender(self, partition_id=None, operation=None, send_timeout=None): + # type: (str, str, float) -> EventSender """ - Add a sender to the client to send EventData object to an EventHub. + Create a sender to the client to send EventData object to an EventHub. - :param partition: Optionally specify a particular partition to send to. + :param partition_id: Optionally specify a particular partition to send to. If omitted, the events will be distributed to available partitions via round-robin. - :type parition: str - :operation: An optional operation to be appended to the hostname in the target URL. + :type partition_id: str + :param operation: An optional operation to be appended to the hostname in the target URL. The value must start with `/` character. :type operation: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: int - :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during - periods of inactivity. The default value is 30 seconds. If set to `None`, the connection will not - be pinged. - :type keep_alive: int - :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. - Default value is `True`. - :rtype: ~azure.eventhub.sender.Sender + :rtype: ~azure.eventhub.sender.EventSender Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -258,6 +261,6 @@ def create_sender(self, partition_id=None, operation=None, send_timeout=None): target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - handler = Sender( + handler = EventSender( self, target, partition=partition_id, send_timeout=send_timeout) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 26435fd93635..5534a848c640 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -20,7 +20,7 @@ from azure.eventhub import __version__ from azure.eventhub.configuration import Configuration from azure.eventhub import constants -from .common import SASTokenCredentials, SharedKeyCredentials, Address +from .common import EventHubSharedKeyCredential, _Address log = logging.getLogger(__name__) @@ -82,35 +82,25 @@ def _build_uri(address, entity): class EventHubClientAbstract(object): """ - The EventHubClient class defines a high level interface for sending + The EventHubClientAbstract class defines a high level interface for sending events to and receiving events from the Azure Event Hubs service. - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client] - :end-before: [END create_eventhub_client] - :language: python - :dedent: 4 - :caption: Create a new instance of the Event Hub client - """ def __init__(self, host, event_hub_path, credential, **kwargs): """ - Constructs a new EventHubClient with the given address URL. - - :param address: The full URI string of the Event Hub. This can optionally - include URL-encoded access name and key. - :type address: str - :param username: The name of the shared access policy. This must be supplied - if not encoded into the address. - :type username: str - :param password: The shared access key. This must be supplied if not encoded - into the address. - :type password: str - :param debug: Whether to output network trace logs to the logger. Default + Constructs a new EventHubClient. + + :param host: The hostname URI string of the the Event Hub. + :type host: str + :param event_hub_path: The path/name of the Event Hub + :type event_hub_path: str + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool + :param credential: The credential object used for authentication which implements particular interface + of getting tokens. It accepts ~azure.eventhub.EventHubSharedKeyCredential, + ~azure.eventhub.EventHubSASTokenCredential, credential objects generated by the azure-identity library and + objects that implement get token interface. :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. @@ -118,18 +108,34 @@ def __init__(self, host, event_hub_path, credential, **kwargs): :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. :type auth_timeout: int - :param sas_token: A SAS token or function that returns a SAS token. If a function is supplied, - it will be used to retrieve subsequent tokens in the case of token expiry. The function should - take no arguments. - :type sas_token: str or callable + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: int + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: int """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] - self.address = Address() + self.address = _Address() self.address.hostname = host self.address.path = "/" + event_hub_path if event_hub_path else "" self._auth_config = {} self.credential = credential - if isinstance(credential, SharedKeyCredentials): + if isinstance(credential, EventHubSharedKeyCredential): self.username = credential.policy self.password = credential.key self._auth_config['username'] = self.username @@ -148,24 +154,44 @@ def __init__(self, host, event_hub_path, credential, **kwargs): log.info("%r: Created the Event Hub client", self.container_id) @classmethod - def from_connection_string(cls, conn_str, eventhub=None, **kwargs): + def from_connection_string(cls, conn_str, event_hub_path=None, **kwargs): """Create an EventHubClient from a connection string. :param conn_str: The connection string. :type conn_str: str - :param eventhub: The name of the EventHub, if the EntityName is + :param event_hub_path: The path/name of the Event Hub, if the EntityName is not included in the connection string. - :type eventhub: str - :param debug: Whether to output network trace logs to the logger. Default + :type event_hub_path: str + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. - :type auth_timeout: int + :type auth_timeout: float + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: float + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: float Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -177,13 +203,13 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): """ address, policy, key, entity = _parse_conn_str(conn_str) - entity = eventhub or entity + entity = event_hub_path or entity left_slash_pos = address.find("//") if left_slash_pos != -1: host = address[left_slash_pos + 2:] else: host = address - return cls(host, entity, SharedKeyCredentials(policy, key), **kwargs) + return cls(host, entity, EventHubSharedKeyCredential(policy, key), **kwargs) @classmethod def from_iothub_connection_string(cls, conn_str, **kwargs): @@ -192,16 +218,36 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): :param conn_str: The connection string. :type conn_str: str - :param debug: Whether to output network trace logs to the logger. Default + :param network_tracing: Whether to output network trace logs to the logger. Default is `False`. - :type debug: bool + :type network_tracing: bool :param http_proxy: HTTP proxy settings. This must be a dictionary with the following keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). Additionally the following keys may also be present: 'username', 'password'. :type http_proxy: dict[str, Any] :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. - :type auth_timeout: int + :type auth_timeout: float + :param user_agent: The user agent that needs to be appended to the built in user agent string. + :type user_agent: str + :param max_retries: The max number of attempts to redo the failed operation when an error happened. Default + value is 3. + :type max_retries: int + :param transport_type: The transport protocol type - default is ~uamqp.TransportType.Amqp. + ~uamqp.TransportType.AmqpOverWebsocket is applied when http_proxy is set or the + transport type is explicitly requested. + :type transport_type: ~azure.eventhub.TransportType + :param prefetch: The message prefetch count of the receiver. Default is 300. + :type prefetch: int + :param max_batch_size: Receive a batch of events. Batch size will be up to the maximum specified, but + will return as soon as service returns no new events. Default value is the same as prefetch. + :type max_batch_size: int + :param receive_timeout: The timeout time in seconds to receive a batch of events from an Event Hub. + Default value is 0 seconds. + :type receive_timeout: float + :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is + queued. Default value is 60 seconds. If set to 0, there will be no timeout. + :type send_timeout: float Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -221,7 +267,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): host = address[left_slash_pos + 2:] else: host = address - client = cls(host, "", SharedKeyCredentials(username, password), **kwargs) + client = cls(host, "", EventHubSharedKeyCredential(username, password), **kwargs) client._auth_config = { # pylint: disable=protected-access 'iot_username': policy, 'iot_password': key, @@ -233,7 +279,7 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): def _create_auth(self, username=None, password=None): pass - def create_properties(self, user_agent=None): # pylint: disable=no-self-use + def _create_properties(self, user_agent=None): # pylint: disable=no-self-use """ Format the properties with which to instantiate the connection. This acts like a user agent over HTTP. @@ -270,10 +316,12 @@ def _process_redirect_uri(self, redirect): @abstractmethod def create_receiver( - self, consumer_group, partition, epoch=None, offset=None, prefetch=300, - operation=None): + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, + operation=None, + prefetch=None, + ): pass @abstractmethod - def create_sender(self, partition=None, operation=None, send_timeout=60): + def create_sender(self, partition_id=None, operation=None, send_timeout=None): pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 3af21e5d2e86..80469a229e2d 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -82,7 +82,7 @@ def __init__(self, body=None, to_device=None, message=None): self._annotations = message.annotations self._app_properties = message.application_properties else: - if isinstance(body, list) and body: + if body and isinstance(body, list): self.message = Message(body[0], properties=self.msg_properties) for more in body[1:]: self.message._body.append(more) # pylint: disable=protected-access @@ -105,11 +105,11 @@ def __str__(self): dic['enqueued_time'] = str(self.enqueued_time) if self.device_id: dic['device_id'] = str(self.device_id) - if self._batching_label: - dic['_batching_label'] = str(self._batching_label) + if self.partition_key: + dic['partition_key'] = str(self.partition_key) + return str(dic) - return str(dic) @property def sequence_number(self): @@ -155,7 +155,7 @@ def device_id(self): return self._annotations.get(EventData.PROP_DEVICE_ID, None) @property - def _batching_label(self): + def partition_key(self): """ The partition key of the event data object. @@ -166,8 +166,7 @@ def _batching_label(self): except KeyError: return self._annotations.get(EventData.PROP_PARTITION_KEY, None) - @_batching_label.setter - def _batching_label(self, value): + def _set_partition_key(self, value): """ Set the partition key of the event data object. @@ -256,11 +255,11 @@ def encode_message(self): class _BatchSendEventData(EventData): - def __init__(self, batch_event_data, batching_label=None): + def __init__(self, batch_event_data, partition_key=None): self.message = BatchMessage(data=batch_event_data, multi_messages=False, properties=None) - self.set_batching_label(batching_label) + self._set_partition_key(partition_key) - def set_batching_label(self, value): + def _set_partition_key(self, value): if value: annotations = self.message.annotations if annotations is None: @@ -292,9 +291,9 @@ class EventPosition(object): def __init__(self, value, inclusive=False): """ - Initialize Offset. + Initialize EventPosition. - :param value: The offset value. + :param value: The event position value. :type value: ~datetime.datetime or int or str :param inclusive: Whether to include the supplied value as the start point. :type inclusive: bool @@ -305,7 +304,7 @@ def __init__(self, value, inclusive=False): def __str__(self): return str(self.value) - def selector(self): + def _selector(self): """ Creates a selector expression of the offset. @@ -319,34 +318,81 @@ def selector(self): return ("amqp.annotation.x-opt-sequence-number {} '{}'".format(operator, self.value)).encode('utf-8') return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') - @staticmethod - def first_available(): - return FIRST_AVAILABLE + @classmethod + def first_available_event(cls): + """ + Get the beginning of the event stream. + + :rtype: azure.eventhub.common.EventPosition + """ + + return cls("-1") @classmethod def new_events_only(cls): - return NEW_EVENTS_ONLY + """ + Get the end of the event stream. + + :rtype: azure.eventhub.common.EventPosition + """ - @staticmethod - def from_offset(offset, inclusive=False): - return EventPosition(offset, inclusive) + return cls("@latest") - @staticmethod - def from_sequence(sequence, inclusive=False): - return EventPosition(sequence, inclusive) + @classmethod + def from_offset(cls, offset, inclusive=False): + """ + Get the event position from/after the specified offset. - @staticmethod - def from_enqueued_time(enqueued_time, inclusive=False): - return EventPosition(enqueued_time, inclusive) + :param offset: the offset value + :type offset: str + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ + return cls(offset, inclusive) -FIRST_AVAILABLE = EventPosition("-1") -NEW_EVENTS_ONLY = EventPosition("@latest") + @classmethod + def from_sequence(cls, sequence, inclusive=False): + """ + Get the event position from/after the specified sequence number. + + :param sequence: the sequence number + :type sequence: int, long + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ + + return cls(sequence, inclusive) + + @classmethod + def from_enqueued_time(cls, enqueued_time, inclusive=False): + """ + Get the event position from/after the specified enqueue time. + + :param enqueued_time: the enqueue datetime + :type enqueued_time: datetime.datetime + :param inclusive: Whether to include the supplied value as the start point. + :type inclusive: bool + :rtype: azure.eventhub.common.EventPosition + """ + + return cls(enqueued_time, inclusive) # TODO: move some behaviors to these two classes. -class SASTokenCredentials(object): +class EventHubSASTokenCredential(object): + """ + SAS token used for authentication. + """ def __init__(self, token): + """ + :param token: A SAS token or function that returns a SAS token. If a function is supplied, + it will be used to retrieve subsequent tokens in the case of token expiry. The function should + take no arguments. + :type token: str or callable + """ self.token = token def get_sas_token(self): @@ -356,13 +402,23 @@ def get_sas_token(self): return self.token -class SharedKeyCredentials(object): +class EventHubSharedKeyCredential(object): + """ + The shared access key credential used for authentication. + """ def __init__(self, policy, key): + """ + :param policy: The name of the shared access policy. + :type policy: str + :param key: The shared access key. + :type key: str + """ + self.policy = policy self.key = key -class Address(object): +class _Address(object): def __init__(self, hostname=None, path=None): self.hostname = hostname self.path = path diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py index b6e030c9e3a6..72211d69076e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py @@ -16,6 +16,6 @@ def __init__(self, **kwargs): else kwargs.get("transport_type", TransportType.Amqp) self.auth_timeout = kwargs.get("auth_timeout", 60) self.prefetch = kwargs.get("prefetch", 300) - self.max_batch_size = kwargs.get("max_batch_size") + self.max_batch_size = kwargs.get("max_batch_size", self.prefetch) self.receive_timeout = kwargs.get("receive_timeout", 0) self.send_timeout = kwargs.get("send_timeout", 60) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 69aaa701496b..a93d052af4fe 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -5,7 +5,7 @@ from uamqp import types, constants, errors import six -from azure.core import AzureError +from azure.core.exceptions import AzureError _NO_RETRY_ERRORS = ( b"com.microsoft:argument-out-of-range", @@ -15,6 +15,7 @@ b"com.microsoft:argument-error" ) + def _error_handler(error): """ Called internally when an event has failed to send so we @@ -95,14 +96,46 @@ def _parse_error(self, error_list): self.details = details -class AuthenticationError(EventHubError): +class ConnectionLostError(EventHubError): + """Connection to event hub is lost. SDK will retry. So this shouldn't happen. + + """ pass class ConnectError(EventHubError): + """Fail to connect to event hubs + + """ + pass + + +class AuthenticationError(ConnectError): + """Fail to connect to event hubs because of authentication problem + + + """ pass class EventDataError(EventHubError): + """Problematic event data so the send will fail at client side + + """ + pass + + +class EventDataSendError(EventHubError): + """Service returns error while an event data is being sent + + """ pass +''' +class ConnectionTimeoutError(ConnectError): + """Time out when accessing event hub service + Should retry? + + """ +''' + diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 4643cb29419f..ac122b9e1881 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -9,32 +9,26 @@ import time from uamqp import types, errors +from uamqp import compat from uamqp import ReceiveClient, Source from azure.eventhub.common import EventData -from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) -class Receiver(object): +class EventReceiver(object): """ - Implements a Receiver. - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client_receiver_instance] - :end-before: [END create_eventhub_client_receiver_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Receiver. + Implements a EventReceiver. """ timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, keep_alive=None, auto_reconnect=True): + def __init__(self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, + keep_alive=None, auto_reconnect=True): """ Instantiate a receiver. @@ -45,8 +39,9 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ :param prefetch: The number of events to prefetch from the service for processing. Default is 300. :type prefetch: int - :param epoch: An optional epoch value. - :type epoch: int + :param exclusive_receiver_priority: The priority of the exclusive receiver. It will an exclusive + receiver if exclusive_receiver_priority is set. + :type exclusive_receiver_priority: int """ self.running = False self.client = client @@ -66,7 +61,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) # pylint: disable=protected-access if exclusive_receiver_priority: self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClient( @@ -79,7 +74,7 @@ def __init__(self, client, source, event_position=None, prefetch=300, exclusive_ error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access def __enter__(self): return self @@ -92,7 +87,10 @@ def __iter__(self): def __next__(self): self._open() + max_retries = self.client.config.max_retries + connecting_count = 0 while True: + connecting_count += 1 try: if not self.messages_iter: self.messages_iter = self._handler.receive_messages_iter() @@ -100,59 +98,90 @@ def __next__(self): event_data = EventData(message=message) self.offset = event_data.offset return event_data - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + log.info("EventReceiver detached. Attempting reconnect.") + self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + if connecting_count < max_retries: + log.info("EventReceiver detached. Attempting reconnect.") + self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver timed out. Shutting down.") + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) except StopIteration: raise + except KeyboardInterrupt: + log.info("EventReceiver stops due to keyboard interrupt") + print("EventReceiver stopped") + self.close() + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) raise error + def _check_closed(self): + if self.error: + raise EventHubError("This receiver has been closed. Please create a new receiver to receive event data.", + self.error) + + def _redirect(self, redirect): + self.redirected = redirect + self.running = False + self.messages_iter = None + self._open() + def _open(self): """ - Open the Receiver using the supplied conneciton. + Open the EventReceiver using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.connection.Connection - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_receiver_open] - :end-before: [END eventhub_client_receiver_open] - :language: python - :dedent: 4 - :caption: Open the Receiver using the supplied conneciton. - """ # pylint: disable=protected-access + self._check_closed() if self.redirected: + self.client._process_redirect_uri(self.redirected) self.source = self.redirected.address source = Source(self.source) if self.offset is not None: - source.set_filter(self.offset.selector()) + source.set_filter(self.offset._selector()) + alt_creds = { "username": self.client._auth_config.get("iot_username"), "password":self.client._auth_config.get("iot_password")} @@ -166,105 +195,99 @@ def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access if not self.running: - try: - self._handler.open() - self.running = True - while not self._handler.client_ready(): - time.sleep(0.05) - - except errors.AuthenticationException: - log.info("Receiver failed authentication. Retrying...") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Receiver detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate (%r).", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - raise error - else: - log.info("Receiver connection error (%r).", shutdown) - error = ConnectError(str(shutdown), shutdown) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Receiver connect failed: {}".format(e)) - raise error + self._connect() + self.running = True + + def _connect(self): + connected = self._build_connection() + if not connected: + time.sleep(self.reconnect_backoff) + while not self._build_connection(is_reconnect=True): + time.sleep(self.reconnect_backoff) + + def _build_connection(self, is_reconnect=False): + """ - def _reconnect(self): # pylint: disable=too-many-statements + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ # pylint: disable=protected-access - alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} - self._handler.close() - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset.selector()) - self._handler = ReceiveClient( - source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, - link_properties=self.properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) - self.messages_iter = None + if is_reconnect: + alt_creds = { + "username": self.client._auth_config.get("iot_username"), + "password": self.client._auth_config.get("iot_password")} + self._handler.close() + source = Source(self.source) + if self.offset is not None: + source.set_filter(self.offset._selector()) + self._handler = ReceiveClient( + source, + auth=self.client.get_auth(**alt_creds), + debug=self.client.config.network_tracing, + prefetch=self.prefetch, + link_properties=self.properties, + timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties( + self.client.config.user_agent)) # pylint: disable=protected-access + self.messages_iter = None try: self._handler.open() while not self._handler.client_ready(): time.sleep(0.05) return True except errors.AuthenticationException as shutdown: - log.info("Receiver disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error + if is_reconnect: + log.info("EventReceiver couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") + return False + except errors.LinkRedirect as redirect: + self._redirect(redirect) + return True except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("EventReceiver detached. Attempting reconnect.") return False - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("EventReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + if is_reconnect: + log.info("EventReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventReceiver detached. Attempting reconnect.") return False - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Receiver couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("EventReceiver connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventReceiver couldn't authenticate. Attempting reconnect.") return False - log.info("Receiver connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receiver reconnect failed: {}".format(e)) + error = EventHubError("EventReceiver reconnect failed: {}".format(e)) self.close(exception=error) raise error - def reconnect(self): - """If the Receiver was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not self._reconnect(): - time.sleep(self.reconnect_backoff) + def _reconnect(self): + return self._build_connection(is_reconnect=True) def close(self, exception=None): """ @@ -285,6 +308,9 @@ def close(self, exception=None): :caption: Close down the handler. """ + if self.messages_iter: + self.messages_iter.close() + self.messages_iter = None self.running = False if self.error: return @@ -320,6 +346,11 @@ def receive(self, max_batch_size=None, timeout=None): retrieve before the time, the result will be empty. If no batch size is supplied, the prefetch size will be the maximum. :type max_batch_size: int + :param timeout: The timeout time in seconds to receive a batch of events + from an Event Hub. Results will be returned after timeout. If combined + with max_batch_size, it will return after either the count of received events + reaches the max_batch_size or the operation has timed out. + :type timeout: int :rtype: list[~azure.eventhub.common.EventData] Example: @@ -331,45 +362,80 @@ def receive(self, max_batch_size=None, timeout=None): :caption: Receive events from the EventHub. """ - if self.error: - raise self.error + self._check_closed() self._open() + max_batch_size = min(self.client.config.max_batch_size, self.prefetch) if max_batch_size is None else max_batch_size + timeout = self.client.config.receive_timeout if timeout is None else timeout + data_batch = [] + max_retries = self.client.config.max_retries + connecting_count = 0 while True: + connecting_count += 1 try: timeout_ms = 1000 * timeout if timeout else 0 message_batch = self._handler.receive_message_batch( - max_batch_size=max_batch_size, + max_batch_size=max_batch_size - (len(data_batch) if data_batch else 0), timeout=timeout_ms) for message in message_batch: event_data = EventData(message=message) self.offset = event_data.offset data_batch.append(event_data) return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - self.reconnect() + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventReceiver disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + log.info("EventReceiver detached. Attempting reconnect.") + self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + if connecting_count < max_retries: + log.info("EventReceiver detached. Attempting reconnect.") + self._reconnect() else: - log.info("Receiver detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) + log.info("EventReceiver detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventReceiver timed out receiving event data. Attempting reconnect.") + self._reconnect() + else: + log.info("EventReceiver timed out. Shutting down.") + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) + except KeyboardInterrupt: + log.info("EventReceiver stops due to keyboard interrupt") + print("EventReceiver stopped") + self.close() + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) raise error + + next = __next__ # for python2.7 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index ccb193835c20..5e6754281428 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -9,33 +9,26 @@ import time from uamqp import constants, errors +from uamqp import compat from uamqp import SendClient from uamqp.constants import MessageSendResult from azure.eventhub.common import EventData, _BatchSendEventData from azure.eventhub.error import EventHubError, ConnectError, \ - AuthenticationError, EventDataError, _error_handler + AuthenticationError, EventDataError, EventDataSendError, ConnectionLostError, _error_handler log = logging.getLogger(__name__) -class Sender(object): +class EventSender(object): """ - Implements a Sender. - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client_sender_instance] - :end-before: [END create_eventhub_client_sender_instance] - :language: python - :dedent: 4 - :caption: Create a new instance of the Sender. + Implements a EventSender. """ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=None, auto_reconnect=True): """ - Instantiate an EventHub event Sender handler. + Instantiate an EventHub event EventSender handler. :param client: The parent EventHubClient. :type client: ~azure.eventhub.client.EventHubClient. @@ -46,10 +39,10 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N :type partition: str :param send_timeout: The timeout in seconds for an individual event to be sent from the time that it is queued. Default value is 60 seconds. If set to 0, there will be no timeout. - :type send_timeout: int + :type send_timeout: float :param keep_alive: The time interval in seconds between pinging the connection to keep it alive during periods of inactivity. The default value is None, i.e. no keep alive pings. - :type keep_alive: int + :type keep_alive: float :param auto_reconnect: Whether to automatically reconnect the sender if a retryable error occurs. Default value is `True`. :type auto_reconnect: bool @@ -66,6 +59,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) self.reconnect_backoff = 1 self.name = "EHSender-{}".format(uuid.uuid4()) + self.unsent_events = None if partition: self.target += "/Partitions/" + partition self.name += "-partition{}".format(partition) @@ -77,7 +71,7 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access self._outcome = None self._condition = None @@ -89,22 +83,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _open(self): """ - Open the Sender using the supplied conneciton. + Open the EventSender using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. - :param connection: The underlying client shared connection. - :type: connection: ~uamqp.connection.Connection - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_sender_open] - :end-before: [END eventhub_client_sender_open] - :language: python - :dedent: 4 - :caption: Open the Sender using the supplied conneciton. - """ + # pylint: disable=protected-access + self._check_closed() if self.redirected: self.target = self.redirected.address self._handler = SendClient( @@ -115,98 +100,86 @@ def _open(self): error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + properties=self.client._create_properties(self.client.config.user_agent)) if not self.running: - try: - self._handler.open() - self.running = True - while not self._handler.client_ready(): - time.sleep(0.05) - except errors.AuthenticationException: - log.info("Sender failed authentication. Retrying...") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Failed to connect") - error = ConnectError(str(shutdown), shutdown) - raise error - except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Sender couldn't authenticate.", shutdown) - error = AuthenticationError(str(shutdown), shutdown) - raise error - else: - log.info("Sender connection error (%r).", shutdown) - error = ConnectError(str(shutdown), shutdown) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r)", e) - error = EventHubError("Sender connect failed: {}".format(e)) - raise error + self._connect() + self.running = True - def _reconnect(self): + def _connect(self): + connected = self._build_connection() + if not connected: + time.sleep(self.reconnect_backoff) + while not self._build_connection(is_reconnect=True): + time.sleep(self.reconnect_backoff) + + def _build_connection(self, is_reconnect=False): + """ + + :param is_reconnect: True - trying to reconnect after fail to connect or a connection is lost. + False - the 1st time to connect + :return: True - connected. False - not connected + """ # pylint: disable=protected-access - self._handler.close() - unsent_events = self._handler.pending_messages - self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client.create_properties(self.client.config.user_agent)) + if is_reconnect: + self._handler.close() + self._handler = SendClient( + self.target, + auth=self.client.get_auth(), + debug=self.client.config.network_tracing, + msg_timeout=self.timeout, + error_policy=self.retry_policy, + keep_alive_interval=self.keep_alive, + client_name=self.name, + properties=self.client._create_properties(self.client.config.user_agent)) try: self._handler.open() while not self._handler.client_ready(): time.sleep(0.05) - self._handler.queue_message(*unsent_events) - self._handler.wait() return True except errors.AuthenticationException as shutdown: - log.info("Sender disconnected due to token expiry. Shutting down.") - error = AuthenticationError(str(shutdown), shutdown) - self.close(exception=error) - raise error + if is_reconnect: + log.info("EventSender couldn't authenticate. Shutting down. (%r)", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventSender couldn't authenticate. Attempting reconnect.") + return False except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") + if shutdown.action.retry: + log.info("EventSender detached. Attempting reconnect.") return False - log.info("Sender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("EventSender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") + if is_reconnect: + log.info("EventSender detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventSender detached. Attempting reconnect.") return False - log.info("Sender reconnect failed. Shutting down.") - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except errors.AMQPConnectionError as shutdown: - if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: - log.info("Sender couldn't authenticate. Attempting reconnect.") + if is_reconnect: + log.info("EventSender connection error (%r). Shutting down.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + self.close(exception=error) + raise error + else: + log.info("EventSender couldn't authenticate. Attempting reconnect.") return False - log.info("Sender connection error (%r). Shutting down.", shutdown) - error = ConnectError(str(shutdown), shutdown) - self.close(exception=error) - raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Sender Reconnect failed: {}".format(e)) + error = EventHubError("EventSender Reconnect failed: {}".format(e)) self.close(exception=error) raise error - def reconnect(self): - """If the Sender was disconnected from the service with - a retryable error - attempt to reconnect.""" - while not self._reconnect(): - time.sleep(self.reconnect_backoff) + def _reconnect(self): + return self._build_connection(is_reconnect=True) def close(self, exception=None): """ @@ -240,64 +213,107 @@ def close(self, exception=None): self.error = EventHubError("This send handler is now closed.") self._handler.close() - def _send_event_data(self, event_data): + def _send_event_data(self): self._open() - - try: - self._handler.send_message(event_data.message) - if self._outcome != MessageSendResult.Ok: - raise Sender._error(self._outcome, self._condition) - except errors.MessageException as failed: - error = EventDataError(str(failed), failed) - self.close(exception=error) - raise error - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Sender disconnected due to token error. Attempting reconnect.") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + max_retries = self.client.config.max_retries + connecting_count = 0 + while True: + connecting_count += 1 + try: + if self.unsent_events: + self._handler.queue_message(*self.unsent_events) + self._handler.wait() + self.unsent_events = self._handler.pending_messages + if self._outcome != constants.MessageSendResult.Ok: + EventSender._error(self._outcome, self._condition) + return + except (errors.MessageAccepted, + errors.MessageAlreadySettled, + errors.MessageModified, + errors.MessageRejected, + errors.MessageReleased, + errors.MessageContentTooLarge) as msg_error: + raise EventDataError(str(msg_error), msg_error) + except errors.MessageException as failed: + log.info("Send event data error (%r)", failed) + error = EventDataSendError(str(failed), failed) self.close(exception=error) raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = ConnectError(str(shutdown), shutdown) + except errors.AuthenticationException as auth_error: + if connecting_count < max_retries: + log.info("EventSender disconnected due to token error. Attempting reconnect.") + self._reconnect() + else: + log.info("EventSender authentication failed. Shutting down.") + error = AuthenticationError(str(auth_error), auth_error) + self.close(auth_error) + raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry: + log.info("EventSender detached. Attempting reconnect.") + self._reconnect() + else: + log.info("EventSender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if connecting_count < max_retries: + log.info("EventSender detached. Attempting reconnect.") + self._reconnect() + else: + log.info("EventSender detached. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except errors.AMQPConnectionError as shutdown: + if connecting_count < max_retries: + log.info("EventSender connection lost. Attempting reconnect.") + self._reconnect() + else: + log.info("EventSender connection lost. Shutting down.") + error = ConnectionLostError(str(shutdown), shutdown) + self.close(error) + raise error + except compat.TimeoutException as shutdown: + if connecting_count < max_retries: + log.info("EventSender timed out sending event data. Attempting reconnect.") + self._reconnect() + else: + log.info("EventSender timed out. Shutting down.") + self.close(shutdown) + raise TimeoutError(str(shutdown), shutdown) + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Send failed: {}".format(e)) self.close(exception=error) raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Send failed: {}".format(e)) - self.close(exception=error) - raise error - else: - return self._outcome + + def _check_closed(self): + if self.error: + raise EventHubError("This sender has been closed. Please create a new sender to send event data.", self.error) @staticmethod - def _set_batching_label(event_datas, batching_label): + def _set_partition_key(event_datas, partition_key): ed_iter = iter(event_datas) for ed in ed_iter: - ed._batching_label = batching_label + ed._set_partition_key(partition_key) yield ed - def send(self, event_data, batching_label=None): + def send(self, event_data, partition_key=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. :param event_data: The event to be sent. :type event_data: ~azure.eventhub.common.EventData + :param partition_key: With the given partition_key, event data will land to + a particular partition of the Event Hub decided by the service. + :type batching_label: str :raises: ~azure.eventhub.common.EventHubError if the message fails to send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult + :return: None + :rtype: None Example: .. literalinclude:: ../examples/test_examples_eventhub.py @@ -308,18 +324,18 @@ def send(self, event_data, batching_label=None): :caption: Sends an event data and blocks until acknowledgement is received or operation times out. """ - if self.error: - raise self.error + self._check_closed() if isinstance(event_data, EventData): - if batching_label: - event_data._batching_label = batching_label + if partition_key: + event_data._set_partition_key(partition_key) wrapper_event_data = event_data else: wrapper_event_data = _BatchSendEventData( - self._set_batching_label(event_data, batching_label), - batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) + self._set_partition_key(event_data, partition_key), + partition_key=partition_key) if partition_key else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - self._send_event_data(wrapper_event_data) + self.unsent_events = [wrapper_event_data.message] + self._send_event_data() def _on_outcome(self, outcome, condition): """ @@ -327,10 +343,13 @@ def _on_outcome(self, outcome, condition): :param outcome: The outcome of the message delivery - success or failure. :type outcome: ~uamqp.constants.MessageSendResult + :param condition: Detail information of the outcome. + """ self._outcome = outcome self._condition = condition @staticmethod def _error(outcome, condition): - return None if outcome == MessageSendResult.Ok else EventHubError(outcome, condition) + if outcome != MessageSendResult.Ok: + raise condition diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py index d2c649f9a0a6..84d6a9ae84fe 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/eh_partition_pump.py @@ -5,7 +5,7 @@ import logging import asyncio -from azure.eventhub import EventPosition +from azure.eventhub import EventPosition, EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient from azure.eventprocessorhost.partition_pump import PartitionPump @@ -49,7 +49,6 @@ async def on_open_async(self): if self.pump_status == "Opening": loop = asyncio.get_event_loop() self.set_pump_status("Running") - await self.eh_client.run_async() self.running = loop.create_task(self.partition_receiver.run()) if self.pump_status in ["OpenFailed", "Errored"]: @@ -65,17 +64,19 @@ async def open_clients_async(self): """ await self.partition_context.get_initial_offset_async() # Create event hub client and receive handler and set options + hostname = "{}.{}".format(self.host.eh_config.sb_name, self.host.eh_config.namespace_suffix) + event_hub_path = self.host.eh_config.eh_name + shared_key_cred = EventHubSharedKeyCredential(self.host.eh_config.policy, self.host.eh_config.sas_key) + self.eh_client = EventHubClient( - self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace, + hostname, event_hub_path, shared_key_cred, + network_tracing=self.host.eph_options.debug_trace, http_proxy=self.host.eph_options.http_proxy) - self.partition_receive_handler = self.eh_client.add_async_receiver( - self.partition_context.consumer_group_name, - self.partition_context.partition_id, - EventPosition(self.partition_context.offset), + self.partition_receive_handler = self.eh_client.create_receiver( + partition_id=self.partition_context.partition_id, + consumer_group=self.partition_context.consumer_group_name, + event_position=EventPosition(self.partition_context.offset), prefetch=self.host.eph_options.prefetch_count, - keep_alive=self.host.eph_options.keep_alive_interval, - auto_reconnect=self.host.eph_options.auto_reconnect_on_error, loop=self.loop) self.partition_receiver = PartitionReceiver(self) @@ -85,7 +86,7 @@ async def clean_up_clients_async(self): """ if self.partition_receiver: if self.eh_client: - await self.eh_client.stop_async() + await self.partition_receive_handler.close() self.partition_receiver = None self.partition_receive_handler = None self.eh_client = None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py index d532846a5476..41ffe9d043bd 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/partition_manager.py @@ -9,6 +9,7 @@ from collections import Counter from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential from azure.eventprocessorhost.eh_partition_pump import EventHubPartitionPump from azure.eventprocessorhost.cancellation_token import CancellationToken @@ -36,18 +37,20 @@ async def get_partition_ids_async(self): :rtype: list[str] """ if not self.partition_ids: + hostname = "{}.{}".format(self.host.eh_config.sb_name, self.host.eh_config.namespace_suffix) + event_hub_path = self.host.eh_config.eh_name + shared_key_cred = EventHubSharedKeyCredential(self.host.eh_config.policy, self.host.eh_config.sas_key) + + eh_client = EventHubClient( + hostname, event_hub_path, shared_key_cred, + network_tracing=self.host.eph_options.debug_trace, + # http_proxy=self.host.eph_options.http_proxy, + ) try: - eh_client = EventHubClient( - self.host.eh_config.client_address, - debug=self.host.eph_options.debug_trace, - http_proxy=self.host.eph_options.http_proxy) - try: - eh_info = await eh_client.get_eventhub_info_async() - self.partition_ids = eh_info['partition_ids'] - except Exception as err: # pylint: disable=broad-except - raise Exception("Failed to get partition ids", repr(err)) - finally: - await eh_client.stop_async() + eh_info = await eh_client.get_properties() + self.partition_ids = eh_info['partition_ids'] + except Exception as err: # pylint: disable=broad-except + raise Exception("Failed to get partition ids", repr(err)) return self.partition_ids async def start_async(self): diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index 68a211917f4c..a23dc86be6a8 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -29,7 +29,7 @@ from azure.eventprocessorhost.partition_pump import PartitionPump from azure.eventprocessorhost.partition_manager import PartitionManager -from azure.eventhub import EventHubClient, Receiver, EventPosition +from azure.eventhub import EventHubClient, EventReceiver, EventPosition def get_logger(filename, level=logging.INFO): @@ -162,16 +162,23 @@ def device_id(): pytest.skip("No Iothub device ID found.") +@pytest.fixture() +def aad_credential(): + try: + return os.environ['AAD_CLIENT_ID'], os.environ['AAD_SECRET'], os.environ['AAD_TENANT_ID'] + except KeyError: + pytest.skip('No Azure Active Directory credential found') + + @pytest.fixture() def connstr_receivers(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) partitions = client.get_partition_ids() receivers = [] for p in partitions: - #receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("@latest")) receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("-1")) + receiver._open() receivers.append(receiver) - receiver.receive(timeout=1) yield connection_str, receivers for r in receivers: @@ -180,7 +187,7 @@ def connstr_receivers(connection_str): @pytest.fixture() def connstr_senders(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=True) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) partitions = client.get_partition_ids() senders = [] diff --git a/sdk/eventhub/azure-eventhubs/dev_requirements.txt b/sdk/eventhub/azure-eventhubs/dev_requirements.txt index bd3d6e3bb6e0..fa716ae38ebe 100644 --- a/sdk/eventhub/azure-eventhubs/dev_requirements.txt +++ b/sdk/eventhub/azure-eventhubs/dev_requirements.txt @@ -1,4 +1,5 @@ -e ../../servicebus/azure-servicebus +-e ../../core/azure-core pytest>=3.4.1 pytest-asyncio>=0.8.0; python_version > '3.4' docutils>=0.14 diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py new file mode 100644 index 000000000000..8149c1c2a51d --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show iterator receiver. +""" + +import os +import time +import logging +import asyncio + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventPosition, EventHubSharedKeyCredential, EventData + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +EVENT_POSITION = EventPosition.first_available_event() + + +async def iter_receiver(receiver): + async with receiver: + async for item in receiver: + print(item.body_as_str(), item.offset.value, receiver.name) + + +async def main(): + if not HOSTNAME: + raise ValueError("No EventHubs URL supplied.") + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) + await iter_receiver(receiver) + +if __name__ == '__main__': + asyncio.run(main()) + diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py new file mode 100644 index 000000000000..63ad3428fb40 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show running concurrent receivers. +""" + +import os +import time +import logging +import asyncio + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventPosition, EventHubSharedKeyCredential + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +EVENT_POSITION = EventPosition.first_available_event() + + +async def pump(client, partition): + receiver = client.create_receiver(partition_id=partition, event_position=EVENT_POSITION, prefetch=5) + async with receiver: + total = 0 + start_time = time.time() + for event_data in await receiver.receive(timeout=10): + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset.value, last_sn)) + total += 1 + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) + +try: + if not HOSTNAME: + raise ValueError("No EventHubs URL supplied.") + + loop = asyncio.get_event_loop() + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + tasks = [ + asyncio.ensure_future(pump(client, "0")), + asyncio.ensure_future(pump(client, "1"))] + loop.run_until_complete(asyncio.wait(tasks)) + loop.close() + +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py new file mode 100644 index 000000000000..200a2be8ad98 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show sending events asynchronously to an Event Hub with partition keys. +""" + +# pylint: disable=C0111 + +import logging +import time +import asyncio +import os + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData, EventHubSharedKeyCredential + +import examples +logger = examples.get_logger(logging.INFO) + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + + +async def run(client): + sender = client.create_sender() + await send(sender, 4) + + +async def send(sender, count): + async with sender: + for i in range(count): + logger.info("Sending message: {}".format(i)) + data = EventData(str(i)) + await sender.send(data) + +try: + if not HOSTNAME: + raise ValueError("No EventHubs URL supplied.") + + loop = asyncio.get_event_loop() + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + tasks = asyncio.gather( + run(client), + run(client)) + start_time = time.time() + loop.run_until_complete(tasks) + end_time = time.time() + run_time = end_time - start_time + logger.info("Runtime: {} seconds".format(run_time)) + loop.close() + +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py index 7f18fc97b756..4a36faa88947 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py @@ -11,169 +11,94 @@ import logging import asyncio -from azure.eventhub import EventHubError +from azure.eventhub import EventHubError, EventData @pytest.mark.asyncio async def test_example_eventhub_async_send_and_receive(live_eventhub_config): # [START create_eventhub_client_async] - from azure.eventhub import EventHubClientAsync + from azure.eventhub.aio import EventHubClient import os connection_str = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}".format( os.environ['EVENT_HUB_HOSTNAME'], os.environ['EVENT_HUB_SAS_POLICY'], os.environ['EVENT_HUB_SAS_KEY'], os.environ['EVENT_HUB_NAME']) - client = EventHubClientAsync.from_connection_string(connection_str) + client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_async] - from azure.eventhub import EventData, Offset + from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_async_sender] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add a async sender to the async client object. - sender = client.add_async_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + # Create an async sender. + sender = client.create_sender(partition_id="0") # [END create_eventhub_client_async_sender] # [START create_eventhub_client_async_receiver] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add a async receiver to the async client object. - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str) + # Create an async receiver. + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + # Create an exclusive async receiver. + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest'), exclusive_receiver_priority=1) # [END create_eventhub_client_async_receiver] - # [START create_eventhub_client_async_epoch_receiver] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add a async receiver to the async client object. - epoch_receiver = client.add_async_epoch_receiver(consumer_group="$default", partition="0", epoch=42) - # [END create_eventhub_client_async_epoch_receiver] + client = EventHubClient.from_connection_string(connection_str) + sender = client.create_sender(partition_id="0") + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) - # [START eventhub_client_run_async] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add AsyncSenders/AsyncReceivers - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - # Start sending and receiving - except: - raise - finally: - await client.stop_async() - # [END eventhub_client_run_async] + await receiver.receive(timeout=1) - - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - - # [START eventhub_client_async_send] + # [START eventhub_client_async_send] + async with sender: event_data = EventData(b"A single event") await sender.send(event_data) - # [END eventhub_client_async_send] - time.sleep(1) - # [START eventhub_client_async_receive] - logger = logging.getLogger("azure.eventhub") + # [END eventhub_client_async_send] + + await asyncio.sleep(1) + + # [START eventhub_client_async_receive] + logger = logging.getLogger("azure.eventhub") + async with receiver: received = await receiver.receive(timeout=5) for event_data in received: logger.info("Message received:{}".format(event_data.body_as_str())) - # [END eventhub_client_async_receive] + # [END eventhub_client_async_receive] assert len(received) == 1 assert received[0].body_as_str() == "A single event" assert list(received[-1].body)[0] == b"A single event" - except: - raise - finally: - await client.stop_async() - - # [START eventhub_client_async_stop] - client = EventHubClientAsync.from_connection_string(connection_str) - # Add AsyncSenders/AsyncReceivers - try: - # Opens the connection and starts running all AsyncSender/AsyncReceiver clients. - await client.run_async() - # Start sending and receiving - except: - raise - finally: - await client.stop_async() - # [END eventhub_client_async_stop] @pytest.mark.asyncio async def test_example_eventhub_async_sender_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_async_sender_instance] - from azure.eventhub import EventHubClientAsync - - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - # [END create_eventhub_client_async_sender_instance] - - # [START eventhub_client_async_sender_open] - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") - try: - # Open the Async Sender using the supplied conneciton. - await sender.open_async() - # Start sending - except: - raise - finally: - # Close down the send handler. - await sender.close_async() - # [END eventhub_client_async_sender_open] + from azure.eventhub.aio import EventHubClient + from azure.eventhub import EventData # [START eventhub_client_async_sender_close] - client = EventHubClientAsync.from_connection_string(connection_str) - sender = client.add_async_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + sender = client.create_sender(partition_id="0") try: - # Open the Async Sender using the supplied conneciton. - await sender.open_async() - # Start sending - except: - raise + await sender.send(EventData(b"A single event")) finally: # Close down the send handler. - await sender.close_async() + await sender.close() # [END eventhub_client_async_sender_close] @pytest.mark.asyncio async def test_example_eventhub_async_receiver_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_async_receiver_instance] - from azure.eventhub import EventHubClientAsync, Offset - - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - # [END create_eventhub_client_async_receiver_instance] - - # [START eventhub_client_async_receiver_open] - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - # Open the Async Receiver using the supplied conneciton. - await receiver.open_async() - # Start receiving - except: - raise - finally: - # Close down the receive handler. - await receiver.close_async() - # [END eventhub_client_async_receiver_open] + from azure.eventhub.aio import EventHubClient + from azure.eventhub import EventPosition # [START eventhub_client_async_receiver_close] - client = EventHubClientAsync.from_connection_string(connection_str) - receiver = client.add_async_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) try: - # Open the Async Receiver using the supplied conneciton. - await receiver.open_async() - # Start receiving + # Open and receive + await receiver.receive(timeout=1) except: raise finally: # Close down the receive handler. - await receiver.close_async() - # [END eventhub_client_async_receiver_close] \ No newline at end of file + await receiver.close() + # [END eventhub_client_async_receiver_close] diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_send.py b/sdk/eventhub/azure-eventhubs/examples/batch_send.py index 7cbf6259d661..bf80907a655e 100644 --- a/sdk/eventhub/azure-eventhubs/examples/batch_send.py +++ b/sdk/eventhub/azure-eventhubs/examples/batch_send.py @@ -1,54 +1,49 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show batch sending events to an Event Hub. """ # pylint: disable=C0111 -import sys import logging -import datetime import time import os -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventData, EventHubClient, EventHubSharedKeyCredential + import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -def data_generator(): - for i in range(1500): - logger.info("Yielding message {}".format(i)) - yield b"Hello world" - - try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="1") - client.run() - try: + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) + sender = client.create_sender(partition_id="1") + + event_list = [] + for i in range(1500): + event_list.append('Hello World') + + with sender: start_time = time.time() - data = EventData(batch=data_generator()) + data = EventData(body=event_list) sender.send(data) - except: - raise - finally: end_time = time.time() - client.stop() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py b/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py deleted file mode 100644 index 676ac6c3e2ea..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/batch_transfer.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -""" -An example to show batch sending events to an Event Hub. -""" - -# pylint: disable=C0111 - -import sys -import logging -import datetime -import time -import os - -from azure.eventhub import EventHubClient, Sender, EventData - -import examples -logger = examples.get_logger(logging.INFO) - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - -def data_generator(): - for i in range(1500): - logger.info("Yielding message {}".format(i)) - yield b"Hello world" - - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender() - client.run() - try: - start_time = time.time() - data = EventData(batch=data_generator()) - sender.transfer(data, callback=callback) - sender.wait() - except: - raise - finally: - end_time = time.time() - client.stop() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py new file mode 100644 index 000000000000..39697379f4d2 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +""" +An example to show authentication using aad credentials +""" + +import os +import time +import logging + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData +from azure.identity import ClientSecretCredential + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +CLIENT_ID = os.environ.get('AAD_CLIENT_ID') +SECRET = os.environ.get('AAD_SECRET') +TENANT_ID = os.environ.get('AAD_TENANT_ID') + + +credential = ClientSecretCredential(client_id=CLIENT_ID, secret=SECRET, tenant_id=TENANT_ID) +client = EventHubClient(host=HOSTNAME, + event_hub_path=EVENT_HUB, + credential=credential) +try: + sender = client.create_sender(partition_id='0') + + with sender: + event = EventData(body='A single message') + sender.send(event) + +except KeyboardInterrupt: + pass +except Exception as e: + print(e) diff --git a/sdk/eventhub/azure-eventhubs/examples/eph.py b/sdk/eventhub/azure-eventhubs/examples/eph.py index 39f0fbba4179..66e4e6aa866f 100644 --- a/sdk/eventhub/azure-eventhubs/examples/eph.py +++ b/sdk/eventhub/azure-eventhubs/examples/eph.py @@ -109,7 +109,7 @@ async def wait_and_close(host): EventProcessor, eh_config, storage_manager, - ep_params=["param1","param2"], + ep_params=["param1", "param2"], eph_options=eh_options, loop=loop) diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py index acf88117844e..7961abef5c0a 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # -------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. @@ -6,23 +8,21 @@ """ An example to show receiving events from an IoT Hub partition. """ -from azure import eventhub -from azure.eventhub import EventData, EventHubClient, Offset - import os import logging -logger = logging.getLogger('azure.eventhub') -iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] +from azure.eventhub import EventHubClient -client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) -receiver = client.add_receiver("$default", "0", operation='/messages/events') -try: - client.run() - eh_info = client.get_eventhub_info() - print(eh_info) +logger = logging.getLogger('azure.eventhub') + +iot_connection_str = 'HostName=iothubfortrack2py.azure-devices.net;SharedAccessKeyName=iothubowner;SharedAccessKey=glF9a2n0D9fgmWpfTqjjmvkYt0WaTNqZx9GV/UKwDkQ=' # os.environ['IOTHUB_CONNECTION_STR'] + +client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) +receiver = client.create_receiver(partition_id="0", operation='/messages/events') +with receiver: received = receiver.receive(timeout=5) print(received) -finally: - client.stop() + + eh_info = client.get_properties() + print(eh_info) diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_send.py b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py new file mode 100644 index 000000000000..ab4fcb2adec6 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show receiving events from an IoT Hub partition. +""" +import os +import logging + +from azure.eventhub import EventData, EventHubClient + + +logger = logging.getLogger('azure.eventhub') + +iot_device_id = os.environ['IOTHUB_DEVICE'] +iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] + +client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) +try: + sender = client.create_sender(operation='/messages/devicebound') + with sender: + sender.send(EventData(b"A single event", to_device=iot_device_id)) + +except KeyboardInterrupt: + pass diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py new file mode 100644 index 000000000000..a8dd0a4b4400 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from threading import Thread +import os +import time +import logging + +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential, EventData + +import examples +logger = examples.get_logger(logging.INFO) + + +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') + +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') + +EVENT_POSITION = EventPosition.first_available_event() + + +class PartitionReceiverThread(Thread): + def __init__(self, receiver): + Thread.__init__(self) + self.receiver = receiver + + def run(self): + for item in self.receiver: + print(item) + + +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=True) +receiver = client.create_receiver(partition_id="0", event_position=EVENT_POSITION) +with receiver: + thread = PartitionReceiverThread(receiver) + thread.start() + thread.join(2) # stop after 2 seconds diff --git a/sdk/eventhub/azure-eventhubs/examples/proxy.py b/sdk/eventhub/azure-eventhubs/examples/proxy.py index b4a2d51b0411..0417fe32d0f4 100644 --- a/sdk/eventhub/azure-eventhubs/examples/proxy.py +++ b/sdk/eventhub/azure-eventhubs/examples/proxy.py @@ -1,27 +1,30 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show sending and receiving events behind a proxy """ import os import logging -from azure.eventhub import EventHubClient, EventPosition, EventData +from azure.eventhub import EventHubClient, EventPosition, EventData, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +# Hostname can be .servicebus.windows.net" +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -EVENT_POSITION = EventPosition.first_available() + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" HTTP_PROXY = { 'proxy_hostname': '127.0.0.1', # proxy hostname @@ -31,27 +34,28 @@ } -if not ADDRESS: +if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY, http_proxy=HTTP_PROXY) -sender = client.create_sender(partition=PARTITION) -receiver = client.create_receiver(consumer_group=CONSUMER_GROUP, partition=PARTITION, event_position=EVENT_POSITION) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False, http_proxy=HTTP_PROXY) try: + sender = client.create_sender(partition_id=PARTITION) + receiver = client.create_receiver(partition_id=PARTITION, event_position=EVENT_POSITION) + + receiver.receive(timeout=1) + event_list = [] for i in range(20): event_list.append(EventData("Event Number {}".format(i))) print('Start sending events behind a proxy.') - with sender: - sender.send(list) + sender.send(event_list) print('Start receiving events behind a proxy.') - with receiver: - received = receiver.receive(max_batch_size=50, timeout=5) - -except KeyboardInterrupt: - pass + received = receiver.receive(max_batch_size=50, timeout=5) +finally: + sender.close() + receiver.close() diff --git a/sdk/eventhub/azure-eventhubs/examples/recv.py b/sdk/eventhub/azure-eventhubs/examples/recv.py index f43d03be9ce5..0e85dcb5fb39 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv.py @@ -9,51 +9,46 @@ An example to show receiving events from an Event Hub partition. """ import os -import sys import logging import time -from azure.eventhub import EventHubClient, Receiver, Offset +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" total = 0 last_sn = -1 last_offset = "-1" -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + try: - receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000, offset=OFFSET) - client.run() - start_time = time.time() - batch = receiver.receive(timeout=5000) - while batch: - for event_data in batch: - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset.value, last_sn)) - print(event_data.body_as_str()) - total += 1 + receiver = client.create_receiver(partition_id=PARTITION, prefetch=5000, event_position=EVENT_POSITION) + with receiver: + start_time = time.time() batch = receiver.receive(timeout=5000) - - end_time = time.time() - client.stop() - run_time = end_time - start_time - print("Received {} messages in {} seconds".format(total, run_time)) + while batch: + for event_data in batch: + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset.value, last_sn)) + print(event_data.body_as_str()) + total += 1 + batch = receiver.receive(timeout=5000) + + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) except KeyboardInterrupt: pass -finally: - client.stop() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_async.py b/sdk/eventhub/azure-eventhubs/examples/recv_async.py deleted file mode 100644 index 04d922649b3c..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/recv_async.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -An example to show running concurrent receivers. -""" - -import os -import sys -import time -import logging -import asyncio -from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver - -import examples -logger = examples.get_logger(logging.INFO) - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") - - -async def pump(client, partition): - receiver = client.add_async_receiver(CONSUMER_GROUP, partition, OFFSET, prefetch=5) - await client.run_async() - total = 0 - start_time = time.time() - for event_data in await receiver.receive(timeout=10): - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset.value, last_sn)) - total += 1 - end_time = time.time() - run_time = end_time - start_time - print("Received {} messages in {} seconds".format(total, run_time)) - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY) - tasks = [ - asyncio.ensure_future(pump(client, "0")), - asyncio.ensure_future(pump(client, "1"))] - loop.run_until_complete(asyncio.wait(tasks)) - loop.run_until_complete(client.stop_async()) - loop.close() - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py index 9478f51feb21..0b270454828f 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py @@ -11,45 +11,41 @@ """ import os -import sys import logging -from azure.eventhub import EventHubClient, Receiver, Offset + +from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -OFFSET = Offset("-1") + +EVENT_POSITION = EventPosition.first_available_event() PARTITION = "0" total = 0 last_sn = -1 last_offset = "-1" -client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) + try: - receiver = client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=100, offset=OFFSET) - client.run() - batched_events = receiver.receive(max_batch_size=10) - for event_data in batched_events: - last_offset = event_data.offset.value - last_sn = event_data.sequence_number - total += 1 - print("Partition {}, Received {}, sn={} offset={}".format( - PARTITION, - total, - last_sn, - last_offset)) + receiver = client.create_receiver(partition_id=PARTITION, prefetch=100, event_position=EVENT_POSITION) + with receiver: + batched_events = receiver.receive(max_batch_size=10) + for event_data in batched_events: + last_offset = event_data.offset.value + last_sn = event_data.sequence_number + total += 1 + print("Partition {}, Received {}, sn={} offset={}".format( + PARTITION, + total, + last_sn, + last_offset)) except KeyboardInterrupt: pass -finally: - client.stop() \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py b/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py index f9f291ed6bc3..3f82202dbd5b 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_epoch.py @@ -10,48 +10,46 @@ """ import os -import sys import time import logging import asyncio -from azure.eventhub import Offset, EventHubClientAsync, AsyncReceiver + +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') -CONSUMER_GROUP = "$default" -EPOCH = 42 + +EXCLUSIVE_RECEIVER_PRIORITY = 42 PARTITION = "0" -async def pump(client, epoch): - receiver = client.add_async_epoch_receiver(CONSUMER_GROUP, PARTITION, epoch=epoch) - await client.run_async() - total = 0 - start_time = time.time() - for event_data in await receiver.receive(timeout=5): - last_offset = event_data.offset - last_sn = event_data.sequence_number - total += 1 - end_time = time.time() - run_time = end_time - start_time - await client.stop_async() - print("Received {} messages in {} seconds".format(total, run_time)) +async def pump(client, exclusive_receiver_priority): + receiver = client.create_receiver(partition_id=PARTITION, exclusive_receiver_priority=exclusive_receiver_priority) + async with receiver: + total = 0 + start_time = time.time() + for event_data in await receiver.receive(timeout=5): + last_offset = event_data.offset + last_sn = event_data.sequence_number + total += 1 + end_time = time.time() + run_time = end_time - start_time + print("Received {} messages in {} seconds".format(total, run_time)) try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=False, username=USER, password=KEY) + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) loop.run_until_complete(pump(client, 20)) loop.close() diff --git a/sdk/eventhub/azure-eventhubs/examples/send.py b/sdk/eventhub/azure-eventhubs/examples/send.py index 6881b0d578ee..316a6e2739a5 100644 --- a/sdk/eventhub/azure-eventhubs/examples/send.py +++ b/sdk/eventhub/azure-eventhubs/examples/send.py @@ -1,49 +1,52 @@ #!/usr/bin/env python +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + """ An example to show sending events to an Event Hub partition. """ # pylint: disable=C0111 -import sys import logging -import datetime import time import os -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventHubClient, EventData, EventHubSharedKeyCredential import examples logger = examples.get_logger(logging.INFO) -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') +HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net +EVENT_HUB = os.environ.get('EVENT_HUB_NAME') -# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') try: - if not ADDRESS: + if not HOSTNAME: raise ValueError("No EventHubs URL supplied.") - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="0") - client.run() + client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) + sender = client.create_sender(partition_id="0") + + ed = EventData("msg") + try: start_time = time.time() - for i in range(100): - logger.info("Sending message: {}".format(i)) - sender.send(EventData(str(i))) + with sender: + for i in range(100): + logger.info("Sending message: {}".format(i)) + sender.send(ed) except: raise finally: end_time = time.time() - client.stop() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/send_async.py b/sdk/eventhub/azure-eventhubs/examples/send_async.py deleted file mode 100644 index 248fdcf853b9..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/send_async.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -""" -An example to show sending events asynchronously to an Event Hub with partition keys. -""" - -# pylint: disable=C0111 - -import sys -import logging -import time -import asyncio -import os - -from azure.eventhub import EventData, EventHubClientAsync, AsyncSender - -import examples -logger = examples.get_logger(logging.INFO) - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -async def run(client): - sender = client.add_async_sender() - await client.run_async() - await send(sender, 4) - - -async def send(snd, count): - for i in range(count): - logger.info("Sending message: {}".format(i)) - data = EventData(str(i)) - data.partition_key = b'SamplePartitionKey' - await snd.send(data) - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - loop = asyncio.get_event_loop() - client = EventHubClientAsync(ADDRESS, debug=True, username=USER, password=KEY) - tasks = asyncio.gather( - run(client), - run(client)) - start_time = time.time() - loop.run_until_complete(tasks) - loop.run_until_complete(client.stop_async()) - end_time = time.time() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - loop.close() - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py index ffd53e02fd3e..fd1c1ba1773e 100644 --- a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py +++ b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py @@ -16,34 +16,22 @@ def create_eventhub_client(live_eventhub_config): # [START create_eventhub_client] import os - from azure.eventhub import EventHubClient + from azure.eventhub import EventHubClient, EventHubSharedKeyCredential - address = os.environ['EVENT_HUB_ADDRESS'] + host = os.environ['EVENT_HUB_HOSTNAME'] + event_hub_path = os.environ['EVENT_HUB_NAME'] shared_access_policy = os.environ['EVENT_HUB_SAS_POLICY'] shared_access_key = os.environ['EVENT_HUB_SAS_KEY'] client = EventHubClient( - address=address, - username=shared_access_policy, - password=shared_access_key) + host=host, + event_hub_path=event_hub_path, + credential=EventHubSharedKeyCredential(shared_access_policy, shared_access_key) + ) # [END create_eventhub_client] return client -def create_eventhub_client_from_sas_token(live_eventhub_config): - # [START create_eventhub_client_sas_token] - import os - from azure.eventhub import EventHubClient - - address = os.environ['EVENT_HUB_ADDRESS'] - sas_token = os.environ['EVENT_HUB_SAS_TOKEN'] - - client = EventHubClient.from_sas_token( - address=address, - sas_token=sas_token) - # [END create_eventhub_client_sas_token] - - def create_eventhub_client_from_iothub_connection_string(live_eventhub_config): # [START create_eventhub_client_iot_connstr] import os @@ -67,188 +55,82 @@ def test_example_eventhub_sync_send_and_receive(live_eventhub_config): client = EventHubClient.from_connection_string(connection_str) # [END create_eventhub_client_connstr] - from azure.eventhub import EventData, Offset + from azure.eventhub import EventData, EventPosition # [START create_eventhub_client_sender] - client = EventHubClient.from_connection_string(connection_str) - # Add a sender to the client object. - sender = client.add_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str) + # Create a sender. + sender = client.create_sender(partition_id="0") # [END create_eventhub_client_sender] # [START create_eventhub_client_receiver] - client = EventHubClient.from_connection_string(connection_str) - # Add a receiver to the client object. - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - # [END create_eventhub_client_receiver] - - # [START create_eventhub_client_epoch_receiver] - client = EventHubClient.from_connection_string(connection_str) - # Add a receiver to the client object with an epoch value. - epoch_receiver = client.add_epoch_receiver(consumer_group="$default", partition="0", epoch=42) - # [END create_eventhub_client_epoch_receiver] - - # [START eventhub_client_run] client = EventHubClient.from_connection_string(connection_str) - # Add Senders/Receivers - try: - client.run() - # Start sending and receiving - except: - raise - finally: - client.stop() - # [END eventhub_client_run] + # Create a receiver. + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) + # Create an exclusive receiver object. + exclusive_receiver = client.create_receiver(partition_id="0", exclusive_receiver_priority=1) + # [END create_eventhub_client_receiver] client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + sender = client.create_sender(partition_id="0") + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) try: - # Opens the connection and starts running all Sender/Receiver clients. - client.run() - # Start sending and receiving + receiver.receive(timeout=1) # [START create_event_data] event_data = EventData("String data") event_data = EventData(b"Bytes data") event_data = EventData([b"A", b"B", b"C"]) - def batched(): - for i in range(10): - yield "Batch data, Event number {}".format(i) - - event_data = EventData(batch=batched()) + list_data = ['Message {}'.format(i) for i in range(10)] + event_data = EventData(body=list_data) # [END create_event_data] # [START eventhub_client_sync_send] - event_data = EventData(b"A single event") - sender.send(event_data) + with sender: + event_data = EventData(b"A single event") + sender.send(event_data) # [END eventhub_client_sync_send] time.sleep(1) # [START eventhub_client_sync_receive] - logger = logging.getLogger("azure.eventhub") - received = receiver.receive(timeout=5, max_batch_size=1) - for event_data in received: - logger.info("Message received:{}".format(event_data.body_as_str())) + with receiver: + logger = logging.getLogger("azure.eventhub") + received = receiver.receive(timeout=5, max_batch_size=1) + for event_data in received: + logger.info("Message received:{}".format(event_data.body_as_str())) # [END eventhub_client_sync_receive] - assert len(received) == 1 - assert received[0].body_as_str() == "A single event" - assert list(received[-1].body)[0] == b"A single event" - except: - raise - - finally: - client.stop() - - # [START eventhub_client_stop] - client = EventHubClient.from_connection_string(connection_str) - # Add Senders/Receivers - try: - client.run() - # Start sending and receiving - except: - raise + assert len(received) == 1 + assert received[0].body_as_str() == "A single event" + assert list(received[-1].body)[0] == b"A single event" finally: - client.stop() - # [END eventhub_client_stop] + pass -def test_example_eventhub_transfer(connection_str): - import os +def test_example_eventhub_sender_ops(live_eventhub_config, connection_str): from azure.eventhub import EventHubClient, EventData - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender() - - try: - client.run() - # [START eventhub_client_transfer] - logger = logging.getLogger("azure.eventhub") - def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - event_data = EventData(b"A single event") - sender.transfer(event_data, callback=callback) - sender.wait() - # [END eventhub_client_transfer] - except: - raise - finally: - client.stop() - - -def test_example_eventhub_sync_sender_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_sender_instance] - from azure.eventhub import EventHubClient - - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - # [END create_eventhub_client_sender_instance] - - # [START eventhub_client_sender_open] - client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") - try: - # Open the Sender using the supplied conneciton. - sender.open() - # Start sending - except: - raise - finally: - # Close down the send handler. - sender.close() - # [END eventhub_client_sender_open] - # [START eventhub_client_sender_close] client = EventHubClient.from_connection_string(connection_str) - sender = client.add_sender(partition="0") + sender = client.create_sender(partition_id="0") try: - # Open the Sender using the supplied conneciton. - sender.open() - # Start sending - except: - raise + sender.send(EventData(b"A single event")) finally: # Close down the send handler. sender.close() - # [END eventhub_client_sender_close] - - -def test_example_eventhub_sync_receiver_ops(live_eventhub_config, connection_str): - import os - # [START create_eventhub_client_receiver_instance] - from azure.eventhub import EventHubClient, Offset + # [END eventhub_client_sender_close] - client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - # [END create_eventhub_client_receiver_instance] - # [START eventhub_client_receiver_open] - client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) - try: - # Open the Receiver using the supplied conneciton. - receiver.open() - # Start receiving - except: - raise - finally: - # Close down the receive handler. - receiver.close() - # [END eventhub_client_receiver_open] +def test_example_eventhub_receiver_ops(live_eventhub_config, connection_str): + from azure.eventhub import EventHubClient + from azure.eventhub import EventPosition # [START eventhub_client_receiver_close] client = EventHubClient.from_connection_string(connection_str) - receiver = client.add_receiver(consumer_group="$default", partition="0", offset=Offset('@latest')) + receiver = client.create_receiver(partition_id="0", consumer_group="$default", event_position=EventPosition('@latest')) try: - # Open the Receiver using the supplied conneciton. - receiver.open() - # Start receiving - except: - raise + receiver.receive(timeout=1) finally: # Close down the receive handler. receiver.close() - # [END eventhub_client_receiver_close] \ No newline at end of file + # [END eventhub_client_receiver_close] diff --git a/sdk/eventhub/azure-eventhubs/examples/transfer.py b/sdk/eventhub/azure-eventhubs/examples/transfer.py deleted file mode 100644 index 5190add90f5f..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/transfer.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -""" -An example to show sending events to an Event Hub. -""" - -# pylint: disable=C0111 - -import sys -import logging -import datetime -import time -import os - -from azure.eventhub import EventHubClient, Sender, EventData - -import examples -logger = examples.get_logger(logging.INFO) - - -# Address can be in either of these formats: -# "amqps://:@.servicebus.windows.net/myeventhub" -# "amqps://.servicebus.windows.net/myeventhub" -ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') - -# SAS policy and key are not required if they are encoded in the URL -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -def callback(outcome, condition): - logger.info("Message sent. Outcome: {}, Condition: {}".format( - outcome, condition)) - - -try: - if not ADDRESS: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) - sender = client.add_sender(partition="1") - client.run() - try: - start_time = time.time() - for i in range(100): - sender.transfer(EventData(str(i)), callback=callback) - logger.info("Queued 100 messages.") - sender.wait() - logger.info("Finished processing queue.") - except: - raise - finally: - end_time = time.time() - client.stop() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml new file mode 100644 index 000000000000..e7687fdae93b --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/sdk_packaging.toml @@ -0,0 +1,2 @@ +[packaging] +auto_update = false \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 6f6cf3399021..170aeccceb87 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -41,14 +41,14 @@ 'examples', # Exclude packages that will be covered by PEP420 or nspkg 'azure', - '*.eventprocessorhost', - '*.eventprocessorhost.*' ] -if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): +if sys.version_info < (3, 5, 3): exclude_packages.extend([ '*.aio', - '*.aio.*' + '*.aio.*', + '*.eventprocessorhost', + '*.eventprocessorhost.*' ]) setup( @@ -66,7 +66,6 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - # 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -79,7 +78,7 @@ 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', 'azure-storage-blob~=1.3', - 'azure-core~=1.0', + 'azure-core>=0.0.1', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py new file mode 100644 index 000000000000..4799df1a8634 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_auth_async.py @@ -0,0 +1,46 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time +import asyncio + +from azure.eventhub import EventData, EventPosition +from azure.eventhub.aio import EventHubClient + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_client_secret_credential_async(aad_credential, live_eventhub): + try: + from azure.identity.aio import AsyncClientSecretCredential + except ImportError: + pytest.skip("No azure identity library") + + client_id, secret, tenant_id = aad_credential + credential = AsyncClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) + client = EventHubClient(host=live_eventhub['hostname'], + event_hub_path=live_eventhub['event_hub'], + credential=credential, + user_agent='customized information') + sender = client.create_sender(partition_id='0') + receiver = client.create_receiver(partition_id='0', event_position=EventPosition.new_events_only()) + + async with receiver: + + received = await receiver.receive(timeout=1) + assert len(received) == 0 + + async with sender: + event = EventData(body='A single message') + await sender.send(event) + + await asyncio.sleep(1) + + received = await receiver.receive(timeout=1) + + assert len(received) == 1 + assert list(received[0].body)[0] == 'A single message'.encode('utf-8') diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py index fdb5c1ffea35..c7ebd17a9fbc 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py @@ -34,6 +34,7 @@ async def get_partitions(iot_connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_iothub_receive_multiple_async(iot_connection_str): + pytest.skip("This will get AuthenticationError. We're investigating...") partitions = await get_partitions(iot_connection_str) client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) receivers = [] diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py index 78611b3bf2ef..11fda8eddefa 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py @@ -45,6 +45,7 @@ def get_logger(filename, level=logging.INFO): return azure_logger + logger = get_logger("eph_test_async.log", logging.INFO) @@ -125,13 +126,15 @@ async def pump(pid, sender, duration): try: async with sender: + list = [] while time.time() < deadline: data = EventData(body=b"D" * 512) - sender.queue_message(data) + list.append(data) total += 1 if total % 100 == 0: - await sender.send_pending_messages() - #logger.info("{}: Send total {}".format(pid, total)) + await sender.send(data) + list = [] + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -139,7 +142,8 @@ async def pump(pid, sender, duration): @pytest.mark.liveTest -def test_long_running_eph(live_eventhub): +@pytest.mark.asyncio +async def test_long_running_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) @@ -169,9 +173,9 @@ def test_long_running_eph(live_eventhub): send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.create_sender(partition_id=pid, send_timeout=0, keep_alive=False) + sender = send_client.create_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) - results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) + results = await asyncio.gather(*pumps, return_exceptions=True) assert not any(results) @@ -206,7 +210,7 @@ def test_long_running_eph(live_eventhub): tasks = asyncio.gather( host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) - results = loop.run_until_complete(tasks) + results = await tasks assert not any(results) @@ -219,4 +223,5 @@ def test_long_running_eph(live_eventhub): config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] config['consumer_group'] = "$Default" config['partition'] = "0" - test_long_running_eph(config) + loop = asyncio.get_event_loop() + loop.run_until_complete(test_long_running_eph(config)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py index 7b4a9021db1d..2a6a83048251 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py @@ -130,13 +130,15 @@ async def pump(pid, sender, duration): try: async with sender: + list = [] while time.time() < deadline: data = EventData(body=b"D" * 512) - sender.queue_message(data) + list.append(data) total += 1 if total % 100 == 0: - await sender.send_pending_messages() - #logger.info("{}: Send total {}".format(pid, total)) + await sender.send(list) + list = [] + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -144,7 +146,8 @@ async def pump(pid, sender, duration): @pytest.mark.liveTest -def test_long_running_context_eph(live_eventhub): +@pytest.mark.asyncio +async def test_long_running_context_eph(live_eventhub): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--storage-account", help="Storage account name", default=os.environ.get('AZURE_STORAGE_ACCOUNT')) @@ -174,9 +177,9 @@ def test_long_running_context_eph(live_eventhub): send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.add_async_sender(partition_id=pid, send_timeout=0) + sender = send_client.create_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) - results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) + results = await asyncio.gather(*pumps, return_exceptions=True) assert not any(results) # Eventhub config and storage manager @@ -210,7 +213,7 @@ def test_long_running_context_eph(live_eventhub): tasks = asyncio.gather( host.open_async(), wait_and_close(host, args.duration), return_exceptions=True) - results = loop.run_until_complete(tasks) + results = await tasks assert not any(results) @@ -223,4 +226,5 @@ def test_long_running_context_eph(live_eventhub): config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] config['consumer_group'] = "$Default" config['partition'] = "0" - test_long_running_context_eph(config) + loop = asyncio.get_event_loop() + loop.run_until_complete(test_long_running_context_eph(config)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py index 1f50144674b8..63c7940a539e 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py @@ -18,7 +18,7 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventPosition +from azure.eventhub import EventPosition, EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient @@ -103,13 +103,14 @@ async def test_long_running_receive_async(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, auth_timeout=240, network_tracing=False) + event_hub_path=args.eventhub, auth_timeout=240, network_tracing=False) elif args.address: - client = EventHubClient( - args.address, - auth_timeout=240, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) + else: try: import pytest @@ -117,22 +118,19 @@ async def test_long_running_receive_async(connection_str): except ImportError: raise ValueError("Must specify either '--conn-str' or '--address'") - try: - if not args.partitions: - partitions = await client.get_partition_ids() - else: - partitions = args.partitions.split(",") - pumps = [] - for pid in partitions: - receiver = client.create_receiver( - partition_id=pid, - event_position=EventPosition(args.offset), - prefetch=50, - loop=loop) - pumps.append(pump(pid, receiver, args, args.duration)) - await asyncio.gather(*pumps) - finally: - pass + if not args.partitions: + partitions = await client.get_partition_ids() + else: + partitions = args.partitions.split(",") + pumps = [] + for pid in partitions: + receiver = client.create_receiver( + partition_id=pid, + event_position=EventPosition(args.offset), + prefetch=50, + loop=loop) + pumps.append(pump(pid, receiver, args, args.duration)) + await asyncio.gather(*pumps) if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py index dd87e5324558..98dcccb0ea79 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py @@ -104,13 +104,12 @@ async def test_long_running_partition_send_async(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, network_tracing=True) + event_hub_path=args.eventhub, network_tracing=True) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key, - auth_timeout=500) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + network_tracing=False) else: try: import pytest @@ -134,9 +133,7 @@ async def test_long_running_partition_send_async(connection_str): results = await asyncio.gather(*pumps, return_exceptions=True) assert not results except Exception as e: - logger.error("Sender failed: {}".format(e)) - finally: - pass + logger.error("EventSender failed: {}".format(e)) if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py index 4e904d19453f..75f168027644 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py @@ -15,8 +15,10 @@ EventPosition, EventHubError, ConnectError, + ConnectionLostError, AuthenticationError, EventDataError, + EventDataSendError, ) from azure.eventhub.aio import EventHubClient @@ -95,7 +97,7 @@ async def test_send_partition_key_with_partition_async(connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_sender_async(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) sender = client.create_sender(partition_id="1") with pytest.raises(AuthenticationError): await sender._open() @@ -104,7 +106,7 @@ async def test_non_existing_entity_sender_async(connection_str): @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_receiver_async(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) receiver = client.create_receiver(partition_id="0") with pytest.raises(AuthenticationError): await receiver._open() @@ -147,7 +149,7 @@ async def test_send_too_large_message_async(connection_str): sender = client.create_sender() try: data = EventData(b"A" * 1100000) - with pytest.raises(EventDataError): + with pytest.raises(EventDataSendError): await sender.send(data) finally: await sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py index 20641033e5bb..9401c65515aa 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py @@ -5,7 +5,7 @@ #-------------------------------------------------------------------------- import pytest -from azure.eventhub import SharedKeyCredentials +from azure.eventhub import EventHubSharedKeyCredential from azure.eventhub.aio import EventHubClient @@ -13,7 +13,7 @@ @pytest.mark.asyncio async def test_get_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = await client.get_properties() assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] @@ -23,7 +23,7 @@ async def test_get_properties(live_eventhub): @pytest.mark.asyncio async def test_get_partition_ids(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) partition_ids = await client.get_partition_ids() assert partition_ids == ['0', '1'] @@ -33,7 +33,7 @@ async def test_get_partition_ids(live_eventhub): @pytest.mark.asyncio async def test_get_partition_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = await client.get_partition_properties('0') assert properties['event_hub_path'] == live_eventhub['event_hub'] \ diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py new file mode 100644 index 000000000000..b087bb783bb1 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receiver_iterator_async.py @@ -0,0 +1,30 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os +import asyncio +import pytest +import time + +from azure.eventhub import EventData, EventPosition, EventHubError, TransportType +from azure.eventhub.aio import EventHubClient + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_receive_iterator_async(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) + async with receiver: + received = await receiver.receive(timeout=5) + assert len(received) == 0 + senders[0].send(EventData(b"Receiving only a single event")) + async for item in receiver: + received.append(item) + break + assert len(received) == 1 + assert list(received[-1].body)[0] == b"Receiving only a single event" \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py index ebbace8c8a05..e7a4fcad61f0 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py @@ -15,6 +15,7 @@ EventHubError) from azure.eventhub.aio import EventHubClient +SLEEP = False @pytest.mark.liveTest @pytest.mark.asyncio @@ -25,15 +26,18 @@ async def test_send_with_long_interval_async(connstr_receivers): try: await sender.send(EventData(b"A single event")) for _ in range(1): - #await asyncio.sleep(300) - sender._handler._connection._conn.destroy() + if SLEEP: + await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) finally: await sender.close() received = [] for r in receivers: - r._handler._connection._conn.destroy() + if not SLEEP: # if sender sleeps, the receivers will be disconnected. destroy connection to simulate + r._handler._connection._conn.destroy() received.extend(r.receive(timeout=1)) assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" @@ -58,12 +62,16 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): sender = client.create_sender() try: await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) + if SLEEP: + await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() - await asyncio.sleep(300) + if SLEEP: + await asyncio.sleep(300) + else: + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) finally: @@ -71,6 +79,8 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): received = [] for r in receivers: - received.extend(pump(r)) + if not SLEEP: + r._handler._connection._conn.destroy() + received.extend(pump(r)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py index 9883be044345..fc3a5559415d 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py @@ -30,17 +30,17 @@ async def test_send_with_partition_key_async(connstr_receivers): data = EventData(str(data_val)) # data.partition_key = partition_key data_val += 1 - await sender.send(data, batching_label=partition_key) + await sender.send(data, partition_key=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message._batching_label] + existing = found_partition_keys[message.partition_key] assert existing == index except KeyError: - found_partition_keys[message._batching_label] = index + found_partition_keys[message.partition_key] = index @pytest.mark.liveTest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_auth.py b/sdk/eventhub/azure-eventhubs/tests/test_auth.py new file mode 100644 index 000000000000..aa728d2d54de --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/test_auth.py @@ -0,0 +1,40 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time + +from azure.eventhub import EventData, EventHubClient, EventPosition + + +@pytest.mark.liveTest +def test_client_secret_credential(aad_credential, live_eventhub): + try: + from azure.identity import ClientSecretCredential + except ImportError: + pytest.skip("No azure identity library") + client_id, secret, tenant_id = aad_credential + credential = ClientSecretCredential(client_id=client_id, secret=secret, tenant_id=tenant_id) + client = EventHubClient(host=live_eventhub['hostname'], + event_hub_path=live_eventhub['event_hub'], + credential=credential, + user_agent='customized information') + sender = client.create_sender(partition_id='0') + receiver = client.create_receiver(partition_id='0', event_position=EventPosition.new_events_only()) + + with receiver: + received = receiver.receive(timeout=1) + assert len(received) == 0 + + with sender: + event = EventData(body='A single message') + sender.send(event) + time.sleep(1) + + received = receiver.receive(timeout=1) + + assert len(received) == 1 + assert list(received[0].body)[0] == 'A single message'.encode('utf-8') diff --git a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py index df7a184a227e..75a39eb185bf 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py @@ -20,7 +20,5 @@ def test_iothub_send_single_event(iot_connection_str, device_id): sender = client.create_sender(operation='/messages/devicebound') try: sender.send(EventData(b"A single event", to_device=device_id)) - except: - raise finally: sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py index 7ae53a0b2496..b7477bc3e39f 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py @@ -20,6 +20,7 @@ from azure.eventhub import EventPosition from azure.eventhub import EventHubClient +from azure.eventhub import EventHubSharedKeyCredential def get_logger(filename, level=logging.INFO): azure_logger = logging.getLogger("azure.eventhub") @@ -76,7 +77,7 @@ def pump(receivers, duration): batch[-1].offset.value)) print("Total received {}".format(total)) except Exception as e: - print("Receiver failed: {}".format(e)) + print("EventReceiver failed: {}".format(e)) raise @@ -97,12 +98,13 @@ def test_long_running_receive(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, network_tracing=False) + event_hub_path=args.eventhub, network_tracing=False) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) else: try: import pytest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py index 90c6d0dc3cf9..ead603ed7372 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py @@ -17,7 +17,7 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventHubClient, Sender, EventData +from azure.eventhub import EventHubClient, EventSender, EventData def get_logger(filename, level=logging.INFO): @@ -101,12 +101,13 @@ def test_long_running_send(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub) + event_hub_path=args.eventhub) elif args.address: - client = EventHubClient( - args.address, - username=args.sas_policy, - password=args.sas_key) + client = EventHubClient(host=args.address, + event_hub_path=args.eventhub, + credential=EventHubSharedKeyCredential(args.sas_policy, args.sas_key), + auth_timeout=240, + network_tracing=False) else: try: import pytest diff --git a/sdk/eventhub/azure-eventhubs/tests/test_negative.py b/sdk/eventhub/azure-eventhubs/tests/test_negative.py index 206c5b415002..7111840995f9 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_negative.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_negative.py @@ -16,6 +16,7 @@ AuthenticationError, ConnectError, EventDataError, + EventDataSendError, EventHubClient) @@ -86,7 +87,7 @@ def test_send_partition_key_with_partition_sync(connection_str): @pytest.mark.liveTest def test_non_existing_entity_sender(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) sender = client.create_sender(partition_id="1") with pytest.raises(AuthenticationError): sender._open() @@ -94,7 +95,7 @@ def test_non_existing_entity_sender(connection_str): @pytest.mark.liveTest def test_non_existing_entity_receiver(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + client = EventHubClient.from_connection_string(connection_str, event_hub_path="nemo", network_tracing=False) receiver = client.create_receiver(partition_id="0") with pytest.raises(AuthenticationError): receiver._open() @@ -134,7 +135,7 @@ def test_send_too_large_message(connection_str): sender = client.create_sender() try: data = EventData(b"A" * 1100000) - with pytest.raises(EventDataError): + with pytest.raises(EventDataSendError): sender.send(data) finally: sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_properties.py b/sdk/eventhub/azure-eventhubs/tests/test_properties.py index b1889bdcf179..d16820a00083 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_properties.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_properties.py @@ -5,13 +5,13 @@ #-------------------------------------------------------------------------- import pytest -from azure.eventhub import EventHubClient, SharedKeyCredentials +from azure.eventhub import EventHubClient, EventHubSharedKeyCredential @pytest.mark.liveTest def test_get_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = client.get_properties() assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] @@ -20,7 +20,7 @@ def test_get_properties(live_eventhub): @pytest.mark.liveTest def test_get_partition_ids(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) partition_ids = client.get_partition_ids() assert partition_ids == ['0', '1'] @@ -29,7 +29,7 @@ def test_get_partition_ids(live_eventhub): @pytest.mark.liveTest def test_get_partition_properties(live_eventhub): client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], - SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + EventHubSharedKeyCredential(live_eventhub['key_name'], live_eventhub['access_key']) ) properties = client.get_partition_properties('0') assert properties['event_hub_path'] == live_eventhub['event_hub'] \ diff --git a/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py b/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py new file mode 100644 index 000000000000..6f0dd3456df6 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/test_receiver_iterator.py @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import time +import datetime +from threading import Thread + +from azure.eventhub import EventData, EventHubClient, EventPosition, TransportType + + +@pytest.mark.liveTest +def test_receive_iterator(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) + with receiver: + received = receiver.receive(timeout=5) + assert len(received) == 0 + senders[0].send(EventData(b"Receiving only a single event")) + + for item in receiver: + received.append(item) + break + + assert len(received) == 1 + assert received[0].body_as_str() == "Receiving only a single event" + assert list(received[-1].body)[0] == b"Receiving only a single event" diff --git a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py index b24cca267c82..d65f774108e0 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py @@ -14,6 +14,7 @@ EventHubError, EventHubClient) +SLEEP = False @pytest.mark.liveTest def test_send_with_long_interval_sync(connstr_receivers): @@ -23,12 +24,17 @@ def test_send_with_long_interval_sync(connstr_receivers): with sender: sender.send(EventData(b"A single event")) for _ in range(1): - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) received = [] for r in receivers: - received.extend(r.receive(timeout=1)) + if not SLEEP: + r._handler._connection._conn.destroy() + received.extend(r.receive(timeout=3)) assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" @@ -42,16 +48,23 @@ def test_send_with_forced_conn_close_sync(connstr_receivers): with sender: sender.send(EventData(b"A single event")) sender._handler._connection._conn.destroy() - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) - sender._handler._connection._conn.destroy() - time.sleep(300) + if SLEEP: + time.sleep(300) + else: + sender._handler._connection._conn.destroy() sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) received = [] for r in receivers: - received.extend(r.receive(timeout=1)) + if not SLEEP: + r._handler._connection._conn.destroy() + received.extend(r.receive(timeout=1)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" diff --git a/sdk/eventhub/azure-eventhubs/tests/test_send.py b/sdk/eventhub/azure-eventhubs/tests/test_send.py index 3af0cbed2ef2..5a497f34dcca 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_send.py @@ -26,17 +26,17 @@ def test_send_with_partition_key(connstr_receivers): data = EventData(str(data_val)) #data.partition_key = partition_key data_val += 1 - sender.send(data, batching_label=partition_key) + sender.send(data, partition_key=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message._batching_label] + existing = found_partition_keys[message._partition_key] assert existing == index except KeyError: - found_partition_keys[message._batching_label] = index + found_partition_keys[message._partition_key] = index @pytest.mark.liveTest