diff --git a/pylintrc b/pylintrc index c5cddb6d9030..399344e8ea99 100644 --- a/pylintrc +++ b/pylintrc @@ -2,8 +2,8 @@ ignore-patterns=test_*,conftest,setup reports=no -# PYLINT DIRECTORY BLACKLIST. Ignore eventprocessor temporarily until new eventprocessor code is merged to master -ignore=_generated,samples,examples,test,tests,doc,.tox,eventprocessor +# PYLINT DIRECTORY BLACKLIST. +ignore=_generated,samples,examples,test,tests,doc,.tox init-hook='import sys; sys.path.insert(0, os.path.abspath(os.getcwd().rsplit("azure-sdk-for-python", 1)[0] + "azure-sdk-for-python/scripts/pylint_custom_plugin"))' load-plugins=pylint_guidelines_checker diff --git a/sdk/eventhub/azure-eventhubs/HISTORY.md b/sdk/eventhub/azure-eventhubs/HISTORY.md index b51e636b4628..34e9d718e7f1 100644 --- a/sdk/eventhub/azure-eventhubs/HISTORY.md +++ b/sdk/eventhub/azure-eventhubs/HISTORY.md @@ -1,4 +1,18 @@ # Release History +## 5.0.0b3 (2019-09-10) + +**New features** +- `EventProcessor` has a load balancer that balances load among multiple EventProcessors automatically +- In addition to `SamplePartitionManager`, A new `PartitionManager` implementation that uses Azure Blob Storage is added +to centrally store the checkpoint data for event processors. It's not packaged separately as a plug-in to this package. +Refer to [Azure Blob Storage Partition Manager](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio) for details. + +**Breaking changes** + +- `PartitionProcessor` constructor removed argument "checkpoint_manager". Its methods (initialize, process_events, +process_error, close) added argument "partition_context", which has method update_checkpoint. +- `CheckpointManager` was replaced by `PartitionContext` +- Renamed `Sqlite3PartitionManager` to `SamplePartitionManager` ## 5.0.0b2 (2019-08-06) diff --git a/sdk/eventhub/azure-eventhubs/README.md b/sdk/eventhub/azure-eventhubs/README.md index 621c7ebe8179..94280c2df912 100644 --- a/sdk/eventhub/azure-eventhubs/README.md +++ b/sdk/eventhub/azure-eventhubs/README.md @@ -217,13 +217,16 @@ Using an `EventHubConsumer` to consume events like in the previous examples puts The `EventProcessor` will delegate the processing of events to a `PartitionProcessor` that you provide, allowing you to focus on business logic while the processor holds responsibility for managing the underlying consumer operations including checkpointing and load balancing. -While load balancing is a feature we will be adding in the next update, you can see how to use the `EventProcessor` in the below example, where we use an in memory `PartitionManager` that does checkpointing in memory. +You can see how to use the `EventProcessor` in the below example, where we use an in memory `PartitionManager` that does checkpointing in memory. + +[Azure Blob Storage Partition Manager](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio) is another `PartitionManager` implementation that allows multiple EventProcessors to share the load balancing and checkpoint data in a central storage. + ```python import asyncio from azure.eventhub.aio import EventHubClient -from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager +from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor, SamplePartitionManager connection_str = '<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>' @@ -232,24 +235,16 @@ async def do_operation(event): print(event) class MyPartitionProcessor(PartitionProcessor): - def __init__(self, checkpoint_manager): - super(MyPartitionProcessor, self).__init__(checkpoint_manager) - - async def process_events(self, events): + async def process_events(self, events, partition_context): if events: await asyncio.gather(*[do_operation(event) for event in events]) - await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number) - -def partition_processor_factory(checkpoint_manager): - return MyPartitionProcessor(checkpoint_manager) + await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number) async def main(): client = EventHubClient.from_connection_string(connection_str, receive_timeout=5, retry_total=3) - partition_manager = Sqlite3PartitionManager() # in-memory PartitionManager + partition_manager = SamplePartitionManager() # in-memory PartitionManager. try: event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager) - # You can also define a callable object for creating PartitionProcessor like below: - # event_processor = EventProcessor(client, "$default", partition_processor_factory, partition_manager) asyncio.ensure_future(event_processor.start()) await asyncio.sleep(60) await event_processor.stop() @@ -273,6 +268,7 @@ The Event Hubs APIs generate the following exceptions. - **EventDataError:** The EventData to be sent fails data validation. For instance, this error is raised if you try to send an EventData that is already sent. - **EventDataSendError:** The Eventhubs service responds with an error when an EventData is sent. +- **OperationTimeoutError:** EventHubConsumer.send() times out. - **EventHubError:** All other Eventhubs related errors. It is also the root error class of all the above mentioned errors. ## Next steps diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index 040d00c947d8..dfc198f71fa8 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -__version__ = "5.0.0b2" +__version__ = "5.0.0b3" from uamqp import constants # type: ignore from azure.eventhub.common import EventData, EventDataBatch, EventPosition from azure.eventhub.error import EventHubError, EventDataError, ConnectError, \ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py index fd12b439d3d4..a14da749ee78 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/_consumer_producer_mixin.py @@ -13,29 +13,11 @@ log = logging.getLogger(__name__) -def _retry_decorator(to_be_wrapped_func): - def wrapped_func(self, *args, **kwargs): # pylint:disable=unused-argument # TODO: to refactor - timeout = kwargs.pop("timeout", 100000) - if not timeout: - timeout = 100000 # timeout equals to 0 means no timeout, set the value to be a large number. - timeout_time = time.time() + timeout - max_retries = self.client.config.max_retries - retry_count = 0 - last_exception = None - while True: - try: - return to_be_wrapped_func(self, timeout_time=timeout_time, last_exception=last_exception, **kwargs) - except Exception as exception: # pylint:disable=broad-except - last_exception = self._handle_exception(exception, retry_count, max_retries, timeout_time) # pylint:disable=protected-access - retry_count += 1 - return wrapped_func - - class ConsumerProducerMixin(object): def __init__(self): - self.client = None + self._client = None self._handler = None - self.name = None + self._name = None def __enter__(self): return self @@ -44,59 +26,81 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close(exc_val) def _check_closed(self): - if self.error: - raise EventHubError("{} has been closed. Please create a new one to handle event data.".format(self.name)) + if self._error: + raise EventHubError("{} has been closed. Please create a new one to handle event data.".format(self._name)) def _create_handler(self): pass def _redirect(self, redirect): - self.redirected = redirect - self.running = False + self._redirected = redirect + self._running = False self._close_connection() - def _open(self, timeout_time=None): # pylint:disable=unused-argument # TODO: to refactor + def _open(self): """ - Open the EventHubConsumer using the supplied connection. + Open the EventHubConsumer/EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ # pylint: disable=protected-access - if not self.running: + if not self._running: if self._handler: self._handler.close() - if self.redirected: + if self._redirected: alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} + "username": self._client._auth_config.get("iot_username"), + "password": self._client._auth_config.get("iot_password")} else: alt_creds = {} self._create_handler() - self._handler.open(connection=self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth(**alt_creds) + self._handler.open(connection=self._client._conn_manager.get_connection( # pylint: disable=protected-access + self._client._address.hostname, + self._client._get_auth(**alt_creds) )) while not self._handler.client_ready(): time.sleep(0.05) self._max_message_size_on_link = self._handler.message_handler._link.peer_max_message_size \ or constants.MAX_MESSAGE_LENGTH_BYTES # pylint: disable=protected-access - self.running = True + self._running = True def _close_handler(self): self._handler.close() # close the link (sharing connection) or connection (not sharing) - self.running = False + self._running = False def _close_connection(self): self._close_handler() - self.client._conn_manager.reset_connection_if_broken() # pylint: disable=protected-access + self._client._conn_manager.reset_connection_if_broken() # pylint: disable=protected-access - def _handle_exception(self, exception, retry_count, max_retries, timeout_time): - if not self.running and isinstance(exception, compat.TimeoutException): + def _handle_exception(self, exception): + if not self._running and isinstance(exception, compat.TimeoutException): exception = errors.AuthenticationException("Authorization timeout.") - return _handle_exception(exception, retry_count, max_retries, self, timeout_time) + return _handle_exception(exception, self) + + return _handle_exception(exception, self) + + def _do_retryable_operation(self, operation, timeout=100000, **kwargs): + # pylint:disable=protected-access + timeout_time = time.time() + ( + timeout if timeout else 100000) # timeout equals to 0 means no timeout, set the value to be a large number. + retried_times = 0 + last_exception = kwargs.pop('last_exception', None) + operation_need_param = kwargs.pop('operation_need_param', True) + + while retried_times <= self._client._config.max_retries: # pylint: disable=protected-access + try: + if operation_need_param: + return operation(timeout_time=timeout_time, last_exception=last_exception, **kwargs) + return operation() + except Exception as exception: # pylint:disable=broad-except + last_exception = self._handle_exception(exception) + self._client._try_delay(retried_times=retried_times, last_exception=last_exception, + timeout_time=timeout_time, entity_name=self._name) + retried_times += 1 - return _handle_exception(exception, retry_count, max_retries, self, timeout_time) + log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception) + raise last_exception def close(self, exception=None): # type:(Exception) -> None @@ -118,16 +122,16 @@ def close(self, exception=None): :caption: Close down the handler. """ - self.running = False - if self.error: # type: ignore + self._running = False + if self._error: # type: ignore return if isinstance(exception, errors.LinkRedirect): - self.redirected = exception + self._redirected = exception elif isinstance(exception, EventHubError): - self.error = exception + self._error = exception elif exception: - self.error = EventHubError(str(exception)) + self._error = EventHubError(str(exception)) else: - self.error = EventHubError("{} handler is closed.".format(self.name)) + self._error = EventHubError("{} handler is closed.".format(self._name)) if self._handler: self._handler.close() # this will close link if sharing connection. Otherwise close connection diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py index 53624c36f649..444edd15a8a1 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/_consumer_producer_mixin_async.py @@ -13,32 +13,12 @@ log = logging.getLogger(__name__) -def _retry_decorator(to_be_wrapped_func): - async def wrapped_func(self, *args, **kwargs): # pylint:disable=unused-argument # TODO: to refactor - timeout = kwargs.pop("timeout", 100000) - if not timeout: - timeout = 100000 # timeout equals to 0 means no timeout, set the value to be a large number. - timeout_time = time.time() + timeout - max_retries = self.client.config.max_retries - retry_count = 0 - last_exception = None - while True: - try: - return await to_be_wrapped_func( - self, timeout_time=timeout_time, last_exception=last_exception, **kwargs - ) - except Exception as exception: # pylint:disable=broad-except - last_exception = await self._handle_exception(exception, retry_count, max_retries, timeout_time) # pylint:disable=protected-access - retry_count += 1 - return wrapped_func - - class ConsumerProducerMixin(object): def __init__(self): - self.client = None + self._client = None self._handler = None - self.name = None + self._name = None async def __aenter__(self): return self @@ -47,18 +27,18 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close(exc_val) def _check_closed(self): - if self.error: - raise EventHubError("{} has been closed. Please create a new one to handle event data.".format(self.name)) + if self._error: + raise EventHubError("{} has been closed. Please create a new one to handle event data.".format(self._name)) def _create_handler(self): pass async def _redirect(self, redirect): - self.redirected = redirect - self.running = False + self._redirected = redirect + self._running = False await self._close_connection() - async def _open(self, timeout_time=None): # pylint:disable=unused-argument # TODO: to refactor + async def _open(self): """ Open the EventHubConsumer using the supplied connection. If the handler has previously been redirected, the redirect @@ -66,40 +46,62 @@ async def _open(self, timeout_time=None): # pylint:disable=unused-argument # TO """ # pylint: disable=protected-access - if not self.running: + if not self._running: if self._handler: await self._handler.close_async() - if self.redirected: + if self._redirected: alt_creds = { - "username": self.client._auth_config.get("iot_username"), - "password": self.client._auth_config.get("iot_password")} + "username": self._client._auth_config.get("iot_username"), + "password": self._client._auth_config.get("iot_password")} else: alt_creds = {} self._create_handler() - await self._handler.open_async(connection=await self.client._conn_manager.get_connection( - self.client.address.hostname, - self.client.get_auth(**alt_creds) + await self._handler.open_async(connection=await self._client._conn_manager.get_connection( + self._client._address.hostname, + self._client._get_auth(**alt_creds) )) while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) self._max_message_size_on_link = self._handler.message_handler._link.peer_max_message_size \ or constants.MAX_MESSAGE_LENGTH_BYTES # pylint: disable=protected-access - self.running = True + self._running = True async def _close_handler(self): await self._handler.close_async() # close the link (sharing connection) or connection (not sharing) - self.running = False + self._running = False async def _close_connection(self): await self._close_handler() - await self.client._conn_manager.reset_connection_if_broken() # pylint:disable=protected-access + await self._client._conn_manager.reset_connection_if_broken() # pylint:disable=protected-access - async def _handle_exception(self, exception, retry_count, max_retries, timeout_time): - if not self.running and isinstance(exception, compat.TimeoutException): + async def _handle_exception(self, exception): + if not self._running and isinstance(exception, compat.TimeoutException): exception = errors.AuthenticationException("Authorization timeout.") - return await _handle_exception(exception, retry_count, max_retries, self, timeout_time) + return await _handle_exception(exception, self) + + return await _handle_exception(exception, self) + + async def _do_retryable_operation(self, operation, timeout=100000, **kwargs): + # pylint:disable=protected-access + timeout_time = time.time() + ( + timeout if timeout else 100000) # timeout equals to 0 means no timeout, set the value to be a large number. + retried_times = 0 + last_exception = kwargs.pop('last_exception', None) + operation_need_param = kwargs.pop('operation_need_param', True) + + while retried_times <= self._client._config.max_retries: + try: + if operation_need_param: + return await operation(timeout_time=timeout_time, last_exception=last_exception, **kwargs) + return await operation() + except Exception as exception: # pylint:disable=broad-except + last_exception = await self._handle_exception(exception) + await self._client._try_delay(retried_times=retried_times, last_exception=last_exception, + timeout_time=timeout_time, entity_name=self._name) + retried_times += 1 - return await _handle_exception(exception, retry_count, max_retries, self, timeout_time) + log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception) + raise last_exception async def close(self, exception=None): # type: (Exception) -> None @@ -121,18 +123,18 @@ async def close(self, exception=None): :caption: Close down the handler. """ - self.running = False - if self.error: #type: ignore + self._running = False + if self._error: #type: ignore return if isinstance(exception, errors.LinkRedirect): - self.redirected = exception + self._redirected = exception elif isinstance(exception, EventHubError): - self.error = exception + self._error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = ConnectError(str(exception), exception) + self._error = ConnectError(str(exception), exception) elif exception: - self.error = EventHubError(str(exception)) + self._error = EventHubError(str(exception)) else: - self.error = EventHubError("This receive handler is now closed.") + self._error = EventHubError("This receive handler is now closed.") if self._handler: await self._handler.close_async() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py index 84853ce1534b..67f6ab52dd30 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/client_async.py @@ -4,8 +4,10 @@ # -------------------------------------------------------------------------------------------- import logging import datetime +import time import functools import asyncio + from typing import Any, List, Dict, Union, TYPE_CHECKING from uamqp import authentication, constants # type: ignore @@ -47,6 +49,7 @@ class EventHubClient(EventHubClientAbstract): def __init__(self, host, event_hub_path, credential, **kwargs): # type:(str, str, Union[EventHubSharedKeyCredential, EventHubSASTokenCredential, TokenCredential], Any) -> None super(EventHubClient, self).__init__(host=host, event_hub_path=event_hub_path, credential=credential, **kwargs) + self._lock = asyncio.Lock() self._conn_manager = get_connection_manager(**kwargs) async def __aenter__(self): @@ -65,53 +68,66 @@ def _create_auth(self, username=None, password=None): :param password: The shared access key. :type password: str """ - http_proxy = self.config.http_proxy - transport_type = self.config.transport_type - auth_timeout = self.config.auth_timeout + http_proxy = self._config.http_proxy + transport_type = self._config.transport_type + auth_timeout = self._config.auth_timeout - if isinstance(self.credential, EventHubSharedKeyCredential): # pylint:disable=no-else-return + if isinstance(self._credential, EventHubSharedKeyCredential): # pylint:disable=no-else-return username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: return authentication.SASLPlain( - self.host, username, password, http_proxy=http_proxy, transport_type=transport_type) + self._host, username, password, http_proxy=http_proxy, transport_type=transport_type) return authentication.SASTokenAsync.from_shared_access_key( - self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, + self._auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, EventHubSASTokenCredential): - token = self.credential.get_sas_token() + elif isinstance(self._credential, EventHubSASTokenCredential): + token = self._credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) except (KeyError, TypeError, IndexError): raise ValueError("Supplied SAS token has no valid expiry value.") return authentication.SASTokenAsync( - self.auth_uri, self.auth_uri, token, + self._auth_uri, self._auth_uri, token, expires_at=expiry, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) else: - get_jwt_token = functools.partial(self.credential.get_token, 'https://eventhubs.azure.net//.default') - return authentication.JWTTokenAsync(self.auth_uri, self.auth_uri, + get_jwt_token = functools.partial(self._credential.get_token, 'https://eventhubs.azure.net//.default') + return authentication.JWTTokenAsync(self._auth_uri, self._auth_uri, get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - async def _handle_exception(self, exception, retry_count, max_retries): - await _handle_exception(exception, retry_count, max_retries, self) - async def _close_connection(self): await self._conn_manager.reset_connection_if_broken() + async def _try_delay(self, retried_times, last_exception, timeout_time=None, entity_name=None): + entity_name = entity_name or self._container_id + backoff = self._config.backoff_factor * 2 ** retried_times + if backoff <= self._config.backoff_max and ( + timeout_time is None or time.time() + backoff <= timeout_time): # pylint:disable=no-else-return + await asyncio.sleep(backoff) + log.info("%r has an exception (%r). Retrying...", format(entity_name), last_exception) + else: + log.info("%r operation has timed out. Last exception before timeout is (%r)", + entity_name, last_exception) + raise last_exception + async def _management_request(self, mgmt_msg, op_type): - max_retries = self.config.max_retries - retry_count = 0 - while True: - mgmt_auth = self._create_auth() - mgmt_client = AMQPClientAsync(self.mgmt_target, auth=mgmt_auth, debug=self.config.network_tracing) + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password": self._auth_config.get("iot_password") + } + + retried_times = 0 + while retried_times <= self._config.max_retries: + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = AMQPClientAsync(self._mgmt_target, auth=mgmt_auth, debug=self._config.network_tracing) try: - conn = await self._conn_manager.get_connection(self.host, mgmt_auth) + conn = await self._conn_manager.get_connection(self._host, mgmt_auth) await mgmt_client.open_async(connection=conn) response = await mgmt_client.mgmt_request_async( mgmt_msg, @@ -121,11 +137,24 @@ async def _management_request(self, mgmt_msg, op_type): description_fields=b'status-description') return response except Exception as exception: # pylint:disable=broad-except - await self._handle_exception(exception, retry_count, max_retries) - retry_count += 1 + last_exception = await _handle_exception(exception, self) + await self._try_delay(retried_times=retried_times, last_exception=last_exception) + retried_times += 1 finally: await mgmt_client.close_async() + async def _iothub_redirect(self): + async with self._lock: + if self._is_iothub and not self._iothub_redirect_info: + if not self._redirect_consumer: + self._redirect_consumer = self.create_consumer(consumer_group='$default', + partition_id='0', + event_position=EventPosition('-1'), + operation='/messages/events') + async with self._redirect_consumer: + await self._redirect_consumer._open_with_retry() # pylint: disable=protected-access + self._redirect_consumer = None + async def get_properties(self): # type:() -> Dict[str, Any] """ @@ -139,6 +168,8 @@ async def get_properties(self): :rtype: dict :raises: ~azure.eventhub.ConnectError """ + if self._is_iothub and not self._iothub_redirect_info: + await self._iothub_redirect() mgmt_msg = Message(application_properties={'name': self.eh_name}) response = await self._management_request(mgmt_msg, op_type=b'com.microsoft:eventhub') output = {} @@ -178,6 +209,8 @@ async def get_partition_properties(self, partition): :rtype: dict :raises: ~azure.eventhub.ConnectError """ + if self._is_iothub and not self._iothub_redirect_info: + await self._iothub_redirect() mgmt_msg = Message(application_properties={'name': self.eh_name, 'partition': partition}) response = await self._management_request(mgmt_msg, op_type=b'com.microsoft:partition') @@ -232,12 +265,12 @@ def create_consumer( """ owner_level = kwargs.get("owner_level") operation = kwargs.get("operation") - prefetch = kwargs.get("prefetch") or self.config.prefetch + prefetch = kwargs.get("prefetch") or self._config.prefetch loop = kwargs.get("loop") - path = self.address.path + operation if operation else self.address.path + path = self._address.path + operation if operation else self._address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, path, consumer_group, partition_id) + self._address.hostname, path, consumer_group, partition_id) handler = EventHubConsumer( self, source_url, event_position=event_position, owner_level=owner_level, prefetch=prefetch, loop=loop) @@ -276,10 +309,10 @@ def create_producer( """ - target = "amqps://{}{}".format(self.address.hostname, self.address.path) + target = "amqps://{}{}".format(self._address.hostname, self._address.path) if operation: target = target + operation - send_timeout = self.config.send_timeout if send_timeout is None else send_timeout + send_timeout = self._config.send_timeout if send_timeout is None else send_timeout handler = EventHubProducer( self, target, partition=partition_id, send_timeout=send_timeout, loop=loop) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py index 3747d1af4d9d..3aa1a7d6bbe5 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/consumer_async.py @@ -13,7 +13,7 @@ from azure.eventhub import EventData, EventPosition from azure.eventhub.error import EventHubError, ConnectError, _error_handler -from ._consumer_producer_mixin_async import ConsumerProducerMixin, _retry_decorator +from ._consumer_producer_mixin_async import ConsumerProducerMixin log = logging.getLogger(__name__) @@ -32,9 +32,9 @@ class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instan sometimes referred to as "Non-Epoch Consumers." """ - timeout = 0 - _epoch = b'com.microsoft:epoch' - _timeout = b'com.microsoft:timeout' + _timeout = 0 + _epoch_symbol = b'com.microsoft:epoch' + _timeout_symbol = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, source, **kwargs): @@ -64,77 +64,83 @@ def __init__( # pylint: disable=super-init-not-called loop = kwargs.get("loop", None) super(EventHubConsumer, self).__init__() - self.loop = loop or asyncio.get_event_loop() - self.running = False - self.client = client - self.source = source - self.offset = event_position - self.messages_iter = None - self.prefetch = prefetch - self.owner_level = owner_level - self.keep_alive = keep_alive - self.auto_reconnect = auto_reconnect - self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) - self.reconnect_backoff = 1 - self.redirected = None - self.error = None + self._loop = loop or asyncio.get_event_loop() + self._running = False + self._client = client + self._source = source + self._offset = event_position + self._messages_iter = None + self._prefetch = prefetch + self._owner_level = owner_level + self._keep_alive = keep_alive + self._auto_reconnect = auto_reconnect + self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access + self._reconnect_backoff = 1 + self._redirected = None + self._error = None self._link_properties = {} - partition = self.source.split('/')[-1] - self.partition = partition - self.name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) + partition = self._source.split('/')[-1] + self._partition = partition + self._name = "EHReceiver-{}-partition{}".format(uuid.uuid4(), partition) if owner_level: - self._link_properties[types.AMQPSymbol(self._epoch)] = types.AMQPLong(int(owner_level)) - link_property_timeout_ms = (self.client.config.receive_timeout or self.timeout) * 1000 - self._link_properties[types.AMQPSymbol(self._timeout)] = types.AMQPLong(int(link_property_timeout_ms)) + self._link_properties[types.AMQPSymbol(self._epoch_symbol)] = types.AMQPLong(int(owner_level)) + link_property_timeout_ms = (self._client._config.receive_timeout or self._timeout) * 1000 # pylint:disable=protected-access + self._link_properties[types.AMQPSymbol(self._timeout_symbol)] = types.AMQPLong(int(link_property_timeout_ms)) self._handler = None def __aiter__(self): return self async def __anext__(self): - max_retries = self.client.config.max_retries - retry_count = 0 - while True: + retried_times = 0 + last_exception = None + while retried_times < self._client._config.max_retries: # pylint:disable=protected-access try: await self._open() - if not self.messages_iter: - self.messages_iter = self._handler.receive_messages_iter_async() - message = await self.messages_iter.__anext__() + if not self._messages_iter: + self._messages_iter = self._handler.receive_messages_iter_async() + message = await self._messages_iter.__anext__() event_data = EventData._from_message(message) # pylint:disable=protected-access - self.offset = EventPosition(event_data.offset, inclusive=False) - retry_count = 0 + self._offset = EventPosition(event_data.offset, inclusive=False) + retried_times = 0 return event_data except Exception as exception: # pylint:disable=broad-except - await self._handle_exception(exception, retry_count, max_retries, timeout_time=None) - retry_count += 1 + last_exception = await self._handle_exception(exception) + await self._client._try_delay(retried_times=retried_times, last_exception=last_exception, # pylint:disable=protected-access + entity_name=self._name) + retried_times += 1 + log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception) + raise last_exception def _create_handler(self): alt_creds = { - "username": self.client._auth_config.get("iot_username"), # pylint:disable=protected-access - "password": self.client._auth_config.get("iot_password")} # pylint:disable=protected-access - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint:disable=protected-access + "username": self._client._auth_config.get("iot_username") if self._redirected else None, # pylint:disable=protected-access + "password": self._client._auth_config.get("iot_password") if self._redirected else None # pylint:disable=protected-access + } + + source = Source(self._source) + if self._offset is not None: + source.set_filter(self._offset._selector()) # pylint:disable=protected-access self._handler = ReceiveClientAsync( source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, + auth=self._client._get_auth(**alt_creds), # pylint:disable=protected-access + debug=self._client._config.network_tracing, # pylint:disable=protected-access + prefetch=self._prefetch, link_properties=self._link_properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties( # pylint:disable=protected-access - self.client.config.user_agent), - loop=self.loop) - self.messages_iter = None + timeout=self._timeout, + error_policy=self._retry_policy, + keep_alive_interval=self._keep_alive, + client_name=self._name, + properties=self._client._create_properties( # pylint:disable=protected-access + self._client._config.user_agent), # pylint:disable=protected-access + loop=self._loop) + self._messages_iter = None async def _redirect(self, redirect): - self.messages_iter = None + self._messages_iter = None await super(EventHubConsumer, self)._redirect(redirect) - async def _open(self, timeout_time=None): + async def _open(self): """ Open the EventHubConsumer using the supplied connection. If the handler has previously been redirected, the redirect @@ -142,20 +148,25 @@ async def _open(self, timeout_time=None): """ # pylint: disable=protected-access - if not self.running and self.redirected: - self.client._process_redirect_uri(self.redirected) - self.source = self.redirected.address - await super(EventHubConsumer, self)._open(timeout_time) + self._redirected = self._redirected or self._client._iothub_redirect_info + + if not self._running and self._redirected: + self._client._process_redirect_uri(self._redirected) + self._source = self._redirected.address + await super(EventHubConsumer, self)._open() + + async def _open_with_retry(self): + return await self._do_retryable_operation(self._open, operation_need_param=False) async def _receive(self, timeout_time=None, max_batch_size=None, **kwargs): last_exception = kwargs.get("last_exception") - data_batch = kwargs.get("data_batch") + data_batch = [] - await self._open(timeout_time) + await self._open() remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: - log.info("%r receive operation timed out. (%r)", self.name, last_exception) + log.info("%r receive operation timed out. (%r)", self._name, last_exception) raise last_exception return data_batch @@ -165,13 +176,13 @@ async def _receive(self, timeout_time=None, max_batch_size=None, **kwargs): timeout=remaining_time_ms) for message in message_batch: event_data = EventData._from_message(message) # pylint:disable=protected-access - self.offset = EventPosition(event_data.offset) + self._offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch - @_retry_decorator - async def _receive_with_try(self, timeout_time=None, max_batch_size=None, **kwargs): - return await self._receive(timeout_time=timeout_time, max_batch_size=max_batch_size, **kwargs) + async def _receive_with_retry(self, timeout=None, max_batch_size=None, **kwargs): + return await self._do_retryable_operation(self._receive, timeout=timeout, + max_batch_size=max_batch_size, **kwargs) @property def queue_size(self): @@ -215,11 +226,10 @@ async def receive(self, *, max_batch_size=None, timeout=None): """ self._check_closed() - timeout = timeout or self.client.config.receive_timeout - max_batch_size = max_batch_size or min(self.client.config.max_batch_size, self.prefetch) - data_batch = [] # type: List[EventData] + timeout = timeout or self._client._config.receive_timeout # pylint:disable=protected-access + max_batch_size = max_batch_size or min(self._client._config.max_batch_size, self._prefetch) # pylint:disable=protected-access - return await self._receive_with_try(timeout=timeout, max_batch_size=max_batch_size, data_batch=data_batch) + return await self._receive_with_retry(timeout=timeout, max_batch_size=max_batch_size) async def close(self, exception=None): # type: (Exception) -> None @@ -241,17 +251,18 @@ async def close(self, exception=None): :caption: Close down the handler. """ - self.running = False - if self.error: + self._running = False + if self._error: return if isinstance(exception, errors.LinkRedirect): - self.redirected = exception + self._redirected = exception elif isinstance(exception, EventHubError): - self.error = exception + self._error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = ConnectError(str(exception), exception) + self._error = ConnectError(str(exception), exception) elif exception: - self.error = EventHubError(str(exception)) + self._error = EventHubError(str(exception)) else: - self.error = EventHubError("This receive handler is now closed.") - await self._handler.close_async() + self._error = EventHubError("This receive handler is now closed.") + if self._handler: + await self._handler.close_async() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py index 5d0cff3ebc1d..ae1cd8084f3d 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/error_async.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- import asyncio -import time import logging from uamqp import errors, compat # type: ignore @@ -36,19 +35,19 @@ def _create_eventhub_exception(exception): return error -async def _handle_exception(exception, retry_count, max_retries, closable, timeout_time=None): # pylint:disable=too-many-branches, too-many-statements +async def _handle_exception(exception, closable): # pylint:disable=too-many-branches, too-many-statements if isinstance(exception, asyncio.CancelledError): raise exception try: - name = closable.name + name = closable._name # pylint: disable=protected-access except AttributeError: - name = closable.container_id + name = closable._container_id # pylint: disable=protected-access if isinstance(exception, KeyboardInterrupt): # pylint:disable=no-else-raise log.info("%r stops due to keyboard interrupt", name) - closable.close() + await closable.close() raise exception elif isinstance(exception, EventHubError): - closable.close() + await closable.close() raise exception elif isinstance(exception, ( errors.MessageAccepted, @@ -65,10 +64,6 @@ async def _handle_exception(exception, retry_count, max_retries, closable, timeo log.info("%r Event data send error (%r)", name, exception) error = EventDataSendError(str(exception), exception) raise error - elif retry_count >= max_retries: - error = _create_eventhub_exception(exception) - log.info("%r has exhausted retry. Exception still occurs (%r)", name, exception) - raise error else: if isinstance(exception, errors.AuthenticationException): if hasattr(closable, "_close_connection"): @@ -95,20 +90,4 @@ async def _handle_exception(exception, retry_count, max_retries, closable, timeo else: if hasattr(closable, "_close_connection"): await closable._close_connection() # pylint:disable=protected-access - # start processing retry delay - try: - backoff_factor = closable.client.config.backoff_factor - backoff_max = closable.client.config.backoff_max - except AttributeError: - backoff_factor = closable.config.backoff_factor - backoff_max = closable.config.backoff_max - backoff = backoff_factor * 2 ** retry_count - if backoff <= backoff_max and (timeout_time is None or time.time() + backoff <= timeout_time): # pylint:disable=no-else-return - await asyncio.sleep(backoff) - log.info("%r has an exception (%r). Retrying...", format(name), exception) - return _create_eventhub_exception(exception) - else: - error = _create_eventhub_exception(exception) - log.info("%r operation has timed out. Last exception before timeout is (%r)", name, error) - raise error - # end of processing retry delay + return _create_eventhub_exception(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py similarity index 67% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py index f4b48afac6f3..e3eefa4774f4 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py @@ -5,13 +5,16 @@ from .event_processor import EventProcessor from .partition_processor import PartitionProcessor, CloseReason -from .partition_manager import PartitionManager -from .sqlite3_partition_manager import Sqlite3PartitionManager +from .partition_manager import PartitionManager, OwnershipLostError +from .partition_context import PartitionContext +from .sample_partition_manager import SamplePartitionManager __all__ = [ 'CloseReason', 'EventProcessor', 'PartitionProcessor', 'PartitionManager', - 'Sqlite3PartitionManager', -] \ No newline at end of file + 'OwnershipLostError', + 'PartitionContext', + 'SamplePartitionManager', +] diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/_ownership_manager.py new file mode 100644 index 000000000000..094ca8e0ce39 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/_ownership_manager.py @@ -0,0 +1,133 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# ----------------------------------------------------------------------------------- + +import time +import random +import math +from typing import List +from collections import Counter, defaultdict +from azure.eventhub.aio import EventHubClient +from .partition_manager import PartitionManager + + +class OwnershipManager(object): + """Increases or decreases the number of partitions owned by an EventProcessor + so the number of owned partitions are balanced among multiple EventProcessors + + An EventProcessor calls claim_ownership() of this class every x seconds, + where x is set by keyword argument "polling_interval" in EventProcessor, + to claim the ownership of partitions, create tasks for the claimed ownership, and cancel tasks that no longer belong + to the claimed ownership. + + """ + def __init__( + self, eventhub_client: EventHubClient, consumer_group_name: str, owner_id: str, + partition_manager: PartitionManager, ownership_timeout: float + ): + self.cached_parition_ids = [] # type: List[str] + self.eventhub_client = eventhub_client + self.eventhub_name = eventhub_client.eh_name + self.consumer_group_name = consumer_group_name + self.owner_id = owner_id + self.partition_manager = partition_manager + self.ownership_timeout = ownership_timeout + + async def claim_ownership(self): + """Claims ownership for this EventProcessor + 1. Retrieves all partition ids of an event hub from azure event hub service + 2. Retrieves current ownership list via this EventProcessor's PartitionManager. + 3. Balances number of ownership. Refer to _balance_ownership() for details. + 4. Claims the ownership for the balanced number of partitions. + + :return: List[Dict[Any]] + """ + if not self.cached_parition_ids: + await self._retrieve_partition_ids() + to_claim = await self._balance_ownership(self.cached_parition_ids) + claimed_list = await self.partition_manager.claim_ownership(to_claim) if to_claim else None + return claimed_list + + async def _retrieve_partition_ids(self): + """List all partition ids of the event hub that the EventProcessor is working on. + + :return: List[str] + """ + self.cached_parition_ids = await self.eventhub_client.get_partition_ids() + + async def _balance_ownership(self, all_partition_ids): + """Balances and claims ownership of partitions for this EventProcessor. + The balancing algorithm is: + 1. Find partitions with inactive ownership and partitions that haven never been claimed before + 2. Find the number of active owners, including this EventProcessor, for all partitions. + 3. Calculate the average count of partitions that an owner should own. + (number of partitions // number of active owners) + 4. Calculate the largest allowed count of partitions that an owner can own. + math.ceil(number of partitions / number of active owners). + This should be equal or 1 greater than the average count + 5. Adjust the number of partitions owned by this EventProcessor (owner) + a. if this EventProcessor owns more than largest allowed count, abandon one partition + b. if this EventProcessor owns less than average count, add one from the inactive or unclaimed partitions, + or steal one from another owner that has the largest number of ownership among all owners (EventProcessors) + c. Otherwise, no change to the ownership + + The balancing algorithm adjust one partition at a time to gradually build the balanced ownership. + Ownership must be renewed to keep it active. So the returned result includes both existing ownership and + the newly adjusted ownership. + This method balances but doesn't claim ownership. The caller of this method tries to claim the result ownership + list. But it may not successfully claim all of them because of concurrency. Other EventProcessors may happen to + claim a partition at that time. Since balancing and claiming are run in infinite repeatedly, + it achieves balancing among all EventProcessors after some time of running. + + :return: List[Dict[str, Any]], A list of ownership. + """ + ownership_list = await self.partition_manager.list_ownership( + self.eventhub_name, self.consumer_group_name + ) + now = time.time() + ownership_dict = {x["partition_id"]: x for x in ownership_list} # put the list to dict for fast lookup + not_owned_partition_ids = [pid for pid in all_partition_ids if pid not in ownership_dict] + timed_out_partition_ids = [ownership["partition_id"] for ownership in ownership_list + if ownership["last_modified_time"] + self.ownership_timeout < now] + claimable_partition_ids = not_owned_partition_ids + timed_out_partition_ids + active_ownership = [ownership for ownership in ownership_list + if ownership["last_modified_time"] + self.ownership_timeout >= now] + active_ownership_by_owner = defaultdict(list) + for ownership in active_ownership: + active_ownership_by_owner[ownership["owner_id"]].append(ownership) + active_ownership_self = active_ownership_by_owner[self.owner_id] + + # calculate expected count per owner + all_partition_count = len(all_partition_ids) + # owners_count is the number of active owners. If self.owner_id is not yet among the active owners, + # then plus 1 to include self. This will make owners_count >= 1. + owners_count = len(active_ownership_by_owner) + \ + (0 if self.owner_id in active_ownership_by_owner else 1) + expected_count_per_owner = all_partition_count // owners_count + most_count_allowed_per_owner = math.ceil(all_partition_count / owners_count) + # end of calculating expected count per owner + + to_claim = active_ownership_self + if len(active_ownership_self) > most_count_allowed_per_owner: # needs to abandon a partition + to_claim.pop() # abandon one partition if owned too many + elif len(active_ownership_self) < expected_count_per_owner: + # Either claims an inactive partition, or steals from other owners + if claimable_partition_ids: # claim an inactive partition if there is + random_partition_id = random.choice(claimable_partition_ids) + random_chosen_to_claim = ownership_dict.get(random_partition_id, + {"partition_id": random_partition_id, + "eventhub_name": self.eventhub_name, + "consumer_group_name": self.consumer_group_name + }) + random_chosen_to_claim["owner_id"] = self.owner_id + to_claim.append(random_chosen_to_claim) + else: # steal from another owner that has the most count + active_ownership_count_group_by_owner = Counter( + dict((x, len(y)) for x, y in active_ownership_by_owner.items())) + most_frequent_owner_id = active_ownership_count_group_by_owner.most_common(1)[0][0] + # randomly choose a partition to steal from the most_frequent_owner + to_steal_partition = random.choice(active_ownership_by_owner[most_frequent_owner_id]) + to_steal_partition["owner_id"] = self.owner_id + to_claim.append(to_steal_partition) + return to_claim diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py new file mode 100644 index 000000000000..85f6f1983250 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py @@ -0,0 +1,278 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# ----------------------------------------------------------------------------------- + +from typing import Dict, Type +import uuid +import asyncio +import logging + +from azure.eventhub import EventPosition, EventHubError +from azure.eventhub.aio import EventHubClient +from .partition_context import PartitionContext +from .partition_manager import PartitionManager, OwnershipLostError +from ._ownership_manager import OwnershipManager +from .partition_processor import CloseReason, PartitionProcessor +from .utils import get_running_loop + +log = logging.getLogger(__name__) + +OWNER_LEVEL = 0 + + +class EventProcessor(object): # pylint:disable=too-many-instance-attributes + """ + An EventProcessor constantly receives events from all partitions of the Event Hub in the context of a given + consumer group. The received data will be sent to PartitionProcessor to be processed. + + It provides the user a convenient way to receive events from multiple partitions and save checkpoints. + If multiple EventProcessors are running for an event hub, they will automatically balance load. + + Example: + .. code-block:: python + + import asyncio + import logging + import os + from azure.eventhub.aio import EventHubClient + from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor + from azure.eventhub.aio.eventprocessor import SamplePartitionManager + + RECEIVE_TIMEOUT = 5 # timeout in seconds for a receiving operation. 0 or None means no timeout + RETRY_TOTAL = 3 # max number of retries for receive operations within the receive timeout. + # Actual number of retries clould be less if RECEIVE_TIMEOUT is too small + CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"] + + logging.basicConfig(level=logging.INFO) + + async def do_operation(event): + # do some sync or async operations. If the operation is i/o bound, async will have better performance + print(event) + + + class MyPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + if events: + await asyncio.gather(*[do_operation(event) for event in events]) + await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number) + + async def main(): + client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, + retry_total=RETRY_TOTAL) + partition_manager = SamplePartitionManager(db_filename=":memory:") # a filename to persist checkpoint + try: + event_processor = EventProcessor(client, "$default", MyPartitionProcessor, + partition_manager, polling_interval=10) + asyncio.create_task(event_processor.start()) + await asyncio.sleep(60) + await event_processor.stop() + finally: + await partition_manager.close() + + if __name__ == '__main__': + asyncio.get_event_loop().run_until_complete(main()) + + """ + def __init__( + self, eventhub_client: EventHubClient, consumer_group_name: str, + partition_processor_type: Type[PartitionProcessor], + partition_manager: PartitionManager, *, + initial_event_position: EventPosition = EventPosition("-1"), polling_interval: float = 10.0 + ): + """ + Instantiate an EventProcessor. + + :param eventhub_client: An instance of ~azure.eventhub.aio.EventClient object + :type eventhub_client: ~azure.eventhub.aio.EventClient + :param consumer_group_name: The name of the consumer group this event processor is associated with. Events will + be read only in the context of this group. + :type consumer_group_name: str + :param partition_processor_type: A subclass type of ~azure.eventhub.eventprocessor.PartitionProcessor. + :type partition_processor_type: type + :param partition_manager: Interacts with the storage system, dealing with ownership and checkpoints. + For an easy start, SamplePartitionManager comes with the package. + :type partition_manager: Class implementing the ~azure.eventhub.eventprocessor.PartitionManager. + :param initial_event_position: The event position to start a partition consumer. + if the partition has no checkpoint yet. This will be replaced by "reset" checkpoint in the near future. + :type initial_event_position: EventPosition + :param polling_interval: The interval between any two pollings of balancing and claiming + :type polling_interval: float + + """ + + self._consumer_group_name = consumer_group_name + self._eventhub_client = eventhub_client + self._eventhub_name = eventhub_client.eh_name + self._partition_processor_factory = partition_processor_type + self._partition_manager = partition_manager + self._initial_event_position = initial_event_position # will be replaced by reset event position in preview 4 + self._polling_interval = polling_interval + self._ownership_timeout = self._polling_interval * 2 + self._tasks = {} # type: Dict[str, asyncio.Task] + self._id = str(uuid.uuid4()) + self._running = False + + def __repr__(self): + return 'EventProcessor: id {}'.format(self._id) + + async def start(self): + """Start the EventProcessor. + + 1. Calls the OwnershipManager to keep claiming and balancing ownership of partitions in an + infinitely loop until self.stop() is called. + 2. Cancels tasks for partitions that are no longer owned by this EventProcessor + 3. Creates tasks for partitions that are newly claimed by this EventProcessor + 4. Keeps tasks running for partitions that haven't changed ownership + 5. Each task repeatedly calls EvenHubConsumer.receive() to retrieve events and + call user defined partition processor + + :return: None + + """ + log.info("EventProcessor %r is being started", self._id) + ownership_manager = OwnershipManager(self._eventhub_client, self._consumer_group_name, self._id, + self._partition_manager, self._ownership_timeout) + if not self._running: + self._running = True + while self._running: + try: + claimed_ownership_list = await ownership_manager.claim_ownership() + except Exception as err: # pylint:disable=broad-except + log.warning("An exception (%r) occurred during balancing and claiming ownership for eventhub %r " + "consumer group %r. Retrying after %r seconds", + err, self._eventhub_name, self._consumer_group_name, self._polling_interval) + await asyncio.sleep(self._polling_interval) + continue + + to_cancel_list = self._tasks.keys() + if claimed_ownership_list: + claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list] + to_cancel_list = self._tasks.keys() - claimed_partition_ids + self._create_tasks_for_claimed_ownership(claimed_ownership_list) + else: + log.info("EventProcessor %r hasn't claimed an ownership. It keeps claiming.", self._id) + if to_cancel_list: + self._cancel_tasks_for_partitions(to_cancel_list) + log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list) + await asyncio.sleep(self._polling_interval) + + async def stop(self): + """Stop claiming ownership and all the partition consumers owned by this EventProcessor + + This method stops claiming ownership of owned partitions and cancels tasks that are running + EventHubConsumer.receive() for the partitions owned by this EventProcessor. + + :return: None + + """ + self._running = False + for _ in range(len(self._tasks)): + _, task = self._tasks.popitem() + task.cancel() + log.info("EventProcessor %r has been cancelled", self._id) + await asyncio.sleep(2) # give some time to finish after cancelled. + + def _cancel_tasks_for_partitions(self, to_cancel_partitions): + for partition_id in to_cancel_partitions: + if partition_id in self._tasks: + task = self._tasks.pop(partition_id) + task.cancel() + + def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list): + for ownership in to_claim_ownership_list: + partition_id = ownership["partition_id"] + if partition_id not in self._tasks: + self._tasks[partition_id] = get_running_loop().create_task(self._receive(ownership)) + + async def _receive(self, ownership): + log.info("start ownership, %r", ownership) + partition_processor = self._partition_processor_factory() + partition_id = ownership["partition_id"] + eventhub_name = ownership["eventhub_name"] + consumer_group_name = ownership["consumer_group_name"] + owner_id = ownership["owner_id"] + partition_context = PartitionContext( + eventhub_name, + consumer_group_name, + partition_id, + owner_id, + self._partition_manager + ) + partition_consumer = self._eventhub_client.create_consumer( + consumer_group_name, + partition_id, + EventPosition(ownership.get("offset", self._initial_event_position.value)) + ) + + async def process_error(err): + log.warning( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " has met an error. The exception is %r.", + owner_id, eventhub_name, partition_id, consumer_group_name, err + ) + try: + await partition_processor.process_error(err, partition_context) + except Exception as err_again: # pylint:disable=broad-except + log.warning( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " has another error during running process_error(). The exception is %r.", + owner_id, eventhub_name, partition_id, consumer_group_name, err_again + ) + + async def close(reason): + log.info( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " is being closed. Reason is: %r", + owner_id, eventhub_name, partition_id, consumer_group_name, reason + ) + try: + await partition_processor.close(reason, partition_context) + except Exception as err: # pylint:disable=broad-except + log.warning( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " has an error during running close(). The exception is %r.", + owner_id, eventhub_name, partition_id, consumer_group_name, err + ) + + try: + try: + await partition_processor.initialize(partition_context) + except Exception as err: # pylint:disable=broad-except + log.warning( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " has an error during running initialize(). The exception is %r.", + owner_id, eventhub_name, partition_id, consumer_group_name, err + ) + while True: + try: + events = await partition_consumer.receive() + await partition_processor.process_events(events, partition_context) + except asyncio.CancelledError: + log.info( + "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r" + " is cancelled", + owner_id, + eventhub_name, + partition_id, + consumer_group_name + ) + if self._running is False: + await close(CloseReason.SHUTDOWN) + else: + await close(CloseReason.OWNERSHIP_LOST) + raise + except EventHubError as eh_err: + await process_error(eh_err) + await close(CloseReason.EVENTHUB_EXCEPTION) + # An EventProcessor will pick up this partition again after the ownership is released + break + except OwnershipLostError: + await close(CloseReason.OWNERSHIP_LOST) + break + except Exception as other_error: # pylint:disable=broad-except + await process_error(other_error) + await close(CloseReason.PROCESS_EVENTS_ERROR) + break + finally: + await partition_consumer.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py similarity index 61% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py index 2714f675b28c..6aaf939143a2 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py @@ -7,30 +7,33 @@ from .partition_manager import PartitionManager -class CheckpointManager(object): - """ - CheckpointManager is responsible for the creation of checkpoints. - The interaction with the chosen storage service is done via ~azure.eventhub.eventprocessor.PartitionManager. +class PartitionContext(object): + """Contains partition related context information for a PartitionProcessor instance to use. + Users can use update_checkpoint() of this class to save checkpoint data. """ - def __init__(self, partition_id: str, eventhub_name: str, consumer_group_name: str, owner_id: str, partition_manager: PartitionManager): + def __init__(self, eventhub_name: str, consumer_group_name: str, + partition_id: str, owner_id: str, partition_manager: PartitionManager): self.partition_id = partition_id self.eventhub_name = eventhub_name self.consumer_group_name = consumer_group_name self.owner_id = owner_id - self.partition_manager = partition_manager + self._partition_manager = partition_manager async def update_checkpoint(self, offset, sequence_number=None): """ - Updates the checkpoint using the given information for the associated partition and consumer group in the chosen storage service. + Updates the checkpoint using the given information for the associated partition and consumer group in the + chosen storage service. :param offset: The offset of the ~azure.eventhub.EventData the new checkpoint will be associated with. :type offset: str - :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be associated with. + :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be + associated with. :type sequence_number: int :return: None """ - await self.partition_manager.update_checkpoint( + # TODO: whether change this method to accept event_data as well + await self._partition_manager.update_checkpoint( self.eventhub_name, self.consumer_group_name, self.partition_id, self.owner_id, offset, sequence_number ) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_manager.py similarity index 83% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_manager.py index e4ecb1bec824..4bb84779dd53 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_manager.py @@ -34,15 +34,14 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> last_modified_time etag """ - pass @abstractmethod - async def claim_ownership(self, partitions: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: + async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: """ Tries to claim a list of specified ownership. - :param partitions: Iterable of dictionaries containing all the ownership to claim. - :type partitions: Iterable of dict + :param ownership_list: Iterable of dictionaries containing all the ownership to claim. + :type ownership_list: Iterable of dict :return: Iterable of dictionaries containing the following partition ownership information: eventhub_name consumer_group_name @@ -54,13 +53,13 @@ async def claim_ownership(self, partitions: Iterable[Dict[str, Any]]) -> Iterabl last_modified_time etag """ - pass @abstractmethod async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id, offset, sequence_number) -> None: """ - Updates the checkpoint using the given information for the associated partition and consumer group in the chosen storage service. + Updates the checkpoint using the given information for the associated partition and + consumer group in the chosen storage service. :param eventhub_name: The name of the specific Event Hub the ownership are associated with, relative to the Event Hubs namespace that contains it. @@ -73,11 +72,15 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_ :type owner_id: str :param offset: The offset of the ~azure.eventhub.EventData the new checkpoint will be associated with. :type offset: str - :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be associated with. + :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint + will be associated with. :type sequence_number: int - :return: + :return: None + :raise: `OwnershipLostError`, `CheckpointError` """ - pass - async def close(self): - pass + +class OwnershipLostError(Exception): + """Raises when update_checkpoint detects the ownership has been lost + + """ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py similarity index 52% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py index 10aafc79c492..8b0fb2ca7e5c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py @@ -6,27 +6,33 @@ from typing import List from abc import ABC, abstractmethod from enum import Enum -from .checkpoint_manager import CheckpointManager - from azure.eventhub import EventData +from .partition_context import PartitionContext class CloseReason(Enum): SHUTDOWN = 0 # user call EventProcessor.stop() OWNERSHIP_LOST = 1 # lose the ownership of a partition. EVENTHUB_EXCEPTION = 2 # Exception happens during receiving events + PROCESS_EVENTS_ERROR = 3 # Exception happens during process_events class PartitionProcessor(ABC): """ PartitionProcessor processes events received from the Azure Event Hubs service. A single instance of a class - implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns. + implementing this abstract class will be created for every partition the associated + ~azure.eventhub.eventprocessor.EventProcessor owns. """ - def __init__(self, checkpoint_manager: CheckpointManager): - self._checkpoint_manager = checkpoint_manager - async def close(self, reason): + async def initialize(self, partition_context: PartitionContext): + """ + + :param partition_context: The context information of this partition. + :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext + """ + + async def close(self, reason, partition_context: PartitionContext): """Called when EventProcessor stops processing this PartitionProcessor. There are different reasons to trigger the PartitionProcessor to close. @@ -34,25 +40,31 @@ async def close(self, reason): :param reason: Reason for closing the PartitionProcessor. :type reason: ~azure.eventhub.eventprocessor.CloseReason + :param partition_context: The context information of this partition. + Use its method update_checkpoint to save checkpoint to the data store. + :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext """ - pass @abstractmethod - async def process_events(self, events: List[EventData]): + async def process_events(self, events: List[EventData], partition_context: PartitionContext): """Called when a batch of events have been received. :param events: Received events. :type events: list[~azure.eventhub.common.EventData] + :param partition_context: The context information of this partition. + Use its method update_checkpoint to save checkpoint to the data store. + :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext """ - pass - async def process_error(self, error): + async def process_error(self, error, partition_context: PartitionContext): """Called when an error happens :param error: The error that happens. :type error: Exception + :param partition_context: The context information of this partition. + Use its method update_checkpoint to save checkpoint to the data store. + :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext """ - pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/sample_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/sample_partition_manager.py new file mode 100644 index 000000000000..82559fc8c274 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/sample_partition_manager.py @@ -0,0 +1,144 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# ----------------------------------------------------------------------------------- + +import time +import uuid +import sqlite3 +import logging +from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError + +logger = logging.getLogger(__name__) + + +def _check_table_name(table_name: str): + for c in table_name: + if not (c.isalnum() or c == "_"): + raise ValueError("Table name \"{}\" is not in correct format".format(table_name)) + return table_name + + +class SamplePartitionManager(PartitionManager): + """An implementation of PartitionManager by using the sqlite3 in Python standard library. + Sqlite3 is a mini sql database that runs in memory or files. + Please don't use this PartitionManager for production use. + + + """ + primary_keys_dict = {"eventhub_name": "text", "consumer_group_name": "text", "partition_id": "text"} + other_fields_dict = {"owner_id": "text", "owner_level": "integer", "sequence_number": "integer", "offset": "text", + "last_modified_time": "real", "etag": "text"} + checkpoint_fields = ["sequence_number", "offset"] + fields_dict = {**primary_keys_dict, **other_fields_dict} + primary_keys = list(primary_keys_dict.keys()) + other_fields = list(other_fields_dict.keys()) + fields = primary_keys + other_fields + + def __init__(self, db_filename: str = ":memory:", ownership_table: str = "ownership"): + """ + + :param db_filename: name of file that saves the sql data. + Sqlite3 will run in memory without a file when db_filename is ":memory:". + :param ownership_table: The table name of the sqlite3 database. + """ + super(SamplePartitionManager, self).__init__() + self.ownership_table = _check_table_name(ownership_table) + conn = sqlite3.connect(db_filename) + c = conn.cursor() + try: + sql = "create table if not exists " + _check_table_name(ownership_table)\ + + "("\ + + ",".join([x[0]+" "+x[1] for x in self.fields_dict.items()])\ + + ", constraint pk_ownership PRIMARY KEY ("\ + + ",".join(self.primary_keys)\ + + "))" + c.execute(sql) + finally: + c.close() + self.conn = conn + + async def list_ownership(self, eventhub_name, consumer_group_name): + cursor = self.conn.cursor() + try: + cursor.execute("select " + ",".join(self.fields) + + " from "+_check_table_name(self.ownership_table)+" where eventhub_name=? " + "and consumer_group_name=?", + (eventhub_name, consumer_group_name)) + return [dict(zip(self.fields, row)) for row in cursor.fetchall()] + finally: + cursor.close() + + async def claim_ownership(self, ownership_list): + result = [] + cursor = self.conn.cursor() + try: + for p in ownership_list: + cursor.execute("select etag from " + _check_table_name(self.ownership_table) + + " where "+ " and ".join([field+"=?" for field in self.primary_keys]), + tuple(p.get(field) for field in self.primary_keys)) + cursor_fetch = cursor.fetchall() + if not cursor_fetch: + p["last_modified_time"] = time.time() + p["etag"] = str(uuid.uuid4()) + try: + fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.fields)) + sql = "insert into " + _check_table_name(self.ownership_table) + " (" \ + + ",".join(fields_without_checkpoint) \ + + ") values (?,?,?,?,?,?,?)" + cursor.execute(sql, tuple(p.get(field) for field in fields_without_checkpoint)) + except sqlite3.OperationalError as op_err: + logger.info("EventProcessor %r failed to claim partition %r " + "because it was claimed by another EventProcessor at the same time. " + "The Sqlite3 exception is %r", p["owner_id"], p["partition_id"], op_err) + continue + else: + result.append(p) + else: + if p.get("etag") == cursor_fetch[0][0]: + p["last_modified_time"] = time.time() + p["etag"] = str(uuid.uuid4()) + other_fields_without_checkpoint = list( + filter(lambda x: x not in self.checkpoint_fields, self.other_fields) + ) + sql = "update " + _check_table_name(self.ownership_table) + " set "\ + + ','.join([field+"=?" for field in other_fields_without_checkpoint])\ + + " where "\ + + " and ".join([field+"=?" for field in self.primary_keys]) + + cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint) + + tuple(p.get(field) for field in self.primary_keys)) + result.append(p) + else: + logger.info("EventProcessor %r failed to claim partition %r " + "because it was claimed by another EventProcessor at the same time", p["owner_id"], + p["partition_id"]) + self.conn.commit() + return result + finally: + cursor.close() + + async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id, + offset, sequence_number): + cursor = self.conn.cursor() + try: + cursor.execute("select owner_id from " + _check_table_name(self.ownership_table) + + " where eventhub_name=? and consumer_group_name=? and partition_id=?", + (eventhub_name, consumer_group_name, partition_id)) + cursor_fetch = cursor.fetchall() + if cursor_fetch and owner_id == cursor_fetch[0][0]: + cursor.execute("update " + _check_table_name(self.ownership_table) + + " set offset=?, sequence_number=? " + "where eventhub_name=? and consumer_group_name=? and partition_id=?", + (offset, sequence_number, eventhub_name, consumer_group_name, partition_id)) + self.conn.commit() + else: + logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership", + partition_id) + raise OwnershipLostError() + + finally: + cursor.close() + + async def close(self): + self.conn.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/utils.py similarity index 96% rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/utils.py index 368cd8469f10..1d8add0f49a0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/utils.py @@ -10,7 +10,7 @@ def get_running_loop(): try: return asyncio.get_running_loop() except AttributeError: # 3.5 / 3.6 - loop = asyncio._get_running_loop() # pylint: disable=protected-access + loop = asyncio._get_running_loop() # pylint: disable=protected-access, no-member if loop is None: raise RuntimeError('No running event loop') return loop diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py index 5cf04d26af28..999bdc09c787 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/producer_async.py @@ -14,8 +14,7 @@ from azure.eventhub.common import EventData, EventDataBatch from azure.eventhub.error import _error_handler, OperationTimeoutError, EventDataError from ..producer import _error, _set_partition_key -from ._consumer_producer_mixin_async import ConsumerProducerMixin, _retry_decorator - +from ._consumer_producer_mixin_async import ConsumerProducerMixin log = logging.getLogger(__name__) @@ -28,7 +27,7 @@ class EventHubProducer(ConsumerProducerMixin): # pylint: disable=too-many-insta to a partition. """ - _timeout = b'com.microsoft:timeout' + _timeout_symbol = b'com.microsoft:timeout' def __init__( # pylint: disable=super-init-not-called self, client, target, **kwargs): @@ -61,83 +60,81 @@ def __init__( # pylint: disable=super-init-not-called loop = kwargs.get("loop", None) super(EventHubProducer, self).__init__() - self.loop = loop or asyncio.get_event_loop() + self._loop = loop or asyncio.get_event_loop() self._max_message_size_on_link = None - self.running = False - self.client = client - self.target = target - self.partition = partition - self.keep_alive = keep_alive - self.auto_reconnect = auto_reconnect - self.timeout = send_timeout - self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) - self.reconnect_backoff = 1 - self.name = "EHProducer-{}".format(uuid.uuid4()) - self.unsent_events = None - self.redirected = None - self.error = None + self._running = False + self._client = client + self._target = target + self._partition = partition + self._keep_alive = keep_alive + self._auto_reconnect = auto_reconnect + self._timeout = send_timeout + self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access + self._reconnect_backoff = 1 + self._name = "EHProducer-{}".format(uuid.uuid4()) + self._unsent_events = None + self._redirected = None + self._error = None if partition: - self.target += "/Partitions/" + partition - self.name += "-partition{}".format(partition) + self._target += "/Partitions/" + partition + self._name += "-partition{}".format(partition) self._handler = None self._outcome = None self._condition = None - self._link_properties = {types.AMQPSymbol(self._timeout): types.AMQPLong(int(self.timeout * 1000))} + self._link_properties = {types.AMQPSymbol(self._timeout_symbol): types.AMQPLong(int(self._timeout * 1000))} def _create_handler(self): self._handler = SendClientAsync( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, + self._target, + auth=self._client._get_auth(), # pylint:disable=protected-access + debug=self._client._config.network_tracing, # pylint:disable=protected-access + msg_timeout=self._timeout, + error_policy=self._retry_policy, + keep_alive_interval=self._keep_alive, + client_name=self._name, link_properties=self._link_properties, - properties=self.client._create_properties( # pylint: disable=protected-access - self.client.config.user_agent), - loop=self.loop) + properties=self._client._create_properties( # pylint: disable=protected-access + self._client._config.user_agent), # pylint:disable=protected-access + loop=self._loop) - async def _open(self, timeout_time=None, **kwargs): # pylint:disable=arguments-differ, unused-argument # TODO: to refactor + async def _open(self): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ - if not self.running and self.redirected: - self.client._process_redirect_uri(self.redirected) # pylint: disable=protected-access - self.target = self.redirected.address - await super(EventHubProducer, self)._open(timeout_time) + if not self._running and self._redirected: + self._client._process_redirect_uri(self._redirected) # pylint: disable=protected-access + self._target = self._redirected.address + await super(EventHubProducer, self)._open() - @_retry_decorator - async def _open_with_retry(self, timeout_time=None, **kwargs): - return await self._open(timeout_time=timeout_time, **kwargs) + async def _open_with_retry(self): + return await self._do_retryable_operation(self._open, operation_need_param=False) async def _send_event_data(self, timeout_time=None, last_exception=None): - if self.unsent_events: - await self._open(timeout_time) + if self._unsent_events: + await self._open() remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: error = last_exception else: error = OperationTimeoutError("send operation timed out") - log.info("%r send operation timed out. (%r)", self.name, error) + log.info("%r send operation timed out. (%r)", self._name, error) raise error self._handler._msg_timeout = remaining_time # pylint: disable=protected-access - self._handler.queue_message(*self.unsent_events) + self._handler.queue_message(*self._unsent_events) await self._handler.wait_async() - self.unsent_events = self._handler.pending_messages + self._unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: if self._outcome == constants.MessageSendResult.Timeout: self._condition = OperationTimeoutError("send operation timed out") _error(self._outcome, self._condition) return - @_retry_decorator - async def _send_event_data_with_retry(self, timeout_time=None, last_exception=None): - return await self._send_event_data(timeout_time=timeout_time, last_exception=last_exception) + async def _send_event_data_with_retry(self, timeout=None): + return await self._do_retryable_operation(self._send_event_data, timeout=timeout) def _on_outcome(self, outcome, condition): """ @@ -176,7 +173,7 @@ async def create_batch(self, max_size=None, partition_key=None): """ if not self._max_message_size_on_link: - await self._open_with_retry(timeout=self.client.config.send_timeout) + await self._open_with_retry() if max_size and max_size > self._max_message_size_on_link: raise ValueError('Max message size: {} is too large, acceptable max batch size is: {} bytes.' @@ -231,7 +228,7 @@ async def send( event_data = _set_partition_key(event_data, partition_key) wrapper_event_data = EventDataBatch._from_batch(event_data, partition_key) # pylint: disable=protected-access wrapper_event_data.message.on_send_complete = self._on_outcome - self.unsent_events = [wrapper_event_data.message] + self._unsent_events = [wrapper_event_data.message] await self._send_event_data_with_retry(timeout=timeout) # pylint:disable=unexpected-keyword-arg # TODO: to refactor async def close(self, exception=None): diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 600faaf31041..90a1ac86742f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -6,7 +6,10 @@ import logging import datetime +import time import functools +import threading + from typing import Any, List, Dict, Union, TYPE_CHECKING import uamqp # type: ignore @@ -46,6 +49,7 @@ class EventHubClient(EventHubClientAbstract): def __init__(self, host, event_hub_path, credential, **kwargs): # type:(str, str, Union[EventHubSharedKeyCredential, EventHubSASTokenCredential, TokenCredential], Any) -> None super(EventHubClient, self).__init__(host=host, event_hub_path=event_hub_path, credential=credential, **kwargs) + self._lock = threading.RLock() self._conn_manager = get_connection_manager(**kwargs) def __enter__(self): @@ -64,55 +68,68 @@ def _create_auth(self, username=None, password=None): :param password: The shared access key. :type password: str """ - http_proxy = self.config.http_proxy - transport_type = self.config.transport_type - auth_timeout = self.config.auth_timeout + http_proxy = self._config.http_proxy + transport_type = self._config.transport_type + auth_timeout = self._config.auth_timeout # TODO: the following code can be refactored to create auth from classes directly instead of using if-else - if isinstance(self.credential, EventHubSharedKeyCredential): # pylint:disable=no-else-return + if isinstance(self._credential, EventHubSharedKeyCredential): # pylint:disable=no-else-return username = username or self._auth_config['username'] password = password or self._auth_config['password'] if "@sas.root" in username: return authentication.SASLPlain( - self.host, username, password, http_proxy=http_proxy, transport_type=transport_type) + self._host, username, password, http_proxy=http_proxy, transport_type=transport_type) return authentication.SASTokenAuth.from_shared_access_key( - self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, + self._auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) - elif isinstance(self.credential, EventHubSASTokenCredential): - token = self.credential.get_sas_token() + elif isinstance(self._credential, EventHubSASTokenCredential): + token = self._credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) except (KeyError, TypeError, IndexError): raise ValueError("Supplied SAS token has no valid expiry value.") return authentication.SASTokenAuth( - self.auth_uri, self.auth_uri, token, + self._auth_uri, self._auth_uri, token, expires_at=expiry, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) else: # Azure credential - get_jwt_token = functools.partial(self.credential.get_token, + get_jwt_token = functools.partial(self._credential.get_token, 'https://eventhubs.azure.net//.default') - return authentication.JWTTokenAuth(self.auth_uri, self.auth_uri, + return authentication.JWTTokenAuth(self._auth_uri, self._auth_uri, get_jwt_token, http_proxy=http_proxy, transport_type=transport_type) - def _handle_exception(self, exception, retry_count, max_retries): - _handle_exception(exception, retry_count, max_retries, self) - def _close_connection(self): self._conn_manager.reset_connection_if_broken() + def _try_delay(self, retried_times, last_exception, timeout_time=None, entity_name=None): + entity_name = entity_name or self._container_id + backoff = self._config.backoff_factor * 2 ** retried_times + if backoff <= self._config.backoff_max and ( + timeout_time is None or time.time() + backoff <= timeout_time): # pylint:disable=no-else-return + time.sleep(backoff) + log.info("%r has an exception (%r). Retrying...", format(entity_name), last_exception) + else: + log.info("%r operation has timed out. Last exception before timeout is (%r)", + entity_name, last_exception) + raise last_exception + def _management_request(self, mgmt_msg, op_type): - max_retries = self.config.max_retries - retry_count = 0 - while retry_count <= self.config.max_retries: - mgmt_auth = self._create_auth() - mgmt_client = uamqp.AMQPClient(self.mgmt_target) + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password": self._auth_config.get("iot_password") + } + + retried_times = 0 + while retried_times <= self._config.max_retries: + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = uamqp.AMQPClient(self._mgmt_target) try: - conn = self._conn_manager.get_connection(self.host, mgmt_auth) #pylint:disable=assignment-from-none + conn = self._conn_manager.get_connection(self._host, mgmt_auth) #pylint:disable=assignment-from-none mgmt_client.open(connection=conn) response = mgmt_client.mgmt_request( mgmt_msg, @@ -122,11 +139,24 @@ def _management_request(self, mgmt_msg, op_type): description_fields=b'status-description') return response except Exception as exception: # pylint: disable=broad-except - self._handle_exception(exception, retry_count, max_retries) - retry_count += 1 + last_exception = _handle_exception(exception, self) + self._try_delay(retried_times=retried_times, last_exception=last_exception) + retried_times += 1 finally: mgmt_client.close() + def _iothub_redirect(self): + with self._lock: + if self._is_iothub and not self._iothub_redirect_info: + if not self._redirect_consumer: + self._redirect_consumer = self.create_consumer(consumer_group='$default', + partition_id='0', + event_position=EventPosition('-1'), + operation='/messages/events') + with self._redirect_consumer: + self._redirect_consumer._open_with_retry() # pylint: disable=protected-access + self._redirect_consumer = None + def get_properties(self): # type:() -> Dict[str, Any] """ @@ -140,6 +170,8 @@ def get_properties(self): :rtype: dict :raises: ~azure.eventhub.ConnectError """ + if self._is_iothub and not self._iothub_redirect_info: + self._iothub_redirect() mgmt_msg = Message(application_properties={'name': self.eh_name}) response = self._management_request(mgmt_msg, op_type=b'com.microsoft:eventhub') output = {} @@ -179,6 +211,8 @@ def get_partition_properties(self, partition): :rtype: dict :raises: ~azure.eventhub.ConnectError """ + if self._is_iothub and not self._iothub_redirect_info: + self._iothub_redirect() mgmt_msg = Message(application_properties={'name': self.eh_name, 'partition': partition}) response = self._management_request(mgmt_msg, op_type=b'com.microsoft:partition') @@ -228,11 +262,11 @@ def create_consumer(self, consumer_group, partition_id, event_position, **kwargs """ owner_level = kwargs.get("owner_level") operation = kwargs.get("operation") - prefetch = kwargs.get("prefetch") or self.config.prefetch + prefetch = kwargs.get("prefetch") or self._config.prefetch - path = self.address.path + operation if operation else self.address.path + path = self._address.path + operation if operation else self._address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, path, consumer_group, partition_id) + self._address.hostname, path, consumer_group, partition_id) handler = EventHubConsumer( self, source_url, event_position=event_position, owner_level=owner_level, prefetch=prefetch) @@ -265,10 +299,10 @@ def create_producer(self, partition_id=None, operation=None, send_timeout=None): """ - target = "amqps://{}{}".format(self.address.hostname, self.address.path) + target = "amqps://{}{}".format(self._address.hostname, self._address.path) if operation: target = target + operation - send_timeout = self.config.send_timeout if send_timeout is None else send_timeout + send_timeout = self._config.send_timeout if send_timeout is None else send_timeout handler = EventHubProducer( self, target, partition=partition_id, send_timeout=send_timeout) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 1d33091755d0..7d4c8cd2712e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -11,7 +11,8 @@ import functools from abc import abstractmethod from typing import Dict, Union, Any, TYPE_CHECKING -from azure.eventhub import __version__ + +from azure.eventhub import __version__, EventPosition from azure.eventhub.configuration import _Configuration from .common import EventHubSharedKeyCredential, EventHubSASTokenCredential, _Address @@ -132,29 +133,32 @@ def __init__(self, host, event_hub_path, credential, **kwargs): queued. Default value is 60 seconds. If set to 0, there will be no timeout. :type send_timeout: float """ - self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] - self.address = _Address() - self.address.hostname = host - self.address.path = "/" + event_hub_path if event_hub_path else "" + self.eh_name = event_hub_path + self._host = host + self._container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] + self._address = _Address() + self._address.hostname = host + self._address.path = "/" + event_hub_path if event_hub_path else "" self._auth_config = {} # type:Dict[str,str] - self.credential = credential + self._credential = credential if isinstance(credential, EventHubSharedKeyCredential): - self.username = credential.policy - self.password = credential.key - self._auth_config['username'] = self.username - self._auth_config['password'] = self.password + self._username = credential.policy + self._password = credential.key + self._auth_config['username'] = self._username + self._auth_config['password'] = self._password - self.host = host - self.eh_name = event_hub_path - self.keep_alive = kwargs.get("keep_alive", 30) - self.auto_reconnect = kwargs.get("auto_reconnect", True) - self.mgmt_target = "amqps://{}/{}".format(self.host, self.eh_name) - self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) - self.get_auth = functools.partial(self._create_auth) - self.config = _Configuration(**kwargs) - self.debug = self.config.network_tracing + self._keep_alive = kwargs.get("keep_alive", 30) + self._auto_reconnect = kwargs.get("auto_reconnect", True) + self._mgmt_target = "amqps://{}/{}".format(self._host, self.eh_name) + self._auth_uri = "sb://{}{}".format(self._address.hostname, self._address.path) + self._get_auth = functools.partial(self._create_auth) + self._config = _Configuration(**kwargs) + self._debug = self._config.network_tracing + self._is_iothub = False + self._iothub_redirect_info = None + self._redirect_consumer = None - log.info("%r: Created the Event Hub client", self.container_id) + log.info("%r: Created the Event Hub client", self._container_id) @classmethod def _from_iothub_connection_string(cls, conn_str, **kwargs): @@ -173,6 +177,11 @@ def _from_iothub_connection_string(cls, conn_str, **kwargs): 'iot_password': key, 'username': username, 'password': password} + client._is_iothub = True # pylint: disable=protected-access + client._redirect_consumer = client.create_consumer(consumer_group='$default', # pylint: disable=protected-access, no-member + partition_id='0', + event_position=EventPosition('-1'), + operation='/messages/events') return client @abstractmethod @@ -208,11 +217,13 @@ def _create_properties(self, user_agent=None): # pylint: disable=no-self-use def _process_redirect_uri(self, redirect): redirect_uri = redirect.address.decode('utf-8') auth_uri, _, _ = redirect_uri.partition("/ConsumerGroups") - self.address = urlparse(auth_uri) - self.host = self.address.hostname - self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) - self.eh_name = self.address.path.lstrip('/') - self.mgmt_target = redirect_uri + self._address = urlparse(auth_uri) + self._host = self._address.hostname + self.eh_name = self._address.path.lstrip('/') + self._auth_uri = "sb://{}{}".format(self._address.hostname, self._address.path) + self._mgmt_target = redirect_uri + if self._is_iothub: + self._iothub_redirect_info = redirect @classmethod def from_connection_string(cls, conn_str, **kwargs): diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 3979463eef42..73fed892db11 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -200,6 +200,15 @@ def application_properties(self, value): properties = None if value is None else dict(self._app_properties) self.message.application_properties = properties + @property + def system_properties(self): + """ + Metadata set by the Event Hubs Service associated with the EventData + + :rtype: dict + """ + return self._annotations + @property def body(self): """ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py index 44be38386b73..499c3ba5429e 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/consumer.py @@ -14,7 +14,7 @@ from azure.eventhub.common import EventData, EventPosition from azure.eventhub.error import _error_handler -from ._consumer_producer_mixin import ConsumerProducerMixin, _retry_decorator +from ._consumer_producer_mixin import ConsumerProducerMixin log = logging.getLogger(__name__) @@ -34,9 +34,9 @@ class EventHubConsumer(ConsumerProducerMixin): # pylint:disable=too-many-instan sometimes referred to as "Non-Epoch Consumers." """ - timeout = 0 - _epoch = b'com.microsoft:epoch' - _timeout = b'com.microsoft:timeout' + _timeout = 0 + _epoch_symbol = b'com.microsoft:epoch' + _timeout_symbol = b'com.microsoft:timeout' def __init__(self, client, source, **kwargs): """ @@ -61,75 +61,81 @@ def __init__(self, client, source, **kwargs): auto_reconnect = kwargs.get("auto_reconnect", True) super(EventHubConsumer, self).__init__() - self.running = False - self.client = client - self.source = source - self.offset = event_position - self.messages_iter = None - self.prefetch = prefetch - self.owner_level = owner_level - self.keep_alive = keep_alive - self.auto_reconnect = auto_reconnect - self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) - self.reconnect_backoff = 1 + self._running = False + self._client = client + self._source = source + self._offset = event_position + self._messages_iter = None + self._prefetch = prefetch + self._owner_level = owner_level + self._keep_alive = keep_alive + self._auto_reconnect = auto_reconnect + self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint:disable=protected-access + self._reconnect_backoff = 1 self._link_properties = {} - self.redirected = None - self.error = None - partition = self.source.split('/')[-1] - self.partition = partition - self.name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition) + self._redirected = None + self._error = None + partition = self._source.split('/')[-1] + self._partition = partition + self._name = "EHConsumer-{}-partition{}".format(uuid.uuid4(), partition) if owner_level: - self._link_properties[types.AMQPSymbol(self._epoch)] = types.AMQPLong(int(owner_level)) - link_property_timeout_ms = (self.client.config.receive_timeout or self.timeout) * 1000 - self._link_properties[types.AMQPSymbol(self._timeout)] = types.AMQPLong(int(link_property_timeout_ms)) + self._link_properties[types.AMQPSymbol(self._epoch_symbol)] = types.AMQPLong(int(owner_level)) + link_property_timeout_ms = (self._client._config.receive_timeout or self._timeout) * 1000 # pylint:disable=protected-access + self._link_properties[types.AMQPSymbol(self._timeout_symbol)] = types.AMQPLong(int(link_property_timeout_ms)) self._handler = None def __iter__(self): return self def __next__(self): - max_retries = self.client.config.max_retries - retry_count = 0 - while True: + retried_times = 0 + last_exception = None + while retried_times < self._client._config.max_retries: # pylint:disable=protected-access try: self._open() - if not self.messages_iter: - self.messages_iter = self._handler.receive_messages_iter() - message = next(self.messages_iter) + if not self._messages_iter: + self._messages_iter = self._handler.receive_messages_iter() + message = next(self._messages_iter) event_data = EventData._from_message(message) # pylint:disable=protected-access - self.offset = EventPosition(event_data.offset, inclusive=False) - retry_count = 0 + self._offset = EventPosition(event_data.offset, inclusive=False) + retried_times = 0 return event_data except Exception as exception: # pylint:disable=broad-except - self._handle_exception(exception, retry_count, max_retries, timeout_time=None) - retry_count += 1 + last_exception = self._handle_exception(exception) + self._client._try_delay(retried_times=retried_times, last_exception=last_exception, # pylint:disable=protected-access + entity_name=self._name) + retried_times += 1 + log.info("%r operation has exhausted retry. Last exception: %r.", self._name, last_exception) + raise last_exception def _create_handler(self): alt_creds = { - "username": self.client._auth_config.get("iot_username"), # pylint:disable=protected-access - "password": self.client._auth_config.get("iot_password")} # pylint:disable=protected-access - source = Source(self.source) - if self.offset is not None: - source.set_filter(self.offset._selector()) # pylint:disable=protected-access + "username": self._client._auth_config.get("iot_username") if self._redirected else None, # pylint:disable=protected-access + "password": self._client._auth_config.get("iot_password") if self._redirected else None # pylint:disable=protected-access + } + + source = Source(self._source) + if self._offset is not None: + source.set_filter(self._offset._selector()) # pylint:disable=protected-access self._handler = ReceiveClient( source, - auth=self.client.get_auth(**alt_creds), - debug=self.client.config.network_tracing, - prefetch=self.prefetch, + auth=self._client._get_auth(**alt_creds), # pylint:disable=protected-access + debug=self._client._config.network_tracing, # pylint:disable=protected-access + prefetch=self._prefetch, link_properties=self._link_properties, - timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, - properties=self.client._create_properties( # pylint:disable=protected-access - self.client.config.user_agent)) - self.messages_iter = None + timeout=self._timeout, + error_policy=self._retry_policy, + keep_alive_interval=self._keep_alive, + client_name=self._name, + properties=self._client._create_properties( # pylint:disable=protected-access + self._client._config.user_agent)) # pylint:disable=protected-access + self._messages_iter = None def _redirect(self, redirect): - self.messages_iter = None + self._messages_iter = None super(EventHubConsumer, self)._redirect(redirect) - def _open(self, timeout_time=None): + def _open(self): """ Open the EventHubConsumer using the supplied connection. If the handler has previously been redirected, the redirect @@ -137,35 +143,40 @@ def _open(self, timeout_time=None): """ # pylint: disable=protected-access - if not self.running and self.redirected: - self.client._process_redirect_uri(self.redirected) - self.source = self.redirected.address - super(EventHubConsumer, self)._open(timeout_time) + self._redirected = self._redirected or self._client._iothub_redirect_info + + if not self._running and self._redirected: + self._client._process_redirect_uri(self._redirected) + self._source = self._redirected.address + super(EventHubConsumer, self)._open() + + def _open_with_retry(self): + return self._do_retryable_operation(self._open, operation_need_param=False) def _receive(self, timeout_time=None, max_batch_size=None, **kwargs): last_exception = kwargs.get("last_exception") - data_batch = kwargs.get("data_batch") + data_batch = [] - self._open(timeout_time) + self._open() remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: - log.info("%r receive operation timed out. (%r)", self.name, last_exception) + log.info("%r receive operation timed out. (%r)", self._name, last_exception) raise last_exception return data_batch remaining_time_ms = 1000 * remaining_time message_batch = self._handler.receive_message_batch( - max_batch_size=max_batch_size - (len(data_batch) if data_batch else 0), + max_batch_size=max_batch_size, timeout=remaining_time_ms) for message in message_batch: event_data = EventData._from_message(message) # pylint:disable=protected-access - self.offset = EventPosition(event_data.offset) + self._offset = EventPosition(event_data.offset) data_batch.append(event_data) return data_batch - @_retry_decorator - def _receive_with_try(self, timeout_time=None, max_batch_size=None, **kwargs): - return self._receive(timeout_time=timeout_time, max_batch_size=max_batch_size, **kwargs) + def _receive_with_retry(self, timeout=None, max_batch_size=None, **kwargs): + return self._do_retryable_operation(self._receive, timeout=timeout, + max_batch_size=max_batch_size, **kwargs) @property def queue_size(self): @@ -209,11 +220,10 @@ def receive(self, max_batch_size=None, timeout=None): """ self._check_closed() - timeout = timeout or self.client.config.receive_timeout - max_batch_size = max_batch_size or min(self.client.config.max_batch_size, self.prefetch) - data_batch = [] # type: List[EventData] + timeout = timeout or self._client._config.receive_timeout # pylint:disable=protected-access + max_batch_size = max_batch_size or min(self._client._config.max_batch_size, self._prefetch) # pylint:disable=protected-access - return self._receive_with_try(timeout=timeout, max_batch_size=max_batch_size, data_batch=data_batch) + return self._receive_with_retry(timeout=timeout, max_batch_size=max_batch_size) def close(self, exception=None): # type:(Exception) -> None @@ -235,9 +245,9 @@ def close(self, exception=None): :caption: Close down the handler. """ - if self.messages_iter: - self.messages_iter.close() - self.messages_iter = None + if self._messages_iter: + self._messages_iter.close() + self._messages_iter = None super(EventHubConsumer, self).close(exception) next = __next__ # for python2.7 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py index 72b11f5478ad..129cf14a3842 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -2,7 +2,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import time import logging import six @@ -157,11 +156,11 @@ def _create_eventhub_exception(exception): return error -def _handle_exception(exception, retry_count, max_retries, closable, timeout_time=None): # pylint:disable=too-many-branches, too-many-statements - try: - name = closable.name - except AttributeError: - name = closable.container_id +def _handle_exception(exception, closable): # pylint:disable=too-many-branches, too-many-statements + try: # closable is a producer/consumer object + name = closable._name # pylint: disable=protected-access + except AttributeError: # closable is an client object + name = closable._container_id # pylint: disable=protected-access if isinstance(exception, KeyboardInterrupt): # pylint:disable=no-else-raise log.info("%r stops due to keyboard interrupt", name) closable.close() @@ -184,10 +183,6 @@ def _handle_exception(exception, retry_count, max_retries, closable, timeout_tim log.info("%r Event data send error (%r)", name, exception) error = EventDataSendError(str(exception), exception) raise error - elif retry_count >= max_retries: - error = _create_eventhub_exception(exception) - log.info("%r has exhausted retry. Exception still occurs (%r)", name, exception) - raise error else: if isinstance(exception, errors.AuthenticationException): if hasattr(closable, "_close_connection"): @@ -214,20 +209,4 @@ def _handle_exception(exception, retry_count, max_retries, closable, timeout_tim else: if hasattr(closable, "_close_connection"): closable._close_connection() # pylint:disable=protected-access - # start processing retry delay - try: - backoff_factor = closable.client.config.backoff_factor - backoff_max = closable.client.config.backoff_max - except AttributeError: - backoff_factor = closable.config.backoff_factor - backoff_max = closable.config.backoff_max - backoff = backoff_factor * 2 ** retry_count - if backoff <= backoff_max and (timeout_time is None or time.time() + backoff <= timeout_time): #pylint:disable=no-else-return - time.sleep(backoff) - log.info("%r has an exception (%r). Retrying...", format(name), exception) - return _create_eventhub_exception(exception) - else: - error = _create_eventhub_exception(exception) - log.info("%r operation has timed out. Last exception before timeout is (%r)", name, error) - raise error - # end of processing retry delay + return _create_eventhub_exception(exception) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py deleted file mode 100644 index 85020257df46..000000000000 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py +++ /dev/null @@ -1,218 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# ----------------------------------------------------------------------------------- - -from typing import Callable, List -import uuid -import asyncio -import logging - -from azure.eventhub import EventPosition, EventHubError -from azure.eventhub.aio import EventHubClient -from .checkpoint_manager import CheckpointManager -from .partition_manager import PartitionManager -from .partition_processor import PartitionProcessor, CloseReason -from .utils import get_running_loop - -log = logging.getLogger(__name__) - -OWNER_LEVEL = 0 - - -class EventProcessor(object): - """ - An EventProcessor constantly receives events from all partitions of the Event Hub in the context of a given - consumer group. The received data will be sent to PartitionProcessor to be processed. - - It provides the user a convenient way to receive events from multiple partitions and save checkpoints. - If multiple EventProcessors are running for an event hub, they will automatically balance load. - This load balancing won't be available until preview 3. - - Example: - .. code-block:: python - - class MyPartitionProcessor(PartitionProcessor): - async def process_events(self, events): - if events: - # do something sync or async to process the events - await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number) - - import asyncio - from azure.eventhub.aio import EventHubClient - from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager - client = EventHubClient.from_connection_string("", receive_timeout=5, retry_total=3) - partition_manager = Sqlite3PartitionManager() - try: - event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager) - asyncio.ensure_future(event_processor.start()) - await asyncio.sleep(100) # allow it to run 100 seconds - await event_processor.stop() - finally: - await partition_manager.close() - - """ - def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str, - partition_processor_factory: Callable[[CheckpointManager], PartitionProcessor], - partition_manager: PartitionManager, **kwargs): - """ - Instantiate an EventProcessor. - - :param eventhub_client: An instance of ~azure.eventhub.aio.EventClient object - :type eventhub_client: ~azure.eventhub.aio.EventClient - :param consumer_group_name: The name of the consumer group this event processor is associated with. Events will - be read only in the context of this group. - :type consumer_group_name: str - :param partition_processor_factory: A callable(type or function) object that creates an instance of a class - implementing the ~azure.eventhub.eventprocessor.PartitionProcessor. - :type partition_processor_factory: callable object - :param partition_manager: Interacts with the storage system, dealing with ownership and checkpoints. - For preview 2, sample Sqlite3PartitionManager is provided. - :type partition_manager: Class implementing the ~azure.eventhub.eventprocessor.PartitionManager. - :param initial_event_position: The offset to start a partition consumer if the partition has no checkpoint yet. - :type initial_event_position: int or str - - """ - self._consumer_group_name = consumer_group_name - self._eventhub_client = eventhub_client - self._eventhub_name = eventhub_client.eh_name - self._partition_processor_factory = partition_processor_factory - self._partition_manager = partition_manager - self._initial_event_position = kwargs.get("initial_event_position", "-1") - self._max_batch_size = eventhub_client.config.max_batch_size - self._receive_timeout = eventhub_client.config.receive_timeout - self._tasks = [] # type: List[asyncio.Task] - self._id = str(uuid.uuid4()) - - def __repr__(self): - return 'EventProcessor: id {}'.format(self._id) - - async def start(self): - """Start the EventProcessor. - - 1. retrieve the partition ids from eventhubs. - 2. claim partition ownership of these partitions. - 3. repeatedly call EvenHubConsumer.receive() to retrieve events and call user defined PartitionProcessor.process_events(). - - :return: None - - """ - log.info("EventProcessor %r is being started", self._id) - partition_ids = await self._eventhub_client.get_partition_ids() - claimed_list = await self._claim_partitions(partition_ids) - await self._start_claimed_partitions(claimed_list) - - async def stop(self): - """Stop all the partition consumer - - This method cancels tasks that are running EventHubConsumer.receive() for the partitions owned by this EventProcessor. - - :return: None - - """ - for i in range(len(self._tasks)): - task = self._tasks.pop() - task.cancel() - log.info("EventProcessor %r has been cancelled", self._id) - await asyncio.sleep(2) # give some time to finish after cancelled - - async def _claim_partitions(self, partition_ids): - partitions_ownership = await self._partition_manager.list_ownership(self._eventhub_name, self._consumer_group_name) - partitions_ownership_dict = dict() - for ownership in partitions_ownership: - partitions_ownership_dict[ownership["partition_id"]] = ownership - - to_claim_list = [] - for pid in partition_ids: - p_ownership = partitions_ownership_dict.get(pid) - if p_ownership: - to_claim_list.append(p_ownership) - else: - new_ownership = {"eventhub_name": self._eventhub_name, "consumer_group_name": self._consumer_group_name, - "owner_id": self._id, "partition_id": pid, "owner_level": OWNER_LEVEL} - to_claim_list.append(new_ownership) - claimed_list = await self._partition_manager.claim_ownership(to_claim_list) - return claimed_list - - async def _start_claimed_partitions(self, claimed_partitions): - for partition in claimed_partitions: - partition_id = partition["partition_id"] - offset = partition.get("offset", self._initial_event_position) - consumer = self._eventhub_client.create_consumer(self._consumer_group_name, partition_id, - EventPosition(str(offset))) - partition_processor = self._partition_processor_factory( - checkpoint_manager=CheckpointManager(partition_id, self._eventhub_name, self._consumer_group_name, - self._id, self._partition_manager) - ) - loop = get_running_loop() - task = loop.create_task( - _receive(consumer, partition_processor, self._receive_timeout)) - self._tasks.append(task) - try: - await asyncio.gather(*self._tasks) - finally: - log.info("EventProcessor %r has stopped", self._id) - - -async def _receive(partition_consumer, partition_processor, receive_timeout): - try: - while True: - try: - events = await partition_consumer.receive(timeout=receive_timeout) - except asyncio.CancelledError as cancelled_error: - log.info( - "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r " - "is cancelled", - partition_processor._checkpoint_manager.owner_id, - partition_processor._checkpoint_manager.eventhub_name, - partition_processor._checkpoint_manager.partition_id, - partition_processor._checkpoint_manager.consumer_group_name - ) - await partition_processor.process_error(cancelled_error) - await partition_processor.close(reason=CloseReason.SHUTDOWN) - break - except EventHubError as eh_err: - reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION - log.warning( - "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r " - "has met an exception receiving events. It's being closed. The exception is %r.", - partition_processor._checkpoint_manager.owner_id, - partition_processor._checkpoint_manager.eventhub_name, - partition_processor._checkpoint_manager.partition_id, - partition_processor._checkpoint_manager.consumer_group_name, - eh_err - ) - await partition_processor.process_error(eh_err) - await partition_processor.close(reason=reason) - break - try: - await partition_processor.process_events(events) - except asyncio.CancelledError as cancelled_error: - log.info( - "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r " - "is cancelled.", - partition_processor._checkpoint_manager.owner_id, - partition_processor._checkpoint_manager.eventhub_name, - partition_processor._checkpoint_manager.partition_id, - partition_processor._checkpoint_manager.consumer_group_name - ) - await partition_processor.process_error(cancelled_error) - await partition_processor.close(reason=CloseReason.SHUTDOWN) - break - except Exception as exp: # user code has caused an error - log.warning( - "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r " - "has met an exception from user code process_events. It's being closed. The exception is %r.", - partition_processor._checkpoint_manager.owner_id, - partition_processor._checkpoint_manager.eventhub_name, - partition_processor._checkpoint_manager.partition_id, - partition_processor._checkpoint_manager.consumer_group_name, - exp - ) - await partition_processor.process_error(exp) - await partition_processor.close(reason=CloseReason.EVENTHUB_EXCEPTION) - break - # TODO: will review whether to break and close partition processor after user's code has an exception - # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3? - finally: - await partition_consumer.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py deleted file mode 100644 index eb08e970fa89..000000000000 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py +++ /dev/null @@ -1,110 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# ----------------------------------------------------------------------------------- - -import time -import uuid -import sqlite3 -from .partition_manager import PartitionManager - - -def _check_table_name(table_name: str): - for c in table_name: - if not (c.isalnum() or c == "_"): - raise ValueError("Table name \"{}\" is not in correct format".format(table_name)) - return table_name - - -class Sqlite3PartitionManager(PartitionManager): - """An implementation of PartitionManager by using the sqlite3 in Python standard library. - Sqlite3 is a mini sql database that runs in memory or files. - - - """ - def __init__(self, db_filename: str = ":memory:", ownership_table: str = "ownership"): - """ - - :param db_filename: name of file that saves the sql data. - Sqlite3 will run in memory without a file when db_filename is ":memory:". - :param ownership_table: The table name of the sqlite3 database. - """ - super(Sqlite3PartitionManager, self).__init__() - self.ownership_table = _check_table_name(ownership_table) - conn = sqlite3.connect(db_filename) - c = conn.cursor() - try: - c.execute("create table " + ownership_table + - "(eventhub_name text," - "consumer_group_name text," - "owner_id text," - "partition_id text," - "owner_level integer," - "sequence_number integer," - "offset text," - "last_modified_time integer," - "etag text)") - except sqlite3.OperationalError: - pass - finally: - c.close() - self.conn = conn - - async def list_ownership(self, eventhub_name, consumer_group_name): - cursor = self.conn.cursor() - try: - fields = ["eventhub_name", "consumer_group_name", "owner_id", "partition_id", "owner_level", - "sequence_number", - "offset", "last_modified_time", "etag"] - cursor.execute("select " + ",".join(fields) + - " from "+_check_table_name(self.ownership_table)+" where eventhub_name=? " - "and consumer_group_name=?", - (eventhub_name, consumer_group_name)) - result_list = [] - - for row in cursor.fetchall(): - d = dict(zip(fields, row)) - result_list.append(d) - return result_list - finally: - cursor.close() - - async def claim_ownership(self, partitions): - cursor = self.conn.cursor() - try: - for p in partitions: - cursor.execute("select * from " + _check_table_name(self.ownership_table) + - " where eventhub_name=? " - "and consumer_group_name=? " - "and partition_id =?", - (p["eventhub_name"], p["consumer_group_name"], - p["partition_id"])) - if not cursor.fetchall(): - cursor.execute("insert into " + _check_table_name(self.ownership_table) + - " (eventhub_name,consumer_group_name,partition_id,owner_id,owner_level,last_modified_time,etag) " - "values (?,?,?,?,?,?,?)", - (p["eventhub_name"], p["consumer_group_name"], p["partition_id"], p["owner_id"], p["owner_level"], - time.time(), str(uuid.uuid4()) - )) - else: - cursor.execute("update " + _check_table_name(self.ownership_table) + " set owner_id=?, owner_level=?, last_modified_time=?, etag=? " - "where eventhub_name=? and consumer_group_name=? and partition_id=?", - (p["owner_id"], p["owner_level"], time.time(), str(uuid.uuid4()), - p["eventhub_name"], p["consumer_group_name"], p["partition_id"])) - self.conn.commit() - return partitions - finally: - cursor.close() - - async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id, - offset, sequence_number): - cursor = self.conn.cursor() - try: - cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?", - (offset, sequence_number, eventhub_name, consumer_group_name, partition_id)) - self.conn.commit() - finally: - cursor.close() - - async def close(self): - self.conn.close() diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py index a36541475ceb..8008fac7ecd0 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/producer.py @@ -14,7 +14,7 @@ from azure.eventhub.common import EventData, EventDataBatch from azure.eventhub.error import _error_handler, OperationTimeoutError, EventDataError -from ._consumer_producer_mixin import ConsumerProducerMixin, _retry_decorator +from ._consumer_producer_mixin import ConsumerProducerMixin log = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class EventHubProducer(ConsumerProducerMixin): # pylint:disable=too-many-instan to a partition. """ - _timeout = b'com.microsoft:timeout' + _timeout_symbol = b'com.microsoft:timeout' def __init__(self, client, target, **kwargs): """ @@ -71,79 +71,76 @@ def __init__(self, client, target, **kwargs): super(EventHubProducer, self).__init__() self._max_message_size_on_link = None - self.running = False - self.client = client - self.target = target - self.partition = partition - self.timeout = send_timeout - self.redirected = None - self.error = None - self.keep_alive = keep_alive - self.auto_reconnect = auto_reconnect - self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) - self.reconnect_backoff = 1 - self.name = "EHProducer-{}".format(uuid.uuid4()) - self.unsent_events = None + self._running = False + self._client = client + self._target = target + self._partition = partition + self._timeout = send_timeout + self._redirected = None + self._error = None + self._keep_alive = keep_alive + self._auto_reconnect = auto_reconnect + self._retry_policy = errors.ErrorPolicy(max_retries=self._client._config.max_retries, on_error=_error_handler) # pylint: disable=protected-access + self._reconnect_backoff = 1 + self._name = "EHProducer-{}".format(uuid.uuid4()) + self._unsent_events = None if partition: - self.target += "/Partitions/" + partition - self.name += "-partition{}".format(partition) + self._target += "/Partitions/" + partition + self._name += "-partition{}".format(partition) self._handler = None self._outcome = None self._condition = None - self._link_properties = {types.AMQPSymbol(self._timeout): types.AMQPLong(int(self.timeout * 1000))} + self._link_properties = {types.AMQPSymbol(self._timeout_symbol): types.AMQPLong(int(self._timeout * 1000))} def _create_handler(self): self._handler = SendClient( - self.target, - auth=self.client.get_auth(), - debug=self.client.config.network_tracing, - msg_timeout=self.timeout, - error_policy=self.retry_policy, - keep_alive_interval=self.keep_alive, - client_name=self.name, + self._target, + auth=self._client._get_auth(), # pylint:disable=protected-access + debug=self._client._config.network_tracing, # pylint:disable=protected-access + msg_timeout=self._timeout, + error_policy=self._retry_policy, + keep_alive_interval=self._keep_alive, + client_name=self._name, link_properties=self._link_properties, - properties=self.client._create_properties(self.client.config.user_agent)) # pylint: disable=protected-access + properties=self._client._create_properties(self._client._config.user_agent)) # pylint: disable=protected-access - def _open(self, timeout_time=None, **kwargs): # pylint:disable=unused-argument, arguments-differ # TODO:To refactor + def _open(self): """ Open the EventHubProducer using the supplied connection. If the handler has previously been redirected, the redirect context will be used to create a new handler before opening it. """ + if not self._running and self._redirected: + self._client._process_redirect_uri(self._redirected) # pylint: disable=protected-access + self._target = self._redirected.address + super(EventHubProducer, self)._open() - if not self.running and self.redirected: - self.client._process_redirect_uri(self.redirected) # pylint: disable=protected-access - self.target = self.redirected.address - super(EventHubProducer, self)._open(timeout_time) - - @_retry_decorator - def _open_with_retry(self, timeout_time=None, **kwargs): - return self._open(timeout_time=timeout_time, **kwargs) + def _open_with_retry(self): + return self._do_retryable_operation(self._open, operation_need_param=False) def _send_event_data(self, timeout_time=None, last_exception=None): - if self.unsent_events: - self._open(timeout_time) + if self._unsent_events: + self._open() remaining_time = timeout_time - time.time() if remaining_time <= 0.0: if last_exception: error = last_exception else: error = OperationTimeoutError("send operation timed out") - log.info("%r send operation timed out. (%r)", self.name, error) + log.info("%r send operation timed out. (%r)", self._name, error) raise error self._handler._msg_timeout = remaining_time # pylint: disable=protected-access - self._handler.queue_message(*self.unsent_events) + self._handler.queue_message(*self._unsent_events) self._handler.wait() - self.unsent_events = self._handler.pending_messages + self._unsent_events = self._handler.pending_messages if self._outcome != constants.MessageSendResult.Ok: if self._outcome == constants.MessageSendResult.Timeout: self._condition = OperationTimeoutError("send operation timed out") _error(self._outcome, self._condition) - @_retry_decorator - def _send_event_data_with_retry(self, timeout_time=None, last_exception=None): - return self._send_event_data(timeout_time=timeout_time, last_exception=last_exception) + def _send_event_data_with_retry(self, timeout=None): + return self._do_retryable_operation(self._send_event_data, timeout=timeout) def _on_outcome(self, outcome, condition): """ @@ -182,7 +179,7 @@ def create_batch(self, max_size=None, partition_key=None): """ if not self._max_message_size_on_link: - self._open_with_retry(timeout=self.client.config.send_timeout) + self._open_with_retry() if max_size and max_size > self._max_message_size_on_link: raise ValueError('Max message size: {} is too large, acceptable max batch size is: {} bytes.' @@ -236,8 +233,8 @@ def send(self, event_data, partition_key=None, timeout=None): event_data = _set_partition_key(event_data, partition_key) wrapper_event_data = EventDataBatch._from_batch(event_data, partition_key) # pylint: disable=protected-access wrapper_event_data.message.on_send_complete = self._on_outcome - self.unsent_events = [wrapper_event_data.message] - self._send_event_data_with_retry(timeout=timeout) # pylint:disable=unexpected-keyword-arg # TODO:to refactor + self._unsent_events = [wrapper_event_data.message] + self._send_event_data_with_retry(timeout=timeout) def close(self, exception=None): # pylint:disable=useless-super-delegation # type:(Exception) -> None diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index c424357a77c4..ed0212e85562 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -16,10 +16,11 @@ collect_ignore = [] if sys.version_info < (3, 5): collect_ignore.append("tests/asynctests") + collect_ignore.append("tests/eventprocessor_tests") collect_ignore.append("features") collect_ignore.append("examples/async_examples") -from azure.eventhub import EventHubClient, EventHubConsumer, EventPosition +from azure.eventhub import EventHubClient, EventPosition def pytest_addoption(parser): @@ -202,45 +203,3 @@ def connstr_senders(connection_str): yield connection_str, senders for s in senders: s.close() - - -@pytest.fixture() -def storage_clm(eph): - try: - container = str(uuid.uuid4()) - storage_clm = AzureStorageCheckpointLeaseManager( - os.environ['AZURE_STORAGE_ACCOUNT'], - os.environ['AZURE_STORAGE_ACCESS_KEY'], - container) - except KeyError: - pytest.skip("Live Storage configuration not found.") - try: - storage_clm.initialize(eph) - storage_clm.storage_client.create_container(container) - yield storage_clm - finally: - try: - storage_clm.storage_client.delete_container(container) - except: - warnings.warn(UserWarning("storage container teardown failed")) - -@pytest.fixture() -def eh_partition_pump(eph): - lease = AzureBlobLease() - lease.with_partition_id("1") - partition_pump = EventHubPartitionPump(eph, lease) - return partition_pump - - -@pytest.fixture() -def partition_pump(eph): - lease = Lease() - lease.with_partition_id("1") - partition_pump = PartitionPump(eph, lease) - return partition_pump - - -@pytest.fixture() -def partition_manager(eph): - partition_manager = PartitionManager(eph) - return partition_manager diff --git a/sdk/eventhub/azure-eventhubs/examples/__init__.py b/sdk/eventhub/azure-eventhubs/examples/__init__.py index 94facc8618df..34913fb394d7 100644 --- a/sdk/eventhub/azure-eventhubs/examples/__init__.py +++ b/sdk/eventhub/azure-eventhubs/examples/__init__.py @@ -2,20 +2,3 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - -import sys -import logging - -def get_logger(level): - azure_logger = logging.getLogger("azure.eventhub") - azure_logger.setLevel(level) - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter(logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')) - if not azure_logger.handlers: - azure_logger.addHandler(handler) - - uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.INFO) - if not uamqp_logger.handlers: - uamqp_logger.addHandler(handler) - return azure_logger diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py index fb726d854b8c..53e73228032e 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/iterator_receiver_async.py @@ -10,41 +10,27 @@ """ import os -import time -import logging import asyncio from azure.eventhub.aio import EventHubClient -from azure.eventhub import EventPosition, EventHubSharedKeyCredential, EventData +from azure.eventhub import EventPosition, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") -async def iter_consumer(consumer): - async with consumer: - async for item in consumer: - print(item) - - async def main(): - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EVENT_POSITION) - await iter_consumer(consumer) + async with consumer: + async for item in consumer: + print(item) if __name__ == '__main__': loop = asyncio.get_event_loop() loop.run_until_complete(main()) - diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py index 9f59c0ea7ab6..ba6bf68a9258 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_async.py @@ -11,21 +11,15 @@ import os import time -import logging import asyncio from azure.eventhub.aio import EventHubClient from azure.eventhub import EventPosition, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") @@ -44,18 +38,11 @@ async def pump(client, partition): run_time = end_time - start_time print("Received {} messages in {} seconds".format(total, run_time)) -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - loop = asyncio.get_event_loop() - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) - tasks = [ - asyncio.ensure_future(pump(client, "0")), - asyncio.ensure_future(pump(client, "1"))] - loop.run_until_complete(asyncio.wait(tasks)) - loop.close() - -except KeyboardInterrupt: - pass + +loop = asyncio.get_event_loop() +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +tasks = [ + asyncio.ensure_future(pump(client, "0")), + asyncio.ensure_future(pump(client, "1"))] +loop.run_until_complete(asyncio.wait(tasks)) diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_owner_level.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_owner_level.py similarity index 56% rename from sdk/eventhub/azure-eventhubs/examples/recv_owner_level.py rename to sdk/eventhub/azure-eventhubs/examples/async_examples/recv_owner_level.py index 4217874771ad..384d914ad436 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_owner_level.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/recv_owner_level.py @@ -11,46 +11,37 @@ import os import time -import logging import asyncio from azure.eventhub.aio import EventHubClient from azure.eventhub import EventHubSharedKeyCredential, EventPosition -import examples -logger = examples.get_logger(logging.INFO) +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] PARTITION = "0" async def pump(client, owner_level): - consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, event_position=EventPosition("-1"), owner_level=owner_level) + consumer = client.create_consumer( + consumer_group="$default", partition_id=PARTITION, event_position=EventPosition("-1"), owner_level=owner_level + ) async with consumer: total = 0 start_time = time.time() for event_data in await consumer.receive(timeout=5): last_offset = event_data.offset last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset, last_sn)) total += 1 end_time = time.time() run_time = end_time - start_time print("Received {} messages in {} seconds".format(total, run_time)) -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - loop = asyncio.get_event_loop() - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) - loop.run_until_complete(pump(client, 20)) - loop.close() - -except KeyboardInterrupt: - pass +loop = asyncio.get_event_loop() +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +loop.run_until_complete(pump(client, 20)) diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py index ffae6787628d..d24e73d0bc17 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/send_async.py @@ -6,12 +6,11 @@ # -------------------------------------------------------------------------------------------- """ -An example to show sending events asynchronously to an Event Hub with partition keys. +An example to show sending individual events asynchronously to an Event Hub. """ # pylint: disable=C0111 -import logging import time import asyncio import os @@ -19,14 +18,11 @@ from azure.eventhub.aio import EventHubClient from azure.eventhub import EventData, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] async def run(client): @@ -37,26 +33,16 @@ async def run(client): async def send(producer, count): async with producer: for i in range(count): - logger.info("Sending message: {}".format(i)) + print("Sending message: {}".format(i)) data = EventData(str(i)) await producer.send(data) -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - loop = asyncio.get_event_loop() - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) - tasks = asyncio.gather( - run(client), - run(client)) - start_time = time.time() - loop.run_until_complete(tasks) - end_time = time.time() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - loop.close() - -except KeyboardInterrupt: - pass +loop = asyncio.get_event_loop() +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +tasks = asyncio.gather( + run(client), + run(client)) +start_time = time.time() +loop.run_until_complete(tasks) +print("Runtime: {} seconds".format(time.time() - start_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py index 896f2a007b21..eeb20b3594af 100644 --- a/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py +++ b/sdk/eventhub/azure-eventhubs/examples/async_examples/test_examples_eventhub_async.py @@ -5,14 +5,9 @@ #-------------------------------------------------------------------------- import pytest -import datetime -import os -import time import logging import asyncio -from azure.eventhub import EventHubError, EventData - @pytest.mark.asyncio async def test_example_eventhub_async_send_and_receive(live_eventhub_config): diff --git a/sdk/eventhub/azure-eventhubs/examples/batch_send.py b/sdk/eventhub/azure-eventhubs/examples/batch_send.py deleted file mode 100644 index 3801682f7914..000000000000 --- a/sdk/eventhub/azure-eventhubs/examples/batch_send.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -An example to show batch sending events to an Event Hub. -""" - -# pylint: disable=C0111 - -import logging -import time -import os - -from azure.eventhub import EventData, EventHubClient, EventHubSharedKeyCredential - - -import examples -logger = examples.get_logger(logging.INFO) - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - - -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) - producer = client.create_producer(partition_id="1") - - event_list = [] - for i in range(1500): - event_list.append(EventData('Hello World')) - - with producer: - start_time = time.time() - producer.send(event_list) - end_time = time.time() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass diff --git a/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py index 6c3202162872..079469b89518 100644 --- a/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py +++ b/sdk/eventhub/azure-eventhubs/examples/client_secret_auth.py @@ -9,22 +9,16 @@ """ import os -import time -import logging - from azure.eventhub import EventHubClient from azure.eventhub import EventData from azure.identity import ClientSecretCredential -import examples -logger = examples.get_logger(logging.INFO) - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] CLIENT_ID = os.environ.get('AAD_CLIENT_ID') SECRET = os.environ.get('AAD_SECRET') @@ -35,14 +29,8 @@ client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=credential) -try: - producer = client.create_producer(partition_id='0') - - with producer: - event = EventData(body='A single message') - producer.send(event) -except KeyboardInterrupt: - pass -except Exception as e: - print(e) +producer = client.create_producer(partition_id='0') +with producer: + event = EventData(body='A single message') + producer.send(event) diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py index 8c4c9ced7d29..c0826e274704 100644 --- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py +++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py @@ -2,9 +2,8 @@ import logging import os from azure.eventhub.aio import EventHubClient -from azure.eventhub.eventprocessor import EventProcessor -from azure.eventhub.eventprocessor import PartitionProcessor -from azure.eventhub.eventprocessor import Sqlite3PartitionManager +from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor +from azure.eventhub.aio.eventprocessor import SamplePartitionManager RECEIVE_TIMEOUT = 5 # timeout in seconds for a receiving operation. 0 or None means no timeout RETRY_TOTAL = 3 # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small @@ -19,32 +18,22 @@ async def do_operation(event): class MyPartitionProcessor(PartitionProcessor): - def __init__(self, checkpoint_manager): - super(MyPartitionProcessor, self).__init__(checkpoint_manager) - - async def process_events(self, events): + async def process_events(self, events, partition_context): if events: await asyncio.gather(*[do_operation(event) for event in events]) - await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number) - - -def partition_processor_factory(checkpoint_manager): - return MyPartitionProcessor(checkpoint_manager) - - -async def run_awhile(duration): - client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, - retry_total=RETRY_TOTAL) - partition_manager = Sqlite3PartitionManager() - event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager) - try: - asyncio.ensure_future(event_processor.start()) - await asyncio.sleep(duration) - await event_processor.stop() - finally: - await partition_manager.close() + await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number) + else: + print("empty events received", "partition:", partition_context.partition_id) if __name__ == '__main__': loop = asyncio.get_event_loop() - loop.run_until_complete(run_awhile(60)) + client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL) + partition_manager = SamplePartitionManager(db_filename="eventprocessor_test_db") + event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1) + try: + loop.run_until_complete(event_processor.start()) + except KeyboardInterrupt: + loop.run_until_complete(event_processor.stop()) + finally: + loop.stop() diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py index 08a1c5af32ad..ecc935669d13 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_recv.py @@ -9,20 +9,15 @@ An example to show receiving events from an IoT Hub partition. """ import os -import logging from azure.eventhub import EventHubClient, EventPosition - -logger = logging.getLogger('azure.eventhub') - iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) -consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), operation='/messages/events') +consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), + operation='/messages/events') + with consumer: received = consumer.receive(timeout=5) print(received) - - eh_info = client.get_properties() - print(eh_info) diff --git a/sdk/eventhub/azure-eventhubs/examples/iothub_send.py b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py index 06d35b102647..c2f8f3379259 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iothub_send.py +++ b/sdk/eventhub/azure-eventhubs/examples/iothub_send.py @@ -9,21 +9,12 @@ An example to show receiving events from an IoT Hub partition. """ import os -import logging - from azure.eventhub import EventData, EventHubClient - -logger = logging.getLogger('azure.eventhub') - iot_device_id = os.environ['IOTHUB_DEVICE'] iot_connection_str = os.environ['IOTHUB_CONNECTION_STR'] client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) -try: - producer = client.create_producer(operation='/messages/devicebound') - with producer: - producer.send(EventData(b"A single event", to_device=iot_device_id)) - -except KeyboardInterrupt: - pass +producer = client.create_producer(operation='/messages/devicebound') +with producer: + producer.send(EventData(b"A single event", to_device=iot_device_id)) diff --git a/sdk/eventhub/azure-eventhubs/examples/proxy.py b/sdk/eventhub/azure-eventhubs/examples/proxy.py index 0af2dda1a6ac..61005c347333 100644 --- a/sdk/eventhub/azure-eventhubs/examples/proxy.py +++ b/sdk/eventhub/azure-eventhubs/examples/proxy.py @@ -9,20 +9,15 @@ An example to show sending and receiving events behind a proxy """ import os -import logging - from azure.eventhub import EventHubClient, EventPosition, EventData, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - # Hostname can be .servicebus.windows.net" -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") PARTITION = "0" @@ -33,29 +28,19 @@ 'password': '123456' # password used for proxy authentication if needed } - -if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - -client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False, http_proxy=HTTP_PROXY) +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False, http_proxy=HTTP_PROXY) +producer = client.create_producer(partition_id=PARTITION) +consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, event_position=EVENT_POSITION) try: - producer = client.create_producer(partition_id=PARTITION) - consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, event_position=EVENT_POSITION) - consumer.receive(timeout=1) - event_list = [] for i in range(20): event_list.append(EventData("Event Number {}".format(i))) - print('Start sending events behind a proxy.') - producer.send(event_list) - print('Start receiving events behind a proxy.') - received = consumer.receive(max_batch_size=50, timeout=5) finally: producer.close() consumer.close() - diff --git a/sdk/eventhub/azure-eventhubs/examples/recv.py b/sdk/eventhub/azure-eventhubs/examples/recv.py index 4f64b40e095c..11ed8747fc22 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv.py @@ -9,18 +9,14 @@ An example to show receiving events from an Event Hub partition. """ import os -import logging import time from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") PARTITION = "0" @@ -32,23 +28,17 @@ client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) -try: - consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, event_position=EVENT_POSITION, prefetch=5000) - with consumer: - start_time = time.time() +consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, + event_position=EVENT_POSITION, prefetch=5000) +with consumer: + start_time = time.time() + batch = consumer.receive(timeout=5) + while batch: + for event_data in batch: + last_offset = event_data.offset + last_sn = event_data.sequence_number + print("Received: {}, {}".format(last_offset, last_sn)) + print(event_data.body_as_str()) + total += 1 batch = consumer.receive(timeout=5) - while batch: - for event_data in batch: - last_offset = event_data.offset - last_sn = event_data.sequence_number - print("Received: {}, {}".format(last_offset, last_sn)) - print(event_data.body_as_str()) - total += 1 - batch = consumer.receive(timeout=5) - - end_time = time.time() - run_time = end_time - start_time - print("Received {} messages in {} seconds".format(total, run_time)) - -except KeyboardInterrupt: - pass + print("Received {} messages in {} seconds".format(total, time.time() - start_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py index 9b9edcd03a84..e3255ebe1c3f 100644 --- a/sdk/eventhub/azure-eventhubs/examples/recv_batch.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_batch.py @@ -11,41 +11,31 @@ """ import os -import logging - from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") PARTITION = "0" - total = 0 last_sn = -1 last_offset = "-1" client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), network_tracing=False) -try: - consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, event_position=EVENT_POSITION, prefetch=100) - with consumer: - batched_events = consumer.receive(max_batch_size=10) - for event_data in batched_events: - last_offset = event_data.offset - last_sn = event_data.sequence_number - total += 1 - print("Partition {}, Received {}, sn={} offset={}".format( - PARTITION, - total, - last_sn, - last_offset)) - -except KeyboardInterrupt: - pass +consumer = client.create_consumer(consumer_group="$default", partition_id=PARTITION, + event_position=EVENT_POSITION, prefetch=100) +with consumer: + batched_events = consumer.receive(max_batch_size=10) + for event_data in batched_events: + last_offset = event_data.offset + last_sn = event_data.sequence_number + total += 1 + print("Partition {}, Received {}, sn={} offset={}".format( + PARTITION, + total, + last_sn, + last_offset)) diff --git a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py b/sdk/eventhub/azure-eventhubs/examples/recv_iterator.py similarity index 51% rename from sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py rename to sdk/eventhub/azure-eventhubs/examples/recv_iterator.py index 31f5b804cba3..45068ae2c1ef 100644 --- a/sdk/eventhub/azure-eventhubs/examples/iterator_receiver.py +++ b/sdk/eventhub/azure-eventhubs/examples/recv_iterator.py @@ -5,40 +5,22 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from threading import Thread import os -import time -import logging - from azure.eventhub import EventHubClient, EventPosition, EventHubSharedKeyCredential, EventData -import examples -logger = examples.get_logger(logging.INFO) - - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] EVENT_POSITION = EventPosition("-1") -class PartitionConsumerThread(Thread): - def __init__(self, consumer): - Thread.__init__(self) - self.consumer = consumer - - def run(self): - with consumer: - for item in self.consumer: - print(item) - - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) + network_tracing=False) consumer = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EVENT_POSITION) - -thread = PartitionConsumerThread(consumer) -thread.start() +try: + with consumer: + for item in consumer: + print(item) +except KeyboardInterrupt: + print("Iterator stopped") diff --git a/sdk/eventhub/azure-eventhubs/examples/send.py b/sdk/eventhub/azure-eventhubs/examples/send.py index a1a791a4d5af..219d417447c1 100644 --- a/sdk/eventhub/azure-eventhubs/examples/send.py +++ b/sdk/eventhub/azure-eventhubs/examples/send.py @@ -6,51 +6,31 @@ # -------------------------------------------------------------------------------------------- """ -An example to show sending events to an Event Hub partition. -This is just an example of sending EventData, not performance optimal. -To have the best performance, send a batch EventData with one send() call. +An example to show sending individual events to an Event Hub partition. +Although this works, sending events in batches will get better performance. +See 'send_list_of_event_data.py' and 'send_event_data_batch.py' for an example of batching. """ # pylint: disable=C0111 -import logging import time import os - from azure.eventhub import EventHubClient, EventData, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') - -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') - -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) - producer = client.create_producer(partition_id="0") - - try: - start_time = time.time() - with producer: - # not performance optimal, but works. Please do send events in batch to get much better performance. - for i in range(100): - ed = EventData("msg") - logger.info("Sending message: {}".format(i)) - producer.send(ed) - except: - raise - finally: - end_time = time.time() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass + +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] + +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +producer = client.create_producer(partition_id="0") + +start_time = time.time() +with producer: + for i in range(100): + ed = EventData("msg") + print("Sending message: {}".format(i)) + producer.send(ed) +print("Send 100 messages in {} seconds".format(time.time() - start_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/event_data_batch.py b/sdk/eventhub/azure-eventhubs/examples/send_event_data_batch.py similarity index 50% rename from sdk/eventhub/azure-eventhubs/examples/event_data_batch.py rename to sdk/eventhub/azure-eventhubs/examples/send_event_data_batch.py index 3cf6dc88f177..dfb7b8f3f749 100644 --- a/sdk/eventhub/azure-eventhubs/examples/event_data_batch.py +++ b/sdk/eventhub/azure-eventhubs/examples/send_event_data_batch.py @@ -11,21 +11,16 @@ # pylint: disable=C0111 -import logging import time import os - from azure.eventhub import EventHubClient, EventData, EventHubSharedKeyCredential -import examples -logger = examples.get_logger(logging.INFO) - -HOSTNAME = os.environ.get('EVENT_HUB_HOSTNAME') # .servicebus.windows.net -EVENT_HUB = os.environ.get('EVENT_HUB_NAME') +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] -USER = os.environ.get('EVENT_HUB_SAS_POLICY') -KEY = os.environ.get('EVENT_HUB_SAS_KEY') +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] def create_batch_data(producer): @@ -40,25 +35,11 @@ def create_batch_data(producer): return event_data_batch -try: - if not HOSTNAME: - raise ValueError("No EventHubs URL supplied.") - - client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), - network_tracing=False) - producer = client.create_producer() - - try: - start_time = time.time() - with producer: - event_data_batch = create_batch_data(producer) - producer.send(event_data_batch) - except: - raise - finally: - end_time = time.time() - run_time = end_time - start_time - logger.info("Runtime: {} seconds".format(run_time)) - -except KeyboardInterrupt: - pass +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +producer = client.create_producer() +start_time = time.time() +with producer: + event_data_batch = create_batch_data(producer) + producer.send(event_data_batch) +print("Runtime: {} seconds".format(time.time() - start_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/send_list_of_event_data.py b/sdk/eventhub/azure-eventhubs/examples/send_list_of_event_data.py new file mode 100644 index 000000000000..715c220e6417 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/send_list_of_event_data.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" +An example to show batch sending events to an Event Hub. +""" + +# pylint: disable=C0111 + +import time +import os +from azure.eventhub import EventData, EventHubClient, EventHubSharedKeyCredential + + +HOSTNAME = os.environ['EVENT_HUB_HOSTNAME'] # .servicebus.windows.net +EVENT_HUB = os.environ['EVENT_HUB_NAME'] +USER = os.environ['EVENT_HUB_SAS_POLICY'] +KEY = os.environ['EVENT_HUB_SAS_KEY'] + +client = EventHubClient(host=HOSTNAME, event_hub_path=EVENT_HUB, credential=EventHubSharedKeyCredential(USER, KEY), + network_tracing=False) +producer = client.create_producer(partition_id="1") + +event_list = [] +for i in range(1500): + event_list.append(EventData('Hello World')) +start_time = time.time() +with producer: + producer.send(event_list) +print("Runtime: {} seconds".format(time.time() - start_time)) diff --git a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py index cf8943ef1e6a..52145f9222f0 100644 --- a/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py +++ b/sdk/eventhub/azure-eventhubs/examples/test_examples_eventhub.py @@ -4,14 +4,9 @@ # license information. #-------------------------------------------------------------------------- -import pytest -import datetime -import os import time import logging -from azure.eventhub import EventHubError - def create_eventhub_client(live_eventhub_config): # [START create_eventhub_client] diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 41b8e7c36800..6507991a8eb1 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -80,5 +80,6 @@ ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], + ":python_version<'3.5'": ["typing"], } ) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py index f581f64584ab..4ac63eef5d7f 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py @@ -4,13 +4,11 @@ # license information. #-------------------------------------------------------------------------- -import os import asyncio import pytest -import time from azure.eventhub.aio import EventHubClient -from azure.eventhub import EventData, EventPosition, EventHubError +from azure.eventhub import EventPosition async def pump(receiver, sleep=None): @@ -18,25 +16,16 @@ async def pump(receiver, sleep=None): if sleep: await asyncio.sleep(sleep) async with receiver: - batch = await receiver.receive(timeout=1) + batch = await receiver.receive(timeout=10) messages += len(batch) return messages -async def get_partitions(iot_connection_str): - client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) - receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), prefetch=1000, operation='/messages/events') - async with receiver: - partitions = await client.get_properties() - return partitions["partition_ids"] - - @pytest.mark.liveTest @pytest.mark.asyncio async def test_iothub_receive_multiple_async(iot_connection_str): - pytest.skip("This will get AuthenticationError. We're investigating...") - partitions = await get_partitions(iot_connection_str) client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partitions = await client.get_partition_ids() receivers = [] for p in partitions: receivers.append(client.create_consumer(consumer_group="$default", partition_id=p, event_position=EventPosition("-1"), prefetch=10, operation='/messages/events')) @@ -44,3 +33,53 @@ async def test_iothub_receive_multiple_async(iot_connection_str): assert isinstance(outputs[0], int) and outputs[0] <= 10 assert isinstance(outputs[1], int) and outputs[1] <= 10 + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_iothub_get_properties_async(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + properties = await client.get_properties() + assert properties["partition_ids"] == ["0", "1", "2", "3"] + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_iothub_get_partition_ids_async(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partitions = await client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_iothub_get_partition_properties_async(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partition_properties = await client.get_partition_properties("0") + assert partition_properties["id"] == "0" + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_iothub_receive_after_mgmt_ops_async(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partitions = await client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] + receiver = client.create_consumer(consumer_group="$default", partition_id=partitions[0], event_position=EventPosition("-1"), operation='/messages/events') + async with receiver: + received = await receiver.receive(timeout=10) + assert len(received) == 0 + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_iothub_mgmt_ops_after_receive_async(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), operation='/messages/events') + async with receiver: + received = await receiver.receive(timeout=10) + assert len(received) == 0 + + partitions = await client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] + diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_long_running_eventprocessor.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_long_running_eventprocessor.py index 741a521d8fef..1e3cae9eefa7 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_long_running_eventprocessor.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_long_running_eventprocessor.py @@ -13,7 +13,7 @@ from logging.handlers import RotatingFileHandler from azure.eventhub.aio import EventHubClient -from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager +from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor, SamplePartitionManager from azure.eventhub import EventData @@ -44,23 +44,23 @@ def get_logger(filename, level=logging.INFO): class MyEventProcessor(PartitionProcessor): - async def close(self, reason): + async def close(self, reason, partition_context): logger.info("PartitionProcessor closed (reason {}, id {})".format( reason, - self._checkpoint_manager.partition_id + partition_context.partition_id )) - async def process_events(self, events): + async def process_events(self, events, partition_context): if events: event = events[-1] print("Processing id {}, offset {}, sq_number {})".format( - self._checkpoint_manager.partition_id, + partition_context.partition_id, event.offset, event.sequence_number)) - await self._checkpoint_manager.update_checkpoint(event.offset, event.sequence_number) + await partition_context.update_checkpoint(event.offset, event.sequence_number) - async def process_error(self, error): - logger.info("Event Processor Error for partition {}, {!r}".format(self._checkpoint_manager.partition_id, error)) + async def process_error(self, error, partition_context): + logger.info("Event Processor Error for partition {}, {!r}".format(partition_context.partition_id, error)) async def wait_and_close(host, duration): @@ -133,7 +133,7 @@ async def test_long_running_eph(live_eventhub): client, live_eventhub['consumer_group'], MyEventProcessor, - Sqlite3PartitionManager() + SamplePartitionManager() ) tasks = asyncio.gather( @@ -153,4 +153,4 @@ async def test_long_running_eph(live_eventhub): config['consumer_group'] = "$Default" config['partition'] = "0" loop = asyncio.get_event_loop() - loop.run_until_complete(test_long_running_eph(config)) \ No newline at end of file + loop.run_until_complete(test_long_running_eph(config)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py index 900612684001..50ababacf738 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py @@ -72,7 +72,7 @@ async def pump(_pid, receiver, _args, _dl): total, batch[-1].sequence_number, batch[-1].offset)) - print("{}: Total received {}".format(receiver.partition, total)) + print("{}: Total received {}".format(receiver._partition, total)) except Exception as e: print("Partition {} receiver failed: {}".format(_pid, e)) raise diff --git a/sdk/eventhub/azure-eventhubs/tests/eventprocessor_tests/test_eventprocessor.py b/sdk/eventhub/azure-eventhubs/tests/eventprocessor_tests/test_eventprocessor.py new file mode 100644 index 000000000000..28ff7cd6554b --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/eventprocessor_tests/test_eventprocessor.py @@ -0,0 +1,314 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +import asyncio + +from azure.eventhub import EventData, EventHubError +from azure.eventhub.aio import EventHubClient +from azure.eventhub.aio.eventprocessor import EventProcessor, SamplePartitionManager, PartitionProcessor, \ + CloseReason, OwnershipLostError + + +class LoadBalancerPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + pass + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_loadbalancer_balance(connstr_senders): + + connection_str, senders = connstr_senders + for sender in senders: + sender.send(EventData("EventProcessor Test")) + eventhub_client = EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = SamplePartitionManager() + + event_processor1 = EventProcessor(eventhub_client, "$default", LoadBalancerPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor1.start()) + await asyncio.sleep(5) + assert len(event_processor1._tasks) == 2 # event_processor1 claims two partitions + + event_processor2 = EventProcessor(eventhub_client, "$default", LoadBalancerPartitionProcessor, + partition_manager, polling_interval=1) + + asyncio.ensure_future(event_processor2.start()) + await asyncio.sleep(5) + assert len(event_processor1._tasks) == 1 # two event processors balance. So each has 1 task + assert len(event_processor2._tasks) == 1 + + event_processor3 = EventProcessor(eventhub_client, "$default", LoadBalancerPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor3.start()) + await asyncio.sleep(5) + assert len(event_processor3._tasks) == 0 + await event_processor3.stop() + + await event_processor1.stop() + await asyncio.sleep(5) + assert len(event_processor2._tasks) == 2 # event_procesor2 takes another one after event_processor1 stops + await event_processor2.stop() + + +@pytest.mark.asyncio +async def test_load_balancer_abandon(): + class TestPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + await asyncio.sleep(0.1) + + class MockEventHubClient(object): + eh_name = "test_eh_name" + + def create_consumer(self, consumer_group_name, partition_id, event_position): + return MockEventhubConsumer() + + async def get_partition_ids(self): + return [str(pid) for pid in range(6)] + + class MockEventhubConsumer(object): + async def receive(self): + return [] + + partition_manager = SamplePartitionManager() + + event_processor = EventProcessor(MockEventHubClient(), "$default", TestPartitionProcessor, + partition_manager, polling_interval=0.5) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(5) + + ep_list = [] + for _ in range(2): + ep = EventProcessor(MockEventHubClient(), "$default", TestPartitionProcessor, + partition_manager, polling_interval=0.5) + asyncio.ensure_future(ep.start()) + ep_list.append(ep) + await asyncio.sleep(5) + assert len(event_processor._tasks) == 2 + for ep in ep_list: + await ep.stop() + await event_processor.stop() + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_loadbalancer_list_ownership_error(connstr_senders): + class ErrorPartitionManager(SamplePartitionManager): + async def list_ownership(self, eventhub_name, consumer_group_name): + raise RuntimeError("Test runtime error") + + connection_str, senders = connstr_senders + for sender in senders: + sender.send(EventData("EventProcessor Test")) + eventhub_client = EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = ErrorPartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", LoadBalancerPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(5) + assert event_processor._running is True + assert len(event_processor._tasks) == 0 + await event_processor.stop() + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_partition_processor(connstr_senders): + partition_processor1 = None + partition_processor2 = None + + class TestPartitionProcessor(PartitionProcessor): + def __init__(self): + self.initialize_called = False + self.error = None + self.close_reason = None + self.received_events = [] + self.checkpoint = None + + async def initialize(self, partition_context): + nonlocal partition_processor1, partition_processor2 + if partition_context.partition_id == "1": + partition_processor1 = self + else: + partition_processor2 = self + + async def process_events(self, events, partition_context): + self.received_events.extend(events) + if events: + offset, sn = events[-1].offset, events[-1].sequence_number + await partition_context.update_checkpoint(offset, sn) + self.checkpoint = (offset, sn) + + async def process_error(self, error, partition_context): + self.error = error + assert partition_context is not None + + async def close(self, reason, partition_context): + self.close_reason = reason + assert partition_context is not None + + connection_str, senders = connstr_senders + for sender in senders: + sender.send(EventData("EventProcessor Test")) + eventhub_client = EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = SamplePartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", TestPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(10) + await event_processor.stop() + assert partition_processor1 is not None and partition_processor2 is not None + assert len(partition_processor1.received_events) == 1 and len(partition_processor2.received_events) == 1 + assert partition_processor1.checkpoint is not None + assert partition_processor1.close_reason == CloseReason.SHUTDOWN + assert partition_processor1.error is None + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_partition_processor_process_events_error(connstr_senders): + class ErrorPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + if partition_context.partition_id == "1": + raise RuntimeError("processing events error") + else: + pass + + async def process_error(self, error, partition_context): + if partition_context.partition_id == "1": + assert isinstance(error, RuntimeError) + else: + raise RuntimeError("There shouldn't be an error for partition other than 1") + + async def close(self, reason, partition_context): + if partition_context.partition_id == "1": + assert reason == CloseReason.PROCESS_EVENTS_ERROR + else: + assert reason == CloseReason.SHUTDOWN + + connection_str, senders = connstr_senders + for sender in senders: + sender.send(EventData("EventProcessor Test")) + eventhub_client = EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = SamplePartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", ErrorPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(10) + await event_processor.stop() + + +@pytest.mark.asyncio +async def test_partition_processor_process_eventhub_consumer_error(): + class TestPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + pass + + async def process_error(self, error, partition_context): + assert isinstance(error, EventHubError) + + async def close(self, reason, partition_context): + assert reason == CloseReason.EVENTHUB_EXCEPTION + + class MockEventHubClient(object): + eh_name = "test_eh_name" + + def create_consumer(self, consumer_group_name, partition_id, event_position): + return MockEventhubConsumer() + + async def get_partition_ids(self): + return ["0", "1"] + + class MockEventhubConsumer(object): + async def receive(self): + raise EventHubError("Mock EventHubConsumer EventHubError") + + eventhub_client = MockEventHubClient() + partition_manager = SamplePartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", TestPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(5) + await event_processor.stop() + + +@pytest.mark.asyncio +async def test_partition_processor_process_error_close_error(): + class TestPartitionProcessor(PartitionProcessor): + async def initialize(self, partition_context): + raise RuntimeError("initialize error") + + async def process_events(self, events, partition_context): + raise RuntimeError("process_events error") + + async def process_error(self, error, partition_context): + assert isinstance(error, RuntimeError) + raise RuntimeError("process_error error") + + async def close(self, reason, partition_context): + assert reason == CloseReason.PROCESS_EVENTS_ERROR + raise RuntimeError("close error") + + class MockEventHubClient(object): + eh_name = "test_eh_name" + + def create_consumer(self, consumer_group_name, partition_id, event_position): + return MockEventhubConsumer() + + async def get_partition_ids(self): + return ["0", "1"] + + class MockEventhubConsumer(object): + async def receive(self): + return [EventData("mock events")] + + eventhub_client = MockEventHubClient() #EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = SamplePartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", TestPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(5) + await event_processor.stop() + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_partition_processor_process_update_checkpoint_error(connstr_senders): + class ErrorPartitionManager(SamplePartitionManager): + async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id, + offset, sequence_number): + if partition_id == "1": + raise OwnershipLostError("Mocked ownership lost") + + class TestPartitionProcessor(PartitionProcessor): + async def process_events(self, events, partition_context): + if events: + await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number) + + async def process_error(self, error, partition_context): + assert isinstance(error, OwnershipLostError) + + async def close(self, reason, partition_context): + if partition_context.partition_id == "1": + assert reason == CloseReason.OWNERSHIP_LOST + else: + assert reason == CloseReason.SHUTDOWN + + connection_str, senders = connstr_senders + for sender in senders: + sender.send(EventData("EventProcessor Test")) + eventhub_client = EventHubClient.from_connection_string(connection_str, receive_timeout=3) + partition_manager = ErrorPartitionManager() + + event_processor = EventProcessor(eventhub_client, "$default", TestPartitionProcessor, + partition_manager, polling_interval=1) + asyncio.ensure_future(event_processor.start()) + await asyncio.sleep(10) + await event_processor.stop() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py index ac5787b6b12e..595c822b9cb7 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py @@ -4,23 +4,61 @@ # license information. #-------------------------------------------------------------------------- -import os import pytest -import time -from azure.eventhub import EventData, EventPosition, EventHubClient +from azure.eventhub import EventPosition, EventHubClient @pytest.mark.liveTest def test_iothub_receive_sync(iot_connection_str, device_id): - pytest.skip("current code will cause ErrorCodes.LinkRedirect") client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), operation='/messages/events') - receiver._open() try: - partitions = client.get_properties() - assert partitions["partition_ids"] == ["0", "1", "2", "3"] - received = receiver.receive(timeout=5) + received = receiver.receive(timeout=10) assert len(received) == 0 finally: receiver.close() + + +@pytest.mark.liveTest +def test_iothub_get_properties_sync(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + properties = client.get_properties() + assert properties["partition_ids"] == ["0", "1", "2", "3"] + + +@pytest.mark.liveTest +def test_iothub_get_partition_ids_sync(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partitions = client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] + + +@pytest.mark.liveTest +def test_iothub_get_partition_properties_sync(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partition_properties = client.get_partition_properties("0") + assert partition_properties["id"] == "0" + + +@pytest.mark.liveTest +def test_iothub_receive_after_mgmt_ops_sync(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + partitions = client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] + receiver = client.create_consumer(consumer_group="$default", partition_id=partitions[0], event_position=EventPosition("-1"), operation='/messages/events') + with receiver: + received = receiver.receive(timeout=10) + assert len(received) == 0 + + +@pytest.mark.liveTest +def test_iothub_mgmt_ops_after_receive_sync(iot_connection_str, device_id): + client = EventHubClient.from_connection_string(iot_connection_str, network_tracing=False) + receiver = client.create_consumer(consumer_group="$default", partition_id="0", event_position=EventPosition("-1"), operation='/messages/events') + with receiver: + received = receiver.receive(timeout=10) + assert len(received) == 0 + + partitions = client.get_partition_ids() + assert partitions == ["0", "1", "2", "3"] diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py index 47559b778af3..5a6e42a827e3 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py @@ -62,17 +62,17 @@ def pump(receiver, duration): iteration += 1 if size == 0: print("{}: No events received, queue size {}, delivered {}".format( - receiver.partition, + receiver._partition, receiver.queue_size, total)) elif iteration >= 5: iteration = 0 print("{}: total received {}, last sn={}, last offset={}".format( - receiver.partition, + receiver._partition, total, batch[-1].sequence_number, batch[-1].offset)) - print("{}: Total received {}".format(receiver.partition, total)) + print("{}: Total received {}".format(receiver._partition, total)) except Exception as e: print("EventHubConsumer failed: {}".format(e)) raise