From e503e49c78ac6795a1d1cc01e8694bbab612c3a1 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Wed, 30 May 2018 13:51:55 -0700 Subject: [PATCH 1/6] Initial implementation --- winrm/transport.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/winrm/transport.py b/winrm/transport.py index 430c7612..9ca02d42 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals -import sys -import os +import errno import inspect +import os +import sys +import time is_py2 = sys.version[0] == '2' @@ -260,7 +262,17 @@ def send_message(self, message): def _send_message_request(self, prepared_request, message): try: - response = self.session.send(prepared_request, timeout=self.read_timeout_sec) + + # Retry connection on 'Connection refused' + for attempt in range(5): + try: + response = self.session.send(prepared_request, timeout=self.read_timeout_sec) + break + except requests.exceptions.ConnectionError as e: + if attempt == 4 or 'connection refused' not in str(e).lower(): + raise + time.sleep(5) + response.raise_for_status() return response except requests.HTTPError as ex: From bb250eef2fff2a4f24b602042a2115a5dbc02bfc Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Thu, 7 Jun 2018 20:05:39 +0200 Subject: [PATCH 2/6] Simplify and restore order --- winrm/protocol.py | 12 +++++++++++- winrm/transport.py | 24 +++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/winrm/protocol.py b/winrm/protocol.py index a645e4d6..4f5a4449 100644 --- a/winrm/protocol.py +++ b/winrm/protocol.py @@ -26,6 +26,8 @@ class Protocol(object): DEFAULT_OPERATION_TIMEOUT_SEC = 20 DEFAULT_MAX_ENV_SIZE = 153600 DEFAULT_LOCALE = 'en-US' + DEFAULT_RECONNECTION_RETRIES = 5 + DEFAULT_RECONNECTION_SLEEP = 5 def __init__( self, endpoint, transport='plaintext', username=None, @@ -39,6 +41,8 @@ def __init__( message_encryption='auto', credssp_disable_tlsv1_2=False, send_cbt=True, + reconnection_retries = DEFAULT_RECONNECTION_RETRIES, + reconnection_sleep = DEFAULT_RECONNECTION_SLEEP, ): """ @param string endpoint: the WinRM webservice endpoint @@ -57,6 +61,8 @@ def __init__( @param int operation_timeout_sec: maximum allowed time in seconds for any single wsman HTTP operation (default 20). Note that operation timeouts while receiving output (the only wsman operation that should take any significant time, and where these timeouts are expected) will be silently retried indefinitely. # NOQA @param string kerberos_hostname_override: the hostname to use for the kerberos exchange (defaults to the hostname in the endpoint URL) @param bool message_encryption_enabled: Will encrypt the WinRM messages if set to True and the transport auth supports message encryption (Default True). + @param int reconnection_retries: Number of retries on Connection Refused + @param int reconnection_sleep: Number of seconds to sleep between reconnection attempts """ try: @@ -88,7 +94,9 @@ def __init__( auth_method=transport, message_encryption=message_encryption, credssp_disable_tlsv1_2=credssp_disable_tlsv1_2, - send_cbt=send_cbt + send_cbt=send_cbt, + reconnection_retries=reconnection_retries, + reconnection_sleep=reconnection_sleep, ) self.username = username @@ -100,6 +108,8 @@ def __init__( self.kerberos_delegation = kerberos_delegation self.kerberos_hostname_override = kerberos_hostname_override self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 + self.reconnection_retries = reconnection_retries + self.reconnection_sleep = reconnection_sleep def open_shell(self, i_stream='stdin', o_stream='stdout stderr', working_directory=None, env_vars=None, noprofile=False, diff --git a/winrm/transport.py b/winrm/transport.py index 9ca02d42..b67157f5 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -1,5 +1,4 @@ from __future__ import unicode_literals -import errno import inspect import os import sys @@ -64,7 +63,10 @@ def __init__( auth_method='auto', message_encryption='auto', credssp_disable_tlsv1_2=False, - send_cbt=True): + send_cbt=True, + reconnection_retries=5, + reconnection_sleep=5, + ): self.endpoint = endpoint self.username = username self.password = password @@ -80,6 +82,8 @@ def __init__( self.message_encryption = message_encryption self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 self.send_cbt = send_cbt + self.reconnection_retries = reconnection_retries + self.reconnection_sleep = reconnection_sleep if self.server_cert_validation not in [None, 'validate', 'ignore']: raise WinRMError('invalid server_cert_validation mode: %s' % self.server_cert_validation) @@ -115,7 +119,7 @@ def __init__( from requests.packages.urllib3.exceptions import InsecureRequestWarning warnings.simplefilter('ignore', category=InsecureRequestWarning) except: pass # oh well, we tried... - + try: from urllib3.exceptions import InsecureRequestWarning warnings.simplefilter('ignore', category=InsecureRequestWarning) @@ -264,14 +268,17 @@ def _send_message_request(self, prepared_request, message): try: # Retry connection on 'Connection refused' - for attempt in range(5): + for attempt in range(self.reconnection_retries): try: response = self.session.send(prepared_request, timeout=self.read_timeout_sec) + except requests.packages.urllib3.exceptions.NewConnectionError as e: + time.sleep(self.reconnection_sleep) +# except requests.exceptions.ConnectionError as e: +# if attempt == 4 or 'connection refused' not in str(e).lower(): +# raise +# time.sleep(self.reconnection_sleep) + else: break - except requests.exceptions.ConnectionError as e: - if attempt == 4 or 'connection refused' not in str(e).lower(): - raise - time.sleep(5) response.raise_for_status() return response @@ -283,7 +290,6 @@ def _send_message_request(self, prepared_request, message): else: response_text = '' - raise WinRMTransportError('http', ex.response.status_code, response_text) def _get_message_response_text(self, response): From 47f5ebaaee9545950bbfdf33260f3901e9c1a919 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Sun, 7 Oct 2018 20:25:10 +0200 Subject: [PATCH 3/6] Implement urllib3 native retries --- winrm/protocol.py | 14 +++++++------- winrm/transport.py | 36 ++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/winrm/protocol.py b/winrm/protocol.py index 4f5a4449..7570459a 100644 --- a/winrm/protocol.py +++ b/winrm/protocol.py @@ -26,8 +26,8 @@ class Protocol(object): DEFAULT_OPERATION_TIMEOUT_SEC = 20 DEFAULT_MAX_ENV_SIZE = 153600 DEFAULT_LOCALE = 'en-US' - DEFAULT_RECONNECTION_RETRIES = 5 - DEFAULT_RECONNECTION_SLEEP = 5 + DEFAULT_RECONNECTION_RETRIES = 4 + DEFAULT_RECONNECTION_BACKOFF = 2.0 def __init__( self, endpoint, transport='plaintext', username=None, @@ -42,7 +42,7 @@ def __init__( credssp_disable_tlsv1_2=False, send_cbt=True, reconnection_retries = DEFAULT_RECONNECTION_RETRIES, - reconnection_sleep = DEFAULT_RECONNECTION_SLEEP, + reconnection_backoff = DEFAULT_RECONNECTION_BACKOFF, ): """ @param string endpoint: the WinRM webservice endpoint @@ -61,8 +61,8 @@ def __init__( @param int operation_timeout_sec: maximum allowed time in seconds for any single wsman HTTP operation (default 20). Note that operation timeouts while receiving output (the only wsman operation that should take any significant time, and where these timeouts are expected) will be silently retried indefinitely. # NOQA @param string kerberos_hostname_override: the hostname to use for the kerberos exchange (defaults to the hostname in the endpoint URL) @param bool message_encryption_enabled: Will encrypt the WinRM messages if set to True and the transport auth supports message encryption (Default True). - @param int reconnection_retries: Number of retries on Connection Refused - @param int reconnection_sleep: Number of seconds to sleep between reconnection attempts + @param int reconnection_retries: Number of retries on connection problems + @param float reconnection_backoff: Number of seconds to backoff in between reconnection attempts (first sleeps X, then sleeps 2*X, then sleeps 4*X, ...) """ try: @@ -96,7 +96,7 @@ def __init__( credssp_disable_tlsv1_2=credssp_disable_tlsv1_2, send_cbt=send_cbt, reconnection_retries=reconnection_retries, - reconnection_sleep=reconnection_sleep, + reconnection_backoff=reconnection_backoff, ) self.username = username @@ -109,7 +109,7 @@ def __init__( self.kerberos_hostname_override = kerberos_hostname_override self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 self.reconnection_retries = reconnection_retries - self.reconnection_sleep = reconnection_sleep + self.reconnection_backoff = reconnection_backoff def open_shell(self, i_stream='stdin', o_stream='stdout stderr', working_directory=None, env_vars=None, noprofile=False, diff --git a/winrm/transport.py b/winrm/transport.py index b67157f5..6387daa2 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals -import inspect -import os import sys -import time +import os +import inspect is_py2 = sys.version[0] == '2' @@ -64,8 +63,8 @@ def __init__( message_encryption='auto', credssp_disable_tlsv1_2=False, send_cbt=True, - reconnection_retries=5, - reconnection_sleep=5, + reconnection_retries=4, + reconnection_backoff=2.0, ): self.endpoint = endpoint self.username = username @@ -83,7 +82,7 @@ def __init__( self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 self.send_cbt = send_cbt self.reconnection_retries = reconnection_retries - self.reconnection_sleep = reconnection_sleep + self.reconnection_backoff = reconnection_backoff if self.server_cert_validation not in [None, 'validate', 'ignore']: raise WinRMError('invalid server_cert_validation mode: %s' % self.server_cert_validation) @@ -158,6 +157,16 @@ def build_session(self): settings = session.merge_environment_settings(url=self.endpoint, proxies={}, stream=None, verify=None, cert=None) + # Retry on connection errors, with a backoff factor + retries = requests.packages.urllib3.util.retry.Retry(total=self.reconnection_retries, + connect=self.reconnection_retries, + status=self.reconnection_retries, + read=0, + backoff_factor=self.reconnection_backoff, + status_forcelist=(413, 425, 429, 503)) + session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries)) + session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries)) + # get proxy settings from env # FUTURE: allow proxy to be passed in directly to supersede this value session.proxies = settings['proxies'] @@ -266,20 +275,7 @@ def send_message(self, message): def _send_message_request(self, prepared_request, message): try: - - # Retry connection on 'Connection refused' - for attempt in range(self.reconnection_retries): - try: - response = self.session.send(prepared_request, timeout=self.read_timeout_sec) - except requests.packages.urllib3.exceptions.NewConnectionError as e: - time.sleep(self.reconnection_sleep) -# except requests.exceptions.ConnectionError as e: -# if attempt == 4 or 'connection refused' not in str(e).lower(): -# raise -# time.sleep(self.reconnection_sleep) - else: - break - + response = self.session.send(prepared_request, timeout=self.read_timeout_sec) response.raise_for_status() return response except requests.HTTPError as ex: From cdf25c619590a6f11209e85edba434cd3fba24a6 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Sat, 13 Oct 2018 01:19:24 +0200 Subject: [PATCH 4/6] Fix Python 2.6 issues in Travis CI --- .travis.yml | 2 +- requirements-test.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5b07a4a1..d1e94471 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,10 +51,10 @@ install: - pip install virtualenv - python -m virtualenv ~/.venv - source ~/.venv/bin/activate + - pip install -r requirements-test.txt - pip install coveralls - pip install .[credssp] - pip install .[kerberos] - - pip install -r requirements-test.txt script: - py.test -v --pep8 --cov=winrm --cov-report=term-missing winrm/tests/ diff --git a/requirements-test.txt b/requirements-test.txt index d429d101..cd460172 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,3 +4,4 @@ pytest<=3.2.5 pytest-cov pytest-pep8 mock +pycparser<=2.18; python_version<"2.7" From 0c2483673c73de518eff53462632992276266163 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Sat, 20 Oct 2018 01:31:06 +0200 Subject: [PATCH 5/6] Default to no retries --- winrm/protocol.py | 2 +- winrm/transport.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/winrm/protocol.py b/winrm/protocol.py index 7570459a..c561b4b2 100644 --- a/winrm/protocol.py +++ b/winrm/protocol.py @@ -26,7 +26,7 @@ class Protocol(object): DEFAULT_OPERATION_TIMEOUT_SEC = 20 DEFAULT_MAX_ENV_SIZE = 153600 DEFAULT_LOCALE = 'en-US' - DEFAULT_RECONNECTION_RETRIES = 4 + DEFAULT_RECONNECTION_RETRIES = 0 DEFAULT_RECONNECTION_BACKOFF = 2.0 def __init__( diff --git a/winrm/transport.py b/winrm/transport.py index 6387daa2..8c8dcfab 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -63,7 +63,7 @@ def __init__( message_encryption='auto', credssp_disable_tlsv1_2=False, send_cbt=True, - reconnection_retries=4, + reconnection_retries=0, reconnection_backoff=2.0, ): self.endpoint = endpoint From 22f296684a80ce420bdcd1911ab6373eb0dc8bcd Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Sat, 20 Oct 2018 01:31:37 +0200 Subject: [PATCH 6/6] Remove HTTP 413, likely hypothetical case for WINRM --- winrm/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winrm/transport.py b/winrm/transport.py index 8c8dcfab..2de741c5 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -163,7 +163,7 @@ def build_session(self): status=self.reconnection_retries, read=0, backoff_factor=self.reconnection_backoff, - status_forcelist=(413, 425, 429, 503)) + status_forcelist=(425, 429, 503)) session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries)) session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))