From 2f6f80b53717a72870843361dfe786bdd826a4d3 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Tue, 3 Nov 2020 17:21:51 +0000 Subject: [PATCH 1/5] Set wait timeout to 40 --- homeassistant/components/stream/worker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index 40f9ef873ea24..da03836a61a23 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -2,7 +2,6 @@ from collections import deque import io import logging -import time import av @@ -49,7 +48,6 @@ def stream_worker(hass, stream, quit_event): wait_timeout = 0 while not quit_event.wait(timeout=wait_timeout): - start_time = time.time() try: _stream_worker_internal(hass, stream, quit_event) except av.error.FFmpegError: # pylint: disable=c-extension-no-member @@ -57,7 +55,7 @@ def stream_worker(hass, stream, quit_event): if not stream.keepalive or quit_event.is_set(): break # To avoid excessive restarts, don't restart faster than once every 40 seconds. - wait_timeout = max(40 - (time.time() - start_time), 0) + wait_timeout = 40 _LOGGER.debug( "Restarting stream worker in %d seconds: %s", wait_timeout, From d68fe86bdb17d27f557195e95e44833bbf2913b0 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Thu, 5 Nov 2020 08:25:02 +0000 Subject: [PATCH 2/5] Use dynamic wait_timeout --- homeassistant/components/stream/worker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index da03836a61a23..2046269b0f333 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -2,6 +2,7 @@ from collections import deque import io import logging +import time import av @@ -48,14 +49,20 @@ def stream_worker(hass, stream, quit_event): wait_timeout = 0 while not quit_event.wait(timeout=wait_timeout): + start_time = time.time() try: _stream_worker_internal(hass, stream, quit_event) except av.error.FFmpegError: # pylint: disable=c-extension-no-member _LOGGER.exception("Stream connection failed: %s", stream.source) if not stream.keepalive or quit_event.is_set(): break - # To avoid excessive restarts, don't restart faster than once every 40 seconds. - wait_timeout = 40 + # To avoid excessive restarts, wait before restarting + # As the required recovery time may be different for different setups, start + # with trying a short wait_timeout and increase it on each reconnection attempt. + # Reset the wait_timeout after the worker has been up for several minutes + if time.time() - start_time > 300: + wait_timeout = 0 + wait_timeout += 10 _LOGGER.debug( "Restarting stream worker in %d seconds: %s", wait_timeout, From 4dd17d4a56606bbf298e1dee3ceb24a9077e23e6 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Thu, 5 Nov 2020 08:42:18 +0000 Subject: [PATCH 3/5] Catch error in container open --- homeassistant/components/stream/worker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index 2046269b0f333..b0a5a30d057a4 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -73,7 +73,13 @@ def stream_worker(hass, stream, quit_event): def _stream_worker_internal(hass, stream, quit_event): """Handle consuming streams.""" - container = av.open(stream.source, options=stream.options, timeout=STREAM_TIMEOUT) + try: + container = av.open( + stream.source, options=stream.options, timeout=STREAM_TIMEOUT + ) + except av.AVError: + _LOGGER.error("Error opening stream %s", stream.source) + return try: video_stream = container.streams.video[0] except (KeyError, IndexError): From 2f160d27a52e518d9de874b138161c3dbda3095f Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Fri, 6 Nov 2020 09:40:00 +0000 Subject: [PATCH 4/5] Get restart times from const.py --- homeassistant/components/stream/const.py | 3 +++ homeassistant/components/stream/worker.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/stream/const.py b/homeassistant/components/stream/const.py index 9406c24eaf647..181808e549ed6 100644 --- a/homeassistant/components/stream/const.py +++ b/homeassistant/components/stream/const.py @@ -23,3 +23,6 @@ MAX_MISSING_DTS = 6 # Number of packets missing DTS to allow STREAM_TIMEOUT = 30 # Timeout for reading stream + +STREAM_RESTART_INCREMENT = 10 # Increase wait_timeout by this amount each retry +STREAM_RESTART_RESET_TIME = 300 # Reset wait_timeout after this many seconds diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index b0a5a30d057a4..aa6a5d350a9e1 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -11,6 +11,8 @@ MAX_TIMESTAMP_GAP, MIN_SEGMENT_DURATION, PACKETS_TO_WAIT_FOR_AUDIO, + STREAM_RESTART_INCREMENT, + STREAM_RESTART_RESET_TIME, STREAM_TIMEOUT, ) from .core import Segment, StreamBuffer @@ -60,9 +62,9 @@ def stream_worker(hass, stream, quit_event): # As the required recovery time may be different for different setups, start # with trying a short wait_timeout and increase it on each reconnection attempt. # Reset the wait_timeout after the worker has been up for several minutes - if time.time() - start_time > 300: + if time.time() - start_time > STREAM_RESTART_RESET_TIME: wait_timeout = 0 - wait_timeout += 10 + wait_timeout += STREAM_RESTART_INCREMENT _LOGGER.debug( "Restarting stream worker in %d seconds: %s", wait_timeout, From ae764dc3aca452b5b79c45a374e0d53c346ee817 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Fri, 6 Nov 2020 10:03:24 +0000 Subject: [PATCH 5/5] Fix test_stream_keepalive --- tests/components/stream/test_hls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/components/stream/test_hls.py b/tests/components/stream/test_hls.py index 863513c8157d1..16d2d724f2224 100644 --- a/tests/components/stream/test_hls.py +++ b/tests/components/stream/test_hls.py @@ -147,7 +147,9 @@ def time_side_effect(): with patch("av.open") as av_open, patch( "homeassistant.components.stream.worker.time" - ) as mock_time: + ) as mock_time, patch( + "homeassistant.components.stream.worker.STREAM_RESTART_INCREMENT", 0 + ): av_open.side_effect = av.error.InvalidDataError(-2, "error") mock_time.time.side_effect = time_side_effect # Request stream