From cf15a32f59d44dcf7fd5373aa48fbf9ca6d9c5d1 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Wed, 23 Sep 2020 08:47:23 +0000 Subject: [PATCH 1/6] Create master playlist for cast --- homeassistant/components/stream/core.py | 23 +++ homeassistant/components/stream/hls.py | 166 ++++++++++++++------ homeassistant/components/stream/recorder.py | 2 +- homeassistant/components/stream/worker.py | 7 + 4 files changed, 149 insertions(+), 49 deletions(-) diff --git a/homeassistant/components/stream/core.py b/homeassistant/components/stream/core.py index 5e4e85ceea6ef..74c1eb3efcdbf 100644 --- a/homeassistant/components/stream/core.py +++ b/homeassistant/components/stream/core.py @@ -48,6 +48,9 @@ def __init__(self, stream, timeout: int = 300) -> None: self._event = asyncio.Event() self._segments = deque(maxlen=MAX_SEGMENTS) self._unsub = None + self._video_codec = None + self._audio_codec = None + self._bit_rate = None @property def name(self) -> str: @@ -153,6 +156,26 @@ def cleanup(self): self._segments = deque(maxlen=MAX_SEGMENTS) self._stream.remove_provider(self) + @property + def video_codec(self) -> tuple: + """Video codec getter.""" + return self._video_codec + + @video_codec.setter + def video_codec(self, value): + """Video codec setter.""" + self._video_codec = value + + @property + def audio_codec(self) -> tuple: + """Audio codec getter.""" + return self._audio_codec + + @audio_codec.setter + def audio_codec(self, value): + """Audio codec setter.""" + self._audio_codec = value + class StreamView(HomeAssistantView): """ diff --git a/homeassistant/components/stream/hls.py b/homeassistant/components/stream/hls.py index 816d1231c4cef..0fde59b45f790 100644 --- a/homeassistant/components/stream/hls.py +++ b/homeassistant/components/stream/hls.py @@ -1,4 +1,5 @@ """Provide functionality to stream HLS.""" +import io from typing import Callable from aiohttp import web @@ -16,9 +17,89 @@ def async_setup_hls(hass): hass.http.register_view(HlsPlaylistView()) hass.http.register_view(HlsSegmentView()) hass.http.register_view(HlsInitView()) + hass.http.register_view(HlsMasterPlaylistView()) return "/api/hls/{}/playlist.m3u8" +class HlsMasterPlaylistView(StreamView): + """Stream view used only for Chromecast compatibility.""" + + url = r"/api/hls/{token:[a-f0-9]+}/master_playlist.m3u8" + name = "api:stream:hls:master_playlist" + cors_allowed = True + + @staticmethod + def codec_string(track) -> str: + """Return RFC 6381 codec string.""" + v_name, v_profile = track.video_codec + if v_name == "hevc": + if v_profile == "main10": + codec_string = "hev1.2.6.L150.B0" + else: # HEVC Main level 5 + codec_string = "hev1.1.6.L150.B0" + else: # AVC1 level 4.1 + codec_string = "avc1" + if v_profile == "Constrained Baseline": + codec_string += ".424029" + elif v_profile == "Baseline": + codec_string += ".420029" + elif v_profile == "High": + codec_string += ".640029" + else: # Main profile + codec_string += ".4d0029" + if track.audio_codec is not None: + a_name, a_profile = track.audio_codec + if a_name == "aac": + if a_profile == "LC": + codec_string += ",mp4a.40.2" + else: # aac HE + codec_string += ",mp4a.40.5" + else: # mp3 + codec_string += ",mp4a.69" + return codec_string + + # @staticmethod + # def codec_string(track) -> str: + # """Return RFC 6381 codec string.""" + # v_name = track.video_codec[0] + # if v_name == "hevc": + # codec_string = "hev1" + # else: # AVC1 level 4.1 + # codec_string = "avc1" + # if track.audio_codec is not None: + # a_name = track.audio_codec[0] + # if a_name == "aac": + # codec_string += ",mp4a.40" + # else: # mp3 + # codec_string += ",mp4a.69" + # return codec_string + + def render(self, track): + """Render M3U8 file.""" + # Need to calculate max bandwidth as input_container.bit_rate doesn't seem to work + # Calculate file size / duration and use a multiplier to account for variation + segment = track.get_segment(track.segments[-1]) + bandwidth = round( + segment.segment.seek(0, io.SEEK_END) * 8 / segment.duration * 3 + ) + lines = [ + "#EXTM3U", + f'#EXT-X-STREAM-INF:BANDWIDTH={bandwidth},CODECS="{self.codec_string(track)}"', + "playlist.m3u8", + ] + return "\n".join(lines) + "\n" + + async def handle(self, request, stream, sequence): + """Return m3u8 playlist.""" + track = stream.add_provider("hls") + stream.start() + # Wait for a segment to be ready + if not track.segments: + await track.recv() + headers = {"Content-Type": FORMAT_CONTENT_TYPE["hls"]} + return web.Response(body=self.render(track).encode("utf-8"), headers=headers) + + class HlsPlaylistView(StreamView): """Stream view to serve a M3U8 stream.""" @@ -26,18 +107,50 @@ class HlsPlaylistView(StreamView): name = "api:stream:hls:playlist" cors_allowed = True + @staticmethod + def render_preamble(track): + """Render preamble.""" + return [ + "#EXT-X-VERSION:7", + f"#EXT-X-TARGETDURATION:{track.target_duration}", + '#EXT-X-MAP:URI="init.mp4"', + ] + + @staticmethod + def render_playlist(track): + """Render playlist.""" + segments = track.segments + + if not segments: + return [] + + playlist = ["#EXT-X-MEDIA-SEQUENCE:{}".format(segments[0])] + + for sequence in segments: + segment = track.get_segment(sequence) + playlist.extend( + [ + "#EXTINF:{:.04f},".format(float(segment.duration)), + f"./segment/{segment.sequence}.m4s", + ] + ) + + return playlist + + def render(self, track): + """Render M3U8 file.""" + lines = ["#EXTM3U"] + self.render_preamble(track) + self.render_playlist(track) + return "\n".join(lines) + "\n" + async def handle(self, request, stream, sequence): """Return m3u8 playlist.""" - renderer = M3U8Renderer(stream) track = stream.add_provider("hls") stream.start() # Wait for a segment to be ready if not track.segments: await track.recv() headers = {"Content-Type": FORMAT_CONTENT_TYPE["hls"]} - return web.Response( - body=renderer.render(track).encode("utf-8"), headers=headers - ) + return web.Response(body=self.render(track).encode("utf-8"), headers=headers) class HlsInitView(StreamView): @@ -77,49 +190,6 @@ async def handle(self, request, stream, sequence): ) -class M3U8Renderer: - """M3U8 Render Helper.""" - - def __init__(self, stream): - """Initialize renderer.""" - self.stream = stream - - @staticmethod - def render_preamble(track): - """Render preamble.""" - return [ - "#EXT-X-VERSION:7", - f"#EXT-X-TARGETDURATION:{track.target_duration}", - '#EXT-X-MAP:URI="init.mp4"', - ] - - @staticmethod - def render_playlist(track): - """Render playlist.""" - segments = track.segments - - if not segments: - return [] - - playlist = ["#EXT-X-MEDIA-SEQUENCE:{}".format(segments[0])] - - for sequence in segments: - segment = track.get_segment(sequence) - playlist.extend( - [ - "#EXTINF:{:.04f},".format(float(segment.duration)), - f"./segment/{segment.sequence}.m4s", - ] - ) - - return playlist - - def render(self, track): - """Render M3U8 file.""" - lines = ["#EXTM3U"] + self.render_preamble(track) + self.render_playlist(track) - return "\n".join(lines) + "\n" - - @PROVIDERS.register("hls") class HlsStreamOutput(StreamOutput): """Represents HLS Output formats.""" @@ -137,7 +207,7 @@ def format(self) -> str: @property def audio_codecs(self) -> str: """Return desired audio codecs.""" - return {"aac", "ac3", "mp3"} + return {"aac", "mp3"} @property def video_codecs(self) -> tuple: diff --git a/homeassistant/components/stream/recorder.py b/homeassistant/components/stream/recorder.py index 82b146cc51f54..d0b8789f60248 100644 --- a/homeassistant/components/stream/recorder.py +++ b/homeassistant/components/stream/recorder.py @@ -78,7 +78,7 @@ def format(self) -> str: @property def audio_codecs(self) -> str: """Return desired audio codec.""" - return {"aac", "ac3", "mp3"} + return {"aac", "mp3"} @property def video_codecs(self) -> tuple: diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index 40231d87a533e..9f8cf40b9e24a 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -78,6 +78,13 @@ def _stream_worker_internal(hass, stream, quit_event): if container.format.name in {"hls", "mpegts"}: audio_stream = None + # Store codec metadata on outputs + for stream_output in stream.outputs.values(): + stream_output.video_codec = (video_stream.name, video_stream.profile) + stream_output.audio_codec = ( + (audio_stream.name, audio_stream.profile) if audio_stream else None + ) + # The presentation timestamps of the first packet in each stream we receive # Use to adjust before muxing or outputting, but we don't adjust internally first_pts = {} From da392521916541c0406cd6a16d29bec19d3e899b Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Fri, 25 Sep 2020 07:08:06 +0000 Subject: [PATCH 2/6] Remove alternative codec_string method --- homeassistant/components/stream/hls.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/homeassistant/components/stream/hls.py b/homeassistant/components/stream/hls.py index 0fde59b45f790..d6505d56c17cd 100644 --- a/homeassistant/components/stream/hls.py +++ b/homeassistant/components/stream/hls.py @@ -58,22 +58,6 @@ def codec_string(track) -> str: codec_string += ",mp4a.69" return codec_string - # @staticmethod - # def codec_string(track) -> str: - # """Return RFC 6381 codec string.""" - # v_name = track.video_codec[0] - # if v_name == "hevc": - # codec_string = "hev1" - # else: # AVC1 level 4.1 - # codec_string = "avc1" - # if track.audio_codec is not None: - # a_name = track.audio_codec[0] - # if a_name == "aac": - # codec_string += ",mp4a.40" - # else: # mp3 - # codec_string += ",mp4a.69" - # return codec_string - def render(self, track): """Render M3U8 file.""" # Need to calculate max bandwidth as input_container.bit_rate doesn't seem to work From d0024cabe283d2b82de6554c41422427f79e0ec9 Mon Sep 17 00:00:00 2001 From: Justin Wong <46082645+uvjustin@users.noreply.github.com> Date: Fri, 25 Sep 2020 07:09:16 +0000 Subject: [PATCH 3/6] Reset audio_codec to None if buffer has no audio --- homeassistant/components/stream/worker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index 9f8cf40b9e24a..7ce6d8020d2f0 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -178,6 +178,9 @@ def initialize_segment(video_pts): buffer = create_stream_buffer( stream_output, video_stream, audio_stream, sequence ) + # if the created buffer does not support audio, reset audio_codec to None + if buffer.astream is None: + stream_output.audio_codec = None outputs[stream_output.name] = ( buffer, {video_stream: buffer.vstream, audio_stream: buffer.astream}, From 9b79ed3e7048501a65840f0d82ffb6582133fcba Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Sat, 26 Sep 2020 17:58:16 +0000 Subject: [PATCH 4/6] grab codec string from mp4 header instead of guessing --- homeassistant/components/stream/fmp4utils.py | 111 +++++++++++++++++++ homeassistant/components/stream/hls.py | 37 +------ 2 files changed, 115 insertions(+), 33 deletions(-) diff --git a/homeassistant/components/stream/fmp4utils.py b/homeassistant/components/stream/fmp4utils.py index 00603807215a6..dc929e531c1c6 100644 --- a/homeassistant/components/stream/fmp4utils.py +++ b/homeassistant/components/stream/fmp4utils.py @@ -36,3 +36,114 @@ def get_m4s(segment: io.BytesIO, sequence: int) -> bytes: mfra_location = next(find_box(segment, b"mfra")) segment.seek(moof_location) return segment.read(mfra_location - moof_location) + + +def get_codec_string(segment: io.BytesIO) -> str: + """Get RFC 6381 codec string.""" + codecs = [] + + # Find moov + moov_location = next(find_box(segment, b"moov")) + + # Find tracks + for trak_location in find_box(segment, b"trak", moov_location): + # Drill down to media info + mdia_location = next(find_box(segment, b"mdia", trak_location)) + minf_location = next(find_box(segment, b"minf", mdia_location)) + stbl_location = next(find_box(segment, b"stbl", minf_location)) + stsd_location = next(find_box(segment, b"stsd", stbl_location)) + + # Get stsd box + segment.seek(stsd_location) + stsd_length = int.from_bytes(segment.read(4), byteorder="big") + segment.seek(stsd_location) + stsd_box = segment.read(stsd_length) + + # Base Codec + codec = stsd_box[20:24].decode("utf-8") + + # Handle H264 + if ( + codec in ("avc1", "avc2", "avc3", "avc4") + and stsd_length > 110 + and stsd_box[106:110] == b"avcC" + ): + profile = stsd_box[111:112].hex() + compatibility = stsd_box[112:113].hex() + level = stsd_box[113:114].hex() + codec += "." + profile + compatibility + level + + # Handle H265 + elif ( + codec in ("hev1", "hvc1") + and stsd_length > 110 + and stsd_box[106:110] == b"hvcC" + ): + tmp_byte = int.from_bytes(stsd_box[111:112], byteorder="big") + + # Profile Space + codec += "." + profile_space_map = {0: "", 1: "A", 2: "B", 3: "C"} + profile_space = tmp_byte >> 6 + codec += profile_space_map[profile_space] + general_profile_idc = tmp_byte & 31 + codec += str(general_profile_idc) + + # Compatibility + codec += "." + general_profile_compatibility = int.from_bytes( + stsd_box[112:116], byteorder="big" + ) + reverse = 0 + for i in range(0, 32): + reverse |= general_profile_compatibility & 1 + if i == 31: + break + reverse <<= 1 + general_profile_compatibility >>= 1 + codec += hex(reverse)[2:] + + # Tier Flag + if (tmp_byte & 32) >> 5 == 0: + codec += ".L" + else: + codec += ".H" + codec += str(int.from_bytes(stsd_box[122:123], byteorder="big")) + + # Constraint String + has_byte = False + constraint_string = "" + for i in range(121, 115, -1): + gci = int.from_bytes(stsd_box[i : i + 1], byteorder="big") + if gci or has_byte: + constraint_string = "." + hex(gci)[2:] + constraint_string + has_byte = True + codec += constraint_string + + # Handle Audio + elif codec == "mp4a": + oti = None + dsi = None + + # Parse ES Descriptors + oti_loc = stsd_box.find(b"\x04\x80\x80\x80") + if oti_loc > 0: + oti = stsd_box[oti_loc + 5 : oti_loc + 6].hex() + codec += f".{oti}" + + dsi_loc = stsd_box.find(b"\x05\x80\x80\x80") + if dsi_loc > 0: + dsi_length = int.from_bytes( + stsd_box[dsi_loc + 4 : dsi_loc + 5], byteorder="big" + ) + dsi_data = stsd_box[dsi_loc + 5 : dsi_loc + 5 + dsi_length] + dsi0 = int.from_bytes(dsi_data[0:1], byteorder="big") + dsi = (dsi0 & 248) >> 3 + if dsi == 31 and len(dsi_data) >= 2: + dsi1 = int.from_bytes(dsi_data[1:2], byteorder="big") + dsi = 32 + ((dsi0 & 7) << 3) + ((dsi1 & 224) >> 5) + codec += f".{dsi}" + + codecs.append(codec) + + return ",".join(codecs) diff --git a/homeassistant/components/stream/hls.py b/homeassistant/components/stream/hls.py index d6505d56c17cd..2a51df700d73a 100644 --- a/homeassistant/components/stream/hls.py +++ b/homeassistant/components/stream/hls.py @@ -8,7 +8,7 @@ from .const import FORMAT_CONTENT_TYPE from .core import PROVIDERS, StreamOutput, StreamView -from .fmp4utils import get_init, get_m4s +from .fmp4utils import get_codec_string, get_init, get_m4s @callback @@ -18,7 +18,7 @@ def async_setup_hls(hass): hass.http.register_view(HlsSegmentView()) hass.http.register_view(HlsInitView()) hass.http.register_view(HlsMasterPlaylistView()) - return "/api/hls/{}/playlist.m3u8" + return "/api/hls/{}/master_playlist.m3u8" class HlsMasterPlaylistView(StreamView): @@ -28,36 +28,6 @@ class HlsMasterPlaylistView(StreamView): name = "api:stream:hls:master_playlist" cors_allowed = True - @staticmethod - def codec_string(track) -> str: - """Return RFC 6381 codec string.""" - v_name, v_profile = track.video_codec - if v_name == "hevc": - if v_profile == "main10": - codec_string = "hev1.2.6.L150.B0" - else: # HEVC Main level 5 - codec_string = "hev1.1.6.L150.B0" - else: # AVC1 level 4.1 - codec_string = "avc1" - if v_profile == "Constrained Baseline": - codec_string += ".424029" - elif v_profile == "Baseline": - codec_string += ".420029" - elif v_profile == "High": - codec_string += ".640029" - else: # Main profile - codec_string += ".4d0029" - if track.audio_codec is not None: - a_name, a_profile = track.audio_codec - if a_name == "aac": - if a_profile == "LC": - codec_string += ",mp4a.40.2" - else: # aac HE - codec_string += ",mp4a.40.5" - else: # mp3 - codec_string += ",mp4a.69" - return codec_string - def render(self, track): """Render M3U8 file.""" # Need to calculate max bandwidth as input_container.bit_rate doesn't seem to work @@ -66,9 +36,10 @@ def render(self, track): bandwidth = round( segment.segment.seek(0, io.SEEK_END) * 8 / segment.duration * 3 ) + codecs = get_codec_string(segment.segment) lines = [ "#EXTM3U", - f'#EXT-X-STREAM-INF:BANDWIDTH={bandwidth},CODECS="{self.codec_string(track)}"', + f'#EXT-X-STREAM-INF:BANDWIDTH={bandwidth},CODECS="{codecs}"', "playlist.m3u8", ] return "\n".join(lines) + "\n" From ff1d9c5bb8222cfb52905dcddd2121cbe746c101 Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Sat, 26 Sep 2020 18:04:52 +0000 Subject: [PATCH 5/6] remove dead code --- homeassistant/components/stream/core.py | 23 ----------------------- homeassistant/components/stream/worker.py | 10 ---------- 2 files changed, 33 deletions(-) diff --git a/homeassistant/components/stream/core.py b/homeassistant/components/stream/core.py index 74c1eb3efcdbf..5e4e85ceea6ef 100644 --- a/homeassistant/components/stream/core.py +++ b/homeassistant/components/stream/core.py @@ -48,9 +48,6 @@ def __init__(self, stream, timeout: int = 300) -> None: self._event = asyncio.Event() self._segments = deque(maxlen=MAX_SEGMENTS) self._unsub = None - self._video_codec = None - self._audio_codec = None - self._bit_rate = None @property def name(self) -> str: @@ -156,26 +153,6 @@ def cleanup(self): self._segments = deque(maxlen=MAX_SEGMENTS) self._stream.remove_provider(self) - @property - def video_codec(self) -> tuple: - """Video codec getter.""" - return self._video_codec - - @video_codec.setter - def video_codec(self, value): - """Video codec setter.""" - self._video_codec = value - - @property - def audio_codec(self) -> tuple: - """Audio codec getter.""" - return self._audio_codec - - @audio_codec.setter - def audio_codec(self, value): - """Audio codec setter.""" - self._audio_codec = value - class StreamView(HomeAssistantView): """ diff --git a/homeassistant/components/stream/worker.py b/homeassistant/components/stream/worker.py index 7ce6d8020d2f0..40231d87a533e 100644 --- a/homeassistant/components/stream/worker.py +++ b/homeassistant/components/stream/worker.py @@ -78,13 +78,6 @@ def _stream_worker_internal(hass, stream, quit_event): if container.format.name in {"hls", "mpegts"}: audio_stream = None - # Store codec metadata on outputs - for stream_output in stream.outputs.values(): - stream_output.video_codec = (video_stream.name, video_stream.profile) - stream_output.audio_codec = ( - (audio_stream.name, audio_stream.profile) if audio_stream else None - ) - # The presentation timestamps of the first packet in each stream we receive # Use to adjust before muxing or outputting, but we don't adjust internally first_pts = {} @@ -178,9 +171,6 @@ def initialize_segment(video_pts): buffer = create_stream_buffer( stream_output, video_stream, audio_stream, sequence ) - # if the created buffer does not support audio, reset audio_codec to None - if buffer.astream is None: - stream_output.audio_codec = None outputs[stream_output.name] = ( buffer, {video_stream: buffer.vstream, audio_stream: buffer.astream}, From 1df610cbe637836f3cc3b99960a17db204de93ca Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Sat, 26 Sep 2020 18:25:10 +0000 Subject: [PATCH 6/6] fix lint --- homeassistant/components/stream/hls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/stream/hls.py b/homeassistant/components/stream/hls.py index 2a51df700d73a..09729f79ada09 100644 --- a/homeassistant/components/stream/hls.py +++ b/homeassistant/components/stream/hls.py @@ -28,7 +28,8 @@ class HlsMasterPlaylistView(StreamView): name = "api:stream:hls:master_playlist" cors_allowed = True - def render(self, track): + @staticmethod + def render(track): """Render M3U8 file.""" # Need to calculate max bandwidth as input_container.bit_rate doesn't seem to work # Calculate file size / duration and use a multiplier to account for variation