From dbf7182d27a86e5daeed7f51bbfe7ff9ac520f57 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 16 Jan 2020 12:38:07 -0800 Subject: [PATCH] Fix skipping of raw format streams in HLS Raw, containerless streams can't be played yet (#2337), but our logic for skipping or rejecting them was broken. This broken logic affected the whole v2.5.x series of releases up to and including v2.5.8. This fixes the logic and improves it in several ways: - Skip streams that can't be played, instead of rejecting the whole master playlist - Handle raw AC3 and EC3, in addition to MP3 and AAC - Handle and skip WebM+HLS in the same way - Add the playlist and segment URLs to HLS_COULD_NOT_PARSE_SEGMENT_START_TIME errors This allows us to re-enable the Apple HLS+TS asset as video-only. Backported to v2.5.x Change-Id: Ia00857d87b085aa7e2b810b0b949993cebabe4ba --- demo/common/assets.js | 2 - lib/hls/hls_parser.js | 107 +++++++++++++++++++++++++++--------- lib/util/error.js | 8 +++ test/hls/hls_parser_unit.js | 74 +++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 28 deletions(-) diff --git a/demo/common/assets.js b/demo/common/assets.js index 7e9448e6d6..f77233837f 100644 --- a/demo/common/assets.js +++ b/demo/common/assets.js @@ -999,8 +999,6 @@ shakaAssets.testAssets = [ /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/apple_test_pattern.png', /* manifestUri= */ 'https://storage.googleapis.com/shaka-demo-assets/apple-advanced-stream-ts/master.m3u8', /* source= */ shakaAssets.Source.APPLE) - // Disabled until we support raw AAC: https://github.com/google/shaka-player/issues/2337 - .markAsDisabled() .addFeature(shakaAssets.Feature.HLS) .addFeature(shakaAssets.Feature.MP2TS) .addFeature(shakaAssets.Feature.CAPTIONS) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index c719ec5b8d..e962f12948 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -800,6 +800,10 @@ shaka.hls.HlsParser.prototype.createVariantsForTag_ = */ shaka.hls.HlsParser.prototype.filterLegacyCodecs_ = function(streamInfos) { streamInfos.forEach(function(streamInfo) { + if (!streamInfo) { + return; + } + let codecs = streamInfo.stream.codecs.split(','); codecs = codecs.filter(function(codec) { // mp4a.40.34 is a nonstandard codec string that is sometimes used in HLS @@ -814,8 +818,8 @@ shaka.hls.HlsParser.prototype.filterLegacyCodecs_ = function(streamInfos) { /** - * @param {!Array.} audioInfos - * @param {!Array.} videoInfos + * @param {!Array.} audioInfos + * @param {!Array.} videoInfos * @param {number} bandwidth * @param {?string} width * @param {?string} height @@ -1026,12 +1030,12 @@ shaka.hls.HlsParser.prototype.createStreamInfoFromMediaTag_ = const streamInfo = await this.createStreamInfo_( verbatimMediaPlaylistUri, allCodecs, type, language, primary, name, channelsCount, /* closedCaptions */ null); - if (streamInfo == null) return null; // TODO: This check is necessary because of the possibility of multiple // calls to createStreamInfoFromMediaTag_ before either has resolved. if (this.uriToStreamInfosMap_.has(verbatimMediaPlaylistUri)) { return this.uriToStreamInfosMap_.get(verbatimMediaPlaylistUri); } + if (streamInfo == null) return null; this.mediaTagsToStreamInfosMap_.set(tag.id, streamInfo); this.uriToStreamInfosMap_.set(verbatimMediaPlaylistUri, streamInfo); @@ -1210,8 +1214,8 @@ shaka.hls.HlsParser.prototype.createStreamInfo_ = async function( const mimeType = await this.guessMimeType_(type, codecs, playlist); // MediaSource expects no codec strings combined with raw formats. - // TODO(#2337): Replace with a flag indicating a raw format. - if (mimeType == 'audio/mpeg' || mimeType == 'audio/aac') { + // TODO(#2337): Instead, create a Stream flag indicating a raw format. + if (shaka.hls.HlsParser.RAW_FORMATS_.includes(mimeType)) { codecs = ''; } @@ -1220,8 +1224,20 @@ shaka.hls.HlsParser.prototype.createStreamInfo_ = async function( let startPosition = mediaSequenceTag ? Number(mediaSequenceTag.value) : 0; - const segments = await this.createSegments_( - verbatimMediaPlaylistUri, playlist, startPosition, mimeType, codecs); + let segments; + + try { + segments = await this.createSegments_(verbatimMediaPlaylistUri, + playlist, startPosition, mimeType, codecs); + } catch (error) { + if (error.code == shaka.util.Error.Code.HLS_INTERNAL_SKIP_STREAM) { + shaka.log.alwaysWarn('Skipping unsupported HLS stream', + mimeType, verbatimMediaPlaylistUri); + return null; + } + + throw error; + } let minTimestamp = segments[0].startTime; let lastEndTime = segments[segments.length - 1].endTime; @@ -1646,13 +1662,27 @@ shaka.hls.HlsParser.prototype.getStartTime_ = async function( shaka.log.v1('Fetching segment to find start time'); + if (shaka.hls.HlsParser.RAW_FORMATS_.includes(mimeType)) { + // Raw formats contain no timestamps. Even if there is an ID3 tag with a + // timestamp, that's not going to be honored by MediaSource, which will + // use sequence mode for these segments. We don't yet support sequence + // mode, so we must reject these streams. + // TODO(#2337): Support sequence mode and align raw format timestamps to + // other streams. + shaka.log.alwaysWarn( + 'Raw formats are not yet supported. Skipping ' + mimeType); + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.MANIFEST, + shaka.util.Error.Code.HLS_INTERNAL_SKIP_STREAM); + } - if (mimeType == 'audio/mpeg' || mimeType == 'audio/aac') { - // Raw MP3 and AAC files contain no timestamps. - // Don't return a false timestamp. We want to treat them as aligning to - // their corresponding video segments. - // TODO(#2337): Avoid trying to fetch timestamps for raw formats. - return null; + if (mimeType == 'video/webm') { + shaka.log.alwaysWarn('WebM in HLS is not yet supported. Skipping.'); + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.MANIFEST, + shaka.util.Error.Code.HLS_INTERNAL_SKIP_STREAM); } if (mimeType == 'video/mp4' || mimeType == 'audio/mp4') { @@ -1672,13 +1702,15 @@ shaka.hls.HlsParser.prototype.getStartTime_ = async function( const initSegmentResponse = responses[1] || responses[0]; return this.getStartTimeFromMp4Segment_( + verbatimMediaPlaylistUri, segmentResponse.uri, segmentResponse.data, initSegmentResponse.data); } if (mimeType == 'video/mp2t') { const response = await this.fetchPartialSegment_(segmentRef); goog.asserts.assert(response.data, 'Should have a response body!'); - return this.getStartTimeFromTsSegment_(response.data); + return this.getStartTimeFromTsSegment_( + verbatimMediaPlaylistUri, response.uri, response.data); } if (mimeType == 'application/mp4' || mimeType.startsWith('text/')) { @@ -1687,22 +1719,19 @@ shaka.hls.HlsParser.prototype.getStartTime_ = async function( return this.getStartTimeFromTextSegment_(mimeType, codecs, response.data); } - if (mimeType == 'video/webm') { - shaka.log.warning( - 'Hls+WebM combination is not supported at the moment. Skipping.'); - return null; - } - throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MANIFEST, - shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME); + shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME, + verbatimMediaPlaylistUri); }; /** * Parses an mp4 segment to get its start time. * + * @param {string} playlistUri + * @param {string} segmentUri * @param {!ArrayBuffer} mediaData * @param {!ArrayBuffer} initData * @return {number} @@ -1710,7 +1739,7 @@ shaka.hls.HlsParser.prototype.getStartTime_ = async function( * @private */ shaka.hls.HlsParser.prototype.getStartTimeFromMp4Segment_ = - function(mediaData, initData) { + function(playlistUri, segmentUri, mediaData, initData) { const Mp4Parser = shaka.util.Mp4Parser; let timescale = 0; @@ -1737,7 +1766,8 @@ shaka.hls.HlsParser.prototype.getStartTimeFromMp4Segment_ = throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MANIFEST, - shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME); + shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME, + playlistUri, segmentUri); } let startTime = 0; @@ -1761,7 +1791,8 @@ shaka.hls.HlsParser.prototype.getStartTimeFromMp4Segment_ = throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MANIFEST, - shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME); + shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME, + playlistUri, segmentUri); } return startTime; }; @@ -1770,12 +1801,15 @@ shaka.hls.HlsParser.prototype.getStartTimeFromMp4Segment_ = /** * Parses a TS segment to get its start time. * + * @param {string} playlistUri + * @param {string} segmentUri * @param {!ArrayBuffer} data * @return {number} * @throws {shaka.util.Error} * @private */ -shaka.hls.HlsParser.prototype.getStartTimeFromTsSegment_ = function(data) { +shaka.hls.HlsParser.prototype.getStartTimeFromTsSegment_ = + function(playlistUri, segmentUri, data) { let reader = new shaka.util.DataViewReader( new DataView(data), shaka.util.DataViewReader.Endianness.BIG_ENDIAN); @@ -1783,7 +1817,8 @@ shaka.hls.HlsParser.prototype.getStartTimeFromTsSegment_ = function(data) { throw new shaka.util.Error( shaka.util.Error.Severity.CRITICAL, shaka.util.Error.Category.MANIFEST, - shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME); + shaka.util.Error.Code.HLS_COULD_NOT_PARSE_SEGMENT_START_TIME, + playlistUri, segmentUri); }; let packetStart = 0; @@ -2210,10 +2245,30 @@ shaka.hls.HlsParser.AUDIO_EXTENSIONS_TO_MIME_TYPES_ = { 'm4a': 'audio/mp4', // MPEG2-TS also uses video/ for audio: https://bit.ly/TsMse 'ts': 'video/mp2t', + + // Raw formats: 'aac': 'audio/aac', + 'ac3': 'audio/ac3', + 'ec3': 'audio/ec3', + 'mp3': 'audio/mpeg', }; +/** + * MIME types of raw formats. + * TODO(#2337): Support raw formats and share this list among parsers. + * + * @const {!Array.} + * @private + */ +shaka.hls.HlsParser.RAW_FORMATS_ = [ + 'audio/aac', + 'audio/ac3', + 'audio/ec3', + 'audio/mpeg', +]; + + /** * @const {!Object.} * @private diff --git a/lib/util/error.js b/lib/util/error.js index ec19785859..e73098bc1a 100644 --- a/lib/util/error.js +++ b/lib/util/error.js @@ -591,6 +591,8 @@ shaka.util.Error.Code = { /** * The HLS parser was unable to parse segment start time from the media. + *
error.data[0] is the failed media playlist URI. + *
error.data[1] is the failed media segment URI (if any). */ 'HLS_COULD_NOT_PARSE_SEGMENT_START_TIME': 4030, @@ -619,6 +621,12 @@ shaka.util.Error.Code = { */ 'HLS_AES_128_ENCRYPTION_NOT_SUPPORTED': 4034, + /** + * An internal error code that should never be seen by applications, thrown + * to force the HLS parser to skip an unsupported stream. + */ + 'HLS_INTERNAL_SKIP_STREAM': 4035, + // RETIRED: 'INCONSISTENT_BUFFER_STATE': 5000, // RETIRED: 'INVALID_SEGMENT_INDEX': 5001, // RETIRED: 'SEGMENT_DOES_NOT_EXIST': 5002, diff --git a/test/hls/hls_parser_unit.js b/test/hls/hls_parser_unit.js index 4e1a836467..2c9b5748f4 100644 --- a/test/hls/hls_parser_unit.js +++ b/test/hls/hls_parser_unit.js @@ -20,6 +20,7 @@ describe('HlsParser', function() { const ManifestParser = shaka.test.ManifestParser; const TextStreamKind = shaka.util.ManifestParserUtils.TextStreamKind; const Util = shaka.test.Util; + const originalAlwaysWarn = shaka.log.alwaysWarn; const vttText = [ 'WEBVTT\n', @@ -43,6 +44,10 @@ describe('HlsParser', function() { /** @type {!ArrayBuffer} */ let selfInitializingSegmentData; + afterEach(() => { + shaka.log.alwaysWarn = originalAlwaysWarn; + }); + beforeEach(function() { // TODO: use StreamGenerator? initSegmentData = new Uint8Array([ @@ -2229,4 +2234,73 @@ describe('HlsParser', function() { await testHlsParser(master, media, manifest); }); + + it('skips raw audio formats', async () => { + const master = [ + '#EXTM3U\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",URI="audio1"\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",URI="audio2"\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",URI="audio3"\n', + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",URI="audio4"\n', + '#EXT-X-STREAM-INF:BANDWIDTH=400,CODECS="avc1,mp4a",', + 'RESOLUTION=1280x720,AUDIO="audio"\n', + 'video\n', + ].join(''); + + const videoMedia = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXT-X-MAP:URI="v-init.mp4"\n', + '#EXTINF:5,\n', + 'v1.mp4', + ].join(''); + + const audioMedia1 = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXTINF:5,\n', + 'a1.mp3', + ].join(''); + + const audioMedia2 = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXTINF:5,\n', + 'a1.aac', + ].join(''); + + const audioMedia3 = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXTINF:5,\n', + 'a1.ac3', + ].join(''); + + const audioMedia4 = [ + '#EXTM3U\n', + '#EXT-X-PLAYLIST-TYPE:VOD\n', + '#EXTINF:5,\n', + 'a1.ec3', + ].join(''); + + fakeNetEngine + .setResponseText('test:/master', master) + .setResponseText('test:/video', videoMedia) + .setResponseText('test:/audio1', audioMedia1) + .setResponseText('test:/audio2', audioMedia2) + .setResponseText('test:/audio3', audioMedia3) + .setResponseText('test:/audio4', audioMedia4) + .setResponseValue('test:/v-init.mp4', initSegmentData) + .setResponseValue('test:/v1.mp4', segmentData); + + const alwaysWarnSpy = jasmine.createSpy('shaka.log.alwaysWarn'); + shaka.log.alwaysWarn = shaka.test.Util.spyFunc(alwaysWarnSpy); + + const manifest = await parser.start('test:/master', playerInterface); + expect(manifest.periods[0].variants.length).toBe(1); + expect(manifest.periods[0].variants[0].audio).toBe(null); + + // We should log a warning when this happens. + expect(alwaysWarnSpy).toHaveBeenCalled(); + }); });