Skip to content

Commit

Permalink
fix(DASH): Fix support for multi-mimeType variants (#6348)
Browse files Browse the repository at this point in the history 
A previous PR, #5950, added support for variants that contain multiple
different codecs.
It was supposed to also add support for variants with multiple
mimeTypes, but that part didn't work correctly. This reworks a lot of
#5950 and #6047, to change how they handle such complicated variants.

This has the side-effect of allowing the stream utils to differentiate
between content that has multiple codecs because of type changes, and
content that has multiple codecs because of being muxed video+audio.

Fixes #6010

---------

Co-authored-by: Álvaro Velad Galván <[email protected]>
  • Loading branch information
@theodab @avelad
theodab and avelad authored Apr 10, 2024
1 parent d01bef3 commit 1da5da9
Show file tree
Hide file tree
Showing 31 changed files with 816 additions and 480 deletions.
12 changes: 11 additions & 1 deletion externs/shaka/manifest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,8 @@ shaka.extern.SegmentIndex = class {
* undefined),
* mssPrivateData: (shaka.extern.MssPrivateData|undefined),
* external: boolean,
* fastSwitching: boolean
* fastSwitching: boolean,
* fullMimeTypes: !Set.<string>
* }}
*
* @description
Expand Down Expand Up @@ -478,10 +479,14 @@ shaka.extern.SegmentIndex = class {
* @property {string} mimeType
* <i>Required.</i> <br>
* The Stream's MIME type, e.g., 'audio/mp4', 'video/webm', or 'text/vtt'.
* In the case of a stream that adapts between different periods with
* different MIME types, this represents only the first period.
* @property {string} codecs
* <i>Defaults to '' (i.e., unknown / not needed).</i> <br>
* The Stream's codecs, e.g., 'avc1.4d4015' or 'vp9', which must be
* compatible with the Stream's MIME type. <br>
* In the case of a stream that adapts between different periods with
* different codecs, this represents only the first period.
* See {@link https://tools.ietf.org/html/rfc6381}
* @property {(number|undefined)} frameRate
* <i>Video streams only.</i> <br>
Expand Down Expand Up @@ -583,6 +588,11 @@ shaka.extern.SegmentIndex = class {
* Eg: external text tracks.
* @property {boolean} fastSwitching
* Indicate if the stream should be used for fast switching.
* @property {!Set.<string>} fullMimeTypes
* A set of full MIME types (e.g. MIME types plus codecs information), that
* represents the types used in each period of the original manifest.
* Meant for being used by compatibility checking, such as with
* MediaSource.isTypeSupported.
*
* @exportDoc
*/
Expand Down
8 changes: 7 additions & 1 deletion externs/shaka/offline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,9 @@ shaka.extern.StreamDB;
* tilesLayout: ?string,
* pendingSegmentRefId: (string|undefined),
* pendingInitSegmentRefId: (string|undefined),
* dataKey: number
* dataKey: number,
* mimeType: ?string,
* codecs: ?string
* }}
*
* @property {?number} initSegmentKey
Expand Down Expand Up @@ -263,6 +265,10 @@ shaka.extern.StreamDB;
* downloaded.
* @property {number} dataKey
* The key to the data in storage.
* @property {?string} mimeType
* The mimeType of the segment.
* @property {?string} codecs
* The codecs of the segment.
*/
shaka.extern.SegmentDB;

Expand Down
13 changes: 10 additions & 3 deletions lib/dash/dash_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1504,8 +1504,13 @@ shaka.dash.DashParser = class {
const getBaseUris = context.representation.getBaseUris;
streamInfo = {
generateSegmentIndex: () => {
return Promise.resolve(shaka.media.SegmentIndex.forSingleSegment(
periodStart, duration, getBaseUris()));
const segmentIndex = shaka.media.SegmentIndex.forSingleSegment(
periodStart, duration, getBaseUris());
segmentIndex.forEachTopLevelReference((ref) => {
ref.mimeType = context.representation.mimeType;
ref.codecs = context.representation.codecs;
});
return Promise.resolve(segmentIndex);
},
};
}
Expand Down Expand Up @@ -1604,7 +1609,7 @@ shaka.dash.DashParser = class {
},
segmentIndex: null,
mimeType: context.representation.mimeType,
codecs: context.representation.codecs,
codecs,
frameRate: context.representation.frameRate,
pixelAspectRatio: context.representation.pixelAspectRatio,
bandwidth: context.bandwidth,
Expand Down Expand Up @@ -1634,6 +1639,8 @@ shaka.dash.DashParser = class {
accessibilityPurpose,
external: false,
fastSwitching: false,
fullMimeTypes: new Set([shaka.util.MimeUtils.getFullType(
context.representation.mimeType, context.representation.codecs)]),
};
}

Expand Down
9 changes: 8 additions & 1 deletion lib/dash/segment_base.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,17 @@ shaka.dash.SegmentBase = class {

const getUris = () => resolvedUris;
const qualityInfo = shaka.dash.SegmentBase.createQualityInfo(context);
return new shaka.media.InitSegmentReference(
const ref = new shaka.media.InitSegmentReference(
getUris,
startByte,
endByte,
qualityInfo,
/* timescale= */ null,
/* segmentData= */ null,
aesKey);
ref.codecs = context.representation.codecs;
ref.mimeType = context.representation.mimeType;
return ref;
}

/**
Expand Down Expand Up @@ -188,6 +191,10 @@ shaka.dash.SegmentBase = class {
indexData, initData, uris, initSegmentReference, timestampOffset,
appendWindowStart, appendWindowEnd);
}
for (const ref of references) {
ref.codecs = context.representation.codecs;
ref.mimeType = context.representation.mimeType;
}

presentationTimeline.notifySegments(references);

Expand Down
42 changes: 24 additions & 18 deletions lib/dash/segment_list.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ shaka.dash.SegmentList = class {
const references = SegmentList.createSegmentReferences_(
context.periodInfo.start, context.periodInfo.duration,
info.startNumber, context.representation.getBaseUris, info,
initSegmentReference, aesKey);
initSegmentReference, aesKey, context.representation.mimeType,
context.representation.codecs);

const isNew = !segmentIndex;
if (segmentIndex) {
Expand Down Expand Up @@ -200,12 +201,14 @@ shaka.dash.SegmentList = class {
* @param {shaka.dash.SegmentList.SegmentListInfo} info
* @param {shaka.media.InitSegmentReference} initSegmentReference
* @param {shaka.extern.aesKey|undefined} aesKey
* @param {string} mimeType
* @param {string} codecs
* @return {!Array.<!shaka.media.SegmentReference>}
* @private
*/
static createSegmentReferences_(
periodStart, periodDuration, startNumber, getBaseUris, info,
initSegmentReference, aesKey) {
initSegmentReference, aesKey, mimeType, codecs) {
const ManifestParserUtils = shaka.util.ManifestParserUtils;

let max = info.mediaSegments.length;
Expand Down Expand Up @@ -252,22 +255,25 @@ shaka.dash.SegmentList = class {
}
return uris;
};
references.push(
new shaka.media.SegmentReference(
periodStart + startTime,
periodStart + endTime,
getUris,
segment.start,
segment.end,
initSegmentReference,
timestampOffset,
appendWindowStart, appendWindowEnd,
/* partialReferences= */ [],
/* tilesLayout= */ '',
/* tileDuration= */ null,
/* syncTime= */ null,
shaka.media.SegmentReference.Status.AVAILABLE,
aesKey));

const ref = new shaka.media.SegmentReference(
periodStart + startTime,
periodStart + endTime,
getUris,
segment.start,
segment.end,
initSegmentReference,
timestampOffset,
appendWindowStart, appendWindowEnd,
/* partialReferences= */ [],
/* tilesLayout= */ '',
/* tileDuration= */ null,
/* syncTime= */ null,
shaka.media.SegmentReference.Status.AVAILABLE,
aesKey);
ref.codecs = codecs;
ref.mimeType = mimeType;
references.push(ref);
prevEndTime = endTime;
}

Expand Down
23 changes: 21 additions & 2 deletions lib/dash/segment_template.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,8 @@ shaka.dash.SegmentTemplate = class {
timeline: segmentInfo.timeline,
mediaTemplate: media && StringUtils.htmlUnescape(media),
indexTemplate: index,
mimeType: context.representation.mimeType,
codecs: context.representation.codecs,
};
}

Expand Down Expand Up @@ -467,6 +469,8 @@ shaka.dash.SegmentTemplate = class {
/* syncTime= */ null,
shaka.media.SegmentReference.Status.AVAILABLE,
aesKey);
ref.codecs = context.representation.codecs;
ref.mimeType = context.representation.mimeType;
// This is necessary information for thumbnail streams:
ref.trueEndTime = trueSegmentEnd;
return ref;
Expand Down Expand Up @@ -557,14 +561,17 @@ shaka.dash.SegmentTemplate = class {
return resolvedUris;
};
const qualityInfo = shaka.dash.SegmentBase.createQualityInfo(context);
return new shaka.media.InitSegmentReference(
const ref = new shaka.media.InitSegmentReference(
getUris,
/* startByte= */ 0,
/* endByte= */ null,
qualityInfo,
/* timescale= */ null,
/* segmentData= */ null,
aesKey);
ref.codecs = context.representation.codecs;
ref.mimeType = context.representation.mimeType;
return ref;
}
};

Expand Down Expand Up @@ -840,6 +847,8 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
this.periodEnd_ !== Infinity) {
segmentEnd = this.periodEnd_;
}
const codecs = this.templateInfo_.codecs;
const mimeType = this.templateInfo_.mimeType;

const partialSegmentRefs = [];

Expand Down Expand Up @@ -882,6 +891,8 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
/* syncTime= */ null,
shaka.media.SegmentReference.Status.AVAILABLE,
this.aesKey_);
partial.codecs = codecs;
partial.mimeType = mimeType;
if (this.segmentSequenceCadence_ == 0) {
if (i > 0) {
partial.markAsNonIndependent();
Expand Down Expand Up @@ -925,6 +936,8 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
shaka.media.SegmentReference.Status.AVAILABLE,
this.aesKey_,
/* allPartialSegments= */ range.partialSegments > 0);
ref.codecs = codecs;
ref.mimeType = mimeType;
ref.trueEndTime = trueSegmentEnd;
this.references[correctedPosition] = ref;
}
Expand Down Expand Up @@ -960,7 +973,9 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
* unscaledPresentationTimeOffset: number,
* timeline: Array.<shaka.media.PresentationTimeline.TimeRange>,
* mediaTemplate: ?string,
* indexTemplate: ?string
* indexTemplate: ?string,
* mimeType: string,
* codecs: string
* }}
*
* @description
Expand All @@ -982,5 +997,9 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
* The media URI template, if given.
* @property {?string} indexTemplate
* The index URI template, if given.
* @property {string} mimeType
* The mimeType.
* @property {string} codecs
* The codecs.
*/
shaka.dash.SegmentTemplate.SegmentTemplateInfo;
20 changes: 19 additions & 1 deletion lib/hls/hls_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ goog.require('shaka.util.Error');
goog.require('shaka.util.FakeEvent');
goog.require('shaka.util.LanguageUtils');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.MimeUtils');
goog.require('shaka.util.Networking');
goog.require('shaka.util.OperationManager');
goog.require('shaka.util.Pssh');
Expand Down Expand Up @@ -1260,6 +1261,7 @@ shaka.hls.HlsParser = class {
textStreamInfo.stream.mimeType =
this.guessMimeTypeBeforeLoading_(type, codecs) ||
this.guessMimeTypeFallback_(type);
this.setFullTypeForStream_(textStreamInfo.stream);
}
}
}
Expand All @@ -1269,6 +1271,15 @@ shaka.hls.HlsParser = class {
return textStreams.filter((s) => s);
}

/**
* @param {!shaka.extern.Stream} stream
* @private
*/
setFullTypeForStream_(stream) {
stream.fullMimeTypes = new Set([shaka.util.MimeUtils.getFullType(
stream.mimeType, stream.codecs)]);
}

/**
* @param {!Array.<!shaka.hls.Tag>} imageTags from the playlist.
* @param {!Array.<!shaka.hls.Tag>} iFrameTags from the playlist.
Expand Down Expand Up @@ -2297,6 +2308,7 @@ shaka.hls.HlsParser = class {
stream.mimeType = realStream.mimeType;
stream.bandwidth = stream.bandwidth || realStream.bandwidth;
stream.codecs = stream.codecs || realStream.codecs;
this.setFullTypeForStream_(stream);

// Since we lazy-loaded this content, the player may need to create new
// sessions for the DRM info in this stream.
Expand All @@ -2311,6 +2323,7 @@ shaka.hls.HlsParser = class {
// To aid manifest filtering, assume before loading that all video
// renditions have the same MIME type. (And likewise for audio.)
otherStreamInfo.stream.mimeType = realStream.mimeType;
this.setFullTypeForStream_(otherStreamInfo.stream);
}
}
}
Expand All @@ -2319,6 +2332,7 @@ shaka.hls.HlsParser = class {
const firstSegment = realStream.segmentIndex.get(0);
if (firstSegment && firstSegment.initSegmentReference) {
stream.mimeType = 'application/mp4';
this.setFullTypeForStream_(stream);
}
}

Expand Down Expand Up @@ -2584,6 +2598,7 @@ shaka.hls.HlsParser = class {
stream.drmInfos = drmInfos;
stream.keyIds = keyIds;
stream.mimeType = mimeType;
this.setFullTypeForStream_(stream);

const mediaSequenceToStartTime = this.isLive_() ?
this.mediaSequenceToStartTimeByType_.get(type) : new Map();
Expand Down Expand Up @@ -2740,7 +2755,7 @@ shaka.hls.HlsParser = class {
roles.push(shaka.util.ManifestParserUtils.TextStreamKind.SUBTITLE);
}

return {
const stream = {
id: this.globalId_++,
originalId: name,
groupId: null,
Expand Down Expand Up @@ -2778,7 +2793,10 @@ shaka.hls.HlsParser = class {
accessibilityPurpose: accessibilityPurpose,
external: false,
fastSwitching: false,
fullMimeTypes: new Set(),
};
this.setFullTypeForStream_(stream);
return stream;
}

/**
Expand Down
Loading

0 comments on commit 1da5da9

Please sign in to comment.