feat(dash): Add support for thumbnail tracks (shaka-project#3145)

demo/common/assets.js

@@ -131,6 +131,9 @@ shakaAssets.Feature = {
   DASH: shakaDemo.MessageIds.DASH,
   // Set if the asset is an HLS manifest.
   HLS: shakaDemo.MessageIds.HLS,
+
+  // Set if the asset has at least one image stream.
+  THUMBNAILS: shakaDemo.MessageIds.THUMBNAILS,
 };
 
 
@@ -676,6 +679,17 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.MP4)
       .addFeature(shakaAssets.Feature.SUBTITLES)
       .addFeature(shakaAssets.Feature.OFFLINE),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'Tears of Steel (Thumbnails)',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/tears_of_steel.png',
+      /* manifestUri= */ 'https://demo.unified-streaming.com/video/tears-of-steel/tears-of-steel-tiled-thumbnails-timeline.ism/.mpd',
+      /* source= */ shakaAssets.Source.UNIFIED_STREAMING)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.SUBTITLES)
+      .addFeature(shakaAssets.Feature.OFFLINE)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
   // End Unified Streaming assets }}}
 
   // DASH-IF assets {{{
@@ -776,6 +790,52 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.DASH)
       .addFeature(shakaAssets.Feature.LIVE)
       .addFeature(shakaAssets.Feature.MP4),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'DASH-IF THUMBNAILS - Single adaptation set, 7 tiles at 10x1, each thumb 320x180',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',
+      /* manifestUri= */ 'https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_tiled_thumbnails.mpd',
+      /* source= */ shakaAssets.Source.DASH_IF)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'DASH-IF THUMBNAILS - Single adaptation set, 4 tiles at 10x1, each thumb 205x115',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',
+      /* manifestUri= */ 'https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_4_tiles_thumbnails.mpd',
+      /* source= */ shakaAssets.Source.DASH_IF)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'DASH-IF THUMBNAILS - Single adaptation set, 1 tile at 10x20, each thumb 102x58',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',
+      /* manifestUri= */ 'https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_tiled_thumbnails_2.mpd',
+      /* source= */ shakaAssets.Source.DASH_IF)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'DASH-IF THUMBNAILS - Two adaptation sets with different thumb resolutions',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',
+      /* manifestUri= */ 'https://dash.akamaized.net/akamai/bbb_30fps/bbb_with_multiple_tiled_thumbnails.mpd',
+      /* source= */ shakaAssets.Source.DASH_IF)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'DASH-IF THUMBNAILS - Live stream, Single adaptation set, 1x1 tiles (livesim)',
+      /* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',
+      /* manifestUri= */ 'https://livesim.dashif.org/livesim/testpic_2s/Manifest_thumbs.mpd',
+      /* source= */ shakaAssets.Source.DASH_IF)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.LIVE)
+      .addFeature(shakaAssets.Feature.THUMBNAILS),
   // End DASH-IF Assets }}}
 
   // bitcodin assets {{{

demo/common/message_ids.js

@@ -24,6 +24,7 @@ shakaDemo.MessageIds = {
   STORED: 'DEMO_STORED',
   SUBTITLES: 'DEMO_SUBTITLES',
   SURROUND: 'DEMO_SURROUND',
+  THUMBNAILS: 'DEMO_THUMBNAILS',
   TRICK_MODE: 'DEMO_TRICK_MODE',
   ULTRA_HIGH_DEFINITION: 'DEMO_ULTRA_HIGH_DEFINITION',
   VOD: 'DEMO_VOD',
@@ -155,6 +156,7 @@ shakaDemo.MessageIds = {
   DELAY_LICENSE: 'DEMO_DELAY_LICENSE',
   DISABLE_AUDIO: 'DEMO_DISABLE_AUDIO',
   DISABLE_TEXT: 'DEMO_DISABLE_TEXT',
+  DISABLE_THUMBNAILS: 'DEMO_DISABLE_THUMBNAILS',
   DISABLE_VIDEO: 'DEMO_DISABLE_VIDEO',
   DRM_RETRY_SECTION_HEADER: 'DEMO_DRM_RETRY_SECTION_HEADER',
   DRM_SECTION_HEADER: 'DEMO_DRM_SECTION_HEADER',

demo/config.js

@@ -208,7 +208,9 @@ shakaDemo.Config = class {
         .addBoolInput_(MessageIds.DISABLE_VIDEO,
             'manifest.disableVideo')
         .addBoolInput_(MessageIds.DISABLE_TEXT,
-            'manifest.disableText');
+            'manifest.disableText')
+        .addBoolInput_(MessageIds.DISABLE_THUMBNAILS,
+            'manifest.disableThumbnails');
 
     this.addRetrySection_('manifest', MessageIds.MANIFEST_RETRY_SECTION_HEADER);
   }

demo/locales/en.json

@@ -52,6 +52,7 @@
   "DEMO_DEMO_MODE_HEADER": "DEMO MODE",
   "DEMO_DISABLE_AUDIO": "Disable Audio",
   "DEMO_DISABLE_TEXT": "Disable Text",
+  "DEMO_DISABLE_THUMBNAILS": "Disable Thumbnails",
   "DEMO_DISABLE_VIDEO": "Disable Video",
   "DEMO_DOCUMENTATION": "Documentation",
   "DEMO_DRM_RETRY_SECTION_HEADER": "DRM Retry Parameters",
@@ -176,6 +177,7 @@
   "DEMO_SURROUND_SEARCH": "Filters for assets with at least one surround sound audio track.",
   "DEMO_SWITCH_INTERVAL": "Switch Interval",
   "DEMO_TEXT_LANGUAGE": "Preferred Text Language",
+  "DEMO_THUMBNAILS": "Thumbnails",
   "DEMO_TIMEOUT": "Timeout Factor",
   "DEMO_TRICK_MODE": "Special trick mode track",
   "DEMO_TRICK_MODE_SEARCH": "Filters for assets that have special video tracks to be used in trick mode playback (aka fast-forward).",

demo/locales/source.json

@@ -211,6 +211,10 @@
     "description": "The name of a configuration value.",
     "message": "Disable Text"
   },
+  "DEMO_DISABLE_THUMBNAILS": {
+    "description": "The name of a configuration value.",
+    "message": "Disable Thumbnails"
+  },
   "DEMO_DISABLE_VIDEO": {
     "description": "The name of a configuration value.",
     "message": "Disable Video"
@@ -707,6 +711,10 @@
     "description": "The name of a configuration value.",
     "message": "Preferred Text Language"
   },
+  "DEMO_THUMBNAILS": {
+    "description": "Text that describes an asset that has a thumbnail stream.",
+    "message": "Thumbnails"
+  },
   "DEMO_TIMEOUT": {
     "description": "The name of a configuration value.",
     "message": "Timeout Factor"

demo/search.js

@@ -398,6 +398,8 @@ shakaDemo.Search = class {
         shakaDemo.MessageIds.AD_SEARCH);
     this.makeBooleanInput_(specialContainer, Feature.AUDIO_ONLY, FEATURE,
         shakaDemo.MessageIds.AUDIO_ONLY_SEARCH);
+    this.makeBooleanInput_(specialContainer, Feature.THUMBNAILS, FEATURE,
+        shakaDemo.MessageIds.THUMBNAILS);
 
     container.appendChild(this.resultsDiv_);
   }

externs/shaka/manifest.js

@@ -15,6 +15,7 @@
  *   presentationTimeline: !shaka.media.PresentationTimeline,
  *   variants: !Array.<shaka.extern.Variant>,
  *   textStreams: !Array.<shaka.extern.Stream>,
+ *   imageStreams: !Array.<shaka.extern.Stream>,
  *   offlineSessionIds: !Array.<string>,
  *   minBufferTime: number
  * }}
@@ -60,6 +61,9 @@
  * @property {!Array.<shaka.extern.Stream>} textStreams
  *   <i>Required.</i> <br>
  *   The presentation's text streams.
+ * @property {!Array.<shaka.extern.Stream>} imageStreams
+ *   <i>Required.</i> <br>
+ *   The presentation's image streams
  * @property {!Array.<string>} offlineSessionIds
  *   <i>Defaults to [].</i> <br>
  *   An array of EME sessions to load for offline playback.
@@ -244,7 +248,8 @@ shaka.extern.CreateSegmentIndexFunction;
  *   channelsCount: ?number,
  *   audioSamplingRate: ?number,
  *   spatialAudio: boolean,
- *   closedCaptions: Map.<string, string>
+ *   closedCaptions: Map.<string, string>,
+ *   tilesLayout: (string|undefined)
  * }}
  *
  * @description
@@ -346,6 +351,11 @@ shaka.extern.CreateSegmentIndexFunction;
  *   as the value. If the channel number is not provided by the description,
  *   we'll set an 0-based index as the key.
  *   Example: {'CC1': 'eng'; 'CC3': 'swe'}, or {'1', 'eng'; '2': 'swe'}, etc.
+ * @property {(string|undefined)} tilesLayout
+ *   <i>Image streams only.</i> <br>
+ *   The value is a grid-item-dimension consisting of two positive decimal
+ *   integers in the format: column-x-row ('4x3'). It describes the arrangement
+ *   of Images in a Grid. The minimum valid LAYOUT is '1x1'.
  * @exportDoc
  */
 shaka.extern.Stream;

externs/shaka/offline.js

@@ -120,7 +120,8 @@ shaka.extern.ManifestDB;
  *   channelsCount: ?number,
  *   audioSamplingRate: ?number,
  *   spatialAudio: boolean,
- *   closedCaptions: Map.<string, string>
+ *   closedCaptions: Map.<string, string>,
+ *   tilesLayout: (string|undefined)
  * }}
  *
  * @property {number} id
@@ -168,7 +169,7 @@ shaka.extern.ManifestDB;
  * @property {?number} channelsCount
  *   The channel count information for the audio stream.
  * @property {?number} audioSamplingRate
- *   Specifies the maximum sampling rate of the content
+ *   Specifies the maximum sampling rate of the content.
  * @property {boolean} spatialAudio
  *   Whether the stream set has spatial audio.
  * @property {Map.<string, string>} closedCaptions
@@ -177,6 +178,10 @@ shaka.extern.ManifestDB;
  *   as the value. If the channel number is not provided by the description,
  *   we'll set an 0-based index as the key.
  *   Example: {'CC1': 'eng'; 'CC3': 'swe'}, or {'1', 'eng'; '2': 'swe'}, etc.
+ * @property {(string|undefined)} tilesLayout
+ *   The value is a grid-item-dimension consisting of two positive decimal
+ *   integers in the format: column-x-row ('4x3'). It describes the arrangement
+ *   of Images in a Grid. The minimum valid LAYOUT is '1x1'.
  */
 shaka.extern.StreamDB;
 

externs/shaka/player.js

@@ -219,19 +219,21 @@ shaka.extern.BufferedInfo;
  *   audioId: ?number,
  *   channelsCount: ?number,
  *   audioSamplingRate: ?number,
+ *   tilesLayout: ?string,
  *   audioBandwidth: ?number,
  *   videoBandwidth: ?number,
  *   spatialAudio: boolean,
  *   originalVideoId: ?string,
  *   originalAudioId: ?string,
- *   originalTextId: ?string
+ *   originalTextId: ?string,
+ *   originalImageId: ?string
  * }}
  *
  * @description
  * An object describing a media track.  This object should be treated as
  * read-only as changing any values does not have any effect.  This is the
  * public view of an audio/video paring (variant type) or text track (text
- * type).
+ * type) or image track (image type).
  *
  * @property {number} id
  *   The unique ID of the track.
@@ -240,7 +242,8 @@ shaka.extern.BufferedInfo;
  *   visible/audible in the buffer).
  *
  * @property {string} type
- *   The type of track, either <code>'variant'</code> or <code>'text'</code>.
+ *   The type of track, either <code>'variant'</code> or <code>'text'</code>
+ *   or <code>'image'</code>.
  * @property {number} bandwidth
  *   The bandwidth required to play the track, in bits/sec.
  *
@@ -294,6 +297,10 @@ shaka.extern.BufferedInfo;
  *   The count of the audio track channels.
  * @property {?number} audioSamplingRate
  *   Specifies the maximum sampling rate of the content.
+ * @property {?string} tilesLayout
+ *   The value is a grid-item-dimension consisting of two positive decimal
+ *   integers in the format: column-x-row ('4x3'). It describes the arrangement
+ *   of Images in a Grid. The minimum valid LAYOUT is '1x1'.
  * @property {boolean} spatialAudio
  *   True indicates that the content has spatial audio.
  *   This flag is based on signals from the manifest.
@@ -310,6 +317,9 @@ shaka.extern.BufferedInfo;
  * @property {?string} originalTextId
  *   (text tracks only) The original ID of the text track, if any, as it
  *   appeared in the original manifest.
+ * @property {?string} originalImageId
+ *   (image tracks only) The original ID of the image track, if any, as it
+ *   appeared in the original manifest.
  * @exportDoc
  */
 shaka.extern.Track;
@@ -681,6 +691,7 @@ shaka.extern.HlsManifestConfiguration;
  *   disableAudio: boolean,
  *   disableVideo: boolean,
  *   disableText: boolean,
+ *   disableThumbnails: boolean,
  *   defaultPresentationDelay: number,
  *   dash: shaka.extern.DashManifestConfiguration,
  *   hls: shaka.extern.HlsManifestConfiguration
@@ -702,6 +713,9 @@ shaka.extern.HlsManifestConfiguration;
  * @property {boolean} disableText
  *   If <code>true</code>, the text tracks are ignored.
  *   Defaults to <code>false</code>.
+ * @property {boolean} disableThumbnails
+ *   If <code>true</code>, the image tracks are ignored.
+ *   Defaults to <code>false</code>.
  * @property {number} defaultPresentationDelay
  *   A default <code>presentationDelay</code> value.
  *   For DASH, it's a default <code>presentationDelay</code> value if
@@ -1015,3 +1029,28 @@ shaka.extern.PlayerConfiguration;
  * @exportDoc
  */
 shaka.extern.LanguageRole;
+
+
+/**
+ * @typedef {{
+ *   height: number,
+ *   positionX: number,
+ *   positionY: number,
+ *   uris: !Array.<string>,
+ *   width: number
+ * }}
+ *
+ * @property {number} height
+ *    The thumbnail height in px.
+ * @property {number} positionX
+ *    The thumbnail left position in px.
+ * @property {number} positionY
+ *    The thumbnail top position in px.
+ * @property {!Array.<string>} uris
+ *   An array of URIs to attempt.  They will be tried in the order they are
+ *   given.
+ * @property {number} width
+ *    The thumbnail width in px.
+ * @exportDoc
+ */
+shaka.extern.Thumbnail;

lib/cast/cast_utils.js

@@ -320,6 +320,8 @@ shaka.cast.CastUtils.PlayerGetterMethods = {
   'getTextTracks': 2,
   'getStats': 5,
   'getVariantTracks': 2,
+  'getImageTracks': 2,
+  'getThumbnails': 2,
   'isAudioOnly': 10,
   'isBuffering': 1,
   'isInProgress': 1,