fix(image): Fix HLS image track issues

This makes the HLS parser honor more attributes for image tracks.
It also makes some changes to player.getImageTracks, so that the
returned track shows the size of a single thumbnail rather than the
entire sheet.

Closes shaka-project#3840

Change-Id: I2ae096f455864201e08a85e29f0f02a3e06eb07f

Author: theodab

lib/hls/hls_parser.js

@@ -1281,6 +1281,30 @@ shaka.hls.HlsParser = class {
     if (this.uriToStreamInfosMap_.has(verbatimImagePlaylistUri)) {
       return this.uriToStreamInfosMap_.get(verbatimImagePlaylistUri);
     }
+
+    // Parse misc attributes.
+    const resolution = tag.getAttributeValue('RESOLUTION');
+    if (resolution) {
+      // The RESOLUTION tag represents the resolution of a single thumbnail, not
+      // of the entire sheet at once (like we expect in the output).
+      // So multiply by the layout size.
+
+      const reference = streamInfo.stream.segmentIndex.get(0);
+      const layout = reference.getTilesLayout();
+      if (layout) {
+        streamInfo.stream.width =
+            Number(resolution.split('x')[0]) * Number(layout.split('x')[0]);
+        streamInfo.stream.height =
+            Number(resolution.split('x')[1]) * Number(layout.split('x')[1]);
+        // TODO: What happens if there are multiple grids, with different
+        // layout sizes, inside this image stream?
+      }
+    }
+    const bandwidth = tag.getAttributeValue('BANDWIDTH');
+    if (bandwidth) {
+      streamInfo.stream.bandwidth = Number(bandwidth);
+    }
+
     this.uriToStreamInfosMap_.set(verbatimImagePlaylistUri, streamInfo);
     return streamInfo;
   }
@@ -1814,13 +1838,18 @@ shaka.hls.HlsParser = class {
     }
 
     let tilesLayout = '';
+    let tileDuration = null;
     if (type == shaka.util.ManifestParserUtils.ContentType.IMAGE) {
       // By default in HLS the tilesLayout is 1x1
       tilesLayout = '1x1';
       const tilesTag =
           shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-TILES');
       if (tilesTag) {
         tilesLayout = tilesTag.getRequiredAttrValue('LAYOUT');
+        const duration = tilesTag.getAttributeValue('DURATION');
+        if (duration) {
+          tileDuration = Number(duration);
+        }
       }
     }
 
@@ -1836,6 +1865,7 @@ shaka.hls.HlsParser = class {
         /* appendWindowEnd= */ Infinity,
         partialSegmentRefs,
         tilesLayout,
+        tileDuration,
     );
   }
 

lib/media/segment_index.js

@@ -347,7 +347,8 @@ shaka.media.SegmentIndex = class {
             lastReference.appendWindowStart,
             lastReference.appendWindowEnd,
             lastReference.partialReferences,
-            lastReference.tilesLayout);
+            lastReference.tilesLayout,
+            lastReference.tileDuration);
   }
 
 

lib/media/segment_reference.js

@@ -155,11 +155,15 @@ shaka.media.SegmentReference = class {
    *   The value is a grid-item-dimension consisting of two positive decimal
    *   integers in the format: column-x-row ('4x3'). It describes the
    *   arrangement of Images in a Grid. The minimum valid LAYOUT is '1x1'.
+   * @param {?number=} tileDuration
+   *  The explicit duration of an individual tile within the tiles grid.
+   *  If not provided, the duration should be automatically calculated based on
+   *  the duration of the reference.
    */
   constructor(
       startTime, endTime, uris, startByte, endByte, initSegmentReference,
       timestampOffset, appendWindowStart, appendWindowEnd,
-      partialReferences = [], tilesLayout = '') {
+      partialReferences = [], tilesLayout = '', tileDuration = null) {
     // A preload hinted Partial Segment has the same startTime and endTime.
     goog.asserts.assert(startTime <= endTime,
         'startTime must be less than or equal to endTime');
@@ -206,6 +210,9 @@ shaka.media.SegmentReference = class {
 
     /** @type {?string} */
     this.tilesLayout = tilesLayout;
+
+    /** @type {?number} */
+    this.tileDuration = tileDuration;
   }
 
   /**
@@ -290,6 +297,17 @@ shaka.media.SegmentReference = class {
   getTilesLayout() {
     return this.tilesLayout;
   }
+
+  /**
+   * Returns the segment's explicit tile duration.
+   * Only defined in image segments.
+   *
+   * @return {?number}
+   * @export
+   */
+  getTileDuration() {
+    return this.tileDuration;
+  }
 };
 
 

lib/player.js

@@ -3613,7 +3613,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       const height = fullImageHeight / rows;
       const totalImages = columns * rows;
       const segmentDuration = reference.trueEndTime - reference.startTime;
-      const thumbnailDuration = segmentDuration / totalImages;
+      const thumbnailDuration =
+          reference.getTileDuration() || (segmentDuration / totalImages);
       let thumbnailTime = reference.startTime;
       let positionX = 0;
       let positionY = 0;

lib/util/stream_utils.js

@@ -989,6 +989,25 @@ shaka.util.StreamUtils = class {
   static imageStreamToTrack(stream) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
+    let width = stream.width || null;
+    let height = stream.height || null;
+
+    // The stream width and height represent the size of the entire thumbnail
+    // sheet, so divide by the layout.
+    const reference = stream.segmentIndex.get(0);
+    let layout = stream.tilesLayout;
+    if (reference) {
+      layout = reference.getTilesLayout() || layout;
+    }
+    if (layout && width != null) {
+      width /= Number(layout.split('x')[0]);
+    }
+    if (layout && height != null) {
+      height /= Number(layout.split('x')[1]);
+    }
+    // TODO: What happens if there are multiple grids, with different
+    // layout sizes, inside this image stream?
+
     /** @type {shaka.extern.Track} */
     const track = {
       id: stream.id,
@@ -998,8 +1017,8 @@ shaka.util.StreamUtils = class {
       language: '',
       label: null,
       kind: null,
-      width: stream.width || null,
-      height: stream.height || null,
+      width,
+      height,
       frameRate: null,
       pixelAspectRatio: null,
       hdr: null,

test/player_unit.js

@@ -1092,8 +1092,8 @@ describe('Player', () => {
         // Image tracks
         manifest.addImageStream(53, (stream) => {
           stream.originalId = 'thumbnail';
-          stream.width = 100;
-          stream.height = 200;
+          stream.width = 200;
+          stream.height = 400;
           stream.bandwidth = 10;
           stream.mimeType = 'image/jpeg';
           stream.tilesLayout = '1x1';
@@ -1503,8 +1503,8 @@ describe('Player', () => {
           audioBandwidth: null,
           videoBandwidth: null,
           bandwidth: 10,
-          width: 100,
-          height: 200,
+          width: 200,
+          height: 400,
           frameRate: null,
           pixelAspectRatio: null,
           hdr: null,
@@ -3537,6 +3537,8 @@ describe('Player', () => {
 
         await player.load(fakeManifestUri, 0, fakeMimeType);
 
+        expect(player.getImageTracks()[0].width).toBe(100);
+        expect(player.getImageTracks()[0].height).toBe(50);
         const thumbnail0 = await player.getThumbnails(5, 0);
         const thumbnail1 = await player.getThumbnails(5, 11);
         const thumbnail2 = await player.getThumbnails(5, 21);