feat(hls): Read EXT-X-PROGRAM-DATE-TIME (shaka-project#4034)

This makes the HLS parser read the EXT-X-PROGRAM-DATE-TIME value
on manifests, and use it to make sure that segments are inserted at
the correct place in the timeline, when in sequence mode.

Issue shaka-project#2337

Author: theodab

demo/common/message_ids.js

@@ -188,6 +188,7 @@ shakaDemo.MessageIds = {
   IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY: 'DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY',
   IGNORE_HLS_IMAGE_FAILURES: 'DEMO_IGNORE_HLS_IMAGE_FAILURES',
   IGNORE_HLS_TEXT_FAILURES: 'DEMO_IGNORE_HLS_TEXT_FAILURES',
+  IGNORE_MANIFEST_PROGRAM_DATE_TIME: 'DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME',
   IGNORE_MIN_BUFFER_TIME: 'DEMO_IGNORE_MIN_BUFFER_TIME',
   IGNORE_TEXT_FAILURES: 'DEMO_IGNORE_TEXT_FAILURES',
   INACCURATE_MANIFEST_TOLERANCE: 'DEMO_INACCURATE_MANIFEST_TOLERANCE',

demo/config.js

@@ -213,6 +213,8 @@ shakaDemo.Config = class {
             'manifest.hls.defaultAudioCodec')
         .addTextInput_(MessageIds.DEFAULT_VIDEO_CODEC,
             'manifest.hls.defaultVideoCodec')
+        .addBoolInput_(MessageIds.IGNORE_MANIFEST_PROGRAM_DATE_TIME,
+            'manifest.hls.ignoreManifestProgramDateTime')
         .addNumberInput_(MessageIds.AVAILABILITY_WINDOW_OVERRIDE,
             'manifest.availabilityWindowOverride',
             /* canBeDecimal= */ true,

demo/locales/en.json

@@ -99,6 +99,7 @@
   "DEMO_IMA_ASSET_KEY": "Asset key (for LIVE DAI Content)",
   "DEMO_IMA_CONTENT_SRC_ID": "Content source ID (for VOD DAI Content)",
   "DEMO_IMA_VIDEO_ID": "Video ID (for VOD DAI Content)",
+  "DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME": "Ignore Program Date Time from manifest",
   "DEMO_IGNORE_MIN_BUFFER_TIME": "Ignore Min Buffer Time",
   "DEMO_IGNORE_TEXT_FAILURES": "Ignore Text Stream Failures",
   "DEMO_INACCURATE_MANIFEST_TOLERANCE": "Inaccurate Manifest Tolerance",

demo/locales/source.json

@@ -399,6 +399,10 @@
     "description": "The label on a field that allows users to provide a video id for a custom asset.",
     "message": "Video ID (for VOD DAI Content)"
   },
+  "DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME": {
+    "description": "The name of a configuration value.",
+    "message": "Ignore Program Date Time from manifest"
+  },
   "DEMO_IGNORE_MIN_BUFFER_TIME": {
     "description": "The name of a configuration value.",
     "message": "Ignore Min Buffer Time"

externs/shaka/player.js

@@ -747,7 +747,8 @@ shaka.extern.DashManifestConfiguration;
  *   ignoreTextStreamFailures: boolean,
  *   ignoreImageStreamFailures: boolean,
  *   defaultAudioCodec: string,
- *   defaultVideoCodec: string
+ *   defaultVideoCodec: string,
+ *   ignoreManifestProgramDateTime: boolean
  * }}
  *
  * @property {boolean} ignoreTextStreamFailures
@@ -762,6 +763,11 @@ shaka.extern.DashManifestConfiguration;
  * @property {string} defaultVideoCodec
  *   The default video codec if it is not specified in the HLS playlist.
  *   <i>Defaults to <code>'avc1.42E01E'</code>.</i>
+ * @property {boolean} ignoreManifestProgramDateTime
+ *   If <code>true</code>, the HLS parser will ignore the
+ *   <code>EXT-X-PROGRAM-DATE-TIME</code> tags in the manifest.
+ *   Meant for tags that are incorrect or malformed.
+ *   <i>Defaults to <code>false</code>.</i>
  * @exportDoc
  */
 shaka.extern.HlsManifestConfiguration;

lib/hls/hls_parser.js

@@ -38,6 +38,7 @@ goog.require('shaka.util.OperationManager');
 goog.require('shaka.util.Pssh');
 goog.require('shaka.util.Timer');
 goog.require('shaka.util.Platform');
+goog.require('shaka.util.XmlUtils');
 goog.requireType('shaka.hls.Segment');
 
 
@@ -120,6 +121,15 @@ shaka.hls.HlsParser = class {
      */
     this.updatePlaylistDelay_ = 0;
 
+    /**
+     * A time offset to apply to EXT-X-PROGRAM-DATE-TIME values to normalize
+     * them so that they start at 0. This is necessary because these times will
+     * be used to set presentation times for segments.
+     * null means we don't have enough data yet.
+     * @private {?number}
+     */
+    this.syncTimeOffset_ = null;
+
     /**
      * This timer is used to trigger the start of a manifest update. A manifest
      * update is async. Once the update is finished, the timer will be restarted
@@ -351,6 +361,42 @@ shaka.hls.HlsParser = class {
     // No-op
   }
 
+  /**
+   * If necessary, makes sure that sync times will be normalized to 0, so that
+   * a stream does not start buffering at 50 years in because sync times are
+   * measured in time since 1970.
+   * @private
+   */
+  calculateSyncTimeOffset_() {
+    if (this.syncTimeOffset_ != null) {
+      // The offset was already calculated.
+      return;
+    }
+
+    const segments = new Set();
+    let lowestSyncTime = Infinity;
+    for (const streamInfo of this.uriToStreamInfosMap_.values()) {
+      const segmentIndex = streamInfo.stream.segmentIndex;
+      if (segmentIndex) {
+        segmentIndex.forEachTopLevelReference((segment) => {
+          if (segment.syncTime != null) {
+            lowestSyncTime = Math.min(lowestSyncTime, segment.syncTime);
+            segments.add(segment);
+          }
+        });
+      }
+    }
+    if (segments.size > 0) {
+      this.syncTimeOffset_ = -lowestSyncTime;
+      for (const segment of segments) {
+        segment.syncTime += this.syncTimeOffset_;
+        for (const partial of segment.partialReferences) {
+          partial.syncTime += this.syncTimeOffset_;
+        }
+      }
+    }
+  }
+
   /**
    * Parses the manifest.
    *
@@ -433,6 +479,10 @@ shaka.hls.HlsParser = class {
           shaka.util.Error.Code.OPERATION_ABORTED);
     }
 
+    // Now that we have generated all streams, we can determine the offset to
+    // apply to sync times.
+    this.calculateSyncTimeOffset_();
+
     if (this.aesEncrypted_ && variants.length == 0) {
       // We do not support AES-128 encryption with HLS yet. Variants is null
       // when the playlist is encrypted with AES-128.
@@ -1706,8 +1756,8 @@ shaka.hls.HlsParser = class {
    * @param {number} startTime
    * @param {!Map.<string, string>} variables
    * @param {string} absoluteMediaPlaylistUri
-   * @return {!shaka.media.SegmentReference}
    * @param {string} type
+   * @return {!shaka.media.SegmentReference}
    * @private
    */
   createSegmentReference_(
@@ -1730,6 +1780,23 @@ shaka.hls.HlsParser = class {
                 'true, and see https://bit.ly/3clctcj for details.');
     }
 
+    let syncTime = null;
+    if (!this.config_.hls.ignoreManifestProgramDateTime) {
+      const dateTimeTag =
+          shaka.hls.Utils.getFirstTagWithName(tags, 'EXT-X-PROGRAM-DATE-TIME');
+      if (dateTimeTag && dateTimeTag.value) {
+        const time = shaka.util.XmlUtils.parseDate(dateTimeTag.value);
+        goog.asserts.assert(time != null,
+            'EXT-X-PROGRAM-DATE-TIME format not valid');
+        // Sync time offset is null on the first go-through. This indicates that
+        // we have not yet seen every stream, and thus do not yet have enough
+        // information to determine how to normalize the sync times.
+        // For that first go-through, the sync time will be applied after the
+        // references are all created. Until then, just offset by 0.
+        syncTime = time + (this.syncTimeOffset_ || 0);
+      }
+    }
+
     // Create SegmentReferences for the partial segments.
     const partialSegmentRefs = [];
     if (this.lowLatencyMode_ && hlsSegment.partialSegments.length) {
@@ -1839,6 +1906,7 @@ shaka.hls.HlsParser = class {
         partialSegmentRefs,
         tilesLayout,
         tileDuration,
+        syncTime,
     );
   }
 
@@ -1966,6 +2034,81 @@ shaka.hls.HlsParser = class {
       references.push(reference);
     }
 
+    // If some segments have sync times, but not all, extrapolate the sync
+    // times of the ones with none.
+    const someSyncTime = references.some((ref) => ref.syncTime != null);
+    if (someSyncTime) {
+      for (let i = 0; i < references.length; i++) {
+        const reference = references[i];
+        if (reference.syncTime != null) {
+          // No need to extrapolate.
+          continue;
+        }
+        // Find the nearest segment with syncTime, in either direction.
+        // This looks forward and backward simultaneously, keeping track of what
+        // to offset the syncTime it finds by as it goes.
+        let forwardAdd = 0;
+        let forwardI = i;
+        /**
+         * Look forwards one reference at a time, summing all durations as we
+         * go, until we find a reference with a syncTime to use as a basis.
+         * This DOES count the original reference, but DOESN'T count the first
+         * reference with a syncTime (as we approach it from behind).
+         * @return {?number}
+         */
+        const lookForward = () => {
+          const other = references[forwardI];
+          if (other) {
+            if (other.syncTime != null) {
+              return other.syncTime + forwardAdd;
+            }
+            forwardAdd -= other.endTime - other.startTime;
+            forwardI += 1;
+          }
+          return null;
+        };
+        let backwardAdd = 0;
+        let backwardI = i;
+        /**
+         * Look backwards one reference at a time, summing all durations as we
+         * go, until we find a reference with a syncTime to use as a basis.
+         * This DOESN'T count the original reference, but DOES count the first
+         * reference with a syncTime (as we approach it from ahead).
+         * @return {?number}
+         */
+        const lookBackward = () => {
+          const other = references[backwardI];
+          if (other) {
+            if (other != reference) {
+              backwardAdd += other.endTime - other.startTime;
+            }
+            if (other.syncTime != null) {
+              return other.syncTime + backwardAdd;
+            }
+            backwardI -= 1;
+          }
+          return null;
+        };
+        while (reference.syncTime == null) {
+          reference.syncTime = lookBackward();
+          if (reference.syncTime == null) {
+            reference.syncTime = lookForward();
+          }
+        }
+      }
+    }
+
+    // Split the sync times properly among partial segments.
+    if (someSyncTime) {
+      for (const reference of references) {
+        let syncTime = reference.syncTime;
+        for (const partial of reference.partialReferences) {
+          partial.syncTime = syncTime;
+          syncTime += partial.endTime - partial.startTime;
+        }
+      }
+    }
+
     return references;
   }
 

lib/hls/manifest_text_parser.js

@@ -179,9 +179,8 @@ shaka.hls.ManifestTextParser = class {
           segmentTags.push(currentMapTag);
         }
         // The URI appears after all of the tags describing the segment.
-        const segment =
-            new shaka.hls.Segment(absoluteSegmentUri, segmentTags,
-                partialSegmentTags);
+        const segment = new shaka.hls.Segment(absoluteSegmentUri, segmentTags,
+            partialSegmentTags);
         segments.push(segment);
         segmentTags = [];
         partialSegmentTags = [];

lib/media/media_source_engine.js

@@ -506,7 +506,6 @@ shaka.media.MediaSourceEngine = class {
    * @param {?number} endTime relative to the start of the presentation
    * @param {?boolean} hasClosedCaptions True if the buffer contains CEA closed
    * captions
-
    * @param {boolean=} seeked True if we just seeked
    * @param {boolean=} sequenceMode True if sequence mode
    * @return {!Promise}
@@ -515,11 +514,11 @@ shaka.media.MediaSourceEngine = class {
       seeked, sequenceMode) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
-    // If we just cleared buffer and is on an unbuffered seek, we need to set
-    // the new timestampOffset of the sourceBuffer.
-    // Don't do this for text streams, though, since they don't use MediaSource
-    // anyway.
     if (startTime != null && sequenceMode && contentType != ContentType.TEXT) {
+      // If we just cleared buffer and is on an unbuffered seek, we need to set
+      // the new timestampOffset of the sourceBuffer.
+      // Don't do this for text streams, though, since they don't use
+      // MediaSource anyway.
       if (seeked) {
         const timestampOffset = /** @type {number} */ (startTime);
         this.enqueueOperation_(

lib/media/segment_index.js

@@ -101,6 +101,17 @@ shaka.media.SegmentIndex = class {
   }
 
 
+  /**
+   * Iterates over all top-level segment references in this segment index.
+   * @param {function(!shaka.media.SegmentReference)} fn
+   */
+  forEachTopLevelReference(fn) {
+    for (const reference of this.references) {
+      fn(reference);
+    }
+  }
+
+
   /**
    * Finds the position of the segment for the given time, in seconds, relative
    * to the start of the presentation.  Returns the position of the segment
@@ -348,7 +359,8 @@ shaka.media.SegmentIndex = class {
             lastReference.appendWindowEnd,
             lastReference.partialReferences,
             lastReference.tilesLayout,
-            lastReference.tileDuration);
+            lastReference.tileDuration,
+            lastReference.syncTime);
   }
 
 

lib/media/segment_reference.js

@@ -159,11 +159,15 @@ shaka.media.SegmentReference = class {
    *  The explicit duration of an individual tile within the tiles grid.
    *  If not provided, the duration should be automatically calculated based on
    *  the duration of the reference.
+   * @param {?number=} syncTime
+   *  A time value, expressed in the same scale as the start and end time, which
+   *  is used to synchronize between streams.
    */
   constructor(
       startTime, endTime, uris, startByte, endByte, initSegmentReference,
       timestampOffset, appendWindowStart, appendWindowEnd,
-      partialReferences = [], tilesLayout = '', tileDuration = null) {
+      partialReferences = [], tilesLayout = '', tileDuration = null,
+      syncTime = null) {
     // A preload hinted Partial Segment has the same startTime and endTime.
     goog.asserts.assert(startTime <= endTime,
         'startTime must be less than or equal to endTime');
@@ -213,6 +217,9 @@ shaka.media.SegmentReference = class {
 
     /** @type {?number} */
     this.tileDuration = tileDuration;
+
+    /** @type {?number} */
+    this.syncTime = syncTime;
   }
 
   /**

lib/media/streaming_engine.js

@@ -1595,14 +1595,15 @@ shaka.media.StreamingEngine = class {
 
     await this.evict_(mediaState, presentationTime);
     this.destroyer_.ensureNotDestroyed();
-    shaka.log.v1(logPrefix, 'appending media segment');
+    shaka.log.v1(logPrefix, 'appending media segment at',
+        (reference.syncTime == null ? 'unknown' : reference.syncTime));
 
     const seeked = mediaState.seeked;
     mediaState.seeked = false;
     await this.playerInterface_.mediaSourceEngine.appendBuffer(
         mediaState.type,
         segment,
-        reference.startTime,
+        reference.syncTime == null ? reference.startTime : reference.syncTime,
         reference.endTime,
         hasClosedCaptions,
         seeked,

lib/util/player_configuration.js

@@ -121,6 +121,7 @@ shaka.util.PlayerConfiguration = class {
         ignoreImageStreamFailures: false,
         defaultAudioCodec: 'mp4a.40.2',
         defaultVideoCodec: 'avc1.42E01E',
+        ignoreManifestProgramDateTime: false,
       },
     };
 

lib/util/xml_utils.js

@@ -191,7 +191,7 @@ shaka.util.XmlUtils = class {
     }
 
     const result = Date.parse(dateString);
-    return (!isNaN(result) ? Math.floor(result / 1000.0) : null);
+    return isNaN(result) ? null : (result / 1000.0);
   }