fix: Account for server-side ad cue points in external text tracks. (s…

…haka-project#3617)

When an external subtitle track is added and you are using DAI, the external track does not take into account the ads that the video has, so this PR makes this internally take into account when generating the external track.

externs/shaka/ads.js

@@ -33,6 +33,24 @@
 shaka.extern.AdsStats;
 
 
+/**
+ * @typedef {{
+ *   start: number,
+ *   end: ?number
+ * }}
+ *
+ * @description
+ * Contains the times of a range of an Ad.
+ *
+ * @property {number} start
+ *   The start time of the range, in milliseconds.
+ * @property {number} end
+ *   The end time of the range, in milliseconds.
+ * @exportDoc
+ */
+shaka.extern.AdCuePoint;
+
+
 /**
  * An object that's responsible for all the ad-related logic
  * in the player.
@@ -77,6 +95,11 @@ shaka.extern.IAdManager = class extends EventTarget {
    */
   replaceServerSideAdTagParameters(adTagParameters) {}
 
+  /**
+   * @return {!Array.<!shaka.extern.AdCuePoint>}
+   */
+  getServerSideCuePoints() {}
+
   /**
    * Get statistics for the current playback session. If the player is not
    * playing content, this will return an empty stats object.

lib/ads/ad_manager.js

@@ -6,7 +6,6 @@
 
 
 goog.provide('shaka.ads.AdManager');
-goog.provide('shaka.ads.CuePoint');
 
 goog.require('shaka.Player');
 goog.require('shaka.ads.AdsStats');
@@ -568,6 +567,22 @@ shaka.ads.AdManager = class extends shaka.util.FakeEventTarget {
   }
 
 
+  /**
+   * @return {!Array.<!shaka.extern.AdCuePoint>}
+   * @override
+   * @export
+   */
+  getServerSideCuePoints() {
+    if (!this.ssAdManager_) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.ADS,
+          shaka.util.Error.Code.SS_AD_MANAGER_NOT_INITIALIZED);
+    }
+    return this.ssAdManager_.getCuePoints();
+  }
+
+
   /**
    * @return {shaka.extern.AdsStats}
    * @override
@@ -620,20 +635,6 @@ shaka.ads.AdManager = class extends shaka.util.FakeEventTarget {
   }
 };
 
-
-shaka.ads.CuePoint = class {
-  /**
-   * @param {number} start
-   * @param {?number=} end
-   */
-  constructor(start, end = null) {
-    /** @public {number} */
-    this.start = start;
-    /** @public {?number} */
-    this.end = end;
-  }
-};
-
 /**
  * The event name for when a sequence of ads has been loaded.
  *

lib/ads/client_side_ad_manager.js

@@ -145,10 +145,14 @@ shaka.ads.ClientSideAdManager = class {
 
     const cuePointStarts = this.imaAdsManager_.getCuePoints();
     if (cuePointStarts.length) {
-      /** @type {!Array.<!shaka.ads.CuePoint>} */
+      /** @type {!Array.<!shaka.extern.AdCuePoint>} */
       const cuePoints = [];
       for (const start of cuePointStarts) {
-        const shakaCuePoint = new shaka.ads.CuePoint(start);
+        /** @type {shaka.extern.AdCuePoint} */
+        const shakaCuePoint = {
+          start: start,
+          end: null,
+        };
         cuePoints.push(shakaCuePoint);
       }
 

lib/ads/server_side_ad_manager.js

@@ -62,6 +62,9 @@ shaka.ads.ServerSideAdManager = class {
     /** @private {string} */
     this.backupUrl_ = '';
 
+    /** @private {!Array.<!shaka.extern.AdCuePoint>} */
+    this.currentCuePoints_ = [];
+
     /** @private {shaka.util.EventManager} */
     this.eventManager_ = new shaka.util.EventManager();
 
@@ -212,6 +215,7 @@ shaka.ads.ServerSideAdManager = class {
     // this.streamManager_.reset();
     this.backupUrl_ = '';
     this.snapForwardTime_ = null;
+    this.currentCuePoints_ = [];
   }
 
   /**
@@ -241,6 +245,13 @@ shaka.ads.ServerSideAdManager = class {
     }
   }
 
+  /**
+   * @return {!Array.<!shaka.extern.AdCuePoint>}
+   */
+  getCuePoints() {
+    return this.currentCuePoints_;
+  }
+
   /**
    * If a seek jumped over the ad break, return to the start of the
    * ad break, then complete the seek after the ad played through.
@@ -369,13 +380,19 @@ shaka.ads.ServerSideAdManager = class {
   onCuePointsChanged_(e) {
     const streamData = e.getStreamData();
 
-    /** @type {!Array.<!shaka.ads.CuePoint>} */
+    /** @type {!Array.<!shaka.extern.AdCuePoint>} */
     const cuePoints = [];
     for (const point of streamData.cuepoints) {
-      const shakaCuePoint = new shaka.ads.CuePoint(point.start, point.end);
+      /** @type {shaka.extern.AdCuePoint} */
+      const shakaCuePoint = {
+        start: point.start,
+        end: point.end,
+      };
       cuePoints.push(shakaCuePoint);
     }
 
+    this.currentCuePoints_ = cuePoints;
+
     this.onEvent_(
         new shaka.util.FakeEvent(shaka.ads.AdManager.CUEPOINTS_CHANGED,
             {'cuepoints': cuePoints}));

lib/player.js

@@ -4452,12 +4452,20 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       mimeType = await this.getTextMimetype_(uri);
     }
 
+    let adCuePoints = [];
+    if (this.adManager_) {
+      try {
+        adCuePoints = this.adManager_.getServerSideCuePoints();
+      } catch (error) {}
+    }
+
     if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
       if (forced) {
         // See: https://github.com/whatwg/html/issues/4472
         kind = 'forced';
       }
-      await this.addSrcTrackElement_(uri, language, kind, mimeType, label);
+      await this.addSrcTrackElement_(uri, language, kind, mimeType, label || '',
+          adCuePoints);
       const textTracks = this.getTextTracks();
       const srcTrack = textTracks.find((t) => {
         return t.language == language &&
@@ -4487,6 +4495,18 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
           shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_TEXT_TO_LIVE_STREAM);
     }
 
+    if (adCuePoints.length) {
+      goog.asserts.assert(
+          this.networkingEngine_, 'Need networking engine.');
+      const data = await this.getTextData_(uri,
+          this.networkingEngine_,
+          this.config_.streaming.retryParameters);
+      const vvtText = this.convertToWebVTT_(data, mimeType, adCuePoints);
+      const blob = new Blob([vvtText], {type: 'text/vtt'});
+      uri = shaka.media.MediaSourceEngine.createObjectURL(blob);
+      mimeType = 'text/vtt';
+    }
+
     /** @type {shaka.extern.Stream} */
     const stream = {
       id: this.nextExternalStreamId_++,
@@ -4559,8 +4579,14 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     if (!mimeType) {
       mimeType = await this.getTextMimetype_(uri);
     }
+    let adCuePoints = [];
+    if (this.adManager_) {
+      try {
+        adCuePoints = this.adManager_.getServerSideCuePoints();
+      } catch (error) {}
+    }
     await this.addSrcTrackElement_(uri, language, /* kind= */ 'chapters',
-        mimeType);
+        mimeType, /* label= */ '', adCuePoints);
     const chaptersTracks = this.getChaptersTracks();
     const chaptersTrack = chaptersTracks.find((t) => {
       return t.language == language;
@@ -4623,25 +4649,27 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    * @param {string} language
    * @param {string} kind
    * @param {string} mimeType
-   * @param {string=} label
+   * @param {string} label
+   * @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
    * @private
    */
-  async addSrcTrackElement_(uri, language, kind, mimeType, label) {
-    if (mimeType != 'text/vtt') {
+  async addSrcTrackElement_(uri, language, kind, mimeType, label,
+      adCuePoints) {
+    if (mimeType != 'text/vtt' || adCuePoints.length) {
       goog.asserts.assert(
           this.networkingEngine_, 'Need networking engine.');
       const data = await this.getTextData_(uri,
           this.networkingEngine_,
           this.config_.streaming.retryParameters);
-      const vvtText = this.convertToWebVTT_(data, mimeType);
+      const vvtText = this.convertToWebVTT_(data, mimeType, adCuePoints);
       const blob = new Blob([vvtText], {type: 'text/vtt'});
       uri = shaka.media.MediaSourceEngine.createObjectURL(blob);
       mimeType = 'text/vtt';
     }
     const trackElement =
       /** @type {!HTMLTrackElement} */(document.createElement('track'));
     trackElement.src = uri;
-    trackElement.label = label || '';
+    trackElement.label = label;
     trackElement.kind = kind;
     trackElement.srclang = language;
     // Because we're pulling in the text track file via Javascript, the
@@ -4680,10 +4708,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    *
    * @param {BufferSource} buffer
    * @param {string} mimeType
+   * @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
    * @return {string}
    * @private
    */
-  convertToWebVTT_(buffer, mimeType) {
+  convertToWebVTT_(buffer, mimeType, adCuePoints) {
     const factory = shaka.text.TextEngine.findParser(mimeType);
     if (factory) {
       const obj = factory();
@@ -4694,7 +4723,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       };
       const data = shaka.util.BufferUtils.toUint8(buffer);
       const cues = obj.parseMedia(data, time);
-      return shaka.text.WebVttGenerator.convert(cues);
+      return shaka.text.WebVttGenerator.convert(cues, adCuePoints);
     }
     throw new shaka.util.Error(
         shaka.util.Error.Severity.CRITICAL,

lib/text/web_vtt_generator.js

@@ -17,9 +17,10 @@ goog.require('shaka.text.Cue');
 shaka.text.WebVttGenerator = class {
   /**
    * @param {!Array.<!shaka.text.Cue>} cues
+   * @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
    * @return {string}
    */
-  static convert(cues) {
+  static convert(cues, adCuePoints) {
     // Flatten nested cue payloads recursively.  If a cue has nested cues,
     // their contents should be combined and replace the payload of the parent.
     const flattenPayload = (cue) => {
@@ -64,6 +65,25 @@ shaka.text.WebVttGenerator = class {
       }
     };
 
+    const webvttTimeString = (time) => {
+      let newTime = time;
+      for (const adCuePoint of adCuePoints) {
+        if (adCuePoint.end && adCuePoint.start < time) {
+          const offset = adCuePoint.end - adCuePoint.start;
+          newTime += offset;
+        }
+      }
+      const hours = Math.floor(newTime / 3600);
+      const minutes = Math.floor(newTime / 60 % 60);
+      const seconds = Math.floor(newTime % 60);
+      const milliseconds = Math.floor(newTime * 1000 % 1000);
+      return (hours < 10 ? '0' : '') + hours + ':' +
+          (minutes < 10 ? '0' : '') + minutes + ':' +
+          (seconds < 10 ? '0' : '') + seconds + '.' +
+          (milliseconds < 100 ? (milliseconds < 10 ? '00' : '0') : '') +
+          milliseconds;
+    };
+
     // We don't want to modify the array or objects passed in, since we don't
     // technically own them.  So we build a new array and replace certain items
     // in it if they need to be flattened.
@@ -80,17 +100,6 @@ shaka.text.WebVttGenerator = class {
 
     let webvttString = 'WEBVTT\n\n';
     for (const cue of flattenedCues) {
-      const webvttTimeString = (time) => {
-        const hours = Math.floor(time / 3600);
-        const minutes = Math.floor(time / 60 % 60);
-        const seconds = Math.floor(time % 60);
-        const milliseconds = Math.floor(time * 1000 % 1000);
-        return (hours < 10 ? '0' : '') + hours + ':' +
-            (minutes < 10 ? '0' : '') + minutes + ':' +
-            (seconds < 10 ? '0' : '') + seconds + '.' +
-            (milliseconds < 100 ? (milliseconds < 10 ? '00' : '0') : '') +
-            milliseconds;
-      };
       const webvttSettings = (cue) => {
         const settings = [];
         const Cue = shaka.text.Cue;

test/test/util/fake_ad_manager.js

@@ -50,6 +50,13 @@ shaka.test.FakeAdManager = class extends shaka.util.FakeEventTarget {
   /** @override */
   replaceServerSideAdTagParameters(adTagParameters) {}
 
+  /**
+   * @override
+   */
+  getServerSideCuePoints() {
+    return [];
+  }
+
   /** @override */
   getStats() {
     return this.stats_;