feat(UI): UI support for VR content (#6450)

Co-authored-by: Joey Parrish <[email protected]>

LICENSE

@@ -284,3 +284,30 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+-----
+
+Contains code from https://github.com/toji/gl-matrix
+(See ui/gl_matrix/matrix_4x4.js and ui/gl_matrix/matrix_quaternion.js)
+
+The MIT License (MIT)
+
+Copyright (c) 2015-2021, Brandon Jones, Colin MacKenzie IV.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -317,6 +317,24 @@ Content Steering features **not** supported:
 - PATHWAY-CLONES other replacements than HOST.
 
 
+## VR support
+Shaka Player supports VR when:
+- Content is automatically treated as VR if it fits the following criteria:
+  - HLS or DASH manifest
+  - fMP4 segments
+  - Init segment contains `prji` and `hfov` boxes
+- Or, if it is manually enabled via the UI config.
+
+VR modes supported:
+- Equirectangular projection with 360 degrees of horizontal field of view.
+- Cubemap projection with 360 degrees of horizontal field of view.
+
+
+NOTES:
+  - VR is only supported for clear streams or HLS-AES stream. DRM prevents
+    access to the video pixels for transformation.
+
+
 ## Documentation & Important Links ##
 
  * [Demo](https://shaka-player-demo.appspot.com)([sources](demo/))

build/shaka-lab.yaml

@@ -43,6 +43,8 @@ vars:
         - "--disable-backgrounding-occluded-windows"
         # Disable GPU acceleration to avoid contention for hardware resources
         # during parallel testing and to create more stability for screenshots.
+        # NOTE: This breaks WebGL-based tests, specifically on Mac.  Other
+        # platforms seem to fall back to software GL.
         - "--disable-gpu"
 
       # Instruct chromedriver not to disable component updater. The component

build/types/ui

@@ -28,6 +28,7 @@
 +../../ui/playback_rate_selection.js
 +../../ui/presentation_time.js
 +../../ui/range_element.js
++../../ui/recenter_vr.js
 +../../ui/remote_button.js
 +../../ui/resolution_selection.js
 +../../ui/rewind_button.js
@@ -38,6 +39,13 @@
 +../../ui/spacer.js
 +../../ui/statistics_button.js
 +../../ui/text_selection.js
++../../ui/toggle_stereoscopic.js
 +../../ui/ui.js
 +../../ui/ui_utils.js
 +../../ui/volume_bar.js
++../../ui/vr_manager.js
++../../ui/vr_utils.js
++../../ui/vr_webgl.js
+
++../../ui/gl_matrix/matrix_4x4.js
++../../ui/gl_matrix/matrix_quaternion.js

demo/asset_card.js

@@ -77,7 +77,11 @@ shakaDemo.AssetCard = class {
       img.src = asset.iconUri;
       img.alt = '';  // Not necessary to understand the page
 
-      picture.appendChild(webpSource);
+      // It can only be guaranteed that they have a webp version if they are on
+      // our server.
+      if (asset.iconUri.startsWith('https://storage.googleapis.com')) {
+        picture.appendChild(webpSource);
+      }
       picture.appendChild(pngSource);
       picture.appendChild(img);
 

demo/common/asset.js

@@ -68,6 +68,8 @@ const ShakaDemoAssetInfo = class {
     this.clearKeys = new Map(); // TODO: Setter method?
     /** @type {?Object} */
     this.extraConfig = null;
+    /** @type {?Object} */
+    this.extraUiConfig = null;
     /** @type {?string} */
     this.adTagUri = null;
     /** @type {?string} */
@@ -189,6 +191,15 @@ const ShakaDemoAssetInfo = class {
     return this;
   }
 
+  /**
+   * @param {!Object} extraUiConfig
+   * @return {!ShakaDemoAssetInfo}
+   */
+  setExtraUiConfig(extraUiConfig) {
+    this.extraUiConfig = extraUiConfig;
+    return this;
+  }
+
   /**
    * @param {string} mimeType
    * @return {!ShakaDemoAssetInfo}

demo/common/assets.js

@@ -43,6 +43,8 @@ shakaAssets.Source = {
   BRIGHTCOVE: 'Brightcove',
   BROADPEAK: 'Broadpeak',
   EZDRM: 'EZDRM',
+  THEO_PLAYER: 'THEOplayer',
+  JWPLAYER: 'JW Player',
 };
 
 
@@ -165,6 +167,9 @@ shakaAssets.Feature = {
 
   // Set if the asset has Content Steering.
   CONTENT_STEERING: 'Content Steering',
+
+  // Set if the asset is VR.
+  VR: 'VR',
 };
 
 
@@ -1062,6 +1067,20 @@ shakaAssets.testAssets = [
       .addFeature(shakaAssets.Feature.MP4)
       .addFeature(shakaAssets.Feature.THUMBNAILS)
       .addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'VR Playhouse (DASH, VR equirectangular)',
+      /* iconUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/poster.jpg',
+      /* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/playhouse-vr/mpds/105560.mpd',
+      /* source= */ shakaAssets.Source.BITCODIN)
+      .addFeature(shakaAssets.Feature.DASH)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.VR)
+      .addFeature(shakaAssets.Feature.OFFLINE)
+      .setExtraUiConfig({
+        displayInVrMode: true,
+        defaultVrProjectionMode: 'equirectangular',
+      }),
   // End bitcodin assets }}}
 
   // MetaCDN assets {{{
@@ -1679,5 +1698,51 @@ shakaAssets.testAssets = [
         },
       }),
   // }}}
+
+  // THEOplayer assets {{{
+  /* THEOplayer Contents */
+  new ShakaDemoAssetInfo(
+      /* name= */ 'National Geographic (HLS, VR equirectangular)',
+      /* iconUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/poster.jpg',
+      /* manifestUri= */ 'https://demo.theoplayer.com/hubfs/videos/natgeo/playlist.m3u8',
+      /* source= */ shakaAssets.Source.THEO_PLAYER)
+      .addFeature(shakaAssets.Feature.HLS)
+      .addFeature(shakaAssets.Feature.ULTRA_HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP2TS)
+      .addFeature(shakaAssets.Feature.VR)
+      .addFeature(shakaAssets.Feature.OFFLINE)
+      .setExtraUiConfig({
+        displayInVrMode: true,
+        defaultVrProjectionMode: 'equirectangular',
+      }),
+  // }}}
+
+  // JW Player assets {{{
+  /* JW Player Contents */
+  new ShakaDemoAssetInfo(
+      /* name= */ 'Delta wing (MP4, VR cubemap)',
+      /* iconUri= */ 'https://electroteque.org/plugins/jwplayer/vrvideo/images/previews/cubemap.png',
+      /* manifestUri= */ 'https://videos.electroteque.org/360/ultra_light_flight_cubemap.mp4',
+      /* source= */ shakaAssets.Source.JWPLAYER)
+      .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.VR)
+      .setExtraUiConfig({
+        displayInVrMode: true,
+        defaultVrProjectionMode: 'cubemap',
+      }),
+  new ShakaDemoAssetInfo(
+      /* name= */ 'Dragster (MP4, VR equirectangular)',
+      /* iconUri= */ 'https://electroteque.org/plugins/jwplayer/vrvideo/images/previews/playlists.png',
+      /* manifestUri= */ 'https://videos.electroteque.org/360/dragster_4k_720p.mp4',
+      /* source= */ shakaAssets.Source.JWPLAYER)
+      .addFeature(shakaAssets.Feature.HIGH_DEFINITION)
+      .addFeature(shakaAssets.Feature.MP4)
+      .addFeature(shakaAssets.Feature.VR)
+      .setExtraUiConfig({
+        displayInVrMode: true,
+        defaultVrProjectionMode: 'equirectangular',
+      }),
+  // }}}
 ];
 /* eslint-enable max-len */

demo/index.html

@@ -128,7 +128,7 @@
       <main class="mdl-layout__content" id="main-div">
         <div id="video-bar" class="hidden">
           <div data-shaka-player-container data-shaka-player-cast-receiver-id="07AEE832" class="video-container">
-            <video data-shaka-player autoplay playsinline id="video"></video>
+            <video data-shaka-player autoplay playsinline id="video" crossorigin="anonymous"></video>
           </div>
         </div>
         <div id="visualizer-div" class="hidden should-disable-on-fail">

demo/main.js

@@ -1294,6 +1294,15 @@ shakaDemo.Main = class {
 
       await this.drmConfiguration_(asset);
       this.controls_.getCastProxy().setAppData({'asset': asset});
+      const ui = this.video_['ui'];
+      if (asset.extraUiConfig) {
+        ui.configure(asset.extraUiConfig);
+      } else {
+        const uiConfig = {
+          displayInVrMode: false,
+        };
+        ui.configure(uiConfig);
+      }
 
       // Finally, the asset can be loaded.
       if (asset.preloadManager) {

demo/search.js

@@ -397,6 +397,8 @@ shakaDemo.Search = class {
         'Filters for assets that have an LCEVC enhancement layer.');
     this.makeBooleanInput_(specialContainer, Feature.CONTENT_STEERING, FEATURE,
         'Filters for assets that use Content Steering.');
+    this.makeBooleanInput_(specialContainer, Feature.VR, FEATURE,
+        'Filters for assets that are VR.');
 
     container.appendChild(this.resultsDiv_);
   }

docs/tutorials/ui-customization.md

@@ -74,6 +74,10 @@ The following elements can be added to the UI bar using this configuration value
 * language: adds a button that controls audio language selection.
 * playback_rate: adds a button that controls the playback rate selection.
 * captions: adds a button that controls the current text track selection (including turning it off).
+* recenter_vr: adds a button that recenter the VR view to the initial view. The button is visible
+  only if playing a VR content.
+* toggle_stereoscopic: adds a button that toggle between monoscopic and stereoscopic. The button
+  is visible only if playing a VR content.
 <!-- TODO: If we add more buttons that can be put in the order this way, list them here. -->
 [Document Picture-in-Picture API]: https://developer.chrome.com/docs/web-platform/document-picture-in-picture/
 
@@ -96,6 +100,10 @@ The following buttons can be added to the overflow menu:
 * remote: adds a button that opens a Remote Playback dialog. The button is visible only if the
   browser supports Remote Playback API.
 * Statistics: adds a button that displays statistics of the video.
+* recenter_vr: adds a button that recenter the VR view to the initial view. The button is visible
+  only if playing a VR content.
+* toggle_stereoscopic: adds a button that toggle between monoscopic and stereoscopic. The button
+  is visible only if playing a VR content.
 <!-- TODO: If we add more buttons that can be put in the order this way, list them here. -->
 
 Example:

docs/tutorials/ui.md

@@ -100,6 +100,31 @@ document.addEventListener('shaka-ui-load-failed', initFailed);
 ```
 
 
+#### Enabling VR
+
+To enable the playback of VR content, there are two possibilities:
+
+1. Enable via UI config:
+```js
+const config = {
+  'displayInVrMode': true
+}
+ui.configure(config);
+```
+
+2. Content is automatically treated as VR if it fits the following criteria:
+ - HLS or DASH manifest
+ - fMP4 segments
+ - Init segment contains `prji` and `hfov` boxes
+
+
+If you want the VR to be rendered outside of the main container, add the
+`data-shaka-player-vr-canvas` tag to a canvas element on the page.
+
+Note: VR is only supported for clear streams or HLS-AES stream. DRM prevents
+access to the video pixels for transformation.
+
+
 #### Enabling Chromecast support
 
 If you'd like to take advantage of Shaka's built-in Chromecast support,

externs/device_sensor_event.js

@@ -0,0 +1,18 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @fileoverview Externs for DeviceMotionEvent which were missing in the
+ * Closure compiler.
+ *
+ * @externs
+ */
+
+
+/**
+ * @return {!Promise.<string>}
+ */
+DeviceMotionEvent.requestPermission = function() {};

karma.conf.js

@@ -237,6 +237,7 @@ module.exports = (config) => {
       {pattern: 'test/test/assets/3675/*', included: false},
       {pattern: 'test/test/assets/6339/*', included: false},
       {pattern: 'test/test/assets/dash-aes-128/*', included: false},
+      {pattern: 'test/test/assets/dash-vr/*', included: false},
       {pattern: 'test/test/assets/hls-aes-256/*', included: false},
       {pattern: 'test/test/assets/hls-raw-aac/*', included: false},
       {pattern: 'test/test/assets/hls-raw-ac3/*', included: false},

lib/util/dom_utils.js

@@ -53,6 +53,17 @@ shaka.util.Dom = class {
   }
 
 
+  /**
+   * Cast a Node/Element to an HTMLCanvasElement
+   *
+   * @param {!Node|!Element} original
+   * @return {!HTMLCanvasElement}
+   */
+  static asHTMLCanvasElement(original) {
+    return /** @type {!HTMLCanvasElement}*/ (original);
+  }
+
+
   /**
    * Cast a Node/Element to an HTMLMediaElement
    *

lib/util/platform.js

@@ -442,6 +442,25 @@ shaka.util.Platform = class {
     return shaka.util.Platform.userAgentContains_('Android');
   }
 
+  /**
+   * Return true if the platform is controlled by a remote control.
+   *
+   * @return {boolean}
+   */
+  static isSmartTV() {
+    const Platform = shaka.util.Platform;
+    if (Platform.isTizen() || Platform.isWebOS() ||
+        Platform.isXboxOne() || Platform.isPS4() ||
+        Platform.isPS5() || Platform.isAmazonFireTV() ||
+        Platform.isEOS() || Platform.isAPL() ||
+        Platform.isVirginMedia() || Platform.isOrange() ||
+        Platform.isWPE() || Platform.isChromecast() ||
+        Platform.isHisense()) {
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Check if the user agent contains a key. This is the best way we know of
    * right now to detect platforms. If there is a better way, please send a

roadmap.md

@@ -26,6 +26,7 @@ v4.8 - 2024 Q2
  - AES-256 and AES-256-CTR (HLS)
    https://github.com/shaka-project/shaka-player/issues/6001
  - Detect maximum HW resolution automatically on some platforms
+ - UI support for VR content
 
 =====