From 22683335c0384541a2d532552dfc812773a19206 Mon Sep 17 00:00:00 2001
From: lukasIO <mail@lukasseiler.de>
Date: Thu, 23 Mar 2023 10:01:37 +0100
Subject: [PATCH] Don't pause videos when element is in pictureInPicture mode
 (#622)

* Don't pause videos in adaptiveStream when in pictureInPicture mode

* update tests

* address comments

* update state first

* Create .changeset/eight-years-boil.md

* remove debug logs
---
 .changeset/eight-years-boil.md          |  5 +++
 src/room/track/RemoteVideoTrack.test.ts |  2 +
 src/room/track/RemoteVideoTrack.ts      | 53 ++++++++++++++++++++++---
 3 files changed, 55 insertions(+), 5 deletions(-)
 create mode 100644 .changeset/eight-years-boil.md

diff --git a/.changeset/eight-years-boil.md b/.changeset/eight-years-boil.md
new file mode 100644
index 0000000000..9a9488d38d
--- /dev/null
+++ b/.changeset/eight-years-boil.md
@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Don't auto-pause videos when element is in pictureInPicture mode (only applies when adaptiveStream is enabled)
diff --git a/src/room/track/RemoteVideoTrack.test.ts b/src/room/track/RemoteVideoTrack.test.ts
index cef63ea7dd..b526dac172 100644
--- a/src/room/track/RemoteVideoTrack.test.ts
+++ b/src/room/track/RemoteVideoTrack.test.ts
@@ -130,6 +130,8 @@ class MockElementInfo implements ElementInfo {
 
   visible = false;
 
+  pictureInPicture = false;
+
   setVisible = (visible: boolean) => {
     if (this.visible !== visible) {
       this.visible = visible;
diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts
index 0231ea4a71..9de66e7708 100644
--- a/src/room/track/RemoteVideoTrack.ts
+++ b/src/room/track/RemoteVideoTrack.ts
@@ -3,7 +3,12 @@ import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, VideoReceiverStats } from '../stats';
 import CriticalTimers from '../timers';
-import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import {
+  getIntersectionObserver,
+  getResizeObserver,
+  isWeb,
+  ObservableMediaElement,
+} from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import type { AdaptiveStreamSettings } from './types';
@@ -223,7 +228,9 @@ export default class RemoteVideoTrack extends RemoteTrack {
       this.adaptiveStreamSettings?.pauseVideoInBackground ?? true // default to true
         ? this.isInBackground
         : false;
-    const isVisible = this.elementInfos.some((info) => info.visible) && !backgroundPause;
+    const isPiPMode = this.elementInfos.some((info) => info.pictureInPicture);
+    const isVisible =
+      (this.elementInfos.some((info) => info.visible) && !backgroundPause) || isPiPMode;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -273,6 +280,7 @@ export interface ElementInfo {
   width(): number;
   height(): number;
   visible: boolean;
+  pictureInPicture: boolean;
   visibilityChangedAt: number | undefined;
 
   handleResize?: () => void;
@@ -284,7 +292,13 @@ export interface ElementInfo {
 class HTMLElementInfo implements ElementInfo {
   element: HTMLMediaElement;
 
-  visible: boolean;
+  get visible(): boolean {
+    return this.isPiP || this.isIntersecting;
+  }
+
+  get pictureInPicture(): boolean {
+    return this.isPiP;
+  }
 
   visibilityChangedAt: number | undefined;
 
@@ -292,9 +306,14 @@ class HTMLElementInfo implements ElementInfo {
 
   handleVisibilityChanged?: () => void;
 
+  private isPiP: boolean;
+
+  private isIntersecting: boolean;
+
   constructor(element: HTMLMediaElement, visible?: boolean) {
     this.element = element;
-    this.visible = visible ?? isElementInViewport(element);
+    this.isIntersecting = visible ?? isElementInViewport(element);
+    this.isPiP = isWeb() && document.pictureInPictureElement === element;
     this.visibilityChangedAt = 0;
   }
 
@@ -307,6 +326,10 @@ class HTMLElementInfo implements ElementInfo {
   }
 
   observe() {
+    // make sure we update the current visible state once we start to observe
+    this.isIntersecting = isElementInViewport(this.element);
+    this.isPiP = document.pictureInPictureElement === this.element;
+
     (this.element as ObservableMediaElement).handleResize = () => {
       this.handleResize?.();
     };
@@ -314,20 +337,40 @@ class HTMLElementInfo implements ElementInfo {
 
     getIntersectionObserver().observe(this.element);
     getResizeObserver().observe(this.element);
+    (this.element as HTMLVideoElement).addEventListener('enterpictureinpicture', this.onEnterPiP);
+    (this.element as HTMLVideoElement).addEventListener('leavepictureinpicture', this.onLeavePiP);
   }
 
   private onVisibilityChanged = (entry: IntersectionObserverEntry) => {
     const { target, isIntersecting } = entry;
     if (target === this.element) {
-      this.visible = isIntersecting;
+      this.isIntersecting = isIntersecting;
       this.visibilityChangedAt = Date.now();
       this.handleVisibilityChanged?.();
     }
   };
 
+  private onEnterPiP = () => {
+    this.isPiP = true;
+    this.handleVisibilityChanged?.();
+  };
+
+  private onLeavePiP = () => {
+    this.isPiP = false;
+    this.handleVisibilityChanged?.();
+  };
+
   stopObserving() {
     getIntersectionObserver()?.unobserve(this.element);
     getResizeObserver()?.unobserve(this.element);
+    (this.element as HTMLVideoElement).removeEventListener(
+      'enterpictureinpicture',
+      this.onEnterPiP,
+    );
+    (this.element as HTMLVideoElement).removeEventListener(
+      'leavepictureinpicture',
+      this.onLeavePiP,
+    );
   }
 }