WebCodecs

README.md

@@ -49,7 +49,7 @@ H264-video, which then decoded by one of included decoders:
 
 ##### Mse Player
 
-Formerly "native". Based on [xevokk/h264-converter][xevokk/h264-converter].
+Based on [xevokk/h264-converter][xevokk/h264-converter].
 HTML5 Video.<br>
 Requires [Media Source API][MSE] and `video/mp4; codecs="avc1.42E01E"`
 [support][isTypeSupported]. Creates mp4 containers from NALU, received from a
@@ -60,13 +60,22 @@ hardware acceleration.
 
 Based on [mbebenita/Broadway][broadway] and
 [131/h264-live-player][h264-live-player].<br>
+Software video-decoder compiled into wasm-module.
 Requires [WebAssembly][wasm] and preferably [WebGL][webgl] support.
 
 ##### TinyH264 Player
 
 Based on [udevbe/tinyh264][tinyh264].<br>
+Software video-decoder compiled into wasm-module. A slightly updated version of
+[mbebenita/Broadway][broadway].
 Requires [WebAssembly][wasm], [WebWorkers][workers], [WebGL][webgl] support.
 
+##### WebCodecs Player
+
+Decoding is done by browser built-in (software/hardware) media decoder.
+Requires [WebCodecs][webcodecs] support. At the moment, available only in
+[Chromium](https://www.chromestatus.com/feature/5669293909868544) and derivatives.
+
 #### Remote control
 * Touch events (including multi-touch)
 * Multi-touch emulation: <kbd>CTRL</kbd> to start with center at the center of
@@ -122,6 +131,7 @@ devices
 * `USE_BROADWAY` - include [Broadway Player](#broadway-player)
 * `USE_H264_CONVERTER` - include [Mse Player](#mse-player)
 * `USE_TINY_H264` - include [TinyH264 Player](#tinyh264-player)
+* `USE_WEBCODECS` - include [WebCodecs Player](#webcodecs-player)
 
 ## Run configuration
 
@@ -183,3 +193,4 @@ Currently, support of WebSocket protocol added to v1.17 of scrcpy
 [wasm]: https://developer.mozilla.org/en-US/docs/WebAssembly
 [webgl]: https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API
 [workers]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API
+[webcodecs]: https://w3c.github.io/webcodecs/

package.json

@@ -26,6 +26,7 @@
   },
   "devDependencies": {
     "@types/bluebird": "^3.5.32",
+    "@types/dom-webcodecs": "^0.1.2",
     "@types/express": "^4.17.11",
     "@types/mini-css-extract-plugin": "^1.2.2",
     "@types/node": "^12.20.19",

src/app/applDevice/client/DeviceTracker.ts

@@ -7,6 +7,7 @@ import { DeviceState } from '../../../common/DeviceState';
 import { ParsedUrlQueryInput } from 'querystring';
 import { HostItem } from '../../../types/Configuration';
 import { ChannelCode } from '../../../common/ChannelCode';
+import { StreamClientQVHack } from './StreamClientQVHack';
 
 export class DeviceTracker extends BaseDeviceTracker<ApplDeviceDescriptor, never> {
     public static ACTION = ACTION.APPL_DEVICE_LIST;
@@ -59,15 +60,24 @@ export class DeviceTracker extends BaseDeviceTracker<ApplDeviceDescriptor, never
             return;
         }
 
-        const playerTd = document.createElement('div');
-        playerTd.className = blockClass;
-        const q: ParsedUrlQueryInput = {
-            action: ACTION.STREAM_WS_QVH,
-            udid: device.udid,
-        };
-        const link = DeviceTracker.buildLink(q, 'stream', this.params);
-        playerTd.appendChild(link);
-        services.appendChild(playerTd);
+        const name = `${DeviceTracker.AttributePrefixPlayerFor}${fullName}`;
+        const players = StreamClientQVHack.getPlayers();
+        players.forEach((playerClass) => {
+            const { playerCodeName, playerFullName } = playerClass;
+            const playerTd = document.createElement('div');
+            playerTd.classList.add(blockClass);
+            playerTd.setAttribute('name', encodeURIComponent(name));
+            playerTd.setAttribute(DeviceTracker.AttributePlayerFullName, encodeURIComponent(playerFullName));
+            playerTd.setAttribute(DeviceTracker.AttributePlayerCodeName, encodeURIComponent(playerCodeName));
+            const q: ParsedUrlQueryInput = {
+                action: ACTION.STREAM_WS_QVH,
+                player: playerCodeName,
+                udid: device.udid,
+            };
+            const link = DeviceTracker.buildLink(q, `Stream (${playerFullName})`, this.params);
+            playerTd.appendChild(link);
+            services.appendChild(playerTd);
+        });
         tbody.appendChild(row);
     }
 

src/app/applDevice/client/StreamClientQVHack.ts

@@ -8,7 +8,7 @@ import ScreenInfo from '../../ScreenInfo';
 import { StreamReceiver } from '../../client/StreamReceiver';
 import Position from '../../Position';
 import { MsePlayerForQVHack } from '../../player/MsePlayerForQVHack';
-import { BasePlayer } from '../../player/BasePlayer';
+import { BasePlayer, PlayerClass } from '../../player/BasePlayer';
 import { SimpleTouchHandler, TouchHandlerListener } from '../../touchHandler/SimpleTouchHandler';
 import { ACTION } from '../../../common/Action';
 import { ParsedUrlQuery } from 'querystring';
@@ -22,6 +22,7 @@ const TAG = '[StreamClientQVHack]';
 
 export class StreamClientQVHack extends BaseClient<ParamsStreamQVHack, never> implements TouchHandlerListener {
     public static ACTION = ACTION.STREAM_WS_QVH;
+    private static players: Map<string, PlayerClass> = new Map<string, PlayerClass>();
     private deviceName = '';
     private managerClient: WsQVHackClient;
     private wdaConnection = new WdaConnection();
@@ -31,6 +32,37 @@ export class StreamClientQVHack extends BaseClient<ParamsStreamQVHack, never> im
     private touchHandler?: SimpleTouchHandler;
     private readonly udid: string;
 
+    public static registerPlayer(playerClass: PlayerClass): void {
+        if (playerClass.isSupported()) {
+            this.players.set(playerClass.playerFullName, playerClass);
+        }
+    }
+
+    public static getPlayers(): PlayerClass[] {
+        return Array.from(this.players.values());
+    }
+
+    private static getPlayerClass(playerName?: string): PlayerClass | undefined {
+        let playerClass: PlayerClass | undefined;
+        for (const value of this.players.values()) {
+            if (value.playerFullName === playerName || value.playerCodeName === playerName) {
+                playerClass = value;
+            }
+        }
+        if (!playerClass) {
+            return MsePlayerForQVHack;
+        }
+        return playerClass;
+    }
+
+    public static createPlayer(udid: string, playerName?: string): BasePlayer {
+        const playerClass = this.getPlayerClass(playerName);
+        if (!playerClass) {
+            return new MsePlayerForQVHack(udid);
+        }
+        return new playerClass(udid);
+    }
+
     public static start(params: ParsedUrlQuery | ParamsStreamQVHack): StreamClientQVHack {
         return new StreamClientQVHack(params);
     }
@@ -47,7 +79,7 @@ export class StreamClientQVHack extends BaseClient<ParamsStreamQVHack, never> im
         }
         this.managerClient = new WsQVHackClient({ ...this.params, action: ACTION.PROXY_WDA });
         this.streamReceiver = new StreamReceiverQVHack({ ...this.params, udid });
-        this.startStream(this.udid);
+        this.startStream();
         this.setTitle(`${this.udid} stream`);
         this.setBodyClass('stream');
     }
@@ -58,7 +90,12 @@ export class StreamClientQVHack extends BaseClient<ParamsStreamQVHack, never> im
         if (action !== ACTION.STREAM_WS_QVH) {
             throw Error('Incorrect action');
         }
-        return { ...typedParams, action, udid: Util.parseStringEnv(params.udid) };
+        return {
+            ...typedParams,
+            action,
+            udid: Util.parseStringEnv(params.udid),
+            player: Util.parseStringEnv(params.player),
+        };
     }
 
     private onViewVideoResize = (): void => {
@@ -90,8 +127,17 @@ export class StreamClientQVHack extends BaseClient<ParamsStreamQVHack, never> im
             });
     }
 
-    private startStream(udid: string) {
-        const player = new MsePlayerForQVHack(udid, MsePlayerForQVHack.createElement(`qvh_video`));
+    private startStream(inputPlayer?: BasePlayer) {
+        const { udid, player: playerName } = this.params;
+        if (!udid) {
+            throw Error(`Invalid udid value: "${udid}"`);
+        }
+        let player: BasePlayer;
+        if (inputPlayer) {
+            player = inputPlayer;
+        } else {
+            player = StreamClientQVHack.createPlayer(udid, playerName);
+        }
         this.setTouchListeners(player);
         player.pause();
 

src/app/client/BaseDeviceTracker.ts

@@ -18,6 +18,10 @@ export abstract class BaseDeviceTracker<DD extends BaseDeviceDescriptor, TE> ext
     public static readonly ACTION_LIST = 'devicelist';
     public static readonly ACTION_DEVICE = 'device';
     public static readonly HOLDER_ELEMENT_ID = 'devices';
+    public static readonly AttributePrefixInterfaceSelectFor = 'interface_select_for_';
+    public static readonly AttributePlayerFullName = 'data-player-full-name';
+    public static readonly AttributePlayerCodeName = 'data-player-code-name';
+    public static readonly AttributePrefixPlayerFor = 'player_for_';
     protected static instanceId = 0;
     protected title = 'Device list';
     protected tableId = 'base_device_list';

src/app/googDevice/client/DeviceTracker.ts

@@ -33,10 +33,6 @@ const DESC_COLUMNS: DescriptionColumn[] = [
 export class DeviceTracker extends BaseDeviceTracker<GoogDeviceDescriptor, never> {
     public static readonly ACTION = ACTION.GOOG_DEVICE_LIST;
     public static readonly CREATE_DIRECT_LINKS = true;
-    public static readonly AttributePrefixInterfaceSelectFor = 'interface_select_for_';
-    public static readonly AttributePlayerFullName = 'data-player-full-name';
-    public static readonly AttributePlayerCodeName = 'data-player-code-name';
-    public static readonly AttributePrefixPlayerFor = 'player_for_';
     private static instancesByUrl: Map<string, DeviceTracker> = new Map();
     private static tools: Set<Tool> = new Set();
     protected tableId = 'goog_device_list';

src/app/index.ts

@@ -18,13 +18,23 @@ window.onload = async function (): Promise<void> {
     /// #if USE_H264_CONVERTER
     const { MsePlayer } = await import('./player/MsePlayer');
     StreamClientScrcpy.registerPlayer(MsePlayer);
+
+    const { MsePlayerForQVHack } = await import('./player/MsePlayerForQVHack');
+    StreamClientQVHack.registerPlayer(MsePlayerForQVHack);
     /// #endif
 
     /// #if USE_TINY_H264
     const { TinyH264Player } = await import('./player/TinyH264Player');
     StreamClientScrcpy.registerPlayer(TinyH264Player);
     /// #endif
 
+    /// #if USE_WEBCODECS
+    const { WebCodecsPlayer } = await import('./player/WebCodecsPlayer');
+    StreamClientScrcpy.registerPlayer(WebCodecsPlayer);
+
+    StreamClientQVHack.registerPlayer(WebCodecsPlayer);
+    /// #endif
+
     if (action === StreamClientScrcpy.ACTION && typeof parsedQuery.udid === 'string') {
         StreamClientScrcpy.start(parsedQuery);
         return;

src/app/player/BaseCanvasBasedPlayer.ts

@@ -6,7 +6,7 @@ import { DisplayInfo } from '../DisplayInfo';
 type DecodedFrame = {
     width: number;
     height: number;
-    buffer: Uint8Array;
+    frame: any;
 };
 
 interface CanvasDecoder {
@@ -15,7 +15,7 @@ interface CanvasDecoder {
 
 export abstract class BaseCanvasBasedPlayer extends BasePlayer {
     protected framesList: Uint8Array[] = [];
-    protected lastDecodedFrame?: DecodedFrame;
+    protected decodedFrames: DecodedFrame[] = [];
     protected videoStats: PlaybackQuality[] = [];
     protected animationFrameId?: number;
     protected canvas?: CanvasDecoder;
@@ -64,25 +64,35 @@ export abstract class BaseCanvasBasedPlayer extends BasePlayer {
             return;
         }
         if (this.receivedFirstFrame) {
-            if (this.lastDecodedFrame) {
-                const { buffer, width, height } = this.lastDecodedFrame;
-                this.canvas.decode(buffer, width, height);
+            const data = this.decodedFrames.shift();
+            if (data) {
+                const { frame, width, height } = data;
+                this.canvas.decode(frame, width, height);
             }
         }
-        this.lastDecodedFrame = undefined;
-        this.animationFrameId = undefined;
+        if (this.decodedFrames.length) {
+            this.animationFrameId = requestAnimationFrame(this.drawDecoded);
+        } else {
+            this.animationFrameId = undefined;
+        }
     };
 
-    protected onFrameDecoded(width: number, height: number, buffer: Uint8Array): void {
+    protected onFrameDecoded(width: number, height: number, frame: any): void {
         if (!this.receivedFirstFrame) {
             // decoded frame with previous video settings
             return;
         }
         let dropped = 0;
-        if (this.lastDecodedFrame) {
-            dropped = 1;
+        const maxStored = this.videoSettings.maxFps / 10; // for 100ms
+
+        while (this.decodedFrames.length > maxStored) {
+            const data = this.decodedFrames.shift();
+            if (data) {
+                this.dropFrame(data.frame);
+                dropped++;
+            }
         }
-        this.lastDecodedFrame = { width, height, buffer };
+        this.decodedFrames.push({ width, height, frame });
         this.videoStats.push({
             decodedFrames: 1,
             droppedFrames: dropped,
@@ -95,6 +105,10 @@ export abstract class BaseCanvasBasedPlayer extends BasePlayer {
         }
     }
 
+    protected dropFrame(_frame: any): void {
+        // dispose frame if required
+    }
+
     private shiftFrame(): void {
         if (this.getState() !== BasePlayer.STATE.PLAYING) {
             return;
@@ -160,8 +174,8 @@ export abstract class BaseCanvasBasedPlayer extends BasePlayer {
         this.tag.oncontextmenu = (e: MouseEvent): void => {
             e.preventDefault();
         };
-        this.tag.width = width;
-        this.tag.height = height;
+        this.tag.width = Math.round(width);
+        this.tag.height = Math.round(height);
     }
 
     public play(): void {
@@ -188,6 +202,10 @@ export abstract class BaseCanvasBasedPlayer extends BasePlayer {
         const { width, height } = screenInfo.videoSize;
         this.initCanvas(width, height);
         this.framesList = [];
+        if (this.animationFrameId) {
+            cancelAnimationFrame(this.animationFrameId);
+            this.animationFrameId = undefined;
+        }
     }
 
     public pushFrame(frame: Uint8Array): void {

src/app/player/BasePlayer.ts

@@ -83,6 +83,18 @@ export abstract class BasePlayer extends TypedEmitter<PlayerEvents> {
     private statLines: string[] = [];
     public readonly supportsScreenshot: boolean = false;
 
+    public static storageKeyPrefix = 'BaseDecoder';
+    public static playerFullName = 'BasePlayer';
+    public static playerCodeName = 'baseplayer';
+    public static preferredVideoSettings: VideoSettings = new VideoSettings({
+        lockedVideoOrientation: -1,
+        bitrate: 524288,
+        maxFps: 24,
+        iFrameInterval: 5,
+        bounds: new Size(480, 480),
+        sendFrameMeta: false,
+    });
+
     constructor(
         public readonly udid: string,
         protected displayInfo?: DisplayInfo,
@@ -494,4 +506,25 @@ export abstract class BasePlayer extends TypedEmitter<PlayerEvents> {
     public abstract getFitToScreenStatus(): boolean;
 
     public abstract loadVideoSettings(): VideoSettings;
+
+    public static loadVideoSettings(udid: string, displayInfo?: DisplayInfo): VideoSettings {
+        return this.getVideoSettingFromStorage(this.preferredVideoSettings, this.storageKeyPrefix, udid, displayInfo);
+    }
+
+    public static getFitToScreenStatus(udid: string, displayInfo?: DisplayInfo): boolean {
+        return this.getFitToScreenFromStorage(this.storageKeyPrefix, udid, displayInfo);
+    }
+
+    public static getPreferredVideoSetting(): VideoSettings {
+        return this.preferredVideoSettings;
+    }
+
+    public static saveVideoSettings(
+        udid: string,
+        videoSettings: VideoSettings,
+        fitToScreen: boolean,
+        displayInfo?: DisplayInfo,
+    ): void {
+        this.putVideoSettingsToStorage(this.storageKeyPrefix, udid, videoSettings, fitToScreen, displayInfo);
+    }
 }