Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: WebCodecs videoDecoder #1186

Merged
merged 101 commits into from
May 30, 2024
Merged
Changes from 1 commit
Commits
Show all changes
101 commits
Select commit Hold shift + click to select a range
76b6703
chore: for electron22
guoxianzhe Apr 1, 2024
d7bd5a9
chore: optimize
guoxianzhe Apr 1, 2024
4f5282a
chore: optimize
guoxianzhe Apr 1, 2024
6f5a52f
chore: optimize
guoxianzhe Apr 1, 2024
b1dadac
chore: optimize
guoxianzhe Apr 2, 2024
9576797
chore: optimize
guoxianzhe Apr 2, 2024
32f56ec
chore: optimize
guoxianzhe Apr 2, 2024
efdd91c
chore: optimize
guoxianzhe Apr 2, 2024
9e37e84
chore: optimize
guoxianzhe Apr 3, 2024
f09605d
chore: optimize
guoxianzhe Apr 3, 2024
0118095
chore: optimize
guoxianzhe Apr 3, 2024
fd8bb96
chore: optimize
guoxianzhe Apr 3, 2024
35babb2
chore: optimize
guoxianzhe Apr 3, 2024
d0cc76f
chore: optimize
guoxianzhe Apr 3, 2024
89bc83a
chore: optimize
guoxianzhe Apr 7, 2024
73b6327
chore: optimize
guoxianzhe Apr 7, 2024
1418f3d
chore: optimize
guoxianzhe Apr 7, 2024
6516a7f
chore: optimize
guoxianzhe Apr 7, 2024
5d17e96
chore: optimize
guoxianzhe Apr 8, 2024
fc055fb
chore: optimize
guoxianzhe Apr 8, 2024
179917c
chore: optimize
guoxianzhe Apr 9, 2024
bbbac3a
chore: optimize
guoxianzhe Apr 10, 2024
257c983
chore: optimize
guoxianzhe Apr 10, 2024
28b88bc
chore: optimize
guoxianzhe Apr 10, 2024
c37b338
chore: optimize
guoxianzhe Apr 11, 2024
781faea
chore: optimize
guoxianzhe Apr 11, 2024
ca338ff
chore: optimize
guoxianzhe Apr 11, 2024
01a4137
chore: optimize
guoxianzhe Apr 12, 2024
4838842
chore: optimize
guoxianzhe Apr 12, 2024
594dfb1
chore: optimize
guoxianzhe Apr 12, 2024
2498fd2
chore: optimize
guoxianzhe Apr 12, 2024
954236b
chore: optimize
guoxianzhe Apr 12, 2024
0fd0f81
chore: optimize
guoxianzhe Apr 12, 2024
47c1702
chore: optimize
guoxianzhe Apr 12, 2024
a3ad786
chore: optimize
guoxianzhe Apr 15, 2024
e74389a
chore: optimize
guoxianzhe Apr 15, 2024
fb0d41a
chore: optimize
guoxianzhe Apr 15, 2024
b3ef481
chore: optimize
guoxianzhe Apr 15, 2024
49e6bee
chore: optimize
guoxianzhe Apr 15, 2024
fd065cb
chore: optimize
guoxianzhe Apr 15, 2024
8f70fd0
chore: optimize
guoxianzhe Apr 15, 2024
a0e3f97
chore: optimize
guoxianzhe Apr 15, 2024
a0970f4
chore: optimize
guoxianzhe Apr 15, 2024
c44361b
chore: optimize
guoxianzhe Apr 15, 2024
f32881c
chore: optimize
guoxianzhe Apr 16, 2024
bab682d
chore: optimize
guoxianzhe Apr 17, 2024
a6ebf69
chore: optimize
guoxianzhe Apr 17, 2024
f82b832
chore: optimize
guoxianzhe Apr 17, 2024
7229d08
chore: optimize
guoxianzhe Apr 17, 2024
1149331
chore: optimize
guoxianzhe Apr 17, 2024
b1525a2
chore: optimize
guoxianzhe Apr 17, 2024
39f3c93
chore: optimize
guoxianzhe Apr 17, 2024
a623a47
chore: optimize
guoxianzhe Apr 17, 2024
c0c1d81
chore: optimize
guoxianzhe Apr 18, 2024
5dd41f7
chore: optimize
guoxianzhe Apr 18, 2024
3f81441
chore: optimize
guoxianzhe Apr 18, 2024
b80f380
chore: optimize
guoxianzhe Apr 18, 2024
9039ef0
chore: optimize
guoxianzhe Apr 18, 2024
1890736
chore: optimize
guoxianzhe Apr 18, 2024
cbe5ffc
chore: optimize
guoxianzhe Apr 18, 2024
d73a6f9
chore: optimize
guoxianzhe Apr 18, 2024
5a3db35
chore: optimize
guoxianzhe Apr 18, 2024
5cb72f6
chore: optimize
guoxianzhe Apr 19, 2024
43ef0b5
chore: optimize
guoxianzhe Apr 22, 2024
f5ce76e
chore: optimize
guoxianzhe Apr 23, 2024
2a76c5a
chore: optimize
guoxianzhe Apr 24, 2024
eb04ca4
chore: optimize
guoxianzhe Apr 25, 2024
6141483
chore: optimize
guoxianzhe Apr 28, 2024
0b2109b
chore: optimize
guoxianzhe Apr 28, 2024
ee1c5e8
chore: optimize
guoxianzhe Apr 28, 2024
8670244
chore: optimize
guoxianzhe May 8, 2024
afce122
chore: optimize
guoxianzhe May 8, 2024
95047cb
chore: reconfig when width and height changed
guoxianzhe May 9, 2024
617d160
chore: optimize
guoxianzhe May 9, 2024
4257aa6
chore: optimize
guoxianzhe May 9, 2024
ebc4a07
chore: optimize
guoxianzhe May 9, 2024
53bd304
chore: optimize
guoxianzhe May 9, 2024
1af394c
chore: optimize
guoxianzhe May 9, 2024
83c89fa
chore: optimize
guoxianzhe May 9, 2024
732d62b
chore: optimize
guoxianzhe May 9, 2024
20e28a0
chore: optimize
guoxianzhe May 9, 2024
f135da7
chore: optimize
guoxianzhe May 9, 2024
c8c88f3
chore: optimize
guoxianzhe May 9, 2024
16520be
chore: optimize
guoxianzhe May 9, 2024
29aa87d
chore: optimize
guoxianzhe May 11, 2024
e800bea
chore: optimize
guoxianzhe May 11, 2024
3240684
Revert "chore: optimize"
guoxianzhe May 11, 2024
c844528
Revert "chore: optimize"
guoxianzhe May 11, 2024
06525d7
chore: optimize
guoxianzhe May 11, 2024
69a8f9b
chore: optimize
guoxianzhe May 11, 2024
74fd6c9
chore: optimize
guoxianzhe May 11, 2024
c095a46
chore: optimize
guoxianzhe May 11, 2024
15c3f71
chore: optimize
guoxianzhe May 11, 2024
0e0d394
chore: optimize
guoxianzhe May 13, 2024
5c5b50f
chore: optimize
guoxianzhe May 14, 2024
d651e75
chore: optimize
guoxianzhe May 20, 2024
29aca56
chore: release 4.2.2-build.143-rc.1
sda-rob May 20, 2024
21ab5e2
chore(example): update example to install agora-electron-sdk@4.2.2-bu…
sda-rob May 20, 2024
819bd26
chore: optimize
guoxianzhe May 21, 2024
f685400
chore: release 4.2.2-build.143-rc.2
sda-rob May 22, 2024
ad50e22
chore(example): update example to install agora-electron-sdk@4.2.2-bu…
sda-rob May 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: optimize
guoxianzhe committed Apr 2, 2024
commit b1dadacb2193259614d65c6f5f7a62c7bc28fafa
75 changes: 38 additions & 37 deletions example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import {
AgoraEnv,
ChannelProfileType,
ClientRoleType,
EncodedVideoFrameInfo,
IRtcEngineEventHandler,
IRtcEngineEx,
IVideoEncodedFrameObserver,
RtcConnection,
UserOfflineReasonType,
VideoSourceType,
VideoStreamType,
WebCodecsDecoder,
createAgoraRtcEngine,
} from 'agora-electron-sdk';
import React, { ReactElement } from 'react';
@@ -17,22 +16,21 @@ import {
BaseAudioComponentState,
BaseComponent,
} from '../../../components/BaseComponent';
import { AgoraList } from '../../../components/ui';
import Config from '../../../config/agora.config';
import { askMediaAccess } from '../../../utils/permissions';

interface State extends BaseAudioComponentState {
fps: number;
}

const SCREEN_UID = 7;

export default class VideoDecoder
extends BaseComponent<{}, State>
implements IRtcEngineEventHandler, IVideoEncodedFrameObserver
implements IRtcEngineEventHandler
{
// @ts-ignore
protected engine?: IRtcEngineEx;
private decoder?: WebCodecsDecoder;
// private decoder?: WebCodecsDecoder;

protected createState(): State {
return {
@@ -56,7 +54,7 @@ export default class VideoDecoder
if (!appId) {
this.error(`appId is invalid`);
}

AgoraEnv.enableWebCodecDecode = true;
this.engine = createAgoraRtcEngine() as IRtcEngineEx;
this.engine.initialize({
appId,
@@ -65,7 +63,6 @@ export default class VideoDecoder
channelProfile: ChannelProfileType.ChannelProfileLiveBroadcasting,
});
this.engine.registerEventHandler(this);
this.engine.getMediaEngine().registerVideoEncodedFrameObserver(this);

// Need granted the microphone and camera permission
await askMediaAccess(['microphone', 'camera', 'screen']);
@@ -74,7 +71,6 @@ export default class VideoDecoder
// If you only call `enableAudio`, only relay the audio stream to the target channel
this.engine.enableVideo();
// Start preview before joinChannel
// this.engine.startPreview();
this.setState({ startPreview: true });
}

@@ -102,8 +98,6 @@ export default class VideoDecoder
// Make myself as the broadcaster to send stream to remote
clientRoleType: ClientRoleType.ClientRoleBroadcaster,
});
this.decoder = new WebCodecsDecoder();
this.decoder.enableFps = true;
}

/**
@@ -117,28 +111,10 @@ export default class VideoDecoder
* Step 5: releaseRtcEngine
*/
protected releaseRtcEngine() {
this.engine?.getMediaEngine().unregisterVideoEncodedFrameObserver(this);
this.engine?.unregisterEventHandler(this);
this.engine?.release();
}

onEncodedVideoFrameReceived(
uid: number,
imageBuffer: Uint8Array,
length: number,
videoEncodedFrameInfo: EncodedVideoFrameInfo
) {
if (uid == SCREEN_UID) {
// start decode
this.decoder?.decodeFrame(
imageBuffer,
videoEncodedFrameInfo,
new Date().getTime()
);
this.setState({ fps: this.decoder?.getFps() || 0 });
}
}

onUserJoined(connection: RtcConnection, remoteUid: number, elapsed: number) {
this.engine?.setRemoteVideoSubscriptionOptions(remoteUid, {
type: VideoStreamType.VideoStreamHigh,
@@ -152,19 +128,44 @@ export default class VideoDecoder
remoteUid: number,
reason: UserOfflineReasonType
) {
if (remoteUid == SCREEN_UID) {
// stop decode
this.decoder?.release();
}
// if (remoteUid == SCREEN_UID) {
// // stop decode
// this.decoder?.release();
// }
super.onUserOffline(connection, remoteUid, reason);
}

// protected renderUsers(): ReactElement | undefined {
// let { fps } = this.state;
// return (
// <>
// <p>Current Fps: {fps}</p>
// <canvas />
// {
// <RtcSurfaceView
// canvas={{
// sourceType: VideoSourceType.VideoSourceRemote,
// renderMode: RenderModeType.RenderModeFit,
// }}
// />
// }
// </>
// );
// }

protected renderUsers(): ReactElement | undefined {
let { fps } = this.state;
let { remoteUsers } = this.state;
return (
<>
<p>Current Fps: {fps}</p>
<canvas />
<AgoraList
data={remoteUsers ?? []}
renderItem={(item) =>
this.renderUser({
uid: item,
sourceType: VideoSourceType.VideoSourceRemote,
})
}
/>
</>
);
}
53 changes: 22 additions & 31 deletions ts/Decoder/WebCodecsDecoder.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { EncodedVideoFrameInfo, VideoFrameType } from '../Private/AgoraBase';

import { logDebug, logError } from '../Utils';
import { logDebug } from '../Utils';

import { WebGLRenderer } from './renderer_webgl';
import { WebCodecsRenderer } from './WebCodecsRenderer';

const frameTypeMapping = {
[VideoFrameType.VideoFrameTypeDeltaFrame]: 'delta',
@@ -14,7 +14,7 @@ export class WebCodecsDecoder {

private _decoder: VideoDecoder;
private _startTime: number | null = null;
private renderer: any;
private renderers: WebCodecsRenderer[] = [];
private pendingFrame: VideoFrame | null = null;
private _frameCount = 0;

@@ -23,27 +23,20 @@ export class WebCodecsDecoder {
private _base_ts_ntp = 1;
private _last_ts_ntp = 1;

constructor() {
this.renderer = new WebGLRenderer(
'webgl2',
document.querySelector('canvas')!.transferControlToOffscreen()
);
constructor(renders: WebCodecsRenderer[]) {
this.renderers = renders;
this._decoder = new VideoDecoder({
// @ts-ignore
output: this._output.bind(this),
error: this._error.bind(this),
});
this.decoder!.configure({
this._decoder!.configure({
codec: 'hvc1.1.2.H153.90',
codedWidth: 3840,
codedHeight: 2160,
});
}

getDecoder(): VideoDecoder {
return this._decoder;
}

_output(frame: VideoFrame) {
this.getFps();
// Schedule the frame to be rendered.
@@ -54,11 +47,7 @@ export class WebCodecsDecoder {
console.error('Decoder error:', e);
}

get decoder() {
return this._decoder;
}

getFps(): number {
public getFps(): number {
let fps = 0;
if (!this.enableFps) {
return fps;
@@ -74,7 +63,7 @@ export class WebCodecsDecoder {
return fps;
}

_renderFrame(frame: VideoFrame) {
private _renderFrame(frame: VideoFrame) {
if (!this.pendingFrame) {
// Schedule rendering in the next animation frame.
// eslint-disable-next-line auto-import/auto-import
@@ -88,8 +77,10 @@ export class WebCodecsDecoder {
}

renderAnimationFrame() {
this.renderer.draw(this.pendingFrame);
this.pendingFrame = null;
for (let renderer of this.renderers) {
renderer.drawFrame(this.pendingFrame);
this.pendingFrame = null;
}
}

// @ts-ignore
@@ -99,7 +90,16 @@ export class WebCodecsDecoder {
ts: number
) {
if (!imageBuffer) {
logError('imageBuffer is empty, cannot decode frame');
logDebug('imageBuffer is empty, skip decode frame');
return;
}
let frameType: string | undefined;
if (frameInfo.frameType !== undefined) {
// @ts-ignore
frameType = frameTypeMapping[frameInfo.frameType];
}
if (!frameType) {
logDebug('frameType is not in frameTypeMapping, skip decode frame');
return;
}
this._frame_ts.push(ts);
@@ -116,15 +116,6 @@ export class WebCodecsDecoder {
this._base_ts = ts;
this._last_ts_ntp = 1;
}
let frameType: string | undefined;
if (frameInfo.frameType !== undefined) {
// @ts-ignore
frameType = frameTypeMapping[frameInfo.frameType];
}
if (!frameType) {
logError('frameType is incorrect, cannot decode frame');
return;
}
this._decoder.decode(
new EncodedVideoChunk({
data: imageBuffer,
Loading