diff --git a/.eslintignore b/.eslintignore index 3acd846b5..c972a6245 100644 --- a/.eslintignore +++ b/.eslintignore @@ -4,3 +4,4 @@ build/ js/ types/ ts/Private/ti/ +native/ diff --git a/.eslintrc b/.eslintrc index 746601a68..19d14f897 100644 --- a/.eslintrc +++ b/.eslintrc @@ -84,8 +84,15 @@ "document": false, "HTMLElement": false, "HTMLDivElement": false, + "RenderingContext": false, + "WebGLRenderingContext": false, "WebGL2RenderingContext": false, "WebGLTexture": false, + "WebGLBuffer": false, + "WebGLProgram": false, + "VideoDecoder": false, + "VideoFrame": false, + "EncodedVideoChunk":false, "HTMLCanvasElement": false, "ResizeObserver": false, "name": false, diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f9d5a82e..00492d361 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,17 +97,17 @@ jobs: yarn link agora-electron-sdk yarn dist:mac yarn unlink agora-electron-sdk - env: + env: USE_HARD_LINKS: false working-directory: example - + - uses: actions/upload-artifact@v3 with: name: AgoraRtcNgExample-mac path: | example/dist/Agora-Electron-API-Example-*-mac.zip if-no-files-found: error - + notification: runs-on: ubuntu-latest needs: [ build-windows, build-mac ] diff --git a/.gitignore b/.gitignore index dcc4de47a..7ecb89fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -242,3 +242,4 @@ docs/ types/ iris/ appId.* +native/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 594576810..b2ec787f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ +## [4.2.2-build.143-rc.2](https://github.com/AgoraIO-Extensions/Electron-SDK/compare/v4.2.2-build.143-rc.1...v4.2.2-build.143-rc.2) (2024-05-22) + +## [4.2.2-build.143-rc.1](https://github.com/AgoraIO-Extensions/Electron-SDK/compare/v4.2.2-build.141-rc.2...v4.2.2-build.143-rc.1) (2024-05-20) + + +### Features + +* support native 4.2.2.142 ([#1178](https://github.com/AgoraIO-Extensions/Electron-SDK/issues/1178)) ([473815e](https://github.com/AgoraIO-Extensions/Electron-SDK/commit/473815eb05fc08845e394b4daf9a59c0d5096d6d)) + + +### Reverts + +* Revert "chore: optimize" ([c844528](https://github.com/AgoraIO-Extensions/Electron-SDK/commit/c844528de5cb301091d9bd39c270db86bd21fa57)) +* Revert "chore: optimize" ([3240684](https://github.com/AgoraIO-Extensions/Electron-SDK/commit/3240684eb9e47e464cd321b657797139918653e2)) + ## [4.2.2-build.142-rc.1](https://github.com/AgoraIO-Extensions/Electron-SDK/compare/v4.2.2-build.141-rc.2...v4.2.2-build.142-rc.1) (2024-03-29) ## [4.2.2-build.141-rc.2](https://github.com/AgoraIO-Extensions/Electron-SDK/compare/v4.2.2-build.141-rc.1...v4.2.2-build.141-rc.2) (2023-12-15) diff --git a/example/package.json b/example/package.json index 3f1faa32a..2ebe4b773 100644 --- a/example/package.json +++ b/example/package.json @@ -15,6 +15,7 @@ }, "build": { "appId": "agora.io.ElectronApiExample", + "includePdb": true, "asar": true, "asarUnpack": [ "node_modules/agora-electron-sdk" @@ -70,10 +71,10 @@ ] }, "dependencies": { - "agora-electron-sdk": "4.2.2-build.142-rc.1", + "agora-electron-sdk": "4.2.2-build.143-rc.2", "antd": "^4.20.3", "download": "^8.0.0", - "ffi-napi": "^4.0.3", + "koffi": "^2.8.0", "react": "^18.1.0", "react-color": "^2.19.3", "react-dom": "^18.1.0", @@ -94,7 +95,7 @@ "@types/react-dom": "^18.0.3", "@types/react-router-dom": "^5.1.6", "@types/ref-napi": "^3.0.7", - "electron": "18.2.3", + "electron": "22.0.0", "electron-builder": "^23.1.0", "electron-webpack": "^2.8.2", "fork-ts-checker-webpack-plugin": "^4.1.2", diff --git a/example/src/main/index.js b/example/src/main/index.js index 3497489ba..571c228af 100644 --- a/example/src/main/index.js +++ b/example/src/main/index.js @@ -1,6 +1,7 @@ import path from 'path'; import { format as formatUrl } from 'url'; +import 'agora-electron-sdk/js/Private/ipc/main.js'; import { BrowserWindow, app, ipcMain, systemPreferences } from 'electron'; const isDevelopment = process.env.NODE_ENV !== 'production'; diff --git a/example/src/renderer/components/BaseComponent.tsx b/example/src/renderer/components/BaseComponent.tsx index 1e2ab914f..d8b52a8b7 100644 --- a/example/src/renderer/components/BaseComponent.tsx +++ b/example/src/renderer/components/BaseComponent.tsx @@ -190,7 +190,6 @@ export abstract class BaseComponent< <> {!!startPreview || joinChannelSuccess ? this.renderUser({ - uid: 0, sourceType: VideoSourceType.VideoSourceCamera, }) : undefined} diff --git a/example/src/renderer/components/RtcSurfaceView/index.tsx b/example/src/renderer/components/RtcSurfaceView/index.tsx index 783b2e873..0476ce209 100644 --- a/example/src/renderer/components/RtcSurfaceView/index.tsx +++ b/example/src/renderer/components/RtcSurfaceView/index.tsx @@ -1,10 +1,12 @@ import { + AgoraEnv, IMediaPlayer, IRtcEngineEx, RtcConnection, VideoCanvas, VideoMirrorModeType, VideoSourceType, + VideoViewSetupMode, createAgoraRtcEngine, } from 'agora-electron-sdk'; import React, { Component } from 'react'; @@ -23,6 +25,12 @@ interface State { uniqueId: number; } +type SetupVideoFunc = + | typeof IRtcEngineEx.prototype.setupRemoteVideoEx + | typeof IRtcEngineEx.prototype.setupRemoteVideo + | typeof IRtcEngineEx.prototype.setupLocalVideo + | typeof IMediaPlayer.prototype.setView; + export class RtcSurfaceView extends Component { constructor(props: Props) { super(props); @@ -43,7 +51,16 @@ export class RtcSurfaceView extends Component { }; componentDidMount() { - this.updateRender(); + const { canvas, connection } = this.props; + this.getSetupVideoFunc().call( + this, + { + ...canvas, + setupMode: VideoViewSetupMode.VideoViewSetupAdd, + view: this.getHTMLElement(), + }, + connection! + ); } shouldComponentUpdate( @@ -59,59 +76,59 @@ export class RtcSurfaceView extends Component { } componentDidUpdate() { - this.updateRender(); + this.updateRenderer(); } componentWillUnmount() { - const dom = this.getHTMLElement(); + const { canvas, connection } = this.props; - createAgoraRtcEngine().destroyRendererByView(dom); + this.getSetupVideoFunc().call( + this, + { + ...canvas, + setupMode: VideoViewSetupMode.VideoViewSetupRemove, + view: this.getHTMLElement(), + }, + connection! + ); } - updateRender = () => { + getSetupVideoFunc = (): SetupVideoFunc => { const { canvas, connection } = this.props; - const { isMirror } = this.state; - const dom = this.getHTMLElement(); const engine = createAgoraRtcEngine(); - let funcName: - | typeof IRtcEngineEx.prototype.setupRemoteVideoEx - | typeof IRtcEngineEx.prototype.setupRemoteVideo - | typeof IRtcEngineEx.prototype.setupLocalVideo - | typeof IMediaPlayer.prototype.setView; + let func: SetupVideoFunc; if (canvas.sourceType === undefined) { if (canvas.uid) { - funcName = engine.setupRemoteVideo; + func = engine.setupRemoteVideo; } else { - funcName = engine.setupLocalVideo; + func = engine.setupLocalVideo; } } else if (canvas.sourceType === VideoSourceType.VideoSourceRemote) { - funcName = engine.setupRemoteVideo; + func = engine.setupRemoteVideo; } else { - funcName = engine.setupLocalVideo; + func = engine.setupLocalVideo; } - if (funcName === engine.setupRemoteVideo && connection) { - funcName = engine.setupRemoteVideoEx; + if (func === engine.setupRemoteVideo && connection) { + func = engine.setupRemoteVideoEx; } - try { - engine.destroyRendererByView(dom); - } catch (e) { - console.warn(e); - } - funcName.call( - this, - { - ...canvas, - mirrorMode: isMirror - ? VideoMirrorModeType.VideoMirrorModeEnabled - : VideoMirrorModeType.VideoMirrorModeDisabled, - view: dom, - }, - connection! - ); + return func; + }; + + updateRenderer = () => { + const { canvas, connection } = this.props; + const { isMirror } = this.state; + AgoraEnv.AgoraRendererManager?.setRendererContext({ + ...canvas, + ...connection, + mirrorMode: isMirror + ? VideoMirrorModeType.VideoMirrorModeEnabled + : VideoMirrorModeType.VideoMirrorModeDisabled, + view: this.getHTMLElement(), + }); }; render() { @@ -124,7 +141,7 @@ export class RtcSurfaceView extends Component { onClick={() => { this.setState((preState) => { return { isMirror: !preState.isMirror }; - }, this.updateRender); + }, this.updateRenderer); }} >
) : undefined} diff --git a/example/src/renderer/examples/advanced/MediaPlayer/MediaPlayer.tsx b/example/src/renderer/examples/advanced/MediaPlayer/MediaPlayer.tsx index e4b2fdb13..532d39163 100644 --- a/example/src/renderer/examples/advanced/MediaPlayer/MediaPlayer.tsx +++ b/example/src/renderer/examples/advanced/MediaPlayer/MediaPlayer.tsx @@ -5,6 +5,7 @@ import { MediaPlayerError, MediaPlayerEvent, MediaPlayerState, + RenderModeType, VideoSourceType, createAgoraRtcEngine, } from 'agora-electron-sdk'; @@ -353,6 +354,7 @@ export default class MediaPlayer canvas={{ mediaPlayerId: this.player?.getMediaPlayerId(), sourceType: VideoSourceType.VideoSourceMediaPlayer, + renderMode: RenderModeType.RenderModeFit, }} /> ) : undefined} diff --git a/example/src/renderer/examples/advanced/ProcessVideoRawData/ProcessVideoRawData.tsx b/example/src/renderer/examples/advanced/ProcessVideoRawData/ProcessVideoRawData.tsx index 7e9d041e2..5124f92ae 100644 --- a/example/src/renderer/examples/advanced/ProcessVideoRawData/ProcessVideoRawData.tsx +++ b/example/src/renderer/examples/advanced/ProcessVideoRawData/ProcessVideoRawData.tsx @@ -9,10 +9,7 @@ import { createAgoraRtcEngine, } from 'agora-electron-sdk'; import download from 'download'; -import ffi, { - LibraryObject, - LibraryObjectDefinitionToLibraryDefinition, -} from 'ffi-napi'; +import ffi, { IKoffiLib } from 'koffi'; import React, { ReactElement } from 'react'; import { @@ -33,12 +30,21 @@ if (process.platform === 'darwin') { } pluginName += postfix; -type PluginType = { - EnablePlugin: ['bool', ['uint64']]; - DisablePlugin: ['bool', ['uint64']]; - CreateSamplePlugin: ['uint64', ['uint64']]; - DestroySamplePlugin: ['void', ['uint64']]; - CreateSampleAudioPlugin: ['uint64', ['uint64']]; +type FuncConfig = { + returnType: string; + paramTypes: string[]; +}; + +type PluginConfig = { + [funcName: string]: FuncConfig; +}; + +const pluginConfig: PluginConfig = { + CreateSampleAudioPlugin: { returnType: 'uint64', paramTypes: ['uint64'] }, + CreateSamplePlugin: { returnType: 'uint64', paramTypes: ['uint64'] }, + DestroySamplePlugin: { returnType: 'void', paramTypes: ['uint64'] }, + DisablePlugin: { returnType: 'bool', paramTypes: ['uint64'] }, + EnablePlugin: { returnType: 'bool', paramTypes: ['uint64'] }, }; interface State extends BaseVideoComponentState { @@ -49,9 +55,16 @@ export default class ProcessVideoRawData extends BaseComponent<{}, State> implements IRtcEngineEventHandler { - pluginLibrary?: LibraryObject< - LibraryObjectDefinitionToLibraryDefinition - >; + nativeLib?: IKoffiLib; + pluginLibrary: { + [funcName: string]: Function; + } = { + CreateSampleAudioPlugin: () => {}, + CreateSamplePlugin: () => {}, + DestroySamplePlugin: () => {}, + DisablePlugin: () => {}, + EnablePlugin: () => {}, + }; plugin?: string | number; pluginAudio?: string | number; @@ -137,21 +150,24 @@ export default class ProcessVideoRawData console.log(`download success`); } - const plugin: PluginType = { - CreateSampleAudioPlugin: ['uint64', ['uint64']], - CreateSamplePlugin: ['uint64', ['uint64']], - DestroySamplePlugin: ['void', ['uint64']], - DisablePlugin: ['bool', ['uint64']], - EnablePlugin: ['bool', ['uint64']], - }; - this.pluginLibrary ??= ffi.Library(dllPath, plugin); + this.nativeLib ??= ffi.load(dllPath); + + for (const [funcName, { returnType, paramTypes }] of Object.entries( + pluginConfig + )) { + this.pluginLibrary[funcName] = this.nativeLib.func( + funcName, + returnType, + paramTypes + ); + } const handle = this.engine?.getNativeHandle(); if (handle !== undefined) { - this.plugin = this.pluginLibrary.CreateSamplePlugin(handle); - this.pluginLibrary.EnablePlugin(this.plugin); - this.pluginAudio = this.pluginLibrary.CreateSampleAudioPlugin(handle); - this.pluginLibrary.EnablePlugin(this.pluginAudio); + this.plugin = this.pluginLibrary.CreateSamplePlugin?.(handle); + this.pluginLibrary.EnablePlugin?.(this.plugin); + this.pluginAudio = this.pluginLibrary.CreateSampleAudioPlugin?.(handle); + this.pluginLibrary.EnablePlugin?.(this.pluginAudio); } this.setState({ enablePlugin: true }); }; @@ -161,16 +177,16 @@ export default class ProcessVideoRawData */ disablePlugin = () => { if (this.plugin) { - this.pluginLibrary?.DisablePlugin(this.plugin); - this.pluginLibrary?.DestroySamplePlugin(this.plugin); + this.pluginLibrary.DisablePlugin?.(this.plugin); + this.pluginLibrary.DestroySamplePlugin?.(this.plugin); this.plugin = undefined; } else { this.error('plugin is invalid'); } if (this.pluginAudio) { - this.pluginLibrary?.DisablePlugin(this.pluginAudio); - this.pluginLibrary?.DestroySamplePlugin(this.pluginAudio); + this.pluginLibrary.DisablePlugin?.(this.pluginAudio); + this.pluginLibrary.DestroySamplePlugin?.(this.pluginAudio); this.pluginAudio = undefined; } else { this.error('pluginAudio is invalid'); diff --git a/example/src/renderer/examples/advanced/ScreenShare/ScreenShare.tsx b/example/src/renderer/examples/advanced/ScreenShare/ScreenShare.tsx index 7b3a9feb8..409342095 100644 --- a/example/src/renderer/examples/advanced/ScreenShare/ScreenShare.tsx +++ b/example/src/renderer/examples/advanced/ScreenShare/ScreenShare.tsx @@ -10,6 +10,7 @@ import { RtcStats, ScreenCaptureSourceInfo, ScreenCaptureSourceType, + ScreenScenarioType, UserOfflineReasonType, VideoSourceType, createAgoraRtcEngine, @@ -51,6 +52,8 @@ interface State extends BaseVideoComponentState { highLightWidth: number; highLightColor: number; enableHighLight: boolean; + videoCodec: number; + videoCodecList: { key: string; value: number }[]; startScreenCapture: boolean; publishScreenCapture: boolean; } @@ -73,19 +76,30 @@ export default class ScreenShare remoteUsers: [], startPreview: false, token2: '', - uid2: 0, + uid2: 7, sources: [], targetSource: undefined, - width: 1920, - height: 1080, - frameRate: 15, - bitrate: 0, + width: 3840, + height: 2160, + frameRate: 60, + bitrate: 20000, captureMouseCursor: true, windowFocus: false, excludeWindowList: [], highLightWidth: 0, highLightColor: 0xff8cbf26, enableHighLight: false, + videoCodec: 2, + videoCodecList: [ + { + key: 'h264', + value: 1, + }, + { + key: 'h265', + value: 2, + }, + ], startScreenCapture: false, publishScreenCapture: false, }; @@ -108,6 +122,21 @@ export default class ScreenShare channelProfile: ChannelProfileType.ChannelProfileLiveBroadcasting, }); this.engine.registerEventHandler(this); + this.engine.setScreenCaptureScenario( + ScreenScenarioType.ScreenScenarioGaming + ); + this.engine.setParameters( + JSON.stringify({ 'engine.video.enable_hw_encoder': true }) + ); + this.engine.setParameters( + JSON.stringify({ 'che.video.show_wgc_border': 1 }) + ); + this.engine.setParameters( + JSON.stringify({ 'rtc.win_allow_directx': false }) + ); + this.engine.setParameters( + JSON.stringify({ 'che.video.enable_promote_gpu_priority': true }) + ); // Need granted the microphone and camera permission await askMediaAccess(['microphone', 'camera', 'screen']); @@ -180,6 +209,7 @@ export default class ScreenShare highLightWidth, highLightColor, enableHighLight, + videoCodec, } = this.state; if (!targetSource) { @@ -187,6 +217,10 @@ export default class ScreenShare return; } + this.engine?.setParameters( + JSON.stringify({ 'che.video.videoCodecIndex': videoCodec }) + ); + if ( targetSource.type === ScreenCaptureSourceType.ScreencapturesourcetypeScreen @@ -399,7 +433,6 @@ export default class ScreenShare {startScreenCapture ? ( @@ -570,6 +605,21 @@ export default class ScreenShare this.setState({ enableHighLight: value }); }} /> + { + return { + value: value.value!, + label: value.key!, + }; + })} + value={videoCodec} + onValueChange={(value, index) => { + this.setState((preState) => { + return { videoCodec: preState.videoCodecList?.at(index)?.value! }; + }); + }} + /> {enableHighLight ? ( <> diff --git a/example/src/renderer/examples/advanced/SendMultiVideoStream/SendMultiVideoStream.tsx b/example/src/renderer/examples/advanced/SendMultiVideoStream/SendMultiVideoStream.tsx index 6f986db4b..2a54a1cc0 100644 --- a/example/src/renderer/examples/advanced/SendMultiVideoStream/SendMultiVideoStream.tsx +++ b/example/src/renderer/examples/advanced/SendMultiVideoStream/SendMultiVideoStream.tsx @@ -13,6 +13,7 @@ import { IVideoFrameObserver, MediaPlayerError, MediaPlayerState, + RenderModeType, RtcConnection, UserOfflineReasonType, VideoFrame, @@ -335,6 +336,7 @@ export default class SendMultiVideoStream canvas={{ mediaPlayerId: this.player?.getMediaPlayerId(), sourceType: VideoSourceType.VideoSourceMediaPlayer, + renderMode: RenderModeType.RenderModeFit, }} /> ) : undefined} diff --git a/example/src/renderer/examples/advanced/VideoEncoderConfiguration/VideoEncoderConfiguration.tsx b/example/src/renderer/examples/advanced/VideoEncoderConfiguration/VideoEncoderConfiguration.tsx index 47b2f9885..893435925 100644 --- a/example/src/renderer/examples/advanced/VideoEncoderConfiguration/VideoEncoderConfiguration.tsx +++ b/example/src/renderer/examples/advanced/VideoEncoderConfiguration/VideoEncoderConfiguration.tsx @@ -1,11 +1,9 @@ import { - AgoraEnv, ChannelProfileType, ClientRoleType, DegradationPreference, IRtcEngineEventHandler, OrientationMode, - RENDER_MODE, RenderModeType, VideoCodecType, VideoMirrorModeType, @@ -37,8 +35,7 @@ interface State extends BaseVideoComponentState { bitrate: number; minBitrate: number; orientationMode: OrientationMode; - renderMode: RENDER_MODE; - renderModeType: RenderModeType; + renderMode: RenderModeType; degradationPreference: DegradationPreference; mirrorMode: VideoMirrorModeType; } @@ -64,8 +61,7 @@ export default class VideoEncoderConfiguration bitrate: 0, minBitrate: -1, orientationMode: OrientationMode.OrientationModeAdaptive, - renderMode: RENDER_MODE.WEBGL, - renderModeType: RenderModeType.RenderModeFit, + renderMode: RenderModeType.RenderModeHidden, degradationPreference: DegradationPreference.MaintainQuality, mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled, }; @@ -130,18 +126,9 @@ export default class VideoEncoderConfiguration /** * Step 3-1: setRenderMode,need leave and join channel again */ - setRenderMode = () => { - const { renderMode } = this.state; - // @ts-ignore - AgoraEnv?.AgoraRendererManager?.['setRenderMode'](renderMode); - }; - - /** - * Step 3-2: setVideoRenderMode - */ setVideoRenderMode = () => { - const { renderModeType, mirrorMode } = this.state; - this.engine?.setLocalRenderMode(renderModeType, mirrorMode); + const { renderMode, mirrorMode } = this.state; + this.engine?.setLocalRenderMode(renderMode, mirrorMode); }; /** @@ -194,7 +181,6 @@ export default class VideoEncoderConfiguration codecType, orientationMode, renderMode, - renderModeType, degradationPreference, mirrorMode, } = this.state; @@ -283,16 +269,6 @@ export default class VideoEncoderConfiguration }} /> - { - this.setState({ renderMode: value }); - }} - /> - - { - this.setState({ renderModeType: value }); + this.setState({ renderMode: value }); }} /> + {!!startPreview || joinChannelSuccess + ? this.renderUser({ + sourceType: VideoSourceType.VideoSourceCamera, + }) + : undefined} + {!!startPreview || joinChannelSuccess + ? remoteUsers.map((item) => + this.renderUser({ + uid: item, + enableFps: true, + sourceType: VideoSourceType.VideoSourceRemote, + }) + ) + : undefined} + + ); } protected renderVideo(user: VideoCanvas): ReactElement | undefined { diff --git a/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx b/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx new file mode 100644 index 000000000..86a3610cd --- /dev/null +++ b/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx @@ -0,0 +1,168 @@ +import { + AgoraEnv, + ChannelProfileType, + ClientRoleType, + IRtcEngineEventHandler, + IRtcEngineEx, + LogFilterType, + VideoSourceType, + createAgoraRtcEngine, +} from 'agora-electron-sdk'; +import React, { ReactElement } from 'react'; + +import { + BaseAudioComponentState, + BaseComponent, +} from '../../../components/BaseComponent'; +import { AgoraTextInput } from '../../../components/ui'; +import Config from '../../../config/agora.config'; +import { askMediaAccess } from '../../../utils/permissions'; + +interface State extends BaseAudioComponentState { + fps: number; + decodeRemoteUserUid: number; +} + +export default class VideoDecoder + extends BaseComponent<{}, State> + implements IRtcEngineEventHandler +{ + protected engine?: IRtcEngineEx; + + protected createState(): State { + return { + appId: Config.appId, + fps: 0, + enableVideo: true, + channelId: Config.channelId, + token: Config.token, + uid: Config.uid, + joinChannelSuccess: false, + decodeRemoteUserUid: 7, + remoteUsers: [], + startPreview: false, + }; + } + + /** + * Step 1: initRtcEngine + */ + protected async initRtcEngine() { + const { appId } = this.state; + if (!appId) { + this.error(`appId is invalid`); + } + this.engine = createAgoraRtcEngine() as IRtcEngineEx; + // need to enable WebCodecsDecoder before call engine.initialize + // if enableWebCodecsDecoder is true, the video stream will be decoded by WebCodecs + // will automatically register videoEncodedFrameObserver + // videoEncodedFrameObserver will be released when engine.release + AgoraEnv.enableWebCodecsDecoder = true; + this.engine.initialize({ + appId, + logConfig: { filePath: Config.logFilePath }, + // Should use ChannelProfileLiveBroadcasting on most of cases + channelProfile: ChannelProfileType.ChannelProfileLiveBroadcasting, + }); + this.engine.setLogFilter(LogFilterType.LogFilterDebug); + this.engine.registerEventHandler(this); + + // Need granted the microphone and camera permission + await askMediaAccess(['microphone', 'camera', 'screen']); + + // Need to enable video on this case + // If you only call `enableAudio`, only relay the audio stream to the target channel + this.engine.enableVideo(); + // Start preview before joinChannel + this.engine?.startPreview(); + this.setState({ startPreview: true }); + } + + /** + * Step 2: joinChannel + */ + protected joinChannel() { + const { channelId, token, uid } = this.state; + if (!channelId) { + this.error('channelId is invalid'); + return; + } + if (uid < 0) { + this.error('uid is invalid'); + return; + } + + // start joining channel + // 1. Users can only see each other after they join the + // same channel successfully using the same app id. + // 2. If app certificate is turned on at dashboard, token is needed + // when joining channel. The channel name and uid used to calculate + // the token has to match the ones used for channel join + this.engine?.joinChannel(token, channelId, uid, { + // Make myself as the broadcaster to send stream to remote + clientRoleType: ClientRoleType.ClientRoleBroadcaster, + }); + } + + /** + * Step 4: leaveChannel + */ + protected leaveChannel() { + this.engine?.leaveChannel(); + } + + /** + * Step 5: releaseRtcEngine + */ + protected releaseRtcEngine() { + this.engine?.unregisterEventHandler(this); + this.engine?.release(); + } + + protected renderUsers(): ReactElement | undefined { + let { decodeRemoteUserUid, startPreview, remoteUsers, joinChannelSuccess } = + this.state; + return ( + <> + {!!startPreview || joinChannelSuccess + ? this.renderUser({ + sourceType: VideoSourceType.VideoSourceCamera, + }) + : undefined} + {!!startPreview || joinChannelSuccess + ? remoteUsers.map((item) => + this.renderUser({ + uid: item, + // Use WebCodecs to decode video stream + // only support one remote stream to decode at the same time for now + useWebCodecsDecoder: item === decodeRemoteUserUid, + enableFps: item === decodeRemoteUserUid, + sourceType: VideoSourceType.VideoSourceRemote, + }) + ) + : undefined} + + ); + } + + protected renderConfiguration(): ReactElement | undefined { + return ( + <> + { + if (isNaN(+text)) return; + this.setState({ + decodeRemoteUserUid: + text === '' ? this.createState().decodeRemoteUserUid : +text, + }); + }} + numberKeyboard={true} + placeholder={`useWebCodecsDecoder Uid (defaults: ${ + this.createState().decodeRemoteUserUid + })`} + /> + + ); + } +} diff --git a/example/src/renderer/examples/basic/index.ts b/example/src/renderer/examples/basic/index.ts index 255daface..db9285a32 100644 --- a/example/src/renderer/examples/basic/index.ts +++ b/example/src/renderer/examples/basic/index.ts @@ -1,6 +1,7 @@ import JoinChannelAudio from './JoinChannelAudio/JoinChannelAudio'; import JoinChannelVideo from './JoinChannelVideo/JoinChannelVideo'; import StringUid from './StringUid/StringUid'; +import VideoDecoder from './VideoDecoder/VideoDecoder'; const Basic = { title: 'Basic', @@ -17,6 +18,10 @@ const Basic = { name: 'StringUid', component: StringUid, }, + { + name: 'VideoDecoder', + component: VideoDecoder, + }, ], }; diff --git a/example/src/renderer/examples/hook/ScreenShare/ScreenShare.tsx b/example/src/renderer/examples/hook/ScreenShare/ScreenShare.tsx index 42e446608..17151fe90 100644 --- a/example/src/renderer/examples/hook/ScreenShare/ScreenShare.tsx +++ b/example/src/renderer/examples/hook/ScreenShare/ScreenShare.tsx @@ -404,7 +404,6 @@ export default function ScreenShare() { {startScreenCapture ? ( {!!startPreview || joinChannelSuccess ? renderUser({ - uid: 0, sourceType: VideoSourceType.VideoSourceCamera, }) : undefined} diff --git a/example/webpack.renderer.additions.js b/example/webpack.renderer.additions.js index 8777458fd..558c0ccf1 100644 --- a/example/webpack.renderer.additions.js +++ b/example/webpack.renderer.additions.js @@ -81,7 +81,7 @@ module.exports = function (config) { // ...config.externals, 'webpack', 'agora-electron-sdk', - 'ffi-napi', + 'koffi', 'ref-napi', ]; console.log('config', config.module.rules); diff --git a/example/yarn.lock b/example/yarn.lock index d4eb58293..c70d37f91 100644 --- a/example/yarn.lock +++ b/example/yarn.lock @@ -1052,21 +1052,20 @@ ajv "^6.12.0" ajv-keywords "^3.4.1" -"@electron/get@^1.13.0": - version "1.14.1" - resolved "https://registry.npmjs.org/@electron/get/-/get-1.14.1.tgz#16ba75f02dffb74c23965e72d617adc721d27f40" - integrity sha512-BrZYyL/6m0ZXz/lDxy/nlVhQz+WF+iPS6qXolEU8atw7h6v1aYkjwJZ63m+bJMBTxDE66X+r2tPS4a/8C82sZw== +"@electron/get@^2.0.0": + version "2.0.3" + resolved "https://registry.npmjs.org/@electron/get/-/get-2.0.3.tgz#fba552683d387aebd9f3fcadbcafc8e12ee4f960" + integrity sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ== dependencies: debug "^4.1.1" env-paths "^2.2.0" fs-extra "^8.1.0" - got "^9.6.0" + got "^11.8.5" progress "^2.0.3" semver "^6.2.0" sumchecker "^3.0.1" optionalDependencies: global-agent "^3.0.0" - global-tunnel-ng "^2.7.1" "@electron/rebuild@^3.2.10": version "3.2.13" @@ -1211,11 +1210,6 @@ classnames "^2.3.2" rc-util "^5.24.4" -"@sindresorhus/is@^0.14.0": - version "0.14.0" - resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" - integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== - "@sindresorhus/is@^0.7.0": version "0.7.0" resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" @@ -1226,13 +1220,6 @@ resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== -"@szmarczak/http-timer@^1.1.2": - version "1.1.2" - resolved "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421" - integrity sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA== - dependencies: - defer-to-connect "^1.0.1" - "@szmarczak/http-timer@^4.0.5": version "4.0.6" resolved "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807" @@ -1416,9 +1403,9 @@ integrity sha512-cD2uPTDnQQCVpmRefonO98/PPijuOnnEy5oytWJFPY1N9aJCz2wJ5kSGWO+zJoed2cY2JxQh6yBuUq4vIn61hw== "@types/node@^16.11.26": - version "16.18.25" - resolved "https://registry.npmjs.org/@types/node/-/node-16.18.25.tgz#8863940fefa1234d3fcac7a4b7a48a6c992d67af" - integrity sha512-rUDO6s9Q/El1R1I21HG4qw/LstTHCPO/oQNAwI/4b2f9EWvMnqt4d3HJwPMawfZ3UvodB8516Yg+VAq54YM+eA== + version "16.18.96" + resolved "https://registry.npmjs.org/@types/node/-/node-16.18.96.tgz#eb0012d23ff53d14d64ec8a352bf89792de6aade" + integrity sha512-84iSqGXoO+Ha16j8pRZ/L90vDMKX04QTYMTfYeE1WrjWaZXuchBehGUZEpNgx7JnmlrIHdnABmpjrQjhCnNldQ== "@types/plist@^3.0.1": version "3.0.2" @@ -1586,6 +1573,13 @@ dependencies: "@types/yargs-parser" "*" +"@types/yauzl@^2.9.1": + version "2.10.3" + resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" + integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q== + dependencies: + "@types/node" "*" + "@webassemblyjs/ast@1.9.0": version "1.9.0" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz#bd850604b4042459a5a41cd7d338cbed695ed964" @@ -1783,10 +1777,10 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -agora-electron-sdk@4.2.2-build.142-rc.1: - version "4.2.2-build.142-rc.1" - resolved "https://registry.npmjs.org/agora-electron-sdk/-/agora-electron-sdk-4.2.2-build.142-rc.1.tgz#f8087c597a9e139f90b8f2dc80386f2037725b8b" - integrity sha512-CD+973GkThf+KhSwnzidh0J/3ssbmlOuA+C6HkoVHZSQLG+188T/gEwpcDoBAJndEUfBMILbcuksvkPnM7KIfA== +agora-electron-sdk@4.2.2-build.143-rc.2: + version "4.2.2-build.143-rc.2" + resolved "https://registry.npmjs.org/agora-electron-sdk/-/agora-electron-sdk-4.2.2-build.143-rc.2.tgz#6bbcf4fc086bc3ee08e58a9789b978a76aa50450" + integrity sha512-mBXt7+4Kuz4xzFgxdPzZAgcmpNVnIFsmc8XIoh21WfeExIjgJrB7+8+KPp8/v0ojQIfuXCrgvlOU7iDKXeUIIA== dependencies: buffer "^6.0.3" cross-env "^7.0.3" @@ -1798,6 +1792,7 @@ agora-electron-sdk@4.2.2-build.142-rc.1: jsonfile "^6.1.0" lodash.isequal "^4.5.0" minimist "^1.2.5" + semver "^7.6.0" shelljs "^0.8.4" ts-interface-checker "^1.0.2" winston "^3.3.3" @@ -2159,15 +2154,14 @@ asar@^3.1.0: optionalDependencies: "@types/glob" "^7.1.1" -asn1.js@^5.2.0: - version "5.4.1" - resolved "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" - integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== +asn1.js@^4.10.1: + version "4.10.1" + resolved "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0" + integrity sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw== dependencies: bn.js "^4.0.0" inherits "^2.0.1" minimalistic-assert "^1.0.0" - safer-buffer "^2.1.0" assert-plus@^1.0.0: version "1.0.0" @@ -2175,12 +2169,12 @@ assert-plus@^1.0.0: integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== assert@^1.1.1: - version "1.5.0" - resolved "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz#55c109aaf6e0aefdb3dc4b71240c70bf574b18eb" - integrity sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA== + version "1.5.1" + resolved "https://registry.npmjs.org/assert/-/assert-1.5.1.tgz#038ab248e4ff078e7bc2485ba6e6388466c78f76" + integrity sha512-zzw1uCAgLbsKwBfFc8CX78DDg+xZeBksSO3vwVIDDN5i94eOrPsSSyiVhmsSABFDM/OcpE2aagCat9dnWQLG1A== dependencies: - object-assign "^4.1.1" - util "0.10.3" + object.assign "^4.1.4" + util "^0.10.4" assign-symbols@^1.0.0: version "1.0.0" @@ -2406,7 +2400,7 @@ bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9: resolved "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88" integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA== -bn.js@^5.0.0, bn.js@^5.1.1: +bn.js@^5.0.0, bn.js@^5.2.1: version "5.2.1" resolved "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz#0bc527a6a0d18d0aa8d5b0538ce4a77dccfa7b70" integrity sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ== @@ -2494,7 +2488,7 @@ brorand@^1.0.1, brorand@^1.1.0: resolved "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f" integrity sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w== -browserify-aes@^1.0.0, browserify-aes@^1.0.4: +browserify-aes@^1.0.4, browserify-aes@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz#326734642f403dabc3003209853bb70ad428ef48" integrity sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA== @@ -2525,7 +2519,7 @@ browserify-des@^1.0.0: inherits "^2.0.1" safe-buffer "^5.1.2" -browserify-rsa@^4.0.0, browserify-rsa@^4.0.1: +browserify-rsa@^4.0.0, browserify-rsa@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz#b2fd06b5b75ae297f7ce2dc651f918f5be158c8d" integrity sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog== @@ -2534,19 +2528,20 @@ browserify-rsa@^4.0.0, browserify-rsa@^4.0.1: randombytes "^2.0.1" browserify-sign@^4.0.0: - version "4.2.1" - resolved "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz#eaf4add46dd54be3bb3b36c0cf15abbeba7956c3" - integrity sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg== + version "4.2.3" + resolved "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.3.tgz#7afe4c01ec7ee59a89a558a4b75bd85ae62d4208" + integrity sha512-JWCZW6SKhfhjJxO8Tyiiy+XYB7cqd2S5/+WeYHsKdNKFlCBhKbblba1A/HN/90YwtxKc8tCErjffZl++UNmGiw== dependencies: - bn.js "^5.1.1" - browserify-rsa "^4.0.1" + bn.js "^5.2.1" + browserify-rsa "^4.1.0" create-hash "^1.2.0" create-hmac "^1.1.7" - elliptic "^6.5.3" + elliptic "^6.5.5" + hash-base "~3.0" inherits "^2.0.4" - parse-asn1 "^5.1.5" - readable-stream "^3.6.0" - safe-buffer "^5.2.0" + parse-asn1 "^5.1.7" + readable-stream "^2.3.8" + safe-buffer "^5.2.1" browserify-zlib@^0.2.0: version "0.2.0" @@ -2786,19 +2781,6 @@ cacheable-request@^2.1.1: normalize-url "2.0.1" responselike "1.0.2" -cacheable-request@^6.0.0: - version "6.1.0" - resolved "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" - integrity sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg== - dependencies: - clone-response "^1.0.2" - get-stream "^5.1.0" - http-cache-semantics "^4.0.0" - keyv "^3.0.0" - lowercase-keys "^2.0.0" - normalize-url "^4.1.0" - responselike "^1.0.2" - cacheable-request@^7.0.2: version "7.0.2" resolved "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz#ea0d0b889364a25854757301ca12b2da77f91d27" @@ -2860,7 +2842,7 @@ chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.1: ansi-styles "^4.1.0" supports-color "^7.1.0" -"chokidar@>=3.0.0 <4.0.0", chokidar@^3.4.1: +"chokidar@>=3.0.0 <4.0.0": version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -2894,6 +2876,21 @@ chokidar@^2.0.0, chokidar@^2.1.8: optionalDependencies: fsevents "^1.2.7" +chokidar@^3.4.1: + version "3.6.0" + resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" + integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== + dependencies: + anymatch "~3.1.2" + braces "~3.0.2" + glob-parent "~5.1.2" + is-binary-path "~2.1.0" + is-glob "~4.0.1" + normalize-path "~3.0.0" + readdirp "~3.6.0" + optionalDependencies: + fsevents "~2.3.2" + chownr@^1.1.1, chownr@^1.1.2: version "1.1.4" resolved "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" @@ -3212,7 +3209,7 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -concat-stream@^1.5.0, concat-stream@^1.6.0, concat-stream@^1.6.2: +concat-stream@^1.5.0, concat-stream@^1.6.0: version "1.6.2" resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== @@ -3222,14 +3219,6 @@ concat-stream@^1.5.0, concat-stream@^1.6.0, concat-stream@^1.6.2: readable-stream "^2.2.2" typedarray "^0.0.6" -config-chain@^1.1.11: - version "1.1.13" - resolved "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4" - integrity sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ== - dependencies: - ini "^1.3.4" - proto-list "~1.2.1" - connect-history-api-fallback@^1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz#8b32089359308d111115d81cad3fceab888f97bc" @@ -3485,9 +3474,9 @@ csstype@^3.0.2: integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== cyclist@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" - integrity sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A== + version "1.0.2" + resolved "https://registry.npmjs.org/cyclist/-/cyclist-1.0.2.tgz#673b5f233bf34d8e602b949429f8171d9121bea3" + integrity sha512-0sVXIohTfLqVIW3kb/0n6IiWF3Ifj5nm2XaSrLq2DI6fKIGa2fYAZdk917rUneaeLVpYfFcyXE2ft0fe3remsA== "d3-array@2 - 3", "d3-array@2.10.0 - 3", d3-array@^3.1.6: version "3.2.3" @@ -3580,7 +3569,7 @@ dayjs@1.x: resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2" integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ== -debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9: +debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -3594,7 +3583,7 @@ debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4: dependencies: ms "2.1.2" -debug@^3.1.0, debug@^3.2.7: +debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== @@ -3722,11 +3711,6 @@ defaults@^1.0.3: dependencies: clone "^1.0.2" -defer-to-connect@^1.0.1: - version "1.1.3" - resolved "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" - integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== - defer-to-connect@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" @@ -3796,9 +3780,9 @@ depd@~1.1.2: integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== des.js@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz#5382142e1bdc53f85d86d53e5f4aa7deb91e0843" - integrity sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA== + version "1.1.0" + resolved "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz#1d37f5766f3bbff4ee9638e871a8768c173b81da" + integrity sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg== dependencies: inherits "^2.0.1" minimalistic-assert "^1.0.0" @@ -4130,19 +4114,19 @@ electron-webpack@^2.8.2: webpack-merge "^4.2.2" yargs "^15.3.1" -electron@18.2.3: - version "18.2.3" - resolved "https://registry.npmjs.org/electron/-/electron-18.2.3.tgz#36bcb8f71e41631e6b11179eeff291c8228bfd6a" - integrity sha512-DJWX03hCRKTscsfXxmW4gmgFuseop+g+m4ml7NfOMfankD8uYyr2Xyi3Ui02inL9qZOlbLMeLVCu6jKCKs8p/w== +electron@22.0.0: + version "22.0.0" + resolved "https://registry.npmjs.org/electron/-/electron-22.0.0.tgz#ef84ab9cf23aa3f8c2f42a1e8e000ad7fd941058" + integrity sha512-cgRc4wjyM+81A0E8UGv1HNJjL1HBI5cWNh/DUIjzYvoUuiEM0SS0hAH/zaFQ18xOz2ced6Yih8SybpOiOYJhdg== dependencies: - "@electron/get" "^1.13.0" + "@electron/get" "^2.0.0" "@types/node" "^16.11.26" - extract-zip "^1.0.3" + extract-zip "^2.0.1" -elliptic@^6.5.3: - version "6.5.4" - resolved "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb" - integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== +elliptic@^6.5.3, elliptic@^6.5.5: + version "6.5.5" + resolved "https://registry.npmjs.org/elliptic/-/elliptic-6.5.5.tgz#c715e09f78b6923977610d4c2346d6ce22e6dded" + integrity sha512-7EjbcmUm17NQFu4Pmgmq2olYMj8nwMnpcddByChSUjArp8F5DQWcIcpriwO4ZToLNAJig0yiyjswfyGNje/ixw== dependencies: bn.js "^4.11.9" brorand "^1.1.0" @@ -4172,7 +4156,7 @@ enabled@2.0.x: resolved "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== -encodeurl@^1.0.2, encodeurl@~1.0.2: +encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== @@ -4554,15 +4538,16 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" -extract-zip@^1.0.3: - version "1.7.0" - resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz#556cc3ae9df7f452c493a0cfb51cc30277940927" - integrity sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA== +extract-zip@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== dependencies: - concat-stream "^1.6.2" - debug "^2.6.9" - mkdirp "^0.5.4" + debug "^4.1.1" + get-stream "^5.1.0" yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" extsprintf@^1.2.0: version "1.4.1" @@ -4618,18 +4603,6 @@ fecha@^4.2.0: resolved "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw== -ffi-napi@^4.0.3: - version "4.0.3" - resolved "https://registry.npmjs.org/ffi-napi/-/ffi-napi-4.0.3.tgz#27a8d42a8ea938457154895c59761fbf1a10f441" - integrity sha512-PMdLCIvDY9mS32RxZ0XGb95sonPRal8aqRhLbeEtWKZTe2A87qRFG9HjOhvG8EX2UmQw5XNRMIOT+1MYlWmdeg== - dependencies: - debug "^4.1.1" - get-uv-event-loop-napi-h "^1.0.5" - node-addon-api "^3.0.0" - node-gyp-build "^4.2.1" - ref-napi "^2.0.1 || ^3.0.2" - ref-struct-di "^1.1.0" - figgy-pudding@^3.5.1: version "3.5.2" resolved "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" @@ -5080,18 +5053,11 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" -get-symbol-from-current-process-h@^1.0.1, get-symbol-from-current-process-h@^1.0.2: +get-symbol-from-current-process-h@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz#510af52eaef873f7028854c3377f47f7bb200265" integrity sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw== -get-uv-event-loop-napi-h@^1.0.5: - version "1.0.6" - resolved "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz#42b0b06b74c3ed21fbac8e7c72845fdb7a200208" - integrity sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg== - dependencies: - get-symbol-from-current-process-h "^1.0.1" - get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" @@ -5212,16 +5178,6 @@ global-prefix@^3.0.0: kind-of "^6.0.2" which "^1.3.1" -global-tunnel-ng@^2.7.1: - version "2.7.1" - resolved "https://registry.npmjs.org/global-tunnel-ng/-/global-tunnel-ng-2.7.1.tgz#d03b5102dfde3a69914f5ee7d86761ca35d57d8f" - integrity sha512-4s+DyciWBV0eK148wqXxcmVAbFVPqtc3sEtUE/GTQfuU80rySLcMhUmHKSHI7/LDj8q0gDYI1lIhRRB7ieRAqg== - dependencies: - encodeurl "^1.0.2" - lodash "^4.17.10" - npm-conf "^1.1.3" - tunnel "^0.0.6" - globals@^11.1.0: version "11.12.0" resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" @@ -5259,7 +5215,7 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -got@^11.7.0: +got@^11.7.0, got@^11.8.5: version "11.8.6" resolved "https://registry.npmjs.org/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a" integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g== @@ -5299,23 +5255,6 @@ got@^8.3.1: url-parse-lax "^3.0.0" url-to-options "^1.0.1" -got@^9.6.0: - version "9.6.0" - resolved "https://registry.npmjs.org/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" - integrity sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q== - dependencies: - "@sindresorhus/is" "^0.14.0" - "@szmarczak/http-timer" "^1.1.2" - cacheable-request "^6.0.0" - decompress-response "^3.3.0" - duplexer3 "^0.1.4" - get-stream "^4.1.0" - lowercase-keys "^1.0.1" - mimic-response "^1.0.1" - p-cancelable "^1.0.0" - to-readable-stream "^1.0.0" - url-parse-lax "^3.0.0" - graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.6: version "4.2.11" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -5475,6 +5414,14 @@ hash-base@^3.0.0: readable-stream "^3.6.0" safe-buffer "^5.2.0" +hash-base@~3.0: + version "3.0.4" + resolved "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz#5fc8686847ecd73499403319a6b0a3f3f6ae4918" + integrity sha512-EeeoJKjTyt868liAlVmcv2ZsUfGHlE3Q+BICOXcZiwN3osr5Q/zFGYmTJpoIzuaSTAwndFy+GqhEwlU4L3j4Ow== + dependencies: + inherits "^2.0.1" + safe-buffer "^5.0.1" + hash.js@^1.0.0, hash.js@^1.0.3: version "1.1.7" resolved "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42" @@ -5790,11 +5737,6 @@ inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, i resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -inherits@2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz#b17d08d326b4423e568eff719f91b0b1cbdf69f1" - integrity sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA== - inherits@2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -6434,13 +6376,6 @@ keyv@3.0.0: dependencies: json-buffer "3.0.0" -keyv@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9" - integrity sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA== - dependencies: - json-buffer "3.0.0" - keyv@^4.0.0: version "4.5.2" resolved "https://registry.npmjs.org/keyv/-/keyv-4.5.2.tgz#0e310ce73bf7851ec702f2eaf46ec4e3805cce56" @@ -6482,6 +6417,11 @@ klona@^2.0.4: resolved "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz#85bffbf819c03b2f53270412420a4555ef882e22" integrity sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA== +koffi@^2.8.0: + version "2.8.0" + resolved "https://registry.npmjs.org/koffi/-/koffi-2.8.0.tgz#26cb3a608ef8ce4684ff6ed7096fffeec21da7cb" + integrity sha512-EXhiH9Ya4f+o4+24+uV4vFAMyPEskARVUaY8VHbIYWqkQVPTDyYJCBNfxp0Kxw6WdhaMwXeR8xIUyz8R2H8Rew== + kuler@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" @@ -6606,7 +6546,7 @@ lodash.isequal@^4.5.0: resolved "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== -lodash@^4.0.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.5, lodash@^4.2.0: +lodash@^4.0.1, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.5, lodash@^4.2.0: version "4.17.21" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -6655,7 +6595,7 @@ lowercase-keys@1.0.0: resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306" integrity sha512-RPlX0+PHuvxVDZ7xX+EBVAp4RsVxP/TdDSN2mJYdiq1Lc4Hz7EUSjUI7RZrKKlmrIzVhf6Jo2stj7++gVarS0A== -lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: +lowercase-keys@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" integrity sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA== @@ -6881,7 +6821,7 @@ mimic-fn@^2.1.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -mimic-response@^1.0.0, mimic-response@^1.0.1: +mimic-response@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== @@ -7020,7 +6960,7 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4, mkdirp@^0.5.6: +mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.6: version "0.5.6" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== @@ -7269,11 +7209,6 @@ normalize-url@2.0.1: query-string "^5.0.1" sort-keys "^2.0.0" -normalize-url@^4.1.0: - version "4.5.1" - resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" - integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== - normalize-url@^6.0.1: version "6.1.0" resolved "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" @@ -7286,14 +7221,6 @@ now-and-later@^2.0.0: dependencies: once "^1.3.2" -npm-conf@^1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/npm-conf/-/npm-conf-1.1.3.tgz#256cc47bd0e218c259c4e9550bf413bc2192aff9" - integrity sha512-Yic4bZHJOt9RCFbRP3GgpqhScOY4HH3V2P8yBj6CeYq118Qr+BLXqT2JvpJ00mryLESpgOxf5XlFv4ZjXxLScw== - dependencies: - config-chain "^1.1.11" - pify "^3.0.0" - npm-run-path@^2.0.0: version "2.0.2" resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" @@ -7500,11 +7427,6 @@ p-cancelable@^0.4.0: resolved "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz#35f363d67d52081c8d9585e37bcceb7e0bbcb2a0" integrity sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ== -p-cancelable@^1.0.0: - version "1.1.0" - resolved "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" - integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== - p-cancelable@^2.0.0: version "2.1.1" resolved "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" @@ -7622,16 +7544,17 @@ param-case@^3.0.3: dot-case "^3.0.4" tslib "^2.0.3" -parse-asn1@^5.0.0, parse-asn1@^5.1.5: - version "5.1.6" - resolved "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz#385080a3ec13cb62a62d39409cb3e88844cdaed4" - integrity sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw== +parse-asn1@^5.0.0, parse-asn1@^5.1.7: + version "5.1.7" + resolved "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.7.tgz#73cdaaa822125f9647165625eb45f8a051d2df06" + integrity sha512-CTM5kuWR3sx9IFamcl5ErfPl6ea/N8IYwiJ+vpeB2g+1iknv7zBl5uPwbMbRVznRVbrNY6lGuDoE5b30grmbqg== dependencies: - asn1.js "^5.2.0" - browserify-aes "^1.0.0" - evp_bytestokey "^1.0.0" - pbkdf2 "^3.0.3" - safe-buffer "^5.1.1" + asn1.js "^4.10.1" + browserify-aes "^1.2.0" + evp_bytestokey "^1.0.3" + hash-base "~3.0" + pbkdf2 "^3.1.2" + safe-buffer "^5.2.1" parse-filepath@^1.0.1: version "1.0.2" @@ -7762,7 +7685,7 @@ path-type@^1.0.0: pify "^2.0.0" pinkie-promise "^2.0.0" -pbkdf2@^3.0.3: +pbkdf2@^3.0.3, pbkdf2@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz#dd822aa0887580e52f1a039dc3eda108efae3075" integrity sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA== @@ -7980,11 +7903,6 @@ prop-types@^15.5.10, prop-types@^15.6.2: object-assign "^4.1.1" react-is "^16.13.1" -proto-list@~1.2.1: - version "1.2.4" - resolved "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" - integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA== - proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -8641,7 +8559,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@^2.3.8, readable-stream@~2.3.6: version "2.3.8" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== @@ -8716,7 +8634,7 @@ reduce-css-calc@^2.1.8: css-unit-converter "^1.1.1" postcss-value-parser "^3.3.0" -"ref-napi@^2.0.1 || ^3.0.2", ref-napi@^3.0.3: +ref-napi@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.3.tgz#e259bfc2bbafb3e169e8cd9ba49037dd00396b22" integrity sha512-LiMq/XDGcgodTYOMppikEtJelWsKQERbLQsYm0IOOnzhwE9xYZC7x8txNnFC9wJNOkPferQI4vD4ZkC0mDyrOA== @@ -8726,13 +8644,6 @@ reduce-css-calc@^2.1.8: node-addon-api "^3.0.0" node-gyp-build "^4.2.1" -ref-struct-di@^1.1.0: - version "1.1.1" - resolved "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz#5827b1d3b32372058f177547093db1fe1602dc10" - integrity sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g== - dependencies: - debug "^3.1.0" - regenerate-unicode-properties@^10.1.0: version "10.1.0" resolved "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz#7c3192cab6dd24e21cb4461e5ddd7dd24fa8374c" @@ -8931,7 +8842,7 @@ resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.14.2, resolve@^1.4.0 path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -responselike@1.0.2, responselike@^1.0.2: +responselike@1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7" integrity sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ== @@ -9019,7 +8930,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@^5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -9045,7 +8956,7 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== -"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.1.0: +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -9171,6 +9082,11 @@ semver@^7.1.3, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7: dependencies: lru-cache "^6.0.0" +semver@^7.6.0: + version "7.6.2" + resolved "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" + integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== + semver@~7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" @@ -10016,11 +9932,6 @@ to-object-path@^0.3.0: dependencies: kind-of "^3.0.2" -to-readable-stream@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz#ce0aa0c2f3df6adf852efb404a783e77c0475771" - integrity sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q== - to-regex-range@^2.1.0: version "2.1.1" resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" @@ -10108,11 +10019,6 @@ tty-browserify@0.0.0: resolved "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6" integrity sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw== -tunnel@^0.0.6: - version "0.0.6" - resolved "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" - integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== - type-fest@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" @@ -10376,12 +10282,12 @@ util.promisify@1.0.0: define-properties "^1.1.2" object.getownpropertydescriptors "^2.0.3" -util@0.10.3: - version "0.10.3" - resolved "https://registry.npmjs.org/util/-/util-0.10.3.tgz#7afb1afe50805246489e3db7fe0ed379336ac0f9" - integrity sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ== +util@^0.10.4: + version "0.10.4" + resolved "https://registry.npmjs.org/util/-/util-0.10.4.tgz#3aa0125bfe668a4672de58857d3ace27ecb76901" + integrity sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A== dependencies: - inherits "2.0.1" + inherits "2.0.3" util@^0.11.0: version "0.11.1" @@ -10650,9 +10556,9 @@ webpack-sources@^1.1.0, webpack-sources@^1.4.0, webpack-sources@^1.4.1, webpack- source-map "~0.6.1" webpack@^4.43.0: - version "4.46.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz#bf9b4404ea20a073605e0a011d188d77cb6ad542" - integrity sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q== + version "4.47.0" + resolved "https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz#8b8a02152d7076aeb03b61b47dad2eeed9810ebc" + integrity sha512-td7fYwgLSrky3fI1EuU5cneU4+pbH6GgOfuKNS1tNPcfdGinGELAqsb/BP4nnvZyKSG2i/xFGU7+n2PvZA8HJQ== dependencies: "@webassemblyjs/ast" "1.9.0" "@webassemblyjs/helper-module-context" "1.9.0" diff --git a/package.json b/package.json index d6b35856c..33fc17269 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agora-electron-sdk", - "version": "4.2.2-build.142-rc.1", + "version": "4.2.2-build.143-rc.2", "description": "agora-electron-sdk", "main": "js/AgoraSdk", "types": "types/AgoraSdk.d.ts", @@ -61,6 +61,7 @@ "@commitlint/config-conventional": "^17.0.2", "@evilmartians/lefthook": "^1.2.2", "@release-it/conventional-changelog": "^5.0.0", + "@types/dom-webcodecs": "^0.1.11", "@types/jest": "^28.1.2", "@types/json-bigint": "^1.0.1", "@types/lodash.isequal": "^4.5.6", @@ -129,6 +130,7 @@ "jsonfile": "^6.1.0", "lodash.isequal": "^4.5.0", "minimist": "^1.2.5", + "semver": "^7.6.0", "shelljs": "^0.8.4", "ts-interface-checker": "^1.0.2", "winston": "^3.3.3", @@ -136,7 +138,7 @@ "yuv-canvas": "1.2.6" }, "agora_electron": { - "iris_sdk_win": "https://download.agora.io/sdk/release/iris_4.2.2.142-build.1_DCG_Windows_Video_20240329_1152.zip", + "iris_sdk_win": "https://download.agora.io/sdk/release/iris_4.2.2.143-build.1_DCG_Windows_Video_20240521_0544.zip", "iris_sdk_mac": "https://download.agora.io/sdk/release/iris_4.2.2.136-build.1_DCG_Mac_Video_20230922_0351.zip" } } diff --git a/scripts/downloadPrebuild.js b/scripts/downloadPrebuild.js index 3e6bcb379..ab60e1d26 100644 --- a/scripts/downloadPrebuild.js +++ b/scripts/downloadPrebuild.js @@ -93,7 +93,7 @@ module.exports = async () => { }, }); - if (no_symbol) { - await removeFileByFilter(); - } + // if (no_symbol) { + // await removeFileByFilter(); + // } }; diff --git a/source_code/agora_node_ext/agora_electron_bridge.cpp b/source_code/agora_node_ext/agora_electron_bridge.cpp index 1030bf187..b06b508b4 100644 --- a/source_code/agora_node_ext/agora_electron_bridge.cpp +++ b/source_code/agora_node_ext/agora_electron_bridge.cpp @@ -40,6 +40,7 @@ napi_value AgoraElectronBridge::Init(napi_env env, napi_value exports) { napi_property_descriptor properties[] = { DECLARE_NAPI_METHOD("CallApi", CallApi), DECLARE_NAPI_METHOD("OnEvent", OnEvent), + DECLARE_NAPI_METHOD("UnEvent", UnEvent), DECLARE_NAPI_METHOD("GetBuffer", GetBuffer), DECLARE_NAPI_METHOD("EnableVideoFrameCache", EnableVideoFrameCache), DECLARE_NAPI_METHOD("DisableVideoFrameCache", DisableVideoFrameCache), @@ -163,8 +164,10 @@ napi_value AgoraElectronBridge::CallApi(napi_env env, napi_callback_info info) { } } else { std::smatch output; - std::regex pattern = std::regex( - "^.*(Observer|Handler|Callback|Receiver|DirectCdnStreaming)$"); + std::regex pattern = + std::regex("^.*_.*((EventHandler|Observer|startDirectCdnStreaming|" + "Source|VideoFrameRenderer)(_[a-zA-Z0-9]*)?)$"); + if (std::regex_match(funcName, output, pattern)) { bufferCount = 1; buffer.resize(bufferCount); @@ -248,6 +251,30 @@ napi_value AgoraElectronBridge::OnEvent(napi_env env, napi_callback_info info) { RETURE_NAPI_OBJ(); } +napi_value AgoraElectronBridge::UnEvent(napi_env env, napi_callback_info info) { + napi_status status; + size_t argc = 2; + napi_value args[2]; + napi_value jsthis; + int ret = ERR_FAILED; + status = napi_get_cb_info(env, info, &argc, args, &jsthis, nullptr); + assert(status == napi_ok); + + AgoraElectronBridge *agoraElectronBridge; + status = + napi_unwrap(env, jsthis, reinterpret_cast(&agoraElectronBridge)); + assert(status == napi_ok); + + std::string eventName = ""; + status = napi_get_value_utf8string(env, args[0], eventName); + assert(status == napi_ok); + + agoraElectronBridge->_iris_rtc_event_handler->removeEvent(eventName); + ret = ERR_OK; + + RETURE_NAPI_OBJ(); +} + napi_value AgoraElectronBridge::SetAddonLogFile(napi_env env, napi_callback_info info) { napi_status status; @@ -295,7 +322,7 @@ napi_value AgoraElectronBridge::EnableVideoFrameCache(napi_env env, unsigned int height = 0; napi_obj_get_property(env, obj, "uid", config.uid); - napi_obj_get_property(env, obj, "videoSourceType", config.video_source_type); + napi_obj_get_property(env, obj, "sourceType", config.video_source_type); napi_obj_get_property(env, obj, "channelId", channelId); strcpy(config.channelId, channelId.c_str()); napi_obj_get_property(env, obj, "width", width); @@ -340,7 +367,7 @@ AgoraElectronBridge::DisableVideoFrameCache(napi_env env, std::string channelId = ""; napi_obj_get_property(env, obj, "uid", config.uid); - napi_obj_get_property(env, obj, "videoSourceType", config.video_source_type); + napi_obj_get_property(env, obj, "sourceType", config.video_source_type); napi_obj_get_property(env, obj, "channelId", channelId); strcpy(config.channelId, channelId.c_str()); @@ -368,8 +395,8 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env, napi_callback_info info) { napi_status status; napi_value jsthis; - size_t argc = 1; - napi_value args[1]; + size_t argc = 2; + napi_value args[2]; status = napi_get_cb_info(env, info, &argc, args, &jsthis, nullptr); AgoraElectronBridge *agoraElectronBridge; @@ -377,9 +404,15 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env, napi_unwrap(env, jsthis, reinterpret_cast(&agoraElectronBridge)); IrisRtcVideoFrameConfig config = EmptyIrisRtcVideoFrameConfig; - napi_value obj = args[0]; - int videoSourceType; + napi_value obj0 = args[0]; std::string channel_id; + + napi_obj_get_property(env, obj0, "uid", config.uid); + napi_obj_get_property(env, obj0, "sourceType", config.video_source_type); + napi_obj_get_property(env, obj0, "channelId", channel_id); + strcpy(config.channelId, channel_id.c_str()); + + napi_value obj1 = args[1]; napi_value y_buffer_obj; void *y_buffer; size_t y_length; @@ -389,40 +422,41 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env, napi_value v_buffer_obj; void *v_buffer; size_t v_length; - int height; int width; + int height; int yStride; + int uStride; + int vStride; - napi_obj_get_property(env, obj, "uid", config.uid); - napi_obj_get_property(env, obj, "videoSourceType", config.video_source_type); - napi_obj_get_property(env, obj, "channelId", channel_id); - strcpy(config.channelId, channel_id.c_str()); - - napi_obj_get_property(env, obj, "yBuffer", y_buffer_obj); + napi_obj_get_property(env, obj1, "yBuffer", y_buffer_obj); napi_get_buffer_info(env, y_buffer_obj, &y_buffer, &y_length); - napi_obj_get_property(env, obj, "uBuffer", u_buffer_obj); + napi_obj_get_property(env, obj1, "uBuffer", u_buffer_obj); napi_get_buffer_info(env, u_buffer_obj, &u_buffer, &u_length); - napi_obj_get_property(env, obj, "vBuffer", v_buffer_obj); + napi_obj_get_property(env, obj1, "vBuffer", v_buffer_obj); napi_get_buffer_info(env, v_buffer_obj, &v_buffer, &v_length); - napi_obj_get_property(env, obj, "height", height); - napi_obj_get_property(env, obj, "width", width); - napi_obj_get_property(env, obj, "yStride", yStride); + napi_obj_get_property(env, obj1, "width", width); + napi_obj_get_property(env, obj1, "height", height); + napi_obj_get_property(env, obj1, "yStride", yStride); + napi_obj_get_property(env, obj1, "uStride", uStride); + napi_obj_get_property(env, obj1, "vStride", vStride); IrisCVideoFrame videoFrame; videoFrame.yBuffer = (uint8_t *) y_buffer; videoFrame.uBuffer = (uint8_t *) u_buffer; videoFrame.vBuffer = (uint8_t *) v_buffer; - videoFrame.height = height; videoFrame.width = width; + videoFrame.height = height; videoFrame.yStride = yStride; + videoFrame.uStride = uStride; + videoFrame.vStride = vStride; videoFrame.metadata_buffer = nullptr; videoFrame.metadata_size = 0; videoFrame.alphaBuffer = nullptr; - bool isFresh = false; + bool isNewFrame = false; napi_value retObj; int32_t ret = ERR_NOT_INITIALIZED; status = napi_create_object(env, &retObj); @@ -434,16 +468,23 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env, } ret = agoraElectronBridge->_iris_rendering->GetVideoFrameCache( - config, &videoFrame, isFresh); + config, &videoFrame, isNewFrame); - unsigned int rotation = 0; napi_obj_set_property(env, retObj, "ret", ret); - napi_obj_set_property(env, retObj, "isNewFrame", isFresh); - napi_obj_set_property(env, retObj, "width", videoFrame.width); - napi_obj_set_property(env, retObj, "height", videoFrame.height); - napi_obj_set_property(env, retObj, "yStride", videoFrame.yStride); - napi_obj_set_property(env, retObj, "rotation", rotation); - napi_obj_set_property(env, retObj, "timestamp", videoFrame.renderTimeMs); + napi_obj_set_property(env, retObj, "isNewFrame", isNewFrame); + + napi_obj_set_property(env, obj1, "type", videoFrame.type); + napi_obj_set_property(env, obj1, "width", videoFrame.width); + napi_obj_set_property(env, obj1, "height", videoFrame.height); + napi_obj_set_property(env, obj1, "yStride", videoFrame.yStride); + napi_obj_set_property(env, obj1, "uStride", videoFrame.uStride); + napi_obj_set_property(env, obj1, "vStride", videoFrame.vStride); + napi_obj_set_property(env, obj1, "rotation", videoFrame.rotation); + napi_obj_set_property(env, obj1, "renderTimeMs", videoFrame.renderTimeMs); + napi_obj_set_property(env, obj1, "avsync_type", videoFrame.avsync_type); + napi_obj_set_property(env, obj1, "metadata_size", videoFrame.metadata_size); + // napi_obj_set_property(env, obj1, "textureId", videoFrame.textureId); + return retObj; } @@ -459,6 +500,8 @@ napi_value AgoraElectronBridge::InitializeEnv(napi_env env, status = napi_unwrap(env, jsthis, reinterpret_cast(&agoraElectronBridge)); + napi_value obj0 = args[0]; + agoraElectronBridge->Init(); LOG_F(INFO, __FUNCTION__); napi_value retValue = nullptr; diff --git a/source_code/agora_node_ext/agora_electron_bridge.h b/source_code/agora_node_ext/agora_electron_bridge.h index 8e0064cbd..dcdafcfee 100644 --- a/source_code/agora_node_ext/agora_electron_bridge.h +++ b/source_code/agora_node_ext/agora_electron_bridge.h @@ -18,6 +18,10 @@ namespace electron { class NodeIrisEventHandler; +struct AgoraEnv { + bool enable_web_codecs_decoder = false; +}; + class AgoraElectronBridge { public: explicit AgoraElectronBridge(); @@ -30,6 +34,7 @@ class AgoraElectronBridge { static napi_value CallApi(napi_env env, napi_callback_info info); static napi_value GetBuffer(napi_env env, napi_callback_info info); static napi_value OnEvent(napi_env env, napi_callback_info info); + static napi_value UnEvent(napi_env env, napi_callback_info info); static napi_value EnableVideoFrameCache(napi_env env, napi_callback_info info); static napi_value DisableVideoFrameCache(napi_env env, @@ -43,8 +48,10 @@ class AgoraElectronBridge { void OnApiError(const char *errorMessage); void Init(); void Release(); + void SetAgoraEnv(const AgoraEnv &agoraEnv) { _agoraEnv = agoraEnv; } private: + AgoraEnv _agoraEnv; static const char *_class_name; static napi_ref *_ref_construcotr_ptr; static const char *_ret_code_str; diff --git a/source_code/agora_node_ext/node_api_header.cpp b/source_code/agora_node_ext/node_api_header.cpp index cc000df82..94f6f0bba 100644 --- a/source_code/agora_node_ext/node_api_header.cpp +++ b/source_code/agora_node_ext/node_api_header.cpp @@ -127,6 +127,15 @@ napi_status napi_obj_get_property(napi_env &env, napi_value &object, return status; } +napi_status napi_obj_get_property(napi_env &env, napi_value &object, + const char *utf8name, bool &result) { + napi_status status; + napi_value retValue; + napi_get_named_property(env, object, utf8name, &retValue); + status = napi_get_value_bool(env, retValue, &result); + return status; +} + napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, uint32_t &result) { napi_status status; diff --git a/source_code/agora_node_ext/node_api_header.h b/source_code/agora_node_ext/node_api_header.h index a8e8263d9..d447b54d9 100644 --- a/source_code/agora_node_ext/node_api_header.h +++ b/source_code/agora_node_ext/node_api_header.h @@ -68,6 +68,9 @@ napi_status napi_obj_set_property(napi_env &env, napi_value &object, napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, int &result); +napi_status napi_obj_get_property(napi_env &env, napi_value &object, + const char *utf8name, bool &result); + napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, uint32_t &result); diff --git a/source_code/agora_node_ext/node_iris_event_handler.cpp b/source_code/agora_node_ext/node_iris_event_handler.cpp index 3b45a45c0..33a5dee9c 100644 --- a/source_code/agora_node_ext/node_iris_event_handler.cpp +++ b/source_code/agora_node_ext/node_iris_event_handler.cpp @@ -43,9 +43,24 @@ void NodeIrisEventHandler::addEvent(const std::string &eventName, napi_env &env, env, call_bcak, 1, &(_callbacks[eventName]->call_back_ref)); } +void NodeIrisEventHandler::removeEvent(const std::string &eventName) { + auto it = _callbacks.find(eventName); + if (it != _callbacks.end()) { + napi_delete_reference(it->second->env, it->second->call_back_ref); + delete it->second; + _callbacks.erase(it); + } +} + void NodeIrisEventHandler::OnEvent(EventParam *param) { - fireEvent(_callback_key, param->event, param->data, param->buffer, - param->length, param->buffer_count); + const char *event = "VideoEncodedFrameObserver_onEncodedVideoFrameReceived"; + + if (strcmp(event, param->event) == 0) { + onEncodedVideoFrameReceived(param->data, param->buffer[0], param->length); + } else { + fireEvent(_callback_key, param->event, param->data, param->buffer, + param->length, param->buffer_count); + } } void NodeIrisEventHandler::fireEvent(const char *callback_name, @@ -119,6 +134,42 @@ void NodeIrisEventHandler::fireEvent(const char *callback_name, }); } +void NodeIrisEventHandler::onEncodedVideoFrameReceived(const char *data, + void *buffer, + unsigned int *length) { + std::string eventData = ""; + if (data) { eventData = data; } + std::vector buffer_data(length[0]); + memcpy(buffer_data.data(), buffer, length[0]); + + unsigned int buffer_length = length[0]; + + node_async_call::async_call([this, eventData, buffer_data, buffer_length] { + auto it = _callbacks.find("call_back_with_encoded_video_frame"); + if (it != _callbacks.end()) { + size_t argc = 2; + napi_value args[2]; + napi_value result; + napi_status status; + status = napi_create_string_utf8(it->second->env, eventData.c_str(), + eventData.length(), &args[0]); + + napi_create_buffer_copy(it->second->env, buffer_length, + buffer_data.data(), nullptr, &args[1]); + + napi_value call_back_value; + status = napi_get_reference_value( + it->second->env, it->second->call_back_ref, &call_back_value); + + napi_value recv_value; + status = napi_get_undefined(it->second->env, &recv_value); + + status = napi_call_function(it->second->env, recv_value, call_back_value, + argc, args, &result); + } + }); +} + }// namespace electron }// namespace rtc }// namespace agora diff --git a/source_code/agora_node_ext/node_iris_event_handler.h b/source_code/agora_node_ext/node_iris_event_handler.h index c1e88b85f..863674ebd 100644 --- a/source_code/agora_node_ext/node_iris_event_handler.h +++ b/source_code/agora_node_ext/node_iris_event_handler.h @@ -31,9 +31,14 @@ class NodeIrisEventHandler : public iris::IrisEventHandler { void **buffer, unsigned int *length, unsigned int buffer_count); + void onEncodedVideoFrameReceived(const char *data, void *buffer, + unsigned int *length); + void addEvent(const std::string &eventName, napi_env &env, napi_value &call_bcak, napi_value &global); + void removeEvent(const std::string &eventName); + private: std::unordered_map _callbacks; const char *_callback_key = "call_back_with_buffer"; diff --git a/ts/Decoder/gpu-utils.ts b/ts/Decoder/gpu-utils.ts new file mode 100644 index 000000000..a973b65ef --- /dev/null +++ b/ts/Decoder/gpu-utils.ts @@ -0,0 +1,92 @@ +//@ts-ignore +import { BrowserWindow } from 'electron'; + +/** + * @ignore + */ + +export type VideoDecodeAcceleratorSupportedProfile = { + codec: string; + minWidth: number; + maxWidth: number; + minHeight: number; + maxHeight: number; +}; + +/** + * @ignore + */ +export class GpuInfo { + videoDecodeAcceleratorSupportedProfile: VideoDecodeAcceleratorSupportedProfile[] = + []; +} + +/** + * @ignore + */ +export const getGpuInfoInternal = (callback: any): void => { + //@ts-ignore + if (process.type !== 'browser') { + console.error('getGpuInfoInternal should be called in main process'); + return; + } + const gpuPage = new BrowserWindow({ + show: false, + webPreferences: { offscreen: true }, + }); + gpuPage.loadURL('chrome://gpu'); + let executeJavaScriptText = + `` + + `let videoAccelerationInfo = [];` + + `let nodeList = document.querySelector('info-view')?.shadowRoot?.querySelector('#video-acceleration-info info-view-table')?.shadowRoot?.querySelectorAll('#info-view-table info-view-table-row') || [];` + + `for (node of nodeList) {` + + ` videoAccelerationInfo.push({` + + ` title: node.shadowRoot.querySelector('#title')?.innerText,` + + ` value: node.shadowRoot.querySelector('#value')?.innerText,` + + ` })` + + `}` + + `JSON.stringify(videoAccelerationInfo)`; + gpuPage.webContents + .executeJavaScript(executeJavaScriptText) + .then((result: string) => { + if (!result) { + console.error( + 'Failed to get GPU info, chrome://gpu is not available in this environment.' + ); + } + let filterResult: { title: string; value: string }[] = JSON.parse( + result + ).filter((item: any) => { + return item.title.indexOf('Decode') !== -1; + }); + let convertResult: VideoDecodeAcceleratorSupportedProfile[] = []; + const resolutionPattern = /(\d+)x(\d+) to (\d+)x(\d+)/; + for (const profile of filterResult) { + const match = profile.value.match(resolutionPattern); + if (!match) { + continue; + } + + const [_resolution, minWidth, minHeight, maxWidth, maxHeight] = match; + + convertResult.push({ + codec: profile.title, + minWidth: minWidth ? Number(minWidth) : 0, + maxWidth: maxWidth ? Number(maxWidth) : 0, + minHeight: minHeight ? Number(minHeight) : 0, + maxHeight: maxHeight ? Number(maxHeight) : 0, + }); + } + typeof callback === 'function' && callback(convertResult); + }) + .catch((error: any) => { + console.error( + 'Failed to get GPU info, please import agora-electron-sdk in main process', + error + ); + typeof callback === 'function' && callback(error); + }) + .finally(() => { + gpuPage.close(); + }); +}; diff --git a/ts/Decoder/index.ts b/ts/Decoder/index.ts new file mode 100644 index 000000000..fdec958f7 --- /dev/null +++ b/ts/Decoder/index.ts @@ -0,0 +1,191 @@ +import { + EncodedVideoFrameInfo, + VideoCodecType, + VideoFrameType, +} from '../Private/AgoraBase'; + +import { WebCodecsRenderer } from '../Renderer/WebCodecsRenderer/index'; +import { RendererCacheContext, RendererType } from '../Types'; +import { AgoraEnv, logDebug, logInfo } from '../Utils'; + +const frameTypeMapping = { + [VideoFrameType.VideoFrameTypeDeltaFrame]: 'delta', + [VideoFrameType.VideoFrameTypeKeyFrame]: 'key', + [VideoFrameType.VideoFrameTypeDroppableFrame]: 'delta', // this is a workaround for the issue that the frameType is not correct +}; + +export class WebCodecsDecoder { + private _decoder: VideoDecoder; + private renderers: WebCodecsRenderer[] = []; + private _cacheContext: RendererCacheContext; + private pendingFrame: VideoFrame | null = null; + private _currentCodecConfig: { + codecType: VideoCodecType | undefined; + codedWidth: number | undefined; + codedHeight: number | undefined; + } | null = null; + + private _base_ts = 0; + private _base_ts_ntp = 1; + private _last_ts_ntp = 1; + + constructor( + renders: WebCodecsRenderer[], + onError: (e: any) => void, + context: RendererCacheContext + ) { + this.renderers = renders; + this._cacheContext = context; + this._decoder = new VideoDecoder({ + // @ts-ignore + output: this._output.bind(this), + error: (e) => { + onError(e); + }, + }); + } + + _output(frame: VideoFrame) { + // Schedule the frame to be rendered. + this._renderFrame(frame); + } + + private _renderFrame(frame: VideoFrame) { + if (!this.pendingFrame) { + // Schedule rendering in the next animation frame. + // eslint-disable-next-line auto-import/auto-import + requestAnimationFrame(this.renderAnimationFrame.bind(this)); + } else { + // Close the current pending frame before replacing it. + this.pendingFrame.close(); + } + // Set or replace the pending frame. + this.pendingFrame = frame; + } + + renderAnimationFrame() { + for (let renderer of this.renderers) { + if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) { + continue; + } + renderer.drawFrame(this.pendingFrame); + this.pendingFrame = null; + } + } + + decoderConfigure(frameInfo: EncodedVideoFrameInfo) { + this.pendingFrame = null; + // @ts-ignore + let codec = + AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType!] + ?.codec; + if (!codec) { + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback( + this._cacheContext + ); + throw new Error( + 'codec is not in frameCodecMapping,failed to configure decoder, fallback to native decoder' + ); + } + this._currentCodecConfig = { + codecType: frameInfo.codecType, + codedWidth: frameInfo.width, + codedHeight: frameInfo.height, + }; + this._decoder!.configure({ + codec: codec, + codedWidth: frameInfo.width, + codedHeight: frameInfo.height, + }); + logInfo( + `configure decoder: codedWidth: ${frameInfo.width}, codedHeight: ${frameInfo.height},codec: ${codec}` + ); + } + + updateTimestamps(ts: number) { + if (this._base_ts !== 0) { + if (ts > this._base_ts) { + this._last_ts_ntp = + this._base_ts_ntp + Math.floor(((ts - this._base_ts) * 1000) / 90); + } else { + this._base_ts = ts; + this._last_ts_ntp++; + this._base_ts_ntp = this._last_ts_ntp; + } + } else { + this._base_ts = ts; + this._last_ts_ntp = 1; + } + } + + handleCodecIsChanged(frameInfo: EncodedVideoFrameInfo) { + if ( + this._currentCodecConfig?.codecType !== frameInfo.codecType || + this._currentCodecConfig?.codedWidth !== frameInfo.width || + this._currentCodecConfig?.codedHeight !== frameInfo.height + ) { + logInfo('frameInfo has changed, reconfigure decoder'); + this._decoder.reset(); + this.decoderConfigure(frameInfo); + } + } + + // @ts-ignore + decodeFrame( + imageBuffer: Uint8Array, + frameInfo: EncodedVideoFrameInfo, + ts: number + ) { + try { + this.handleCodecIsChanged(frameInfo); + } catch (error: any) { + logInfo(error); + return; + } + + if (!imageBuffer) { + logDebug('imageBuffer is empty, skip decode frame'); + return; + } + + let frameType: string | undefined; + if (frameInfo.frameType !== undefined) { + // @ts-ignore + frameType = frameTypeMapping[frameInfo.frameType]; + } + if (!frameType) { + logDebug('frameType is not in frameTypeMapping, skip decode frame'); + return; + } + + this.updateTimestamps(ts); + + this._decoder.decode( + new EncodedVideoChunk({ + data: imageBuffer, + timestamp: this._last_ts_ntp, + // @ts-ignore + type: frameType, + // @ts-ignore + transfer: [imageBuffer.buffer], + }) + ); + } + + reset() { + this._base_ts = 0; + this._base_ts_ntp = 1; + this._last_ts_ntp = 1; + this._decoder.reset(); + } + + release() { + try { + if (this.pendingFrame) { + this.pendingFrame.close(); + } + this._decoder.close(); + } catch (e) {} + this.pendingFrame = null; + } +} diff --git a/ts/Private/extension/AgoraBaseExtension.ts b/ts/Private/extension/AgoraBaseExtension.ts index cb0ff5c3b..e85cc857d 100644 --- a/ts/Private/extension/AgoraBaseExtension.ts +++ b/ts/Private/extension/AgoraBaseExtension.ts @@ -1 +1,14 @@ -export {}; +import '../AgoraBase'; + +declare module '../AgoraBase' { + interface VideoCanvas { + /** + * @ignore + */ + useWebCodecsDecoder?: boolean; + /** + * @ignore + */ + enableFps?: boolean; + } +} diff --git a/ts/Private/internal/IrisApiEngine.ts b/ts/Private/internal/IrisApiEngine.ts index 429540576..e382aecfa 100644 --- a/ts/Private/internal/IrisApiEngine.ts +++ b/ts/Private/internal/IrisApiEngine.ts @@ -1,7 +1,9 @@ import EventEmitter from 'eventemitter3'; import JSON from 'json-bigint'; -import { AgoraEnv } from '../../Utils'; +import createAgoraRtcEngine from '../../AgoraSdk'; +import { IAgoraElectronBridge } from '../../Types'; +import { AgoraEnv, logDebug, logError, logInfo, logWarn } from '../../Utils'; import { IAudioEncodedFrameObserver } from '../AgoraBase'; import { AudioFrame, @@ -60,8 +62,11 @@ import { RtcEngineExInternal } from './RtcEngineExInternal'; // @ts-ignore export const DeviceEventEmitter: EventEmitter = new EventEmitter(); -const AgoraRtcNg = AgoraEnv.AgoraElectronBridge; -AgoraRtcNg.OnEvent('call_back_with_buffer', (...params: any) => { +const AgoraNode = require('../../../build/Release/agora_node_ext'); +export const AgoraElectronBridge: IAgoraElectronBridge = + new AgoraNode.AgoraElectronBridge(); + +AgoraElectronBridge.OnEvent('call_back_with_buffer', (...params: any) => { try { handleEvent(...params); } catch (e) { @@ -355,7 +360,7 @@ export const EVENT_PROCESSORS: EventProcessors = { function handleEvent(...[event, data, buffers]: any) { if (isDebuggable()) { - console.info('onEvent', event, data, buffers); + logInfo('onEvent', event, data, buffers); } let _event: string = event; @@ -414,7 +419,7 @@ function handleEvent(...[event, data, buffers]: any) { export function callIrisApi(funcName: string, params: any): any { try { const buffers: Uint8Array[] = []; - + const rtcEngine = createAgoraRtcEngine(); if (funcName.startsWith('MediaEngine_')) { switch (funcName) { case 'MediaEngine_pushAudioFrame': @@ -459,16 +464,16 @@ export function callIrisApi(funcName: string, params: any): any { } else if (funcName.startsWith('RtcEngine_')) { switch (funcName) { case 'RtcEngine_initialize': - AgoraRtcNg.InitializeEnv(); + AgoraElectronBridge.InitializeEnv(); break; case 'RtcEngine_release': - AgoraRtcNg.CallApi( + AgoraElectronBridge.CallApi( funcName, JSON.stringify(params), buffers, buffers.length ); - AgoraRtcNg.ReleaseEnv(); + AgoraElectronBridge.ReleaseEnv(); return; case 'RtcEngine_sendMetaData': // metadata.buffer @@ -494,8 +499,17 @@ export function callIrisApi(funcName: string, params: any): any { break; } } + if (funcName.indexOf('joinChannel') != -1) { + if (AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + rtcEngine.getMediaEngine().registerVideoEncodedFrameObserver({}); + } + } else if (funcName.indexOf('leaveChannel') != -1) { + if (AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + rtcEngine.getMediaEngine().unregisterVideoEncodedFrameObserver({}); + } + } - let { callApiReturnCode, callApiResult } = AgoraRtcNg.CallApi( + let { callApiReturnCode, callApiResult } = AgoraElectronBridge.CallApi( funcName, JSON.stringify(params), buffers, @@ -506,35 +520,30 @@ export function callIrisApi(funcName: string, params: any): any { const retObj = JSON.parse(ret); if (isDebuggable()) { if (typeof retObj.result === 'number' && retObj.result < 0) { - console.error('callApi', funcName, JSON.stringify(params), ret); + logError('callApi', funcName, JSON.stringify(params), ret); } else { - console.debug('callApi', funcName, JSON.stringify(params), ret); + logDebug('callApi', funcName, JSON.stringify(params), ret); } } return retObj; } else { if (isDebuggable()) { - console.error( + logError( 'callApi', funcName, JSON.stringify(params), callApiReturnCode ); } else { - console.warn( - 'callApi', - funcName, - JSON.stringify(params), - callApiReturnCode - ); + logWarn('callApi', funcName, JSON.stringify(params), callApiReturnCode); } return { result: callApiReturnCode }; } } catch (e) { if (isDebuggable()) { - console.error('callApi', funcName, JSON.stringify(params), e); + logError('callApi', funcName, JSON.stringify(params), e); } else { - console.warn('callApi', funcName, JSON.stringify(params), e); + logWarn('callApi', funcName, JSON.stringify(params), e); } } return {}; diff --git a/ts/Private/internal/MediaEngineInternal.ts b/ts/Private/internal/MediaEngineInternal.ts index e8f81d299..2a467c4e1 100644 --- a/ts/Private/internal/MediaEngineInternal.ts +++ b/ts/Private/internal/MediaEngineInternal.ts @@ -1,5 +1,6 @@ import { createCheckers } from 'ts-interface-checker'; +import { AgoraEnv } from '../../Utils'; import { IAudioFrameObserver, IVideoEncodedFrameObserver, @@ -83,6 +84,14 @@ export class MediaEngineInternal extends IMediaEngineImpl { } override release() { + if (AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + if (MediaEngineInternal._video_frame_observers.length > 0) { + this.unregisterVideoFrameObserver({}); + } + if (MediaEngineInternal._video_encoded_frame_observers.length > 0) { + this.unregisterVideoEncodedFrameObserver({}); + } + } MediaEngineInternal._audio_frame_observers = []; MediaEngineInternal._video_frame_observers = []; MediaEngineInternal._video_encoded_frame_observers = []; diff --git a/ts/Private/internal/MediaPlayerInternal.ts b/ts/Private/internal/MediaPlayerInternal.ts index 888a513e7..140ad74ef 100644 --- a/ts/Private/internal/MediaPlayerInternal.ts +++ b/ts/Private/internal/MediaPlayerInternal.ts @@ -1,6 +1,6 @@ import { createCheckers } from 'ts-interface-checker'; -import { AgoraEnv, logWarn } from '../../Utils'; +import { AgoraEnv, logError } from '../../Utils'; import { ErrorCodeType } from '../AgoraBase'; import { @@ -105,7 +105,7 @@ export class MediaPlayerInternal extends IMediaPlayerImpl { MediaPlayerInternal._audio_spectrum_observers.get(this._mediaPlayerId) ?.length === 0 ) { - console.error( + logError( 'Please call `registerMediaPlayerAudioSpectrumObserver` before you want to receive event by `addListener`' ); return false; @@ -303,32 +303,24 @@ export class MediaPlayerInternal extends IMediaPlayerImpl { } override setView(view: HTMLElement): number { - logWarn('Also can use other api setupLocalVideo'); - return ( - AgoraEnv.AgoraRendererManager?.setupVideo({ - videoSourceType: VideoSourceType.VideoSourceMediaPlayer, - uid: this._mediaPlayerId, - view, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const renderer = AgoraEnv.AgoraRendererManager.addOrRemoveRenderer({ + sourceType: VideoSourceType.VideoSourceMediaPlayer, + mediaPlayerId: this._mediaPlayerId, + view, + }); + if (!renderer) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setRenderMode(renderMode: RenderModeType): number { - logWarn( - 'Also can use other api setRenderOption or setRenderOptionByConfig' - ); - return ( - AgoraEnv.AgoraRendererManager?.setRenderOptionByConfig({ - videoSourceType: VideoSourceType.VideoSourceMediaPlayer, - uid: this._mediaPlayerId, - rendererOptions: { - contentMode: - renderMode === RenderModeType.RenderModeFit - ? RenderModeType.RenderModeFit - : RenderModeType.RenderModeHidden, - mirror: true, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const renderer = AgoraEnv.AgoraRendererManager.setRendererContext({ + sourceType: VideoSourceType.VideoSourceMediaPlayer, + mediaPlayerId: this._mediaPlayerId, + renderMode, + }); + if (!renderer) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } } diff --git a/ts/Private/internal/RtcEngineExInternal.ts b/ts/Private/internal/RtcEngineExInternal.ts index 4598255ca..6ed5be909 100644 --- a/ts/Private/internal/RtcEngineExInternal.ts +++ b/ts/Private/internal/RtcEngineExInternal.ts @@ -1,7 +1,7 @@ import { createCheckers } from 'ts-interface-checker'; -import { Channel } from '../../Types'; -import { AgoraEnv } from '../../Utils'; +import { AgoraElectronBridge } from '../../Private/internal/IrisApiEngine'; +import { AgoraEnv, logError } from '../../Utils'; import { AudioEncodedFrameObserverConfig, AudioRecordingConfiguration, @@ -89,6 +89,10 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { new LocalSpatialAudioEngineInternal(); override initialize(context: RtcEngineContext): number { + const ret = super.initialize(context); + callIrisApi.call(this, 'RtcEngine_setAppType', { + appType: 3, + }); if (AgoraEnv.webEnvReady) { // @ts-ignore window.AgoraEnv = AgoraEnv; @@ -96,23 +100,27 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { const { RendererManager } = require('../../Renderer/RendererManager'); AgoraEnv.AgoraRendererManager = new RendererManager(); } + if (AgoraEnv.CapabilityManager === undefined) { + const { + CapabilityManager, + } = require('../../Renderer/CapabilityManager'); + AgoraEnv.CapabilityManager = new CapabilityManager(); + } } - AgoraEnv.AgoraRendererManager?.enableRender(); - const ret = super.initialize(context); - callIrisApi.call(this, 'RtcEngine_setAppType', { - appType: 3, - }); return ret; } override release(sync: boolean = false) { - AgoraEnv.AgoraElectronBridge.ReleaseRenderer(); - AgoraEnv.AgoraRendererManager?.clear(); + AgoraElectronBridge.ReleaseRenderer(); + AgoraEnv.AgoraRendererManager?.release(); AgoraEnv.AgoraRendererManager = undefined; this._audio_device_manager.release(); this._video_device_manager.release(); this._media_engine.release(); this._local_spatial_audio_engine.release(); + RtcEngineExInternal._event_handlers.map((it) => { + super.unregisterEventHandler(it); + }); RtcEngineExInternal._event_handlers = []; RtcEngineExInternal._direct_cdn_streaming_event_handler = []; RtcEngineExInternal._metadata_observer = []; @@ -124,6 +132,8 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { MediaPlayerInternal._audio_spectrum_observers.clear(); MediaRecorderInternal._observers.clear(); this.removeAllListeners(); + AgoraEnv.CapabilityManager?.release(); + AgoraEnv.CapabilityManager = undefined; super.release(sync); } @@ -145,7 +155,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { if ( RtcEngineExInternal._direct_cdn_streaming_event_handler.length === 0 ) { - console.error( + logError( 'Please call `startDirectCdnStreaming` before you want to receive event by `addListener`' ); return false; @@ -157,7 +167,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { }) ) { if (RtcEngineExInternal._metadata_observer.length === 0) { - console.error( + logError( 'Please call `registerMediaMetadataObserver` before you want to receive event by `addListener`' ); return false; @@ -169,7 +179,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { }) ) { if (RtcEngineExInternal._audio_encoded_frame_observers.length === 0) { - console.error( + logError( 'Please call `registerAudioEncodedFrameObserver` before you want to receive event by `addListener`' ); return false; @@ -190,7 +200,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { addListener( eventType: EventType, listener: IRtcEngineEvent[EventType] - ): void { + ) { this._addListenerPreCheck(eventType); const callback = (eventProcessor: EventProcessor, data: any) => { if (eventProcessor.type(data) !== EVENT_TYPE.IRtcEngine) { @@ -233,6 +243,8 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { } override registerEventHandler(eventHandler: IRtcEngineEventHandler): boolean { + // only call iris when no event handler registered + let callIris = RtcEngineExInternal._event_handlers.length === 0; if ( !RtcEngineExInternal._event_handlers.find( (value) => value === eventHandler @@ -240,7 +252,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { ) { RtcEngineExInternal._event_handlers.push(eventHandler); } - return super.registerEventHandler(eventHandler); + return callIris ? super.registerEventHandler(eventHandler) : true; } override unregisterEventHandler( @@ -250,7 +262,9 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { RtcEngineExInternal._event_handlers.filter( (value) => value !== eventHandler ); - return super.unregisterEventHandler(eventHandler); + // only call iris when no event handler registered + let callIris = RtcEngineExInternal._event_handlers.length === 0; + return callIris ? super.unregisterEventHandler(eventHandler) : true; } override createMediaPlayer(): IMediaPlayer { @@ -326,7 +340,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { options: ChannelMediaOptions ): string { if (AgoraEnv.AgoraRendererManager) { - AgoraEnv.AgoraRendererManager.defaultRenderConfig.channelId = channelId; + AgoraEnv.AgoraRendererManager.defaultChannelId = channelId; } return 'RtcEngine_joinChannel2'; } @@ -426,7 +440,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { options?: ChannelMediaOptions ): string { if (AgoraEnv.AgoraRendererManager) { - AgoraEnv.AgoraRendererManager.defaultRenderConfig.channelId = channelId; + AgoraEnv.AgoraRendererManager.defaultChannelId = channelId; } return options === undefined ? 'RtcEngine_joinChannelWithUserAccount' @@ -556,7 +570,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { if (!value.thumbImage?.buffer || !value.thumbImage?.length) { value.thumbImage!.buffer = undefined; } else { - value.thumbImage!.buffer = AgoraEnv.AgoraElectronBridge.GetBuffer( + value.thumbImage!.buffer = AgoraElectronBridge.GetBuffer( value.thumbImage!.buffer as unknown as number, value.thumbImage.length! ); @@ -564,7 +578,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { if (!value.iconImage?.buffer || !value.iconImage?.length) { value.iconImage!.buffer = undefined; } else { - value.iconImage.buffer = AgoraEnv.AgoraElectronBridge.GetBuffer( + value.iconImage.buffer = AgoraElectronBridge.GetBuffer( value.iconImage!.buffer as unknown as number, value.iconImage.length! ); @@ -579,98 +593,44 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { } override setupLocalVideo(canvas: VideoCanvas): number { - let { - sourceType = VideoSourceType.VideoSourceCamera, - uid, - mediaPlayerId, - view, - renderMode, - mirrorMode, - } = canvas; - if ( - sourceType === VideoSourceType.VideoSourceMediaPlayer && - mediaPlayerId !== undefined - ) { - uid = mediaPlayerId; - } - return ( - AgoraEnv.AgoraRendererManager?.setupLocalVideo({ - videoSourceType: sourceType, - channelId: '', - uid, - view, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const renderer = AgoraEnv.AgoraRendererManager.addOrRemoveRenderer(canvas); + if (!renderer) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setupRemoteVideo(canvas: VideoCanvas): number { - const { - sourceType = VideoSourceType.VideoSourceRemote, - uid, - view, - renderMode, - mirrorMode, - } = canvas; - return ( - AgoraEnv.AgoraRendererManager?.setupRemoteVideo({ - videoSourceType: sourceType, - channelId: - AgoraEnv.AgoraRendererManager?.defaultRenderConfig?.channelId, - uid, - view, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const renderer = AgoraEnv.AgoraRendererManager.addOrRemoveRenderer(canvas); + if (!renderer) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setupRemoteVideoEx( canvas: VideoCanvas, connection: RtcConnection ): number { - const { - sourceType = VideoSourceType.VideoSourceRemote, - uid, - view, - renderMode, - mirrorMode, - } = canvas; - const { channelId } = connection; - return ( - AgoraEnv.AgoraRendererManager?.setupRemoteVideo({ - videoSourceType: sourceType, - channelId, - uid, - view, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const renderer = AgoraEnv.AgoraRendererManager.addOrRemoveRenderer({ + ...canvas, + ...connection, + }); + if (!renderer) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setLocalRenderMode( renderMode: RenderModeType, mirrorMode: VideoMirrorModeType = VideoMirrorModeType.VideoMirrorModeAuto ): number { - return ( - AgoraEnv.AgoraRendererManager?.setRenderOptionByConfig({ - videoSourceType: VideoSourceType.VideoSourceCamera, - channelId: '', - uid: 0, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const result = AgoraEnv.AgoraRendererManager.setRendererContext({ + sourceType: VideoSourceType.VideoSourceCamera, + renderMode, + mirrorMode, + }); + if (!result) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setRemoteRenderMode( @@ -678,17 +638,15 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { renderMode: RenderModeType, mirrorMode: VideoMirrorModeType ): number { - return ( - AgoraEnv.AgoraRendererManager?.setRenderOptionByConfig({ - videoSourceType: VideoSourceType.VideoSourceRemote, - channelId: AgoraEnv.AgoraRendererManager?.defaultRenderConfig.channelId, - uid, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const result = AgoraEnv.AgoraRendererManager.setRendererContext({ + sourceType: VideoSourceType.VideoSourceRemote, + uid, + renderMode, + mirrorMode, + }); + if (!result) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setRemoteRenderModeEx( @@ -697,46 +655,41 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { mirrorMode: VideoMirrorModeType, connection: RtcConnection ): number { - const { channelId } = connection; - return ( - AgoraEnv.AgoraRendererManager?.setRenderOptionByConfig({ - videoSourceType: VideoSourceType.VideoSourceRemote, - channelId, - uid, - rendererOptions: { - contentMode: renderMode, - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const result = AgoraEnv.AgoraRendererManager.setRendererContext({ + sourceType: VideoSourceType.VideoSourceRemote, + ...connection, + uid, + renderMode, + mirrorMode, + }); + if (!result) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override setLocalVideoMirrorMode(mirrorMode: VideoMirrorModeType): number { - return ( - AgoraEnv.AgoraRendererManager?.setRenderOptionByConfig({ - videoSourceType: VideoSourceType.VideoSourceCamera, - channelId: '', - uid: 0, - rendererOptions: { - mirror: mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled, - }, - }) ?? -ErrorCodeType.ErrNotInitialized - ); + if (!AgoraEnv.AgoraRendererManager) return -ErrorCodeType.ErrNotInitialized; + const result = AgoraEnv.AgoraRendererManager.setRendererContext({ + sourceType: VideoSourceType.VideoSourceCamera, + mirrorMode, + }); + if (!result) return -ErrorCodeType.ErrFailed; + return ErrorCodeType.ErrOk; } override destroyRendererByView(view: any) { - AgoraEnv.AgoraRendererManager?.destroyRendererByView(view); + AgoraEnv.AgoraRendererManager?.removeRendererFromCache({ view }); } override destroyRendererByConfig( - videoSourceType: VideoSourceType, - channelId?: Channel, + sourceType: VideoSourceType, + channelId?: string, uid?: number ) { - AgoraEnv.AgoraRendererManager?.destroyRenderersByConfig( - videoSourceType, + AgoraEnv.AgoraRendererManager?.removeRendererFromCache({ + sourceType, channelId, - uid - ); + uid, + }); } } diff --git a/ts/Private/ipc/main.ts b/ts/Private/ipc/main.ts new file mode 100644 index 000000000..d4e46ced6 --- /dev/null +++ b/ts/Private/ipc/main.ts @@ -0,0 +1,22 @@ +//@ts-ignore +import { app, ipcMain } from 'electron'; + +import { getGpuInfoInternal } from '../../Decoder/gpu-utils'; + +import { IPCMessageType } from '../../Types'; +//@ts-ignore +if (process.type === 'browser') { + ipcMain.handle(IPCMessageType.AGORA_IPC_GET_GPU_INFO, () => { + return new Promise((resolve) => { + getGpuInfoInternal((result: any) => { + resolve(result); + }); + }); + }); + console.log('main process AgoraIPCMain handler registered'); + + app.on('quit', () => { + ipcMain.removeHandler(IPCMessageType.AGORA_IPC_GET_GPU_INFO); + console.log('main process AgoraIPCMain handler removed'); + }); +} diff --git a/ts/Private/ipc/renderer.ts b/ts/Private/ipc/renderer.ts new file mode 100644 index 000000000..a6b14a026 --- /dev/null +++ b/ts/Private/ipc/renderer.ts @@ -0,0 +1,21 @@ +//@ts-ignore +import { ipcRenderer } from 'electron'; + +import { IPCMessageType } from '../../Types'; +import { logError } from '../../Utils'; + +export async function ipcSend( + channel: IPCMessageType, + ...args: any[] +): Promise { + if (!Object.values(IPCMessageType).includes(channel)) { + logError('Invalid IPCMessageType'); + return; + } + //@ts-ignore + if (process.type === 'renderer') { + return await ipcRenderer.invoke(channel, ...args); + } else { + logError('Not in renderer process, cannot send ipc message'); + } +} diff --git a/ts/Renderer/AgoraView.ts b/ts/Renderer/AgoraView.ts index 29f126474..b0ffae770 100644 --- a/ts/Renderer/AgoraView.ts +++ b/ts/Renderer/AgoraView.ts @@ -1,3 +1,4 @@ +import { VideoMirrorModeType, VideoViewSetupMode } from '../Private/AgoraBase'; import { RenderModeType, VideoSourceType } from '../Private/AgoraMediaBase'; import { AgoraEnv } from '../Utils'; @@ -70,12 +71,12 @@ export default class AgoraView extends HTMLElement { return observedAttributes; } - get videoSourceType(): VideoSourceType { + get sourceType(): VideoSourceType { const number = Number(this.getAttribute(VIDEO_SOURCE_TYPE_STRING)); return isNaN(number) ? 0 : number; } - set videoSourceType(val) { + set sourceType(val) { if (val) { this.setAttribute(VIDEO_SOURCE_TYPE_STRING, String(val)); } else { @@ -108,7 +109,7 @@ export default class AgoraView extends HTMLElement { } } - get renderContentMode(): RenderModeType { + get renderMode(): RenderModeType { const number = Number( this.getAttribute(RENDERER_CONTENT_MODE_STRING) || RenderModeType.RenderModeFit @@ -116,7 +117,7 @@ export default class AgoraView extends HTMLElement { return isNaN(number) ? RenderModeType.RenderModeFit : number; } - set renderContentMode(val) { + set renderMode(val) { if (val) { this.setAttribute(RENDERER_CONTENT_MODE_STRING, String(val)); } else { @@ -141,21 +142,28 @@ export default class AgoraView extends HTMLElement { } initializeRender = () => { - AgoraEnv.AgoraRendererManager?.destroyRendererByView(this); - AgoraEnv.AgoraRendererManager?.setupVideo({ - videoSourceType: this.videoSourceType, + const { channelId, uid, sourceType, renderMode, renderMirror } = this; + AgoraEnv.AgoraRendererManager?.addOrRemoveRenderer({ + sourceType, view: this, - uid: this.uid, - channelId: this.channelId, - rendererOptions: { - mirror: this.renderMirror, - contentMode: this.renderContentMode, - }, + uid, + channelId, + renderMode, + mirrorMode: renderMirror + ? VideoMirrorModeType.VideoMirrorModeEnabled + : VideoMirrorModeType.VideoMirrorModeDisabled, + setupMode: VideoViewSetupMode.VideoViewSetupReplace, }); }; destroyRender = () => { - AgoraEnv.AgoraRendererManager?.destroyRendererByView(this); + const { channelId, uid, sourceType } = this; + AgoraEnv.AgoraRendererManager?.removeRendererFromCache({ + channelId, + uid, + sourceType, + view: this, + }); }; connectedCallback() { @@ -173,11 +181,13 @@ export default class AgoraView extends HTMLElement { ].includes(attrName); if (isSetRenderOption) { - AgoraEnv.AgoraRendererManager?.setRenderOption( - this, - this.renderContentMode, - this.renderMirror - ); + AgoraEnv.AgoraRendererManager?.setRendererContext({ + view: this, + renderMode: this.renderMode, + mirrorMode: this.renderMirror + ? VideoMirrorModeType.VideoMirrorModeEnabled + : VideoMirrorModeType.VideoMirrorModeDisabled, + }); return; } const isNeedReInitialize = observedAttributes.includes(attrName); diff --git a/ts/Renderer/CapabilityManager.ts b/ts/Renderer/CapabilityManager.ts new file mode 100644 index 000000000..360208fd2 --- /dev/null +++ b/ts/Renderer/CapabilityManager.ts @@ -0,0 +1,126 @@ +import semver from 'semver'; + +import createAgoraRtcEngine from '../AgoraSdk'; + +import { + GpuInfo, + VideoDecodeAcceleratorSupportedProfile, +} from '../Decoder/gpu-utils'; +import { VideoCodecType } from '../Private/AgoraBase'; +import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx'; +import { ipcSend } from '../Private/ipc/renderer'; + +import { IPCMessageType, VideoFallbackStrategy, codecMapping } from '../Types'; +import { AgoraEnv, logError, logInfo } from '../Utils'; + +/** + * @ignore + */ +export class CapabilityManager { + gpuInfo: GpuInfo = new GpuInfo(); + frameCodecMapping: { + [key in VideoCodecType]?: VideoDecodeAcceleratorSupportedProfile; + } = {}; + webCodecsDecoderEnabled: boolean = AgoraEnv.enableWebCodecsDecoder; + private _engine: IRtcEngineEx; + + setWebCodecsDecoderEnabled(enabled: boolean): void { + this.webCodecsDecoderEnabled = enabled; + } + + constructor() { + this._engine = createAgoraRtcEngine(); + if (AgoraEnv.enableWebCodecsDecoder) { + this.getGpuInfo(() => { + if ( + AgoraEnv.videoFallbackStrategy === + VideoFallbackStrategy.PerformancePriority + ) { + if (!this.isSupportedH265()) { + if (this.isSupportedH264()) { + this._engine.setParameters( + JSON.stringify({ 'che.video.h265_dec_enable': false }) + ); + logInfo( + 'the videoFallbackStrategy is PerformancePriority, H265 is not supported, fallback to H264' + ); + } else { + this.webCodecsDecoderEnabled = false; + logInfo( + 'the videoFallbackStrategy is PerformancePriority, H264 and H265 are not supported, fallback to native decoder' + ); + } + } + } else if ( + AgoraEnv.videoFallbackStrategy === + VideoFallbackStrategy.BandwidthPriority + ) { + if (!this.isSupportedH265()) { + this.webCodecsDecoderEnabled = false; + logInfo( + 'the videoFallbackStrategy is BandwidthPriority, H265 is not supported, fallback to native decoder' + ); + } + } + }); + } + } + + public getGpuInfo(callback?: () => void): void { + //getGpuInfo and videoDecoder is not supported in electron version < 22.0.0 + //@ts-ignore + if (semver.lt(process.versions.electron, '22.0.0')) { + logError( + 'WebCodecsDecoder is not supported in electron version < 22.0.0, please upgrade electron to 22.0.0 or later.' + ); + return; + } + //@ts-ignore + if (process.type === 'renderer') { + ipcSend(IPCMessageType.AGORA_IPC_GET_GPU_INFO) + .then((result) => { + this.gpuInfo.videoDecodeAcceleratorSupportedProfile = result; + this.webCodecsDecoderEnabled = (AgoraEnv.enableWebCodecsDecoder && + this.gpuInfo.videoDecodeAcceleratorSupportedProfile.length > 0)!; + + result.forEach((profile: VideoDecodeAcceleratorSupportedProfile) => { + const match = codecMapping.find((item) => + profile.codec.includes(item.profile) + ); + if (match) { + //Normally, the range of compatible widths and heights should be the same under the same codec. + //there is no need to differentiate between different profiles. This could be optimized in the future. + this.frameCodecMapping[match.type] = { + codec: match.codec, + minWidth: profile.minWidth, + minHeight: profile.minHeight, + maxWidth: profile.maxWidth, + maxHeight: profile.maxHeight, + }; + } + }); + callback && callback(); + }) + .catch((error) => { + logError( + 'Failed to get GPU info, please check if you are already import agora-electron-sdk in the main process.', + error + ); + }); + } else { + logError('This function only works in renderer process'); + } + } + + public isSupportedH264(): boolean { + return this.frameCodecMapping[VideoCodecType.VideoCodecH264] !== undefined; + } + + public isSupportedH265(): boolean { + return this.frameCodecMapping[VideoCodecType.VideoCodecH265] !== undefined; + } + + release(): void { + AgoraEnv.enableWebCodecsDecoder = false; + } +} diff --git a/ts/Renderer/IRenderer.ts b/ts/Renderer/IRenderer.ts index 1c0318964..3fa41df8a 100644 --- a/ts/Renderer/IRenderer.ts +++ b/ts/Renderer/IRenderer.ts @@ -1,55 +1,155 @@ -import { EventEmitter } from 'events'; +import { VideoMirrorModeType } from '../Private/AgoraBase'; +import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase'; +import { RendererContext, RendererType } from '../Types'; -import { RenderModeType } from '../Private/AgoraMediaBase'; -import { RendererOptions, ShareVideoFrame } from '../Types'; - -export type RenderFailCallback = - | ((obj: { error: string }) => void) - | undefined - | null; +import { frameSize } from './WebCodecsRenderer'; export abstract class IRenderer { parentElement?: HTMLElement; + container?: HTMLElement; canvas?: HTMLCanvasElement; - event?: EventEmitter; - contentMode = RenderModeType.RenderModeFit; - mirror?: boolean; + rendererType: RendererType | undefined; + context: RendererContext = {}; + private _frameCount = 0; + private _startTime: number | null = null; - public snapshot(fileType = 'image/png') { - if (this.canvas && this.canvas.toDataURL) { - return this.canvas.toDataURL(fileType); + public bind(element: HTMLElement, _frameSize?: frameSize) { + this.parentElement = element; + this.container = document.createElement('div'); + Object.assign(this.container.style, { + width: '100%', + height: '100%', + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + overflow: 'hidden', + }); + this.parentElement.appendChild(this.container); + this.canvas = document.createElement('canvas'); + this.canvas.style.display = 'none'; + this.container.appendChild(this.canvas); + } + + public unbind() { + if (this.container && this.canvas?.parentNode === this.container) { + this.container.removeChild(this.canvas); + } + if ( + this.parentElement && + this.container?.parentNode === this.parentElement + ) { + this.parentElement.removeChild(this.container); } - return null; + + this.canvas = undefined; + this.container = undefined; + this.parentElement = undefined; } - public bind(element: HTMLElement) { - if (!element) { - throw new Error('You have pass a element'); + public drawFrame(_videoFrame?: VideoFrame): void { + if (!this.canvas) return; + if (this.canvas.style.display !== '') { + this.canvas.style.display = ''; } - this.parentElement = element; } - abstract unbind(): void; + public setContext(context: RendererContext) { + if (this.context.renderMode !== context.renderMode) { + this.updateRenderMode(); + } - public equalsElement(element: Element): boolean { - if (!element) { - throw new Error('You have pass a element'); + if (this.context.mirrorMode !== context.mirrorMode) { + this.updateMirrorMode(); } - if (!this.parentElement) { - throw new Error('parentElement is null'); + this.context = context; + } + + protected updateRenderMode() { + if (!this.canvas || !this.container) return; + + const { clientWidth, clientHeight } = this.container; + const { width, height } = this.canvas; + + const containerAspectRatio = clientWidth / clientHeight; + const canvasAspectRatio = width / height; + const widthScale = clientWidth / width; + const heightScale = clientHeight / height; + const isHidden = + this.context?.renderMode === RenderModeType.RenderModeHidden; + + let scale = 1; + // If container's aspect ratio is larger than canvas's aspect ratio + if (containerAspectRatio > canvasAspectRatio) { + // Scale canvas to fit container's width on hidden mode + // Scale canvas to fit container's height on fit mode + scale = isHidden ? widthScale : heightScale; + } else { + // Scale canvas to fit container's height on hidden mode + // Scale canvas to fit container's width on fit mode + scale = isHidden ? heightScale : widthScale; } - return element === this.parentElement; + this.canvas.style.transform = `scale(${scale})`; } - abstract drawFrame(imageData: ShareVideoFrame): void; + protected updateMirrorMode(): void { + if (!this.parentElement) return; - public setRenderOption({ contentMode, mirror }: RendererOptions) { - this.contentMode = contentMode ?? RenderModeType.RenderModeFit; - this.mirror = mirror; - Object.assign(this.parentElement!.style, { - transform: mirror ? 'rotateY(180deg)' : '', + Object.assign(this.parentElement.style, { + transform: + this.context.mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled + ? 'rotateY(180deg)' + : '', }); } - abstract refreshCanvas(): void; + protected rotateCanvas({ width, height, rotation }: VideoFrame): void { + if (!this.canvas) return; + + if (rotation === 0 || rotation === 180) { + this.canvas.width = width!; + this.canvas.height = height!; + } else if (rotation === 90 || rotation === 270) { + this.canvas.height = width!; + this.canvas.width = height!; + } else { + throw new Error( + `Invalid rotation: ${rotation}, only 0, 90, 180, 270 are supported` + ); + } + } + + public getFps(): number { + let fps = 0; + if (!this.context.enableFps || !this.container) { + return fps; + } + if (this._startTime == null) { + this._startTime = performance.now(); + } else { + const elapsed = (performance.now() - this._startTime) / 1000; + fps = ++this._frameCount / elapsed; + } + + let span = this.container.querySelector('span'); + if (!span) { + span = document.createElement('span'); + + Object.assign(span.style, { + position: 'absolute', + bottom: '0', + left: '0', + zIndex: '10', + width: '55px', + background: '#fff', + }); + + this.container.style.position = 'relative'; + + this.container.appendChild(span); + } + + span.innerText = `fps: ${fps.toFixed(0)}`; + + return fps; + } } diff --git a/ts/Renderer/IRendererCache.ts b/ts/Renderer/IRendererCache.ts new file mode 100644 index 000000000..ff024aa01 --- /dev/null +++ b/ts/Renderer/IRendererCache.ts @@ -0,0 +1,81 @@ +import { RendererCacheContext, RendererContext } from '../Types'; + +import { IRenderer } from './IRenderer'; + +export function generateRendererCacheKey({ + channelId, + uid, + sourceType, +}: RendererContext): string { + return `${channelId}_${uid}_${sourceType}`; +} + +export abstract class IRendererCache { + renderers: IRenderer[]; + cacheContext: RendererCacheContext; + + constructor({ + channelId, + uid, + useWebCodecsDecoder, + enableFps, + sourceType, + }: RendererContext) { + this.renderers = []; + this.cacheContext = { + channelId, + uid, + useWebCodecsDecoder, + enableFps, + sourceType, + }; + } + + public get key(): string { + return generateRendererCacheKey(this.cacheContext); + } + + public abstract draw(): void; + + public findRenderer(view: Element): IRenderer | undefined { + return this.renderers.find((renderer) => renderer.parentElement === view); + } + + public addRenderer(renderer: IRenderer): void { + this.renderers.push(renderer); + } + + /** + * Remove the specified renderer if it is specified, otherwise remove all renderers + */ + public removeRenderer(renderer?: IRenderer): void { + let start = 0; + let deleteCount = this.renderers.length; + if (renderer) { + start = this.renderers.indexOf(renderer); + if (start < 0) return; + deleteCount = 1; + } + this.renderers.splice(start, deleteCount).forEach((it) => it.unbind()); + } + + public setRendererContext(context: RendererContext): boolean { + if (context.view) { + const renderer = this.findRenderer(context.view); + if (renderer) { + renderer.context = context; + return true; + } + return false; + } else { + this.renderers.forEach((it) => { + it.context = context; + }); + return this.renderers.length > 0; + } + } + + public release(): void { + this.removeRenderer(); + } +} diff --git a/ts/Renderer/IRendererManager.ts b/ts/Renderer/IRendererManager.ts deleted file mode 100644 index cba695edc..000000000 --- a/ts/Renderer/IRendererManager.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { RenderModeType, VideoSourceType } from '../Private/AgoraMediaBase'; -import { Channel, RendererVideoConfig } from '../Types'; - -/** - * @ignore - */ -export abstract class IRendererManager { - abstract get defaultRenderConfig(): RendererVideoConfig; - - abstract enableRender(enabled?: boolean): void; - - abstract clear(): void; - - abstract setupVideo(rendererVideoConfig: RendererVideoConfig): number; - - abstract setupLocalVideo(rendererConfig: RendererVideoConfig): number; - - abstract setupRemoteVideo(rendererConfig: RendererVideoConfig): number; - - abstract setRenderOptionByConfig(rendererConfig: RendererVideoConfig): number; - - abstract destroyRendererByView(view: Element): void; - - abstract destroyRenderersByConfig( - videoSourceType: VideoSourceType, - channelId?: Channel, - uid?: number - ): void; - - abstract setRenderOption( - view: HTMLElement, - contentMode?: RenderModeType, - mirror?: boolean - ): void; -} diff --git a/ts/Renderer/RendererCache.ts b/ts/Renderer/RendererCache.ts new file mode 100644 index 000000000..e67e0f8d2 --- /dev/null +++ b/ts/Renderer/RendererCache.ts @@ -0,0 +1,108 @@ +import { VideoFrame } from '../Private/AgoraMediaBase'; +import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine'; + +import { RendererContext } from '../Types'; +import { logDebug } from '../Utils'; + +import { IRenderer } from './IRenderer'; +import { IRendererCache } from './IRendererCache'; + +export class RendererCache extends IRendererCache { + private videoFrame: VideoFrame; + private _enabled: boolean; + + constructor(context: RendererContext) { + super(context); + this.videoFrame = { + yBuffer: Buffer.alloc(0), + uBuffer: Buffer.alloc(0), + vBuffer: Buffer.alloc(0), + width: 0, + height: 0, + yStride: 0, + uStride: 0, + vStride: 0, + rotation: 0, + }; + this._enabled = false; + } + + /** + * @deprecated Use videoFrame instead + */ + public get shareVideoFrame(): VideoFrame | undefined { + return this.videoFrame; + } + + private enable() { + if (this._enabled) return; + AgoraElectronBridge.EnableVideoFrameCache(this.cacheContext); + this._enabled = true; + } + + private disable() { + if (!this._enabled) return; + AgoraElectronBridge.DisableVideoFrameCache(this.cacheContext); + this._enabled = false; + } + + private shouldEnable() { + if (this.renderers.length > 0) { + this.enable(); + } else { + this.disable(); + } + } + + override draw() { + let { ret, isNewFrame } = AgoraElectronBridge.GetVideoFrame( + this.cacheContext, + this.videoFrame + ); + + switch (ret) { + case 0: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::OK = 0 + // + break; + case 1: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::RESIZED = 1 + const { yStride, uStride, vStride, height } = this.videoFrame; + this.videoFrame.yBuffer = Buffer.alloc(yStride! * height!); + this.videoFrame.uBuffer = Buffer.alloc(uStride! * height!); + this.videoFrame.vBuffer = Buffer.alloc(vStride! * height!); + + const result = AgoraElectronBridge.GetVideoFrame( + this.cacheContext, + this.videoFrame + ); + ret = result.ret; + isNewFrame = result.isNewFrame; + break; + case 2: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::NO_CACHE = 2 + logDebug('No renderer cache, please enable cache first'); + return; + } + + if (isNewFrame) { + this.renderers.forEach((renderer) => { + renderer.drawFrame(this.videoFrame); + }); + } + } + + override addRenderer(renderer: IRenderer): void { + super.addRenderer(renderer); + this.shouldEnable(); + } + + /** + * Remove the specified renderer if it is specified, otherwise remove all renderers + */ + override removeRenderer(renderer?: IRenderer): void { + super.removeRenderer(renderer); + this.shouldEnable(); + } + + public release(): void { + super.release(); + } +} diff --git a/ts/Renderer/RendererManager.ts b/ts/Renderer/RendererManager.ts index aea4635df..6d48448a6 100644 --- a/ts/Renderer/RendererManager.ts +++ b/ts/Renderer/RendererManager.ts @@ -1,630 +1,372 @@ -import { ErrorCodeType } from '../Private/AgoraBase'; +import createAgoraRtcEngine from '../AgoraSdk'; +import { + VideoMirrorModeType, + VideoStreamType, + VideoViewSetupMode, +} from '../Private/AgoraBase'; import { RenderModeType, VideoSourceType } from '../Private/AgoraMediaBase'; import { - AgoraElectronBridge, - Channel, - ChannelIdMap, - FormatRendererVideoConfig, - RENDER_MODE, - RenderConfig, - RenderMap, - RendererVideoConfig, - ShareVideoFrame, - UidMap, - VideoFrameCacheConfig, + RendererCacheContext, + RendererCacheType, + RendererContext, + RendererType, } from '../Types'; -import { - AgoraEnv, - formatConfigByVideoSourceType, - getDefaultRendererVideoConfig, - logDebug, - logError, - logInfo, - logWarn, -} from '../Utils'; - -import { IRenderer, RenderFailCallback } from './IRenderer'; -import { IRendererManager } from './IRendererManager'; -import WebGLRenderer from './WebGLRenderer'; +import { AgoraEnv, isSupportWebGL, logDebug } from '../Utils'; + +import { IRenderer } from './IRenderer'; +import { generateRendererCacheKey } from './IRendererCache'; +import { RendererCache } from './RendererCache'; +import { WebCodecsRenderer } from './WebCodecsRenderer'; +import { WebCodecsRendererCache } from './WebCodecsRendererCache'; +import { WebGLFallback, WebGLRenderer } from './WebGLRenderer'; import { YUVCanvasRenderer } from './YUVCanvasRenderer'; /** * @ignore */ -export class RendererManager extends IRendererManager { +export class RendererManager { + /** + * @ignore + */ + private renderingFps: number; /** * @ignore */ - isRendering = false; - renderFps: number; + private _currentFrameCount: number; /** * @ignore */ - videoFrameUpdateInterval?: NodeJS.Timer; + private _previousFirstFrameTime: number; /** * @ignore */ - renderers: RenderMap; + private _renderingTimer?: number; /** * @ignore */ - renderMode?: RENDER_MODE; + private _rendererCaches: RendererCacheType[]; /** * @ignore */ - msgBridge: AgoraElectronBridge; + private _context: RendererContext; + /** * @ignore */ - defaultRenderConfig: RendererVideoConfig; + private rendererType: RendererType; constructor() { - super(); - this.renderFps = 15; - this.renderers = new Map(); - this.setRenderMode(); - this.msgBridge = AgoraEnv.AgoraElectronBridge; - this.defaultRenderConfig = { - rendererOptions: { - contentMode: RenderModeType.RenderModeFit, - mirror: false, - }, + this.renderingFps = 15; + this._currentFrameCount = 0; + this._previousFirstFrameTime = 0; + this._rendererCaches = []; + this._context = { + renderMode: RenderModeType.RenderModeHidden, + mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled, }; + this.rendererType = isSupportWebGL() + ? RendererType.WEBGL + : RendererType.SOFTWARE; } - /** - * Sets the channel mode of the current audio file. - * In a stereo music file, the left and right channels can store different audio data. According to your needs, you can set the channel mode to original mode, left channel mode, right channel mode, or mixed channel mode. For example, in the KTV scenario, the left channel of the music file stores the musical accompaniment, and the right channel stores the singing voice. If you only need to listen to the accompaniment, call this method to set the channel mode of the music file to left channel mode; if you need to listen to the accompaniment and the singing voice at the same time, call this method to set the channel mode to mixed channel mode.Call this method after calling open .This method only applies to stereo audio files. - * - * @param mode The channel mode. See AudioDualMonoMode . - * - * @returns - * 0: Success.< 0: Failure. - */ - public setRenderMode(mode?: RENDER_MODE) { - if (mode === undefined) { - this.renderMode = this.checkWebglEnv() - ? RENDER_MODE.WEBGL - : RENDER_MODE.SOFTWARE; - return; - } - - if (mode !== this.renderMode) { - this.renderMode = mode; - logInfo( - 'setRenderMode: new render mode will take effect only if new view bind to render' - ); + public setRenderingFps(fps: number) { + this.renderingFps = fps; + if (this._renderingTimer) { + this.stopRendering(); + this.startRendering(); } } - /** - * @ignore - */ - public setFPS(fps: number) { - this.renderFps = fps; - this.restartRender(); + public set defaultChannelId(channelId: string) { + this._context.channelId = channelId; } - /** - * @ignore - */ - public setRenderOption( - view: HTMLElement, - contentMode = RenderModeType.RenderModeFit, - mirror: boolean = false - ): void { - if (!view) { - logError('setRenderOption: view not exist', view); - } - this.forEachStream(({ renders }) => { - renders?.forEach((render) => { - if (render.equalsElement(view)) { - render.setRenderOption({ contentMode, mirror }); - } - }); - }); + public get defaultChannelId(): string { + return this._context.channelId ?? ''; } - /** - * @ignore - */ - public setRenderOptionByConfig(rendererConfig: RendererVideoConfig): number { - const { - uid, - channelId, - rendererOptions, - videoSourceType, - }: FormatRendererVideoConfig = - getDefaultRendererVideoConfig(rendererConfig); - - const renderList = this.getRenderers({ uid, channelId, videoSourceType }); - renderList - ? renderList - .filter((renderItem) => { - if (rendererConfig.view) { - return renderItem.equalsElement(rendererConfig.view); - } else { - return true; - } - }) - .forEach((renderItem) => renderItem.setRenderOption(rendererOptions)) - : logWarn( - `RenderStreamType: ${videoSourceType} channelId:${channelId} uid:${uid} have no render view, you need to call this api after setView` - ); - return ErrorCodeType.ErrOk; + public get defaultRenderMode(): RenderModeType { + return this._context.renderMode!; } - /** - * @ignore - */ - public checkWebglEnv(): boolean { - let gl; - let canvas: HTMLCanvasElement = document.createElement('canvas'); - - try { - gl = - canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); - logInfo('Your browser support webGL'); - } catch (e) { - logWarn('Your browser may not support webGL'); - return false; - } - - return !!gl; + public get defaultMirrorMode(): VideoMirrorModeType { + return this._context.mirrorMode!; } - /** - * @ignore - */ - public setupVideo(rendererVideoConfig: RendererVideoConfig): number { - const formatConfig = getDefaultRendererVideoConfig(rendererVideoConfig); + public release(): void { + this.stopRendering(); + this.clearRendererCache(); + } - const { uid, channelId, videoSourceType, rendererOptions, view } = - formatConfig; + private presetRendererContext(context: RendererContext): RendererContext { + //this is for preset default value + context.renderMode = context.renderMode || this.defaultRenderMode; + context.mirrorMode = context.mirrorMode || this.defaultMirrorMode; + context.useWebCodecsDecoder = context.useWebCodecsDecoder || false; + context.enableFps = context.enableFps || false; - if (!formatConfig.view) { - logWarn('setupVideo->destroyRenderersByConfig, because of view is null'); - this.destroyRenderersByConfig(videoSourceType, channelId, uid); - return -ErrorCodeType.ErrInvalidArgument; + if (!AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + context.useWebCodecsDecoder = false; } - // ensure a render to RenderMap - const render = this.bindHTMLElementToRender(formatConfig, view!); - - // render config - render?.setRenderOption(rendererOptions); - - // enable iris videoFrame - this.enableVideoFrameCache({ - uid, - channelId, - videoSourceType, - }); - - // enable render - this.enableRender(true); - return ErrorCodeType.ErrOk; - } - - /** - * @ignore - */ - public setupLocalVideo(rendererConfig: RendererVideoConfig): number { - const { videoSourceType } = rendererConfig; - if (videoSourceType === VideoSourceType.VideoSourceRemote) { - logError('setupLocalVideo videoSourceType error', videoSourceType); - return -ErrorCodeType.ErrInvalidArgument; + switch (context.sourceType) { + case VideoSourceType.VideoSourceRemote: + if (context.uid === undefined) { + throw new Error('uid is required'); + } + context.channelId = context.channelId ?? this.defaultChannelId; + break; + case VideoSourceType.VideoSourceMediaPlayer: + if (context.mediaPlayerId === undefined) { + throw new Error('mediaPlayerId is required'); + } + context.channelId = ''; + context.uid = context.mediaPlayerId; + break; + case undefined: + if (context.uid) { + context.sourceType = VideoSourceType.VideoSourceRemote; + } + break; + default: + context.channelId = ''; + context.uid = 0; + break; } - this.setupVideo({ ...rendererConfig }); - return ErrorCodeType.ErrOk; + return context; } - /** - * @ignore - */ - public setupRemoteVideo(rendererConfig: RendererVideoConfig): number { - const { videoSourceType } = rendererConfig; - if (videoSourceType !== VideoSourceType.VideoSourceRemote) { - logError('setupRemoteVideo videoSourceType error', videoSourceType); - return -ErrorCodeType.ErrInvalidArgument; + public addOrRemoveRenderer( + context: RendererContext + ): RendererCacheType | undefined { + // To be compatible with the old API + let { setupMode = VideoViewSetupMode.VideoViewSetupAdd } = context; + if (!context.view) setupMode = VideoViewSetupMode.VideoViewSetupRemove; + switch (setupMode) { + case VideoViewSetupMode.VideoViewSetupAdd: + return this.addRendererToCache(context); + case VideoViewSetupMode.VideoViewSetupRemove: + this.removeRendererFromCache(context); + return undefined; + case VideoViewSetupMode.VideoViewSetupReplace: + this.removeRendererFromCache(context); + return this.addRendererToCache(context); } - this.setupVideo({ ...rendererConfig }); - return ErrorCodeType.ErrOk; } - /** - * Destroys a video renderer object. - * - * @param view The HTMLElement object to be destroyed. - */ - public destroyRendererByView(view: Element): void { - const renders = this.renderers; - renders.forEach((channelMap, videoSourceType) => { - channelMap.forEach((uidMap, channelId) => { - uidMap.forEach((renderConfig, uid) => { - let hasRender = false; - const remainRenders = renderConfig.renders?.filter((render) => { - const isFilter = render.equalsElement(view); - - if (isFilter) { - hasRender = true; - render.unbind(); - } - return !isFilter; - }); - if (!hasRender) { - return; - } - - if (remainRenders?.length === 0 || !remainRenders) { - this.disableVideoFrameCache({ uid, channelId, videoSourceType }); - } - renderConfig.renders = remainRenders; - }); - }); - }); - } + private addRendererToCache( + context: RendererContext + ): RendererCacheType | undefined { + const checkedContext = this.presetRendererContext(context); - /** - * @ignore - */ - public destroyRenderersByConfig( - videoSourceType: VideoSourceType, - channelId?: Channel, - uid?: number - ): void { - const config = formatConfigByVideoSourceType( - videoSourceType, - channelId, - uid - ); - videoSourceType = config.videoSourceType; - channelId = config.channelId; - uid = config.uid; - - this.disableVideoFrameCache(config); - const uidMap = this.renderers.get(videoSourceType)?.get(channelId); - const renderMap = uidMap?.get(uid); - if (!renderMap) { - return; + if (!checkedContext.view) return undefined; + + if (this.findRenderer(checkedContext.view)) { + throw new Error('You have already added this view to the renderer'); } - renderMap.renders?.forEach((renderItem) => { - renderItem.unbind(); - }); - renderMap.renders = []; - } - /** - * @ignore - */ - public removeAllRenderer(): void { - const renderMap = this.forEachStream( - (renderConfig, videoFrameCacheConfig) => { - this.disableVideoFrameCache(videoFrameCacheConfig); - renderConfig.renders?.forEach((renderItem) => { - renderItem.unbind(); - }); - renderConfig.renders = []; + let rendererCache = this.getRendererCache(checkedContext); + if (!rendererCache) { + if (context.useWebCodecsDecoder) { + rendererCache = new WebCodecsRendererCache(checkedContext); + } else { + rendererCache = new RendererCache(checkedContext); } - ); - renderMap.clear(); + this._rendererCaches.push(rendererCache); + } + rendererCache.addRenderer(this.createRenderer(checkedContext)); + if (!context.useWebCodecsDecoder) { + this.startRendering(); + } + return rendererCache; } - /** - * @ignore - */ - public clear(): void { - this.stopRender(); - this.removeAllRenderer(); - } + public removeRendererFromCache(context: RendererContext): void { + const checkedContext = this.presetRendererContext(context); - /** - * Enables/Disables the local video capture. - * This method disables or re-enables the local video capture, and does not affect receiving the remote video stream.After calling enableVideo , the local video capture is enabled by default. You can call enableLocalVideo (false) to disable the local video capture. If you want to re-enable the local video capture, call enableLocalVideo(true).After the local video capturer is successfully disabled or re-enabled, the SDK triggers the onRemoteVideoStateChanged callback on the remote client.You can call this method either before or after joining a channel.This method enables the internal engine and is valid after leaving the channel. - * - * @param enabled Whether to enable the local video capture.true: (Default) Enable the local video capture.false: Disable the local video capture. Once the local video is disabled, the remote users cannot receive the video stream of the local user, while the local user can still receive the video streams of remote users. When set to false, this method does not require a local camera. - * - * @returns - * 0: Success.< 0: Failure. - */ - public enableRender(enabled = true): void { - if (enabled && this.isRendering) { - //is already _isRendering - } else if (enabled && !this.isRendering) { - this.startRenderer(); + const rendererCache = this.getRendererCache(checkedContext); + if (!rendererCache) return; + if (checkedContext.view) { + const renderer = rendererCache.findRenderer(checkedContext.view); + if (!renderer) return; + rendererCache.removeRenderer(renderer); } else { - this.stopRender(); + rendererCache.removeRenderer(); } - } - - /** - * @ignore - */ - public startRenderer(): void { - this.isRendering = true; - const renderFunc = ( - rendererItem: RenderConfig, - { videoSourceType, channelId, uid }: VideoFrameCacheConfig - ) => { - const { renders } = rendererItem; - if (!renders || renders?.length === 0) { - return; - } - let finalResult = this.msgBridge.GetVideoFrame( - rendererItem.shareVideoFrame + if (rendererCache.renderers.length === 0) { + rendererCache.release(); + this._rendererCaches.splice( + this._rendererCaches.indexOf(rendererCache), + 1 ); - - switch (finalResult.ret) { - case 0: - // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::OK = 0, - // everything is ok - break; - case 1: { - // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::RESIZED - const { width, height, yStride } = finalResult; - const newShareVideoFrame = this.resizeShareVideoFrame( - videoSourceType, - channelId, - uid, - width, - height, - yStride - ); - rendererItem.shareVideoFrame = newShareVideoFrame; - finalResult = this.msgBridge.GetVideoFrame(newShareVideoFrame); - break; - } - case 2: - // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::NO_CACHE - // setupVideo/AgoraView render before initialize - // this.enableVideoFrameCache({ videoSourceType, channelId, uid }); - break; - default: - break; - } - if (finalResult.ret !== 0) { - logDebug('GetVideoFrame ret is', finalResult.ret, rendererItem); - return; - } - if (!finalResult.isNewFrame) { - logDebug('GetVideoFrame isNewFrame is false', rendererItem); - return; - } - const renderVideoFrame = rendererItem.shareVideoFrame; - if (renderVideoFrame.width > 0 && renderVideoFrame.height > 0) { - renders.forEach((renderItem) => { - renderItem.drawFrame(rendererItem.shareVideoFrame); - }); - } - }; - const render = () => { - this.forEachStream(renderFunc); - this.videoFrameUpdateInterval = setTimeout(render, 1000 / this.renderFps); - }; - render(); - } - - /** - * @ignore - */ - public stopRender(): void { - this.isRendering = false; - if (this.videoFrameUpdateInterval) { - clearTimeout(this.videoFrameUpdateInterval); - this.videoFrameUpdateInterval = undefined; } } - /** - * @ignore - */ - public restartRender(): void { - if (this.videoFrameUpdateInterval) { - this.stopRender(); - this.startRenderer(); - logInfo(`restartRender: Fps: ${this.renderFps} restartInterval`); + public clearRendererCache(): void { + for (const rendererCache of this._rendererCaches) { + rendererCache.release(); } + this._rendererCaches.splice(0); } - /** - * @ignore - */ - private createRenderer(failCallback?: RenderFailCallback): IRenderer { - if (this.renderMode === RENDER_MODE.SOFTWARE) { - return new YUVCanvasRenderer(); - } else { - return new WebGLRenderer(failCallback); - } + public getRendererCache( + context: RendererContext + ): RendererCacheType | undefined { + return this._rendererCaches.find( + (cache) => cache.key === generateRendererCacheKey(context) + ); } - /** - * @ignore - */ - private getRender({ - videoSourceType, - channelId, - uid, - }: VideoFrameCacheConfig) { - return this.renderers.get(videoSourceType)?.get(channelId)?.get(uid); + public getRenderers(context: RendererContext): IRenderer[] { + return this.getRendererCache(context)?.renderers || []; } - /** - * @ignore - */ - private getRenderers({ - videoSourceType, - channelId, - uid, - }: VideoFrameCacheConfig): IRenderer[] { - return this.getRender({ videoSourceType, channelId, uid })?.renders || []; + public findRenderer(view: Element): IRenderer | undefined { + for (const rendererCache of this._rendererCaches) { + const renderer = rendererCache.findRenderer(view); + if (renderer) return renderer; + } + return undefined; } - /** - * @ignore - */ - private bindHTMLElementToRender( - config: FormatRendererVideoConfig, - view: HTMLElement - ): IRenderer | undefined { - this.ensureRendererConfig(config); - const renders = this.getRenderers(config); - const filterRenders = - renders?.filter((render) => render.equalsElement(view)) || []; - const hasBeenAdd = filterRenders.length > 0; - if (hasBeenAdd) { - logWarn( - 'bindHTMLElementToRender: this view has bind to render', - filterRenders - ); - return filterRenders[0]; + protected createRenderer( + context: RendererContext, + rendererType: RendererType = this.rendererType + ): IRenderer { + let renderer: IRenderer; + switch (rendererType) { + case RendererType.WEBGL: + if (context.useWebCodecsDecoder) { + renderer = new WebCodecsRenderer(); + } else { + renderer = new WebGLRenderer( + this.handleWebGLFallback(context).bind(this) + ); + renderer.bind(context.view); + } + break; + case RendererType.SOFTWARE: + renderer = new YUVCanvasRenderer(); + renderer.bind(context.view); + break; + default: + throw new Error('Unknown renderer type'); } - const renderer = this.createRenderer(() => { - const { contentMode, mirror } = renderer; - renderer.unbind(); - renders.splice(renders.indexOf(renderer), 1); - this.setRenderMode(); - const newRender = this.createRenderer(); - newRender.bind(view); - newRender.setRenderOption({ contentMode, mirror }); - renders.push(newRender); - }); - renderer.bind(view); - renders.push(renderer); + + renderer.setContext(context); return renderer; } - /** - * @ignore - */ - private forEachStream( - callbackfn: ( - renderConfig: RenderConfig, - videoFrameCacheConfig: VideoFrameCacheConfig, - maps: { - channelMap: ChannelIdMap; - uidMap: UidMap; + public startRendering(): void { + if (this._renderingTimer) return; + + const renderingLooper = () => { + if (this._previousFirstFrameTime === 0) { + // Get the current time as the time of the first frame of per second + this._previousFirstFrameTime = performance.now(); + // Reset the frame count + this._currentFrameCount = 0; } - ) => void - ): RenderMap { - const renders = this.renderers; - renders.forEach((channelMap, videoSourceType) => { - channelMap.forEach((uidMap, channelId) => { - uidMap.forEach((renderConfig, uid) => { - callbackfn( - renderConfig, - { videoSourceType, channelId, uid }, - { channelMap, uidMap } - ); - }); - }); - }); - return renders; - } - /** - * @ignore - */ - private enableVideoFrameCache( - videoFrameCacheConfig: VideoFrameCacheConfig - ): void { - logDebug(`enableVideoFrameCache ${JSON.stringify(videoFrameCacheConfig)}`); - this.msgBridge.EnableVideoFrameCache(videoFrameCacheConfig); + // Increase the frame count + ++this._currentFrameCount; + + // Get the current time + const currentFrameTime = performance.now(); + // Calculate the time difference between the current frame and the previous frame + const deltaTime = currentFrameTime - this._previousFirstFrameTime; + // Calculate the expected time of the current frame + const expectedTime = (this._currentFrameCount * 1000) / this.renderingFps; + logDebug( + new Date().toLocaleTimeString(), + 'currentFrameCount', + this._currentFrameCount, + 'expectedTime', + expectedTime, + 'deltaTime', + deltaTime + ); + + if (this._rendererCaches.length === 0) { + // If there is no renderer, stop rendering + this.stopRendering(); + return; + } + + // Render all renderers that do not use WebCodecs + for (const rendererCache of this._rendererCaches.filter( + (cache) => cache instanceof RendererCache + )) { + this.doRendering(rendererCache); + } + + if (this._currentFrameCount >= this.renderingFps) { + this._previousFirstFrameTime = 0; + } + + if (deltaTime < expectedTime) { + // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference + this._renderingTimer = window.setTimeout( + renderingLooper, + expectedTime - deltaTime + ); + } else { + // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately + renderingLooper(); + } + }; + renderingLooper(); } - /** - * @ignore - */ - private disableVideoFrameCache( - videoFrameCacheConfig: VideoFrameCacheConfig - ): void { - logDebug(`disableVideoFrameCache ${JSON.stringify(videoFrameCacheConfig)}`); - this.msgBridge.DisableVideoFrameCache(videoFrameCacheConfig); + public doRendering(rendererCache: RendererCacheType): void { + rendererCache.draw(); } - /** - * @ignore - */ - private ensureRendererConfig(config: VideoFrameCacheConfig): - | Map< - number, - { - shareVideoFrame: ShareVideoFrame; - renders: IRenderer[]; - } - > - | undefined { - const { videoSourceType, uid, channelId } = config; - const emptyRenderConfig = { - renders: [], - shareVideoFrame: this.resizeShareVideoFrame( - videoSourceType, - channelId, - uid - ), + private handleWebGLFallback(context: RendererContext): WebGLFallback { + return (renderer: WebGLRenderer) => { + const renderers = this.getRenderers(context); + renderer.unbind(); + const newRenderer = this.createRenderer(context, RendererType.SOFTWARE); + renderers.splice(renderers.indexOf(renderer), 1, newRenderer); }; - const emptyUidMap = new Map([[uid, emptyRenderConfig]]); - const emptyChannelMap = new Map([[channelId, emptyUidMap]]); - - const renderers = this.renderers; - const videoSourceMap = renderers.get(videoSourceType); - if (!videoSourceMap) { - renderers.set(videoSourceType, emptyChannelMap); - return emptyUidMap; - } - const channelMap = videoSourceMap.get(channelId); - if (!channelMap) { - videoSourceMap.set(channelId, emptyUidMap); - return emptyUidMap; + } + + public handleWebCodecsFallback(context: RendererCacheContext): void { + let engine = createAgoraRtcEngine(); + engine.getMediaEngine().unregisterVideoEncodedFrameObserver({}); + if (context.uid) { + engine.setRemoteVideoSubscriptionOptions(context.uid, { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: false, + }); } - const renderConfig = channelMap?.get(uid); - if (!renderConfig) { - channelMap?.set(uid, emptyRenderConfig); - logWarn( - `ensureRendererMap uid map for channelId:${channelId} uid:${uid}` - ); - return emptyUidMap; + AgoraEnv.enableWebCodecsDecoder = false; + AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false); + let renderers = this.getRenderers(context); + for (let renderer of renderers) { + this.addOrRemoveRenderer({ + ...renderer.context, + setupMode: VideoViewSetupMode.VideoViewSetupReplace, + }); } - return channelMap; } - /** - * @ignore - */ - private resizeShareVideoFrame( - videoSourceType: VideoSourceType, - channelId: string, - uid: number, - width = 0, - height = 0, - yStride = 0 - ): ShareVideoFrame { - return { - videoSourceType, - channelId, - uid, - yBuffer: Buffer.alloc(yStride * height), - uBuffer: Buffer.alloc((yStride * height) / 4), - vBuffer: Buffer.alloc((yStride * height) / 4), - width, - height, - yStride, - }; + public stopRendering(): void { + if (this._renderingTimer) { + window.clearTimeout(this._renderingTimer); + this._renderingTimer = undefined; + } } - /** - * @ignore - */ - public updateVideoFrameCacheInMap( - config: VideoFrameCacheConfig, - shareVideoFrame: ShareVideoFrame - ): void { - let rendererConfigMap = this.ensureRendererConfig(config); - rendererConfigMap - ? Object.assign(rendererConfigMap.get(config.uid) ?? {}, { - shareVideoFrame, - }) - : logWarn( - `updateVideoFrameCacheInMap videoSourceType:${config.videoSourceType} channelId:${config.channelId} uid:${config.uid} rendererConfigMap is null` - ); + public setRendererContext(context: RendererContext): boolean { + const checkedContext = this.presetRendererContext(context); + + for (const rendererCache of this._rendererCaches) { + const result = rendererCache.setRendererContext(checkedContext); + if (result) { + return true; + } + } + return false; } } diff --git a/ts/Renderer/WebCodecsRenderer/index.ts b/ts/Renderer/WebCodecsRenderer/index.ts new file mode 100755 index 000000000..06aaacc4c --- /dev/null +++ b/ts/Renderer/WebCodecsRenderer/index.ts @@ -0,0 +1,141 @@ +import { RendererType } from '../../Types'; +import { getContextByCanvas } from '../../Utils'; +import { IRenderer } from '../IRenderer'; + +export type frameSize = { + width: number; + height: number; +}; + +export class WebCodecsRenderer extends IRenderer { + gl?: WebGLRenderingContext | WebGL2RenderingContext | null; + // eslint-disable-next-line auto-import/auto-import + offscreenCanvas: OffscreenCanvas | undefined; + + constructor() { + super(); + this.rendererType = RendererType.WEBCODECSRENDERER; + } + + static vertexShaderSource = ` + attribute vec2 xy; + varying highp vec2 uv; + void main(void) { + gl_Position = vec4(xy, 0.0, 1.0); + // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1). + // UV coordinates are Y-flipped relative to vertex coordinates. + uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0); + } + `; + static fragmentShaderSource = ` + varying highp vec2 uv; + uniform sampler2D texture; + void main(void) { + gl_FragColor = texture2D(texture, uv); + } + `; + + bind(element: HTMLElement, frameSize: frameSize) { + super.bind(element); + if (!this.canvas) return; + this.canvas.width = frameSize.width; + this.canvas.height = frameSize.height; + this.offscreenCanvas = this.canvas.transferControlToOffscreen(); + this.gl = getContextByCanvas(this.offscreenCanvas); + if (!this.gl) return; + const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER); + if (!vertexShader) return; + this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource); + this.gl.compileShader(vertexShader); + if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) { + throw this.gl.getShaderInfoLog(vertexShader); + } + const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER); + if (!fragmentShader) return; + this.gl.shaderSource( + fragmentShader, + WebCodecsRenderer.fragmentShaderSource + ); + this.gl.compileShader(fragmentShader); + if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) { + throw this.gl.getShaderInfoLog(fragmentShader); + } + const shaderProgram = this.gl.createProgram(); + if (!shaderProgram) return; + this.gl.attachShader(shaderProgram, vertexShader); + this.gl.attachShader(shaderProgram, fragmentShader); + this.gl.linkProgram(shaderProgram); + if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) { + throw this.gl.getProgramInfoLog(shaderProgram); + } + this.gl.useProgram(shaderProgram); + // Vertex coordinates, clockwise from bottom-left. + const vertexBuffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer); + this.gl.bufferData( + this.gl.ARRAY_BUFFER, + new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), + this.gl.STATIC_DRAW + ); + const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy'); + this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0); + this.gl.enableVertexAttribArray(xyLocation); + // Create one texture to upload frames to. + const texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MAG_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MIN_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_S, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_T, + this.gl.CLAMP_TO_EDGE + ); + } + + drawFrame(frame: any) { + if (!this.offscreenCanvas || !frame) return; + this.offscreenCanvas.width = frame.displayWidth; + this.offscreenCanvas.height = frame.displayHeight; + this.updateRenderMode(); + if (!this.gl) return; + + if (this.gl) { + // Upload the frame. + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.RGBA, + this.gl.RGBA, + this.gl.UNSIGNED_BYTE, + frame + ); + frame.close(); + // Configure and clear the drawing area. + this.gl.viewport( + 0, + 0, + this.gl.drawingBufferWidth, + this.gl.drawingBufferHeight + ); + this.gl.clearColor(1.0, 0.0, 0.0, 1.0); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + // Draw the frame. + this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4); + } + super.drawFrame(); + this.getFps(); + } +} diff --git a/ts/Renderer/WebCodecsRendererCache.ts b/ts/Renderer/WebCodecsRendererCache.ts new file mode 100644 index 000000000..b50c6e434 --- /dev/null +++ b/ts/Renderer/WebCodecsRendererCache.ts @@ -0,0 +1,124 @@ +import createAgoraRtcEngine from '../AgoraSdk'; +import { WebCodecsDecoder } from '../Decoder/index'; +import { EncodedVideoFrameInfo, VideoStreamType } from '../Private/AgoraBase'; +import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx'; +import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine'; + +import { RendererContext, RendererType } from '../Types'; +import { AgoraEnv, logInfo } from '../Utils'; + +import { IRendererCache } from './IRendererCache'; +import { WebCodecsRenderer } from './WebCodecsRenderer/index'; + +export class WebCodecsRendererCache extends IRendererCache { + private _decoder?: WebCodecsDecoder | null; + private _engine?: IRtcEngineEx; + private _firstFrame = true; + + constructor(context: RendererContext) { + super(context); + this._engine = createAgoraRtcEngine(); + this._decoder = new WebCodecsDecoder( + this.renderers as WebCodecsRenderer[], + this.onDecoderError.bind(this), + context + ); + this.draw(); + } + + onDecoderError(e: any) { + logInfo('webCodecsDecoder decode failed, fallback to native decoder', e); + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext); + } + + onEncodedVideoFrameReceived(...[data, buffer]: any) { + let _data: any; + try { + _data = JSON.parse(data) ?? {}; + } catch (e) { + _data = {}; + } + if ( + Object.keys(_data).length === 0 || + !this._decoder || + this.cacheContext.uid !== _data.uid + ) + return; + if (this._firstFrame) { + for (let renderer of this.renderers) { + if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) { + continue; + } + renderer.bind(renderer.context.view, { + width: _data.videoEncodedFrameInfo.width!, + height: _data.videoEncodedFrameInfo.height!, + }); + } + + try { + this._decoder.decoderConfigure(_data.videoEncodedFrameInfo); + } catch (error: any) { + logInfo(error); + return; + } + this._firstFrame = false; + } + if (this.shouldFallback(_data.videoEncodedFrameInfo)) { + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext); + } else { + this._decoder.decodeFrame( + buffer, + _data.videoEncodedFrameInfo, + new Date().getTime() + ); + } + } + + public draw() { + this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid!, { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: true, + }); + AgoraElectronBridge.OnEvent( + 'call_back_with_encoded_video_frame', + (...params: any) => { + try { + this.onEncodedVideoFrameReceived(...params); + } catch (e) { + console.error(e); + } + } + ); + } + + public shouldFallback(frameInfo: EncodedVideoFrameInfo): boolean { + let shouldFallback = false; + if (!frameInfo.codecType) { + shouldFallback = true; + logInfo('codecType is not supported, fallback to native decoder'); + } else { + const mapping = + AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType]; + if (mapping === undefined) { + shouldFallback = true; + logInfo('codecType is not supported, fallback to native decoder'); + } else if ( + mapping.minWidth >= frameInfo.width! && + mapping.minHeight >= frameInfo.height! && + mapping.maxWidth <= frameInfo.width! && + mapping.maxHeight <= frameInfo.height! + ) { + shouldFallback = true; + logInfo('frame size is not supported, fallback to native decoder'); + } + } + return shouldFallback; + } + + public release(): void { + AgoraElectronBridge.UnEvent('call_back_with_encoded_video_frame'); + this._decoder?.release(); + this._decoder = null; + super.release(); + } +} diff --git a/ts/Renderer/WebGLRenderer/index.ts b/ts/Renderer/WebGLRenderer/index.ts index 3e0552d0d..0c0f32c3f 100644 --- a/ts/Renderer/WebGLRenderer/index.ts +++ b/ts/Renderer/WebGLRenderer/index.ts @@ -1,10 +1,9 @@ -import { EventEmitter } from 'events'; +import { VideoFrame } from '../../Private/AgoraMediaBase'; +import { RendererType } from '../../Types'; +import { logWarn } from '../../Utils'; +import { IRenderer } from '../IRenderer'; -import { RenderModeType } from '../../Private/AgoraMediaBase'; - -import { ShareVideoFrame } from '../../Types'; -import { logError, logWarn } from '../../Utils'; -import { IRenderer, RenderFailCallback } from '../IRenderer'; +export type WebGLFallback = (renderer: WebGLRenderer, error: Error) => void; const createProgramFromSources = require('./webgl-utils').createProgramFromSources; @@ -44,181 +43,246 @@ const yuvShaderSource = ' gl_FragColor=vec4(r,g,b,1.0);' + '}'; -export class GlRenderer extends IRenderer { - gl: WebGL2RenderingContext | undefined | null; - program: any; - positionLocation: any; - texCoordLocation: any; - yTexture: WebGLTexture | undefined | null; - uTexture: WebGLTexture | undefined | null; - vTexture: WebGLTexture | undefined | null; - texCoordBuffer: any; - surfaceBuffer: any; - - // @ts-ignore - parentElement: HTMLElement | undefined; - container: HTMLElement | undefined; - // @ts-ignore - canvas: HTMLCanvasElement | undefined; - renderImageCount = 0; - initWidth = 0; - initHeight = 0; - initRotation = 0; - clientWidth = 0; - clientHeight = 0; - contentMode = 0; - event = new EventEmitter(); - firstFrameRender = false; - lastImageWidth = 0; - lastImageHeight = 0; - lastImageRotation = 0; - videoBuffer = {}; - - observer?: ResizeObserver; - - failInitRenderCB: RenderFailCallback; - - constructor(failCallback: RenderFailCallback) { +export class WebGLRenderer extends IRenderer { + gl?: WebGLRenderingContext | WebGL2RenderingContext | null; + program?: WebGLProgram; + positionLocation?: number; + texCoordLocation?: number; + yTexture: WebGLTexture | null; + uTexture: WebGLTexture | null; + vTexture: WebGLTexture | null; + texCoordBuffer: WebGLBuffer | null; + surfaceBuffer: WebGLBuffer | null; + fallback?: WebGLFallback; + + constructor(fallback?: WebGLFallback) { super(); - this.failInitRenderCB = failCallback; + this.gl = undefined; + this.rendererType = RendererType.WEBGL; + this.yTexture = null; + this.uTexture = null; + this.vTexture = null; + this.texCoordBuffer = null; + this.surfaceBuffer = null; + this.fallback = fallback; } - public bind(view: HTMLElement) { + public override bind(view: HTMLElement) { super.bind(view); - } - public unbind() { - this.observer?.unobserve && this.observer.disconnect(); - this.program = undefined; - this.positionLocation = undefined; - this.texCoordLocation = undefined; + this.canvas?.addEventListener( + 'webglcontextlost', + this.handleContextLost, + false + ); + this.canvas?.addEventListener( + 'webglcontextrestored', + this.handleContextRestored, + false + ); - this.deleteTexture(this.yTexture); - this.deleteTexture(this.uTexture); - this.deleteTexture(this.vTexture); - this.yTexture = undefined; - this.uTexture = undefined; - this.vTexture = undefined; + const getContext = ( + contextNames = ['webgl2', 'webgl', 'experimental-webgl'] + ): WebGLRenderingContext | WebGLRenderingContext | null => { + for (let i = 0; i < contextNames.length; i++) { + const contextName = contextNames[i]!; + const context = this.canvas?.getContext(contextName, { + depth: true, + stencil: true, + alpha: false, + antialias: false, + premultipliedAlpha: true, + preserveDrawingBuffer: true, + powerPreference: 'default', + failIfMajorPerformanceCaveat: false, + }); + if (context) { + return context as WebGLRenderingContext | WebGLRenderingContext; + } + } + return null; + }; + this.gl ??= getContext(); - this.deleteBuffer(this.texCoordBuffer); - this.deleteBuffer(this.surfaceBuffer); - this.texCoordBuffer = undefined; - this.surfaceBuffer = undefined; + if (!this.gl) { + this.fallback?.call( + null, + this, + new Error('Browser not support! No WebGL detected.') + ); + return; + } - this.gl = undefined; + // Set clear color to black, fully opaque + this.gl.clearColor(0.0, 0.0, 0.0, 1.0); + // Enable depth testing + this.gl.enable(this.gl.DEPTH_TEST); + // Near things obscure far things + this.gl.depthFunc(this.gl.LEQUAL); + // Clear the color as well as the depth buffer. + this.gl.clear( + this.gl.COLOR_BUFFER_BIT | + this.gl.DEPTH_BUFFER_BIT | + this.gl.STENCIL_BUFFER_BIT + ); - try { - if ( - this.container && - this.canvas && - this.canvas.parentNode === this.container - ) { - this.container.removeChild(this.canvas); - } - if ( - this.parentElement && - this.container && - this.container.parentNode === this.parentElement - ) { - this.parentElement.removeChild(this.container); - } - } catch (e) { - logWarn('webgl renderer unbind happen some error', e); - } + // Setup GLSL program + this.program = createProgramFromSources(this.gl, [ + vertexShaderSource, + yuvShaderSource, + ]) as WebGLProgram; + this.gl.useProgram(this.program); - this.canvas && - this.canvas.removeEventListener( - 'webglcontextlost', - this.handleContextLost, - false - ); - this.canvas = undefined; - this.container = undefined; - this.parentElement = undefined; + this.initTextures(); } - private updateViewZoomLevel(rotation: number, width: number, height: number) { - if (!this.parentElement || !this.canvas) { - return; - } - this.clientWidth = this.parentElement.clientWidth; - this.clientHeight = this.parentElement.clientHeight; - - try { - if (this.contentMode === RenderModeType.RenderModeHidden) { - // Cover - if (rotation === 0 || rotation === 180) { - if (this.clientWidth / this.clientHeight > width / height) { - this.canvas.style.transform = `scale(${this.clientWidth / width})`; - } else { - this.canvas.style.transform = `scale(${ - this.clientHeight / height - })`; - } - } else { - // 90, 270 - if (this.clientHeight / this.clientWidth > width / height) { - this.canvas.style.transform = `scale(${this.clientHeight / width})`; - } else { - this.canvas.style.transform = `scale(${this.clientWidth / height})`; - } - } - // Contain - } else if (rotation === 0 || rotation === 180) { - if (this.clientWidth / this.clientHeight > width / height) { - this.canvas.style.transform = `scale(${this.clientHeight / height})`; - } else { - this.canvas.style.transform = `scale(${this.clientWidth / width})`; - } - } else { - // 90, 270 - if (this.clientHeight / this.clientWidth > width / height) { - this.canvas.style.transform = `scale(${this.clientWidth / height})`; - } else { - this.canvas.style.transform = `scale(${this.clientHeight / width})`; - } - } - } catch (e) { - logError('webgl updateViewZoomLevel', e); - return false; - } + public override unbind() { + this.canvas?.removeEventListener( + 'webglcontextlost', + this.handleContextLost, + false + ); + this.canvas?.removeEventListener( + 'webglcontextrestored', + this.handleContextRestored, + false + ); + + this.releaseTextures(); + this.gl = undefined; - return true; + super.unbind(); } - private updateCanvas(rotation: number, width: number, height: number) { - // if (this.canvasUpdated) { - // return; - // } - if (width || height) { - this.lastImageWidth = width; - this.lastImageHeight = height; - this.lastImageRotation = rotation; - } else { - width = this.lastImageWidth; - height = this.lastImageHeight; - rotation = this.lastImageRotation; - } - if (!this.updateViewZoomLevel(rotation, width, height)) { - return; - } - let gl = this.gl; - if (!gl) { - return; - } - gl.bindBuffer(gl.ARRAY_BUFFER, this.surfaceBuffer); - gl.enableVertexAttribArray(this.positionLocation); - gl.vertexAttribPointer(this.positionLocation, 2, gl.FLOAT, false, 0, 0); + public override drawFrame({ + width, + height, + yStride, + uStride, + vStride, + yBuffer, + uBuffer, + vBuffer, + rotation, + }: VideoFrame) { + this.rotateCanvas({ width, height, rotation }); + this.updateRenderMode(); + + if (!this.gl || !this.program) return; + + const left = 0, + top = 0, + right = yStride! - width!, + bottom = 0; + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texCoordBuffer); + const xWidth = width! + left + right; + const xHeight = height! + top + bottom; + this.gl.bufferData( + this.gl.ARRAY_BUFFER, + new Float32Array([ + left / xWidth, + bottom / xHeight, + 1 - right / xWidth, + bottom / xHeight, + left / xWidth, + 1 - top / xHeight, + left / xWidth, + 1 - top / xHeight, + 1 - right / xWidth, + bottom / xHeight, + 1 - right / xWidth, + 1 - top / xHeight, + ]), + this.gl.STATIC_DRAW + ); + this.gl.enableVertexAttribArray(this.texCoordLocation!); + this.gl.vertexAttribPointer( + this.texCoordLocation!, + 2, + this.gl.FLOAT, + false, + 0, + 0 + ); + + this.gl.pixelStorei(this.gl.UNPACK_ALIGNMENT, 1); + + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.yTexture); + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.LUMINANCE, + // Should use xWidth instead of width here (yStide) + xWidth, + height!, + 0, + this.gl.LUMINANCE, + this.gl.UNSIGNED_BYTE, + yBuffer! + ); + + this.gl.activeTexture(this.gl.TEXTURE1); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.uTexture); + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.LUMINANCE, + uStride!, + height! / 2, + 0, + this.gl.LUMINANCE, + this.gl.UNSIGNED_BYTE, + uBuffer! + ); + + this.gl.activeTexture(this.gl.TEXTURE2); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.vTexture); + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.LUMINANCE, + vStride!, + height! / 2, + 0, + this.gl.LUMINANCE, + this.gl.UNSIGNED_BYTE, + vBuffer! + ); + + this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); + super.drawFrame(); + this.getFps(); + } + + protected override rotateCanvas({ width, height, rotation }: VideoFrame) { + super.rotateCanvas({ width, height, rotation }); + + if (!this.gl) return; + + this.gl.viewport(0, 0, width!, height!); + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.surfaceBuffer); + this.gl.enableVertexAttribArray(this.positionLocation!); + this.gl.vertexAttribPointer( + this.positionLocation!, + 2, + this.gl.FLOAT, + false, + 0, + 0 + ); // 4 vertex, 1(x1,y1), 2(x2,y1), 3(x2,y2), 4(x1,y2) - // 0: 1,2,4/4,2,3 + // 0: 1,2,4/4,2,3 // 90: 2,3,1/1,3,4 // 180: 3,4,2/2,4,1 // 270: 4,1,3/3,1,2 const p1 = { x: 0, y: 0 }; - const p2 = { x: width, y: 0 }; - const p3 = { x: width, y: height }; - const p4 = { x: 0, y: height }; + const p2 = { x: width!, y: 0 }; + const p3 = { x: width!, y: height! }; + const p4 = { x: 0, y: height! }; let pp1 = p1, pp2 = p2, pp3 = p3, @@ -247,8 +311,8 @@ export class GlRenderer extends IRenderer { break; default: } - gl.bufferData( - gl.ARRAY_BUFFER, + this.gl.bufferData( + this.gl.ARRAY_BUFFER, new Float32Array([ pp1.x, pp1.y, @@ -263,341 +327,120 @@ export class GlRenderer extends IRenderer { pp3.x, pp3.y, ]), - gl.STATIC_DRAW + this.gl.STATIC_DRAW ); - const resolutionLocation = gl.getUniformLocation( - this.program, + const resolutionLocation = this.gl.getUniformLocation( + this.program!, 'u_resolution' ); - gl.uniform2f(resolutionLocation, width, height); - } - - public drawFrame(videoFrame: ShareVideoFrame) { - let error; - try { - this.renderImage({ - width: videoFrame.width, - height: videoFrame.height, - left: 0, - top: 0, - right: videoFrame.yStride - videoFrame.width, - bottom: 0, - rotation: videoFrame.rotation || 0, - yplane: videoFrame.yBuffer, - uplane: videoFrame.uBuffer, - vplane: videoFrame.vBuffer, - }); - } catch (err) { - error = err; - } - if (!this.gl || error) { - this.failInitRenderCB && - this.failInitRenderCB({ - error: 'webgl lost or webgl initialize failed', - }); - this.failInitRenderCB = null; - return; - } - } - - public refreshCanvas() { - if (this.lastImageWidth) { - this.updateViewZoomLevel( - this.lastImageRotation, - this.lastImageWidth, - this.lastImageHeight - ); - } + this.gl.uniform2f(resolutionLocation, width!, height!); } - private renderImage(image: { - width: number; - height: number; - left: number; - top: number; - right: number; - bottom: number; - rotation: number; - yplane: Uint8Array; - uplane: Uint8Array; - vplane: Uint8Array; - }) { - // Rotation, width, height, left, top, right, bottom, yplane, uplane, vplane - - if ( - image.width != this.initWidth || - image.height != this.initHeight || - image.rotation != this.initRotation - ) { - const view = this.parentElement!; - this.unbind(); - - this.initCanvas(view, image.width, image.height, image.rotation); - const ResizeObserver = window.ResizeObserver; - if (ResizeObserver) { - this.observer = new ResizeObserver(() => { - this.refreshCanvas && this.refreshCanvas(); - }); - this.observer.observe(view); - } - } - let gl = this.gl; - if (!gl) { - return; - } + private initTextures() { + if (!this.gl) return; - gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer); - const xWidth = image.width + image.left + image.right; - const xHeight = image.height + image.top + image.bottom; - gl.bufferData( - gl.ARRAY_BUFFER, - new Float32Array([ - image.left / xWidth, - image.bottom / xHeight, - 1 - image.right / xWidth, - image.bottom / xHeight, - image.left / xWidth, - 1 - image.top / xHeight, - image.left / xWidth, - 1 - image.top / xHeight, - 1 - image.right / xWidth, - image.bottom / xHeight, - 1 - image.right / xWidth, - 1 - image.top / xHeight, - ]), - gl.STATIC_DRAW + this.positionLocation = this.gl.getAttribLocation( + this.program!, + 'a_position' + ); + this.texCoordLocation = this.gl.getAttribLocation( + this.program!, + 'a_texCoord' ); - gl.enableVertexAttribArray(this.texCoordLocation); - gl.vertexAttribPointer(this.texCoordLocation, 2, gl.FLOAT, false, 0, 0); - this.uploadYuv(xWidth, xHeight, image.yplane, image.uplane, image.vplane); + this.surfaceBuffer = this.gl.createBuffer(); + this.texCoordBuffer = this.gl.createBuffer(); + + const createTexture = (textureIndex: number) => { + if (!this.gl) return null; + + // Create a texture. + this.gl.activeTexture(textureIndex); + const texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + // Set the parameters so we can render any size + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_S, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_T, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MIN_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MAG_FILTER, + this.gl.NEAREST + ); + return texture; + }; - this.updateCanvas(image.rotation, image.width, image.height); - gl.drawArrays(gl.TRIANGLES, 0, 6); - this.renderImageCount += 1; + this.yTexture = createTexture(this.gl.TEXTURE0); + this.uTexture = createTexture(this.gl.TEXTURE1); + this.vTexture = createTexture(this.gl.TEXTURE2); - if (!this.firstFrameRender) { - this.firstFrameRender = true; - this.event.emit('ready'); - } - } + const y = this.gl.getUniformLocation(this.program!, 'Ytex'); + this.gl.uniform1i(y, 0); /* Bind Ytex to texture unit 0 */ - private uploadYuv( - width: number, - height: number, - yplane: Uint8Array, - uplane: Uint8Array, - vplane: Uint8Array - ) { - let gl = this.gl; - if (!gl || !this.yTexture || !this.uTexture || !this.vTexture) { - return; - } + const u = this.gl.getUniformLocation(this.program!, 'Utex'); + this.gl.uniform1i(u, 1); /* Bind Utex to texture unit 1 */ - gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, this.yTexture); + const v = this.gl.getUniformLocation(this.program!, 'Vtex'); + this.gl.uniform1i(v, 2); /* Bind Vtex to texture unit 2 */ + } - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.LUMINANCE, - width, - height, - 0, - gl.LUMINANCE, - gl.UNSIGNED_BYTE, - yplane - ); + private releaseTextures() { + this.gl?.deleteProgram(this.program!); + this.program = undefined; - gl.activeTexture(gl.TEXTURE1); - gl.bindTexture(gl.TEXTURE_2D, this.uTexture); - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.LUMINANCE, - width / 2, - height / 2, - 0, - gl.LUMINANCE, - gl.UNSIGNED_BYTE, - uplane - ); + this.positionLocation = undefined; + this.texCoordLocation = undefined; - gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, this.vTexture); - (''); - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.LUMINANCE, - width / 2, - height / 2, - 0, - gl.LUMINANCE, - gl.UNSIGNED_BYTE, - vplane - ); + this.gl?.deleteTexture(this.yTexture); + this.gl?.deleteTexture(this.uTexture); + this.gl?.deleteTexture(this.vTexture); + this.yTexture = null; + this.uTexture = null; + this.vTexture = null; + + this.gl?.deleteBuffer(this.texCoordBuffer); + this.gl?.deleteBuffer(this.surfaceBuffer); + this.texCoordBuffer = null; + this.surfaceBuffer = null; } - private deleteBuffer(buffer: any) { - if (buffer && this.gl) { - this.gl.deleteBuffer(buffer); - } - } + private handleContextLost = (event: Event) => { + event.preventDefault(); + logWarn('webglcontextlost', event); - private deleteTexture(texture: any) { - if (texture && this.gl) { - this.gl.deleteTexture(texture); - } - } + this.releaseTextures(); - private handleContextLost = (event: Event) => { - console.warn('webglcontextlost', event); - try { - this.canvas && - this.canvas.removeEventListener( - 'webglcontextlost', - this.handleContextLost, - false - ); - } catch (error) { - logWarn('webglcontextlost error', error); - } finally { - this.gl = undefined; - this.failInitRenderCB && - this.failInitRenderCB({ - error: 'Browser not support! No WebGL detected.', - }); - } + this.fallback?.call( + null, + this, + new Error('Browser not support! No WebGL detected.') + ); }; - private initCanvas( - view: HTMLElement, - width: number, - height: number, - rotation: number - ) { - this.clientWidth = view.clientWidth; - this.clientHeight = view.clientHeight; - - this.parentElement = view; - - this.container = document.createElement('div'); - this.container.style.width = '100%'; - this.container.style.height = '100%'; - this.container.style.display = 'flex'; - this.container.style.justifyContent = 'center'; - this.container.style.alignItems = 'center'; - this.container.style.overflow = 'hidden'; - this.parentElement.appendChild(this.container); - - this.canvas = document.createElement('canvas'); - if (rotation == 0 || rotation == 180) { - this.canvas.width = width; - this.canvas.height = height; - } else { - this.canvas.width = height; - this.canvas.height = width; - } - this.initWidth = width; - this.initHeight = height; - this.initRotation = rotation; - - this.container.appendChild(this.canvas); - try { - // Try to grab the standard context. If it fails, fallback to experimental. - this.gl = this.canvas.getContext('webgl2', { - preserveDrawingBuffer: true, - }); - // context list after toggle resolution on electron 12.0.6 - this.canvas.addEventListener( - 'webglcontextlost', - this.handleContextLost, - false - ); - } catch (e) { - logWarn('webgl create happen some warming', this.gl, this.canvas); - } - if (!this.gl) { - this.failInitRenderCB && - this.failInitRenderCB({ - error: 'Browser not support! No WebGL detected.', - }); - return; - } - const gl = this.gl as WebGL2RenderingContext; - - // Set clear color to black, fully opaque - gl.clearColor(0.0, 0.0, 0.0, 1.0); - // Enable depth testing - gl.enable(gl.DEPTH_TEST); - // Near things obscure far things - gl.depthFunc(gl.LEQUAL); - // Clear the color as well as the depth buffer. - gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + private handleContextRestored = (event: Event) => { + event.preventDefault(); + logWarn('webglcontextrestored', event); // Setup GLSL program - this.program = createProgramFromSources(gl, [ + this.program = createProgramFromSources(this.gl, [ vertexShaderSource, yuvShaderSource, - ]); - this.gl.useProgram(this.program); + ]) as WebGLProgram; + this.gl?.useProgram(this.program); this.initTextures(); - } - - private initTextures() { - let gl = this.gl; - if (!gl) { - return; - } - let program = this.program; - - this.positionLocation = gl.getAttribLocation(program, 'a_position'); - this.texCoordLocation = gl.getAttribLocation(program, 'a_texCoord'); - - this.surfaceBuffer = gl.createBuffer(); - this.texCoordBuffer = gl.createBuffer(); - - // Create a texture. - gl.activeTexture(gl.TEXTURE0); - this.yTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this.yTexture); - // Set the parameters so we can render any size image. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - - gl.activeTexture(gl.TEXTURE1); - this.uTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this.uTexture); - // Set the parameters so we can render any size image. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - - gl.activeTexture(gl.TEXTURE2); - this.vTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this.vTexture); - // Set the parameters so we can render any size image. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - - const y = gl.getUniformLocation(program, 'Ytex'); - gl.uniform1i(y, 0); /* Bind Ytex to texture unit 0 */ - - const u = gl.getUniformLocation(program, 'Utex'); - gl.uniform1i(u, 1); /* Bind Utex to texture unit 1 */ - - const v = gl.getUniformLocation(program, 'Vtex'); - gl.uniform1i(v, 2); /* Bind Vtex to texture unit 2 */ - } + }; } - -export default GlRenderer; diff --git a/ts/Renderer/YUVCanvasRenderer/index.ts b/ts/Renderer/YUVCanvasRenderer/index.ts index 8c4974559..083fef087 100644 --- a/ts/Renderer/YUVCanvasRenderer/index.ts +++ b/ts/Renderer/YUVCanvasRenderer/index.ts @@ -1,243 +1,65 @@ -import isEqual from 'lodash.isequal'; - -import { RenderModeType } from '../../Private/AgoraMediaBase'; -import { CanvasOptions, ShareVideoFrame } from '../../Types'; +import { VideoFrame } from '../../Private/AgoraMediaBase'; import { IRenderer } from '../IRenderer'; const YUVBuffer = require('yuv-buffer'); const YUVCanvas = require('yuv-canvas'); export class YUVCanvasRenderer extends IRenderer { - private _cacheCanvasOptions?: CanvasOptions; - private _yuvCanvasSink?: any; - private _container?: HTMLElement; - private _videoFrame: ShareVideoFrame; - - constructor() { - super(); - this._videoFrame = { - rotation: 0, - width: 0, - height: 0, - yStride: 0, - yBuffer: new Uint8Array(0), - uBuffer: new Uint8Array(0), - vBuffer: new Uint8Array(0), - videoSourceType: -1, - }; - } + private frameSink?: any; - public bind(element: HTMLElement) { + public override bind(element: HTMLElement) { super.bind(element); - let container = document.createElement('div'); - Object.assign(container.style, { - width: '100%', - height: '100%', - display: 'flex', - justifyContent: 'center', - alignItems: 'center', - overflow: 'hidden', - }); - this._container = container; - this.parentElement!.appendChild(this._container); - this.canvas = document.createElement('canvas'); - this._container.appendChild(this.canvas); - - this._yuvCanvasSink = YUVCanvas.attach(this.canvas, { + this.frameSink = YUVCanvas.attach(this.canvas, { webGL: false, }); } - public unbind() { - if (this._container) { - this._container.replaceChildren(); - this._container = undefined; - } - if (this.parentElement) { - this.parentElement.replaceChildren(); - this.parentElement = undefined; - } - if (this.canvas) { - this.canvas = undefined; - } - if (this._yuvCanvasSink && this._yuvCanvasSink?.loseContext) { - this._yuvCanvasSink?.loseContext(); - } - } - - private zoom( - vertical: boolean, - contentMode: RenderModeType = RenderModeType.RenderModeFit, - width: number, - height: number, - clientWidth: number, - clientHeight: number - ): number { - let localRatio = clientWidth / clientHeight; - let tempRatio = width / height; - if (isNaN(localRatio) || isNaN(tempRatio)) { - return 1; - } - - if (contentMode === RenderModeType.RenderModeHidden) { - if (vertical) { - return clientHeight / clientWidth < width / height - ? clientWidth / height - : clientHeight / width; - } else { - return clientWidth / clientHeight > width / height - ? clientWidth / width - : clientHeight / height; - } - } else { - if (vertical) { - return clientHeight / clientWidth < width / height - ? clientHeight / width - : clientWidth / height; - } else { - return clientWidth / clientHeight > width / height - ? clientHeight / height - : clientWidth / width; - } - } - } - - private updateCanvas( - options: CanvasOptions = { - frameWidth: 0, - frameHeight: 0, - rotation: 0, - contentMode: 0, - clientWidth: 0, - clientHeight: 0, - } - ) { - if (this._cacheCanvasOptions) { - if (isEqual(this._cacheCanvasOptions, options)) { - return; - } - } - - this._cacheCanvasOptions = Object.assign({}, options); - - if (this.canvas) { - if (options.rotation === 0 || options.rotation === 180) { - this.canvas.width = options.frameWidth; - this.canvas.height = options.frameHeight; - Object.assign(this.canvas.style, { - 'width': options.frameWidth + 'px', - 'height': options.frameHeight + 'px', - 'object-fit': 'cover', - }); - } else if (options.rotation === 90 || options.rotation === 270) { - this.canvas.height = options.frameWidth; - this.canvas.width = options.frameHeight; - } else { - throw new Error( - 'Invalid value for rotation. Only support 0, 90, 180, 270' - ); - } - - let transformItems = []; - transformItems.push(`rotateZ(${options.rotation}deg)`); - - let scale = this.zoom( - options.rotation === 90 || options.rotation === 270, - options.contentMode, - options.frameWidth, - options.frameHeight, - options.clientWidth, - options.clientHeight - ); - - this.canvas.style.transform = `scale(${scale.toString()})`; - - if (transformItems.length > 0) { - this.canvas.style.transform += ` ${transformItems.join(' ')}`; - } - } - } - - public drawFrame(frame: ShareVideoFrame) { - if (!this._container || !this._yuvCanvasSink) { - return; - } - - let frameWidth = frame.width; - let frameHeight = frame.height; - - if ( - this._videoFrame.yStride === 0 || - this._videoFrame.height === 0 || - this._videoFrame.yStride != frame.yStride || - this._videoFrame.height != frame.height - ) { - this._videoFrame.yBuffer = new Uint8Array(frame.yStride * frameHeight); - this._videoFrame.uBuffer = new Uint8Array( - (frame.yStride * frameHeight) / 4 - ); - this._videoFrame.vBuffer = new Uint8Array( - (frame.yStride * frameHeight) / 4 - ); - } - - this._videoFrame.yBuffer.set(frame.yBuffer); - this._videoFrame.uBuffer.set(frame.uBuffer); - this._videoFrame.vBuffer.set(frame.vBuffer); - - this._videoFrame.width = frame.width; - this._videoFrame.height = frame.height; - this._videoFrame.yStride = frame.yStride; - this._videoFrame.rotation = frame.rotation; - - let options: CanvasOptions = { - frameWidth, - frameHeight, - rotation: frame.rotation ? frame.rotation : 0, - contentMode: this.contentMode, - clientWidth: this._container.clientWidth, - clientHeight: this._container.clientHeight, - }; - - this.updateCanvas(options); - - let format = YUVBuffer.format({ - width: frameWidth, - height: frameHeight, - chromaWidth: frameWidth / 2, - chromaHeight: frameHeight / 2, - cropLeft: frame.yStride - frameWidth, - }); - - let yuvBufferFrame = YUVBuffer.frame( - format, - { - bytes: this._videoFrame.yBuffer, - stride: frame.yStride, - }, - { - bytes: this._videoFrame.uBuffer, - stride: frame.yStride / 2, - }, - { - bytes: this._videoFrame.vBuffer, - stride: frame.yStride / 2, - } + public override drawFrame({ + width, + height, + yStride, + uStride, + vStride, + yBuffer, + uBuffer, + vBuffer, + rotation, + }: VideoFrame) { + this.rotateCanvas({ width, height, rotation }); + this.updateRenderMode(); + + if (!this.frameSink) return; + + this.frameSink.drawFrame( + YUVBuffer.frame( + YUVBuffer.format({ + width, + height, + chromaWidth: width! / 2, + chromaHeight: height! / 2, + cropLeft: yStride! - width!, + }), + { + bytes: yBuffer, + stride: yStride, + }, + { + bytes: uBuffer, + stride: uStride, + }, + { + bytes: vBuffer, + stride: vStride, + } + ) ); - this._yuvCanvasSink.drawFrame(yuvBufferFrame); + super.drawFrame(); } - public refreshCanvas() { - if (this._cacheCanvasOptions) { - this.zoom( - this._cacheCanvasOptions.rotation === 90 || - this._cacheCanvasOptions.rotation === 270, - this._cacheCanvasOptions.contentMode, - this._cacheCanvasOptions.frameWidth, - this._cacheCanvasOptions.frameHeight, - this._cacheCanvasOptions.clientWidth, - this._cacheCanvasOptions.clientHeight - ); - } + protected override rotateCanvas({ width, height, rotation }: VideoFrame) { + super.rotateCanvas({ width, height, rotation }); + + if (!this.canvas) return; + this.canvas.style.transform += ` rotateZ(${rotation}deg)`; } } diff --git a/ts/Renderer/index.ts b/ts/Renderer/index.ts index 26cd4c4db..0fa58f585 100644 --- a/ts/Renderer/index.ts +++ b/ts/Renderer/index.ts @@ -1,2 +1 @@ export * from './IRenderer'; -export * from './IRendererManager'; diff --git a/ts/Types.ts b/ts/Types.ts index 5d018ca60..2752ccc3f 100644 --- a/ts/Types.ts +++ b/ts/Types.ts @@ -1,86 +1,66 @@ -import { RenderModeType, VideoSourceType } from './Private/AgoraMediaBase'; -import { IRenderer, IRendererManager } from './Renderer'; +import { VideoCanvas, VideoCodecType } from './Private/AgoraBase'; +import { VideoFrame } from './Private/AgoraMediaBase'; +import { RtcConnection } from './Private/IAgoraRtcEngineEx'; +import { CapabilityManager } from './Renderer/CapabilityManager'; +import { RendererCache } from './Renderer/RendererCache'; +import { RendererManager } from './Renderer/RendererManager'; +import { WebCodecsRendererCache } from './Renderer/WebCodecsRendererCache'; -/** - * @ignore - */ -export interface AgoraEnvOptions { +export enum VideoFallbackStrategy { /** * @ignore */ - enableLogging?: boolean; - /** - * @ignore - */ - enableDebugLogging?: boolean; + PerformancePriority = 0, /** * @ignore */ - webEnvReady?: boolean; + BandwidthPriority = 1, } /** * @ignore */ -export interface AgoraEnvType extends AgoraEnvOptions { - /** - * @ignore - */ - AgoraElectronBridge: AgoraElectronBridge; - /** - * @ignore - */ - AgoraRendererManager?: IRendererManager; -} - -/** - * @ignore - */ -export interface CanvasOptions { - /** - * @ignore - */ - frameWidth: number; +export interface AgoraEnvOptions { /** * @ignore */ - frameHeight: number; + enableLogging?: boolean; /** * @ignore */ - rotation: number; + enableDebugLogging?: boolean; /** * @ignore */ - contentMode: RenderModeType; + webEnvReady?: boolean; /** * @ignore */ - clientWidth: number; + enableWebCodecsDecoder: boolean; /** * @ignore */ - clientHeight: number; + videoFallbackStrategy: VideoFallbackStrategy; } /** * @ignore */ -export interface RendererOptions { +export interface AgoraEnvType extends AgoraEnvOptions { /** * @ignore */ - contentMode?: RenderModeType; + AgoraRendererManager?: RendererManager; /** * @ignore */ - mirror?: boolean; + CapabilityManager?: CapabilityManager; } /** * @ignore */ -export enum RENDER_MODE { +export enum RendererType { /** * @ignore */ @@ -89,131 +69,21 @@ export enum RENDER_MODE { * @ignore */ SOFTWARE = 2, -} - -export type User = 'local' | 'videoSource' | number | string; - -export type Channel = '' | string; - -/** - * @ignore - */ -export interface RendererVideoConfig { - /** - * @ignore - */ - videoSourceType?: VideoSourceType; - /** - * @ignore - */ - channelId?: Channel; - /** - * @ignore - */ - uid?: number; - /** - * @ignore - */ - view?: HTMLElement; /** * @ignore */ - rendererOptions?: RendererOptions; + WEBCODECSRENDERER = 3, } -/** - * @ignore - */ -export interface FormatRendererVideoConfig { - /** - * @ignore - */ - videoSourceType: VideoSourceType; - /** - * @ignore - */ - channelId: Channel; - /** - * @ignore - */ - uid: number; - /** - * @ignore - */ - view?: HTMLElement; - /** - * @ignore - */ - rendererOptions: RendererOptions; -} +export type RENDER_MODE = RendererType; -/** - * @ignore - */ -export interface VideoFrameCacheConfig { - /** - * @ignore - */ - uid: number; - /** - * @ignore - */ - channelId: string; - /** - * @ignore - */ - videoSourceType: VideoSourceType; -} +export type RendererContext = VideoCanvas & RtcConnection; +export type RendererCacheType = RendererCache | WebCodecsRendererCache; -/** - * @ignore - */ -export interface ShareVideoFrame { - /** - * @ignore - */ - width: number; - /** - * @ignore - */ - height: number; - /** - * @ignore - */ - yStride: number; - /** - * @ignore - */ - yBuffer: Buffer | Uint8Array; - /** - * @ignore - */ - uBuffer: Buffer | Uint8Array; - /** - * @ignore - */ - vBuffer: Buffer | Uint8Array; - /** - * @ignore - */ - mirror?: boolean; - /** - * @ignore - */ - rotation?: number; - /** - * @ignore - */ - uid?: number; - /** - * @ignore - */ - channelId?: string; - /** - * @ignore - */ - videoSourceType: VideoSourceType; -} +export type RendererCacheContext = Pick< + RendererContext, + 'channelId' | 'uid' | 'sourceType' | 'useWebCodecsDecoder' | 'enableFps' +>; /** * @ignore @@ -232,7 +102,7 @@ export interface Result { /** * @ignore */ -export interface AgoraElectronBridge { +export interface IAgoraElectronBridge { /** * @ignore */ @@ -247,6 +117,8 @@ export interface AgoraElectronBridge { ) => void ): void; + UnEvent(callbackName: string): void; + CallApi( funcName: string, params: any, @@ -260,20 +132,18 @@ export interface AgoraElectronBridge { ReleaseRenderer(): void; - EnableVideoFrameCache(config: VideoFrameCacheConfig): void; + EnableVideoFrameCache(context: RendererCacheContext): void; - DisableVideoFrameCache(config: VideoFrameCacheConfig): void; + DisableVideoFrameCache(context: RendererCacheContext): void; GetBuffer(ptr: number, length: number): Buffer; - GetVideoFrame(streamInfo: ShareVideoFrame): { + GetVideoFrame( + context: RendererCacheContext, + videoFrame: VideoFrame + ): { ret: number; isNewFrame: boolean; - yStride: number; - width: number; - height: number; - rotation: number; - timestamp: number; }; sendMsg: ( @@ -287,17 +157,30 @@ export interface AgoraElectronBridge { /** * @ignore */ -export interface RenderConfig { - /** - * @ignore - */ - renders: IRenderer[]; - /** - * @ignore - */ - shareVideoFrame: ShareVideoFrame; +export enum IPCMessageType { + AGORA_IPC_GET_GPU_INFO = 'AGORA_IPC_GET_GPU_INFO', +} + +interface CodecMappingItem { + codec: string; + type: VideoCodecType; + profile: string; } -export type UidMap = Map; -export type ChannelIdMap = Map; -export type RenderMap = Map; +/** + * @ignore + */ +export const codecMapping: CodecMappingItem[] = [ + { + codec: 'avc1.64e01f', + type: VideoCodecType.VideoCodecH264, + profile: 'h264', + }, + { + codec: 'hvc1.1.6.L5.90', + type: VideoCodecType.VideoCodecH265, + profile: 'hevc', + }, + { codec: 'vp8', type: VideoCodecType.VideoCodecVp8, profile: 'vp8' }, + { codec: 'vp9', type: VideoCodecType.VideoCodecVp9, profile: 'vp9' }, +]; diff --git a/ts/Utils.ts b/ts/Utils.ts index f0b4adc7f..b3346d7db 100644 --- a/ts/Utils.ts +++ b/ts/Utils.ts @@ -1,9 +1,4 @@ -import { VideoSourceType } from './Private/AgoraMediaBase'; -import { - AgoraEnvType, - FormatRendererVideoConfig, - RendererVideoConfig, -} from './Types'; +import { AgoraEnvType } from './Types'; /** * @ignore @@ -14,15 +9,6 @@ export const TAG = '[Agora]: '; */ export const DEBUG_TAG = '[Agora Debug]: '; -/** - * @ignore - */ -export const deprecate = (originApi?: string, replaceApi?: string) => - logError( - `${TAG} This method ${originApi} will be deprecated soon. `, - replaceApi ? `Please use ${replaceApi} instead` : '' - ); - /** * @ignore */ @@ -43,6 +29,21 @@ export const logError = (msg: string, ...optParams: any[]) => { console.error(`${TAG} ${msg}`, ...optParams); }; +const getCurrentTime = () => { + const date = new Date(); + + const year = date.getFullYear().toString().slice(-2); + const month = (date.getMonth() + 1).toString().padStart(2, '0'); + const day = date.getDate().toString().padStart(2, '0'); + + const hours = date.getHours().toString().padStart(2, '0'); + const minutes = date.getMinutes().toString().padStart(2, '0'); + const seconds = date.getSeconds().toString().padStart(2, '0'); + const milliseconds = date.getMilliseconds().toString().padStart(3, '0'); + + return `${month}/${day}/${year} ${hours}:${minutes}:${seconds}:${milliseconds}`; +}; + /** * @ignore */ @@ -50,7 +51,7 @@ export const logInfo = (msg: string, ...optParams: any[]) => { if (!AgoraEnv.enableLogging) { return; } - console.log(`${TAG} ${msg}`, ...optParams); + console.info(`[${getCurrentTime()}]${TAG} ${msg}`, ...optParams); }; /** @@ -60,23 +61,7 @@ export const logDebug = (msg: string, ...optParams: any[]) => { if (!AgoraEnv.enableLogging || !AgoraEnv.enableDebugLogging) { return; } - console.warn(`${DEBUG_TAG} ${msg}`, ...optParams); -}; - -/** - * @ignore - */ -export const parseJSON = (jsonString: string) => { - if (jsonString === '') { - return jsonString; - } - let obj; - try { - obj = JSON.parse(jsonString); - } catch (error) { - logError('parseJSON', error); - } - return obj || jsonString; + console.debug(`${DEBUG_TAG} ${msg}`, ...optParams); }; /** @@ -93,71 +78,6 @@ export const objsKeysToLowerCase = (array: Array) => { }); }; -/** - * @ignore - */ -export const formatConfigByVideoSourceType = ( - videoSourceType?: VideoSourceType, - originChannelId = '', - originUid = 0 -): { - uid: number; - channelId: string; - videoSourceType: VideoSourceType; -} => { - if (videoSourceType === undefined || videoSourceType === null) { - throw new Error(`must set videoSourceType:${videoSourceType}`); - } - let uid = originUid; - let channelId = originChannelId; - - switch (videoSourceType) { - case VideoSourceType.VideoSourceCamera: - case VideoSourceType.VideoSourceCameraPrimary: - case VideoSourceType.VideoSourceScreen: - case VideoSourceType.VideoSourceScreenSecondary: - case VideoSourceType.VideoSourceTranscoded: - channelId = ''; - uid = 0; - break; - case VideoSourceType.VideoSourceRemote: - if (!uid || !channelId) { - throw new Error(`must set uid:${uid} and channelId:${channelId}`); - } - break; - case VideoSourceType.VideoSourceMediaPlayer: - channelId = ''; - if (!uid) { - throw new Error(`must set mediaPlayerId:${uid}`); - } - break; - default: - break; - } - return { uid, channelId, videoSourceType }; -}; - -/** - * @ignore - */ -export const getDefaultRendererVideoConfig = ( - config: RendererVideoConfig -): FormatRendererVideoConfig => { - const rendererOptions = Object.assign( - {}, - AgoraEnv.AgoraRendererManager?.defaultRenderConfig?.rendererOptions, - config.rendererOptions - ); - - const { uid, channelId, videoSourceType } = formatConfigByVideoSourceType( - config.videoSourceType, - config.channelId, - config.uid - ); - - return { ...config, uid, channelId, videoSourceType, rendererOptions }; -}; - /** * @ignore */ @@ -187,7 +107,66 @@ function copyProperties(target: T, source: any) { } } -const agora = require('../build/Release/agora_node_ext'); +/** + * @ignore + */ +export function isSupportWebGL(): boolean { + let flag = false; + const canvas: HTMLCanvasElement = document.createElement('canvas'); + try { + const getContext = ( + contextNames = ['webgl2', 'webgl', 'experimental-webgl'] + ): WebGLRenderingContext | WebGLRenderingContext | null => { + for (let i = 0; i < contextNames.length; i++) { + const contextName = contextNames[i]!; + const context = canvas?.getContext(contextName); + if (context) { + return context as WebGLRenderingContext | WebGLRenderingContext; + } + } + return null; + }; + let gl = getContext(); + flag = !!gl; + gl?.getExtension('WEBGL_lose_context')?.loseContext(); + gl = null; + logInfo('Your browser support webGL'); + } catch (e) { + logWarn('Your browser may not support webGL'); + flag = false; + } + return flag; +} + +/** + * @ignore + */ +export function getContextByCanvas( + // eslint-disable-next-line auto-import/auto-import + canvas: OffscreenCanvas +): WebGLRenderingContext | WebGL2RenderingContext | null { + const contextNames = ['webgl2', 'webgl', 'experimental-webgl']; + + for (const contextName of contextNames) { + //@ts-ignore + const context = canvas.getContext(contextName, { + depth: true, + stencil: true, + alpha: false, + antialias: false, + premultipliedAlpha: true, + preserveDrawingBuffer: true, + powerPreference: 'default', + failIfMajorPerformanceCaveat: false, + }) as WebGLRenderingContext | WebGL2RenderingContext | null; + + if (context) { + return context; + } + } + + return null; +} /** * @ignore @@ -196,5 +175,6 @@ export const AgoraEnv: AgoraEnvType = { enableLogging: true, enableDebugLogging: false, webEnvReady: true, - AgoraElectronBridge: new agora.AgoraElectronBridge(), + enableWebCodecsDecoder: false, + videoFallbackStrategy: 0, }; diff --git a/yarn.lock b/yarn.lock index 1583ad924..12f0c9d76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1733,6 +1733,11 @@ dependencies: "@babel/types" "^7.3.0" +"@types/dom-webcodecs@^0.1.11": + version "0.1.11" + resolved "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.11.tgz#2e36e5cc71789551f107e2fe15d956845fa19567" + integrity sha512-yPEZ3z7EohrmOxbk/QTAa0yonMFkNkjnVXqbGb7D4rMr+F1dGQ8ZUFxXkyLLJuiICPejZ0AZE9Rrk9wUCczx4A== + "@types/graceful-fs@^4.1.3": version "4.1.6" resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.6.tgz#e14b2576a1c25026b7f02ede1de3b84c3a1efeae" @@ -8669,6 +8674,13 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.6.0: + version "7.6.0" + resolved "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"