From 50a08b80f92fa2f6da598566721990cb6c3a6f78 Mon Sep 17 00:00:00 2001 From: Gabe Espinosa Date: Mon, 2 Aug 2021 15:58:58 -0700 Subject: [PATCH 1/2] wait to load image before removing processor --- .../useBackgroundSettings.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts index 165d95d7f..fbe0b238d 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts @@ -128,12 +128,14 @@ export const backgroundConfig = { const virtualBackgroundAssets = '/virtualbackground'; let blurProcessor: GaussianBlurBackgroundProcessor; let virtualBackgroundProcessor: VirtualBackgroundProcessor; +let currentProcessor: GaussianBlurBackgroundProcessor | VirtualBackgroundProcessor | null; export default function useBackgroundSettings(videoTrack: LocalVideoTrack | undefined, room?: Room | null) { const [backgroundSettings, setBackgroundSettings] = useState(() => { const localStorageSettings = window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY); return localStorageSettings ? JSON.parse(localStorageSettings) : { type: 'none', index: 0 }; }); + useEffect(() => { if (!isSupported) { return; @@ -141,6 +143,11 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde // make sure localParticipant has joined room before applying video processors // this ensures that the video processors are not applied on the LocalVideoPreview const handleProcessorChange = async () => { + const processors = { + none: null, + blur: blurProcessor, + image: virtualBackgroundProcessor, + }; if (!blurProcessor) { blurProcessor = new GaussianBlurBackgroundProcessor({ assetsPath: virtualBackgroundAssets, @@ -156,14 +163,17 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde await virtualBackgroundProcessor.loadModel(); } if (videoTrack && room?.localParticipant) { + currentProcessor = processors[backgroundSettings.type]; + // load image if needed before removing the processor + if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { + virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index); + } + // remove previous processor and add the new one if (videoTrack.processor) { videoTrack.removeProcessor(videoTrack.processor); } - if (backgroundSettings.type === 'blur') { - videoTrack.addProcessor(blurProcessor); - } else if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { - virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index); - videoTrack.addProcessor(virtualBackgroundProcessor); + if (currentProcessor) { + videoTrack.addProcessor(currentProcessor); } } }; From ba1f7151165d1184a066eeb6f27572bcb1494d3c Mon Sep 17 00:00:00 2001 From: Gabe Espinosa Date: Tue, 3 Aug 2021 11:36:19 -0700 Subject: [PATCH 2/2] refactored implementation --- .../useBackgroundSettings.ts | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts index fbe0b238d..82f163ba8 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts @@ -1,5 +1,5 @@ import { LocalVideoTrack, Room } from 'twilio-video'; -import { useState, useEffect } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import { SELECTED_BACKGROUND_SETTINGS_KEY } from '../../../constants'; import { GaussianBlurBackgroundProcessor, @@ -128,7 +128,6 @@ export const backgroundConfig = { const virtualBackgroundAssets = '/virtualbackground'; let blurProcessor: GaussianBlurBackgroundProcessor; let virtualBackgroundProcessor: VirtualBackgroundProcessor; -let currentProcessor: GaussianBlurBackgroundProcessor | VirtualBackgroundProcessor | null; export default function useBackgroundSettings(videoTrack: LocalVideoTrack | undefined, room?: Room | null) { const [backgroundSettings, setBackgroundSettings] = useState(() => { @@ -136,6 +135,23 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde return localStorageSettings ? JSON.parse(localStorageSettings) : { type: 'none', index: 0 }; }); + const removeProcessor = useCallback(() => { + if (videoTrack && videoTrack.processor) { + videoTrack.removeProcessor(videoTrack.processor); + } + }, [videoTrack]); + + const addProcessor = useCallback( + (processor: GaussianBlurBackgroundProcessor | VirtualBackgroundProcessor) => { + if (!videoTrack || videoTrack.processor === processor) { + return; + } + removeProcessor(); + videoTrack.addProcessor(processor); + }, + [videoTrack, removeProcessor] + ); + useEffect(() => { if (!isSupported) { return; @@ -143,11 +159,6 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde // make sure localParticipant has joined room before applying video processors // this ensures that the video processors are not applied on the LocalVideoPreview const handleProcessorChange = async () => { - const processors = { - none: null, - blur: blurProcessor, - image: virtualBackgroundProcessor, - }; if (!blurProcessor) { blurProcessor = new GaussianBlurBackgroundProcessor({ assetsPath: virtualBackgroundAssets, @@ -162,24 +173,22 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde }); await virtualBackgroundProcessor.loadModel(); } - if (videoTrack && room?.localParticipant) { - currentProcessor = processors[backgroundSettings.type]; - // load image if needed before removing the processor - if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { - virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index); - } - // remove previous processor and add the new one - if (videoTrack.processor) { - videoTrack.removeProcessor(videoTrack.processor); - } - if (currentProcessor) { - videoTrack.addProcessor(currentProcessor); - } + if (!room?.localParticipant) { + return; + } + + if (backgroundSettings.type === 'blur') { + addProcessor(blurProcessor); + } else if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { + virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index); + addProcessor(virtualBackgroundProcessor); + } else { + removeProcessor(); } }; handleProcessorChange(); window.localStorage.setItem(SELECTED_BACKGROUND_SETTINGS_KEY, JSON.stringify(backgroundSettings)); - }, [backgroundSettings, videoTrack, room]); + }, [backgroundSettings, videoTrack, room, addProcessor, removeProcessor]); return [backgroundSettings, setBackgroundSettings] as const; }