VIDEO-5734/blur background feature

src/components/BackgroundSelectionDialog/BackgroundThumbnail/BackgroundThumbnail.test.tsx

@@ -1,5 +1,7 @@
 import React from 'react';
 import BackgroundThumbnail from './BackgroundThumbnail';
+import BlurIcon from '@material-ui/icons/BlurOnOutlined';
+import NoneIcon from '@material-ui/icons/NotInterestedOutlined';
 import { shallow } from 'enzyme';
 import useVideoContext from '../../../hooks/useVideoContext/useVideoContext';
 
@@ -50,24 +52,25 @@ describe('The BackgroundThumbanil component', () => {
         index: 1,
       },
       setBackgroundSettings: mockSetBackgroundSettings,
+      updateBackgroundSettings: jest.fn(),
     }));
     const wrapper = shallow(<BackgroundThumbnail thumbnail={'image'} index={5} />);
     expect(wrapper.find('.selected').exists()).toBe(false);
   });
 
   it("should contain the NoneIcon when thumbnail is set to 'none'", () => {
     const wrapper = shallow(<BackgroundThumbnail thumbnail={'none'} />);
-    expect(wrapper.find('NotInterestedOutlinedIcon').exists()).toBe(true);
+    expect(wrapper.containsMatchingElement(<NoneIcon />)).toBe(true);
   });
 
   it("should contain the BlurIcon when thumbnail is set to 'blur'", () => {
     const wrapper = shallow(<BackgroundThumbnail thumbnail={'blur'} />);
-    expect(wrapper.find('BlurOnOutlinedIcon').exists()).toBe(true);
+    expect(wrapper.containsMatchingElement(<BlurIcon />)).toBe(true);
   });
 
   it("should not have any icons when thumbnail is set to 'image'", () => {
     const wrapper = shallow(<BackgroundThumbnail thumbnail={'image'} />);
-    expect(wrapper.find('BlurOnOutlinedIcon').exists()).toBe(false);
-    expect(wrapper.find('NotInterestedOutlinedIcon').exists()).toBe(false);
+    expect(wrapper.containsMatchingElement(<BlurIcon />)).toBe(false);
+    expect(wrapper.containsMatchingElement(<NoneIcon />)).toBe(false);
   });
 });

src/components/Buttons/EndCallButton/EndCallButton.test.tsx

@@ -7,6 +7,7 @@ const mockVideoContext = {
   room: {
     disconnect: jest.fn(),
   },
+  removeProcessor: jest.fn(),
 };
 
 jest.mock('../../../hooks/useVideoContext/useVideoContext', () => () => mockVideoContext);
@@ -15,6 +16,7 @@ describe('End Call button', () => {
   it('should disconnect from the room when clicked', () => {
     const wrapper = shallow(<EndCallButton />);
     wrapper.simulate('click');
+    expect(mockVideoContext.removeProcessor).toHaveBeenCalled();
     expect(mockVideoContext.room.disconnect).toHaveBeenCalled();
   });
 });

src/components/Buttons/EndCallButton/EndCallButton.tsx

@@ -20,10 +20,15 @@ const useStyles = makeStyles((theme: Theme) =>
 
 export default function EndCallButton(props: { className?: string }) {
   const classes = useStyles();
-  const { room } = useVideoContext();
+  const { room, removeProcessor } = useVideoContext();
+
+  const handleDisconnect = (): void => {
+    removeProcessor();
+    room!.disconnect();
+  };
 
   return (
-    <Button onClick={() => room!.disconnect()} className={clsx(classes.button, props.className)} data-cy-disconnect>
+    <Button onClick={handleDisconnect} className={clsx(classes.button, props.className)} data-cy-disconnect>
       Disconnect
     </Button>
   );

src/components/ChatWindow/MessageList/__snapshots__/MessageList.test.tsx.snap

@@ -155,10 +155,10 @@ exports[`the messageList component should render correctly 1`] = `
           </div>
         </div>
         <div
-          class="makeStyles-messageContainer-45"
+          class="makeStyles-messageContainer-9"
         >
           <div
-            class="makeStyles-iconContainer-46"
+            class="makeStyles-iconContainer-10"
           >
             <svg
               fill="none"
@@ -212,15 +212,15 @@ exports[`the messageList component should render correctly 1`] = `
             </svg>
           </div>
           <div
-            class="makeStyles-mediaInfo-47"
+            class="makeStyles-mediaInfo-11"
           >
             <p
-              class="makeStyles-filename-48"
+              class="makeStyles-filename-12"
             >
               test1.txt
             </p>
             <p
-              class="makeStyles-size-49"
+              class="makeStyles-size-13"
             >
               120.56 KB
                - Click to open
@@ -243,7 +243,6 @@ exports[`the messageList component should render correctly 1`] = `
                 aria-hidden="true"
                 class="MuiSvgIcon-root"
                 focusable="false"
-                role="presentation"
                 viewBox="0 0 24 24"
               >
                 <path

src/components/MenuBar/Menu/Menu.test.tsx

@@ -15,22 +15,26 @@ import useChatContext from '../../../hooks/useChatContext/useChatContext';
 import useFlipCameraToggle from '../../../hooks/useFlipCameraToggle/useFlipCameraToggle';
 import useMediaQuery from '@material-ui/core/useMediaQuery';
 import useIsRecording from '../../../hooks/useIsRecording/useIsRecording';
+import useLocalVideoToggle from '../../../hooks/useLocalVideoToggle/useLocalVideoToggle';
 
 jest.mock('../../../hooks/useFlipCameraToggle/useFlipCameraToggle');
 jest.mock('@material-ui/core/useMediaQuery');
 jest.mock('../../../state');
 jest.mock('../../../hooks/useVideoContext/useVideoContext', () => () => ({ room: { sid: 'mockRoomSid' } }));
 jest.mock('../../../hooks/useIsRecording/useIsRecording');
 jest.mock('../../../hooks/useChatContext/useChatContext');
+jest.mock('../../../hooks/useLocalVideoToggle/useLocalVideoToggle');
 
 const mockUseFlipCameraToggle = useFlipCameraToggle as jest.Mock<any>;
 const mockUseMediaQuery = useMediaQuery as jest.Mock<boolean>;
 const mockUseAppState = useAppState as jest.Mock<any>;
 const mockUseIsRecording = useIsRecording as jest.Mock<boolean>;
 const mockUseChatContext = useChatContext as jest.Mock<any>;
+const mockUseLocalVideoToggle = useLocalVideoToggle as jest.Mock<any>;
 
 const mockToggleChatWindow = jest.fn();
 mockUseChatContext.mockImplementation(() => ({ setIsChatWindowOpen: mockToggleChatWindow }));
+mockUseLocalVideoToggle.mockImplementation(() => [true, () => {}]);
 
 describe('the Menu component', () => {
   let mockUpdateRecordingRules: jest.Mock<any>;

src/components/Snackbar/__snapshots__/Snackbar.test.tsx.snap

@@ -47,7 +47,7 @@ exports[`the Snackbar component should render correctly with "error" variant 1`]
         onClick={[Function]}
         size="small"
       >
-        <CloseIcon
+        <Memo(ForwardRef(CloseIcon))
           fontSize="small"
         />
       </WithStyles(ForwardRef(IconButton))>
@@ -103,7 +103,7 @@ exports[`the Snackbar component should render correctly with "info" variant 1`]
         onClick={[Function]}
         size="small"
       >
-        <CloseIcon
+        <Memo(ForwardRef(CloseIcon))
           fontSize="small"
         />
       </WithStyles(ForwardRef(IconButton))>
@@ -159,7 +159,7 @@ exports[`the Snackbar component should render correctly with "warning" variant 1
         onClick={[Function]}
         size="small"
       >
-        <CloseIcon
+        <Memo(ForwardRef(CloseIcon))
           fontSize="small"
         />
       </WithStyles(ForwardRef(IconButton))>

src/components/VideoProvider/index.test.tsx

@@ -25,6 +25,15 @@ jest.mock('./useLocalTracks/useLocalTracks', () =>
 jest.mock('./useHandleRoomDisconnection/useHandleRoomDisconnection');
 jest.mock('./useHandleTrackPublicationFailed/useHandleTrackPublicationFailed');
 jest.mock('./useRestartAudioTrackOnDeviceChange/useRestartAudioTrackOnDeviceChange');
+jest.mock('@twilio/video-processors', () => {
+  return {
+    GaussianBlurBackgroundProcessor: jest.fn().mockImplementation(() => {
+      return {
+        loadModel: jest.fn(),
+      };
+    }),
+  };
+});
 
 describe('the VideoProvider component', () => {
   it('should correctly return the Video Context object', () => {
@@ -34,6 +43,10 @@ describe('the VideoProvider component', () => {
       </VideoProvider>
     );
     const { result } = renderHook(useVideoContext, { wrapper });
+    const expectedSettings = {
+      type: 'none',
+      index: 0,
+    };
     expect(result.current).toMatchObject({
       isConnecting: false,
       localTracks: [{ name: 'mockTrack' }],
@@ -47,6 +60,9 @@ describe('the VideoProvider component', () => {
       toggleScreenShare: expect.any(Function),
       isBackgroundSelectionOpen: false,
       setIsBackgroundSelectionOpen: expect.any(Function),
+      backgroundSettings: expectedSettings,
+      setBackgroundSettings: expect.any(Function),
+      removeProcessor: expect.any(Function),
     });
     expect(useRoom).toHaveBeenCalledWith([{ name: 'mockTrack' }], expect.any(Function), {
       dominantSpeaker: true,

src/components/VideoProvider/index.tsx

@@ -36,6 +36,7 @@ export interface IVideoContext {
   setIsBackgroundSelectionOpen: (value: boolean) => void;
   backgroundSettings: BackgroundSettings;
   setBackgroundSettings: (settings: BackgroundSettings) => void;
+  removeProcessor: () => void;
 }
 
 export const VideoContext = createContext<IVideoContext>(null!);
@@ -79,9 +80,12 @@ export function VideoProvider({ options, children, onError = () => {} }: VideoPr
   );
   useHandleTrackPublicationFailed(room, onError);
   useRestartAudioTrackOnDeviceChange(localTracks);
+  console.log('tracks: ');
+  console.log(localTracks);
 
   const [isBackgroundSelectionOpen, setIsBackgroundSelectionOpen] = useState(false);
-  const [backgroundSettings, setBackgroundSettings] = useBackgroundSettings();
+  const videoTrack = localTracks.find(track => track.name.includes('camera')) as LocalVideoTrack | undefined;
+  const [backgroundSettings, setBackgroundSettings, removeProcessor] = useBackgroundSettings(videoTrack);
 
   return (
     <VideoContext.Provider
@@ -102,6 +106,7 @@ export function VideoProvider({ options, children, onError = () => {} }: VideoPr
         setIsBackgroundSelectionOpen,
         backgroundSettings,
         setBackgroundSettings,
+        removeProcessor,
       }}
     >
       <SelectedParticipantProvider room={room}>{children}</SelectedParticipantProvider>

src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx

@@ -0,0 +1,66 @@
+import { act, renderHook } from '@testing-library/react-hooks';
+import useBackgroundSettings, { BackgroundSettings } from './useBackgroundSettings';
+const mockLoadModel = jest.fn();
+
+jest.mock('@twilio/video-processors', () => {
+  return {
+    GaussianBlurBackgroundProcessor: jest.fn().mockImplementation(() => {
+      return {
+        loadModel: mockLoadModel,
+      };
+    }),
+  };
+});
+
+const defaultSettings = {
+  type: 'none',
+  index: 0,
+};
+
+const blurSettings = {
+  type: 'blur',
+};
+
+describe('The useBackgroundSettings hook ', () => {
+  let mockVideoTrack: any;
+  beforeEach(() => {
+    mockVideoTrack = {
+      kind: 'video',
+      processor: '',
+      addProcessor: jest.fn(),
+      removeProcessor: jest.fn(),
+    };
+  });
+
+  it('should return the backgroundsettings and update function.', () => {
+    const { result } = renderHook(() => useBackgroundSettings(mockVideoTrack as any));
+    expect(result.current).toEqual([defaultSettings, expect.any(Function), expect.any(Function)]);
+  });
+
+  it('should set the background settings correctly and set the video processor when "blur" is selected', async () => {
+    const { result } = renderHook(() => useBackgroundSettings(mockVideoTrack as any));
+    await act(async () => {
+      result.current[1](blurSettings as BackgroundSettings);
+    });
+    expect(result.current[0].type).toEqual('blur');
+    expect(mockVideoTrack.addProcessor).toHaveBeenCalled();
+  });
+
+  it('should set the background settings correctly and remove the video processor when "none" is selected', async () => {
+    const { result } = renderHook(() => useBackgroundSettings(mockVideoTrack as any));
+    // set video processor to non-null value
+    await act(async () => {
+      result.current[1](blurSettings as BackgroundSettings);
+    });
+    // set video processor to none
+    await act(async () => {
+      result.current[1](defaultSettings as BackgroundSettings);
+    });
+    expect(mockVideoTrack.addProcessor).toHaveBeenCalled();
+    expect(result.current[0].type).toEqual('none');
+  });
+
+  it('should set the background settings correctly and set the video processor when "image" is selected', () => {
+    // TODO add test after implementing virtual background feature/logic
+  });
+});

src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts

@@ -1,4 +1,7 @@
-import { useState } from 'react';
+import { LocalVideoTrack } from 'twilio-video';
+import { useState, useEffect } from 'react';
+import { SELECTED_BACKGROUND_SETTINGS_KEY } from '../../../constants';
+import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors';
 import AbstractThumb from '../../../images/thumb/Abstract.jpg';
 import BohoHomeThumb from '../../../images/thumb/BohoHome.jpg';
 import BookshelfThumb from '../../../images/thumb/Bookshelf.jpg';
@@ -65,13 +68,44 @@ export const backgroundConfig = {
   images,
 };
 
-// TODO : Add video processing logic after backgroundSettings change
-// useEffect hooks, etc ...
+const virtualBackgroundAssets = '/virtualbackground';
+let blurProcessor: GaussianBlurBackgroundProcessor;
 
-export default function useBackgroundSettings() {
-  const [backgroundSettings, setBackgroundSettings] = useState({
-    type: 'none',
-    index: 0,
-  } as BackgroundSettings);
-  return [backgroundSettings, setBackgroundSettings] as const;
+export default function useBackgroundSettings(videoTrack: LocalVideoTrack | undefined) {
+  const [backgroundSettings, setBackgroundSettings] = useState<BackgroundSettings>(() => {
+    const localStorageSettings = window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY);
+    return localStorageSettings ? JSON.parse(localStorageSettings) : { type: 'none', index: 0 };
+  });
+
+  // Specifically used for handling room disconnection
+  const removeProcessor = (): void => {
+    if (videoTrack && videoTrack.processor) {
+      videoTrack.removeProcessor(videoTrack.processor);
+    }
+  };
+
+  useEffect(() => {
+    if (!blurProcessor) {
+      blurProcessor = new GaussianBlurBackgroundProcessor({
+        assetsPath: virtualBackgroundAssets,
+      });
+      blurProcessor.loadModel();
+    }
+  }, []);
+
+  useEffect(() => {
+    if (videoTrack) {
+      if (videoTrack.processor) {
+        videoTrack.removeProcessor(videoTrack.processor);
+      }
+      if (backgroundSettings.type === 'blur') {
+        videoTrack.addProcessor(blurProcessor);
+      } else if (backgroundSettings.type === 'image') {
+        // TODO implement image background replacement logic
+      }
+    }
+    window.localStorage.setItem(SELECTED_BACKGROUND_SETTINGS_KEY, JSON.stringify(backgroundSettings));
+  }, [backgroundSettings, videoTrack]);
+
+  return [backgroundSettings, setBackgroundSettings, removeProcessor] as const;
 }

src/constants.ts

@@ -8,3 +8,6 @@ export const DEFAULT_VIDEO_CONSTRAINTS: MediaStreamConstraints['video'] = {
 export const SELECTED_AUDIO_INPUT_KEY = 'TwilioVideoApp-selectedAudioInput';
 export const SELECTED_AUDIO_OUTPUT_KEY = 'TwilioVideoApp-selectedAudioOutput';
 export const SELECTED_VIDEO_INPUT_KEY = 'TwilioVideoApp-selectedVideoInput';
+
+// This is used to store the current background settings in localStorage
+export const SELECTED_BACKGROUND_SETTINGS_KEY = 'TwilioVideoApp-selectedBackgroundSettings';