VIDEO-5735/Add virtual background feature

package.json

@@ -8,7 +8,7 @@
     "@material-ui/icons": "^4.9.1",
     "@twilio-labs/plugin-rtc": "^0.8.2",
     "@twilio/conversations": "^1.1.0",
-    "@twilio/video-processors": "1.0.0",
+    "@twilio/video-processors": "^1.0.1",
     "@types/d3-timer": "^1.0.9",
     "@types/dotenv": "^8.2.0",
     "@types/express": "^4.17.11",
@@ -42,7 +42,7 @@
     "strip-color": "^0.1.0",
     "ts-node": "^9.1.1",
     "twilio": "3.63.1",
-    "twilio-video": "^2.14.0",
+    "twilio-video": "^2.15.2",
     "typescript": "^3.8.3"
   },
   "devDependencies": {

src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx

@@ -12,6 +12,16 @@ jest.mock('@twilio/video-processors', () => {
         name: 'GaussianBlurBackgroundProcessor',
       };
     }),
+    VirtualBackgroundProcessor: jest.fn().mockImplementation(() => {
+      return {
+        loadModel: mockLoadModel,
+        backgroundImage: '',
+        name: 'VirtualBackgroundProcessor',
+      };
+    }),
+    ImageFit: {
+      Cover: 'Cover',
+    },
   };
 });
 
@@ -25,6 +35,19 @@ const blurSettings = {
   index: 0,
 };
 
+const imgSettings = {
+  type: 'image',
+  index: 2,
+};
+
+global.Image = jest.fn().mockImplementation(() => {
+  return {
+    set src(newSrc: String) {
+      this.onload();
+    },
+  };
+});
+
 let mockVideoTrack: any;
 let mockRoom: any;
 let backgroundSettings: any;
@@ -60,7 +83,10 @@ describe('The useBackgroundSettings hook ', () => {
     });
     backgroundSettings = renderResult.current[0];
     expect(backgroundSettings.type).toEqual('blur');
-    expect(mockVideoTrack.addProcessor).toHaveBeenCalled();
+    expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({
+      loadModel: mockLoadModel,
+      name: 'GaussianBlurBackgroundProcessor',
+    });
   });
 
   it('should set the background settings correctly and remove the video processor when "none" is selected', async () => {
@@ -75,8 +101,18 @@ describe('The useBackgroundSettings hook ', () => {
     expect(backgroundSettings.type).toEqual('none');
   });
 
-  it('should set the background settings correctly and set the video processor when "image" is selected', () => {
-    // TODO add test after implementing virtual background feature/logic
+  it('should set the background settings correctly and set the video processor when "image" is selected', async () => {
+    await act(async () => {
+      setBackgroundSettings(imgSettings as BackgroundSettings);
+    });
+    backgroundSettings = renderResult.current[0];
+    expect(backgroundSettings.type).toEqual('image');
+    expect(backgroundSettings.index).toEqual(2);
+    expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({
+      backgroundImage: expect.any(Object),
+      loadModel: mockLoadModel,
+      name: 'VirtualBackgroundProcessor',
+    });
   });
 
   describe('The setBackgroundSettings function ', () => {
@@ -107,10 +143,6 @@ describe('The useBackgroundSettings hook ', () => {
 
     it("should not call videoTrack.addProcessor with a param of blurProcessor if backgroundSettings.type is not equal to 'blur'", async () => {
       mockVideoTrack.addProcessor.mockReset();
-      const imgSettings = {
-        type: 'image',
-        index: 2,
-      } as BackgroundSettings;
       await act(async () => {
         setBackgroundSettings(imgSettings);
       });
@@ -121,6 +153,19 @@ describe('The useBackgroundSettings hook ', () => {
       expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(imgSettings));
     });
 
+    it("should not call videoTrack.addProcessor with a param of virtualBackgroundProcessor if backgroundSettings.type is not equal to 'image'", async () => {
+      mockVideoTrack.addProcessor.mockReset();
+      await act(async () => {
+        setBackgroundSettings(blurSettings);
+      });
+      expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith({
+        loadModel: mockLoadModel,
+        backgroundImage: expect.any(Object),
+        name: 'VirtualBackgroundProcessor',
+      });
+      expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(blurSettings));
+    });
+
     it('should not error when videoTrack does not exist and sets the local storage item', async () => {
       const { result } = renderHook(() => useBackgroundSettings({} as any, mockRoom));
       renderResult = result;

src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts

@@ -1,22 +1,38 @@
 import { LocalVideoTrack, Room } from 'twilio-video';
 import { useState, useEffect } from 'react';
 import { SELECTED_BACKGROUND_SETTINGS_KEY } from '../../../constants';
-import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors';
+import { GaussianBlurBackgroundProcessor, VirtualBackgroundProcessor, ImageFit } from '@twilio/video-processors';
+import Abstract from '../../../images/Abstract.jpg';
 import AbstractThumb from '../../../images/thumb/Abstract.jpg';
+import BohoHome from '../../../images/BohoHome.jpg';
 import BohoHomeThumb from '../../../images/thumb/BohoHome.jpg';
+import Bookshelf from '../../../images/Bookshelf.jpg';
 import BookshelfThumb from '../../../images/thumb/Bookshelf.jpg';
+import CoffeeShop from '../../../images/CoffeeShop.jpg';
 import CoffeeShopThumb from '../../../images/thumb/CoffeeShop.jpg';
+import Contemporary from '../../../images/Contemporary.jpg';
 import ContemporaryThumb from '../../../images/thumb/Contemporary.jpg';
+import CozyHome from '../../../images/CozyHome.jpg';
 import CozyHomeThumb from '../../../images/thumb/CozyHome.jpg';
+import Desert from '../../../images/Desert.jpg';
 import DesertThumb from '../../../images/thumb/Desert.jpg';
+import Fishing from '../../../images/Fishing.jpg';
 import FishingThumb from '../../../images/thumb/Fishing.jpg';
+import Flower from '../../../images/Flower.jpg';
 import FlowerThumb from '../../../images/thumb/Flower.jpg';
+import Kitchen from '../../../images/Kitchen.jpg';
 import KitchenThumb from '../../../images/thumb/Kitchen.jpg';
+import ModernHome from '../../../images/ModernHome.jpg';
 import ModernHomeThumb from '../../../images/thumb/ModernHome.jpg';
+import Nature from '../../../images/Nature.jpg';
 import NatureThumb from '../../../images/thumb/Nature.jpg';
+import Ocean from '../../../images/Ocean.jpg';
 import OceanThumb from '../../../images/thumb/Ocean.jpg';
+import Patio from '../../../images/Patio.jpg';
 import PatioThumb from '../../../images/thumb/Patio.jpg';
+import Plant from '../../../images/Plant.jpg';
 import PlantThumb from '../../../images/thumb/Plant.jpg';
+import SanFrancisco from '../../../images/SanFrancisco.jpg';
 import SanFranciscoThumb from '../../../images/thumb/SanFrancisco.jpg';
 import { Thumbnail } from '../../BackgroundSelectionDialog/BackgroundThumbnail/BackgroundThumbnail';
 
@@ -63,13 +79,50 @@ const images = [
   SanFranciscoThumb,
 ];
 
+const rawImagePaths = [
+  Abstract,
+  BohoHome,
+  Bookshelf,
+  CoffeeShop,
+  Contemporary,
+  CozyHome,
+  Desert,
+  Fishing,
+  Flower,
+  Kitchen,
+  ModernHome,
+  Nature,
+  Ocean,
+  Patio,
+  Plant,
+  SanFrancisco,
+];
+
+let imageElements = new Map();
+
+const getImage = (index: number): Promise<HTMLImageElement> => {
+  return new Promise((resolve, reject) => {
+    if (imageElements.has(index)) {
+      return resolve(imageElements.get(index));
+    }
+    const img = new Image();
+    img.onload = () => {
+      imageElements.set(index, img);
+      resolve(img);
+    };
+    img.onerror = reject;
+    img.src = rawImagePaths[index];
+  });
+};
+
 export const backgroundConfig = {
   imageNames,
   images,
 };
 
 const virtualBackgroundAssets = '/virtualbackground';
 let blurProcessor: GaussianBlurBackgroundProcessor;
+let virtualBackgroundProcessor: VirtualBackgroundProcessor;
 
 export default function useBackgroundSettings(videoTrack: LocalVideoTrack | undefined, room?: Room | null) {
   const [backgroundSettings, setBackgroundSettings] = useState<BackgroundSettings>(() => {
@@ -78,27 +131,43 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde
   });
 
   useEffect(() => {
-    if (!blurProcessor) {
-      blurProcessor = new GaussianBlurBackgroundProcessor({
-        assetsPath: virtualBackgroundAssets,
-      });
-      blurProcessor.loadModel();
-    }
+    const loadProcessors = async () => {
+      if (!blurProcessor) {
+        blurProcessor = new GaussianBlurBackgroundProcessor({
+          assetsPath: virtualBackgroundAssets,
+        });
+        await blurProcessor.loadModel();
+      }
+
+      if (!virtualBackgroundProcessor) {
+        virtualBackgroundProcessor = new VirtualBackgroundProcessor({
+          assetsPath: virtualBackgroundAssets,
+          backgroundImage: await getImage(0),
+          fitType: ImageFit.Cover,
+        });
+        await virtualBackgroundProcessor.loadModel();
+      }
+    };
+    loadProcessors();
   }, []);
 
   useEffect(() => {
     // make sure localParticipant has joined room before applying video processors
     // this ensures that the video processors are not applied on the LocalVideoPreview
-    if (videoTrack && room?.localParticipant) {
-      if (videoTrack.processor) {
-        videoTrack.removeProcessor(videoTrack.processor);
-      }
-      if (backgroundSettings.type === 'blur') {
-        videoTrack.addProcessor(blurProcessor);
-      } else if (backgroundSettings.type === 'image') {
-        // TODO implement image background replacement logic
+    const handleProcessorChange = async () => {
+      if (videoTrack && room?.localParticipant) {
+        if (videoTrack.processor) {
+          videoTrack.removeProcessor(videoTrack.processor);
+        }
+        if (backgroundSettings.type === 'blur') {
+          videoTrack.addProcessor(blurProcessor);
+        } else if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') {
+          virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index);
+          videoTrack.addProcessor(virtualBackgroundProcessor);
+        }
       }
-    }
+    };
+    handleProcessorChange();
     window.localStorage.setItem(SELECTED_BACKGROUND_SETTINGS_KEY, JSON.stringify(backgroundSettings));
   }, [backgroundSettings, videoTrack, room]);