diff --git a/package-lock.json b/package-lock.json index ef73e9b4f..60676558a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,9 +11,9 @@ "dependencies": { "@material-ui/core": "^4.9.1", "@material-ui/icons": "^4.9.1", - "@twilio-labs/plugin-rtc": "0.8.2", - "@twilio/conversations": "1.1.0", - "@twilio/video-processors": "1.0.0", + "@twilio-labs/plugin-rtc": "^0.8.2", + "@twilio/conversations": "^1.1.0", + "@twilio/video-processors": "^1.0.1", "@types/d3-timer": "^1.0.9", "@types/dotenv": "^8.2.0", "@types/express": "^4.17.11", @@ -47,7 +47,7 @@ "strip-color": "^0.1.0", "ts-node": "^9.1.1", "twilio": "3.63.1", - "twilio-video": "^2.14.0", + "twilio-video": "^2.15.2", "typescript": "^3.8.3" }, "devDependencies": { @@ -4986,9 +4986,9 @@ } }, "node_modules/@twilio/video-processors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.0.tgz", - "integrity": "sha512-+Fxrw0B3kfeC4A6YCYTQsU9ZhPMosCadW8u39lU9q1Wqj3pmdQtcekNu33h1dmRCMQn0Z+jCsX+/Ek7idZl4QQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.1.tgz", + "integrity": "sha512-e1gu0Sc0UmpwYyP4cfe0pWTztY6Bu1pCp8Y/apWKvxrlu9UVL2pSpGrlf/QUVP+HUJfWIi1TqG9V/ACWH16pWw==", "dependencies": { "@tensorflow-models/body-pix": "^2.1.0", "@tensorflow/tfjs-backend-cpu": "^3.3.0", @@ -27481,9 +27481,9 @@ } }, "node_modules/twilio-video": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.15.1.tgz", - "integrity": "sha512-3CcYhLeej2Sg6eqxc0IOz8WKcqso3oVRH68xEipvqT0mQ69XyF59nkckj2zR0Pagt41/1Tug8ZPZVvrvT3F7BQ==", + "version": "2.15.2", + "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.15.2.tgz", + "integrity": "sha512-A7I2i5ZZ5hoIU4VLjHLKi83U8D7Es7ElB6BM2RjLuYJZypTFbZxllxMMjW0J683nf6kNdcC/rInxygLHG28CYw==", "dependencies": { "@twilio/webrtc": "4.4.0", "backoff": "^2.5.0", @@ -33563,9 +33563,9 @@ } }, "@twilio/video-processors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.0.tgz", - "integrity": "sha512-+Fxrw0B3kfeC4A6YCYTQsU9ZhPMosCadW8u39lU9q1Wqj3pmdQtcekNu33h1dmRCMQn0Z+jCsX+/Ek7idZl4QQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@twilio/video-processors/-/video-processors-1.0.1.tgz", + "integrity": "sha512-e1gu0Sc0UmpwYyP4cfe0pWTztY6Bu1pCp8Y/apWKvxrlu9UVL2pSpGrlf/QUVP+HUJfWIi1TqG9V/ACWH16pWw==", "requires": { "@tensorflow-models/body-pix": "^2.1.0", "@tensorflow/tfjs-backend-cpu": "^3.3.0", @@ -51260,9 +51260,9 @@ } }, "twilio-video": { - "version": "2.15.1", - "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.15.1.tgz", - "integrity": "sha512-3CcYhLeej2Sg6eqxc0IOz8WKcqso3oVRH68xEipvqT0mQ69XyF59nkckj2zR0Pagt41/1Tug8ZPZVvrvT3F7BQ==", + "version": "2.15.2", + "resolved": "https://registry.npmjs.org/twilio-video/-/twilio-video-2.15.2.tgz", + "integrity": "sha512-A7I2i5ZZ5hoIU4VLjHLKi83U8D7Es7ElB6BM2RjLuYJZypTFbZxllxMMjW0J683nf6kNdcC/rInxygLHG28CYw==", "requires": { "@twilio/webrtc": "4.4.0", "backoff": "^2.5.0", diff --git a/package.json b/package.json index 0069f9a06..f9cf7967a 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "@material-ui/icons": "^4.9.1", "@twilio-labs/plugin-rtc": "^0.8.2", "@twilio/conversations": "^1.1.0", - "@twilio/video-processors": "1.0.0", + "@twilio/video-processors": "^1.0.1", "@types/d3-timer": "^1.0.9", "@types/dotenv": "^8.2.0", "@types/express": "^4.17.11", @@ -42,7 +42,7 @@ "strip-color": "^0.1.0", "ts-node": "^9.1.1", "twilio": "3.63.1", - "twilio-video": "^2.14.0", + "twilio-video": "^2.15.2", "typescript": "^3.8.3" }, "devDependencies": { diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx index 5b0585a9c..4e0dd3c1c 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.test.tsx @@ -12,6 +12,16 @@ jest.mock('@twilio/video-processors', () => { name: 'GaussianBlurBackgroundProcessor', }; }), + VirtualBackgroundProcessor: jest.fn().mockImplementation(() => { + return { + loadModel: mockLoadModel, + backgroundImage: '', + name: 'VirtualBackgroundProcessor', + }; + }), + ImageFit: { + Cover: 'Cover', + }, }; }); @@ -25,6 +35,19 @@ const blurSettings = { index: 0, }; +const imgSettings = { + type: 'image', + index: 2, +}; + +global.Image = jest.fn().mockImplementation(() => { + return { + set src(newSrc: String) { + this.onload(); + }, + }; +}); + let mockVideoTrack: any; let mockRoom: any; let backgroundSettings: any; @@ -60,7 +83,10 @@ describe('The useBackgroundSettings hook ', () => { }); backgroundSettings = renderResult.current[0]; expect(backgroundSettings.type).toEqual('blur'); - expect(mockVideoTrack.addProcessor).toHaveBeenCalled(); + expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({ + loadModel: mockLoadModel, + name: 'GaussianBlurBackgroundProcessor', + }); }); it('should set the background settings correctly and remove the video processor when "none" is selected', async () => { @@ -75,8 +101,18 @@ describe('The useBackgroundSettings hook ', () => { expect(backgroundSettings.type).toEqual('none'); }); - it('should set the background settings correctly and set the video processor when "image" is selected', () => { - // TODO add test after implementing virtual background feature/logic + it('should set the background settings correctly and set the video processor when "image" is selected', async () => { + await act(async () => { + setBackgroundSettings(imgSettings as BackgroundSettings); + }); + backgroundSettings = renderResult.current[0]; + expect(backgroundSettings.type).toEqual('image'); + expect(backgroundSettings.index).toEqual(2); + expect(mockVideoTrack.addProcessor).toHaveBeenCalledWith({ + backgroundImage: expect.any(Object), + loadModel: mockLoadModel, + name: 'VirtualBackgroundProcessor', + }); }); describe('The setBackgroundSettings function ', () => { @@ -107,10 +143,6 @@ describe('The useBackgroundSettings hook ', () => { it("should not call videoTrack.addProcessor with a param of blurProcessor if backgroundSettings.type is not equal to 'blur'", async () => { mockVideoTrack.addProcessor.mockReset(); - const imgSettings = { - type: 'image', - index: 2, - } as BackgroundSettings; await act(async () => { setBackgroundSettings(imgSettings); }); @@ -121,6 +153,19 @@ describe('The useBackgroundSettings hook ', () => { expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(imgSettings)); }); + it("should not call videoTrack.addProcessor with a param of virtualBackgroundProcessor if backgroundSettings.type is not equal to 'image'", async () => { + mockVideoTrack.addProcessor.mockReset(); + await act(async () => { + setBackgroundSettings(blurSettings); + }); + expect(mockVideoTrack.addProcessor).not.toHaveBeenCalledWith({ + loadModel: mockLoadModel, + backgroundImage: expect.any(Object), + name: 'VirtualBackgroundProcessor', + }); + expect(window.localStorage.getItem(SELECTED_BACKGROUND_SETTINGS_KEY)).toEqual(JSON.stringify(blurSettings)); + }); + it('should not error when videoTrack does not exist and sets the local storage item', async () => { const { result } = renderHook(() => useBackgroundSettings({} as any, mockRoom)); renderResult = result; diff --git a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts index e9fa45036..618827e39 100644 --- a/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts +++ b/src/components/VideoProvider/useBackgroundSettings/useBackgroundSettings.ts @@ -1,22 +1,38 @@ import { LocalVideoTrack, Room } from 'twilio-video'; import { useState, useEffect } from 'react'; import { SELECTED_BACKGROUND_SETTINGS_KEY } from '../../../constants'; -import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors'; +import { GaussianBlurBackgroundProcessor, VirtualBackgroundProcessor, ImageFit } from '@twilio/video-processors'; +import Abstract from '../../../images/Abstract.jpg'; import AbstractThumb from '../../../images/thumb/Abstract.jpg'; +import BohoHome from '../../../images/BohoHome.jpg'; import BohoHomeThumb from '../../../images/thumb/BohoHome.jpg'; +import Bookshelf from '../../../images/Bookshelf.jpg'; import BookshelfThumb from '../../../images/thumb/Bookshelf.jpg'; +import CoffeeShop from '../../../images/CoffeeShop.jpg'; import CoffeeShopThumb from '../../../images/thumb/CoffeeShop.jpg'; +import Contemporary from '../../../images/Contemporary.jpg'; import ContemporaryThumb from '../../../images/thumb/Contemporary.jpg'; +import CozyHome from '../../../images/CozyHome.jpg'; import CozyHomeThumb from '../../../images/thumb/CozyHome.jpg'; +import Desert from '../../../images/Desert.jpg'; import DesertThumb from '../../../images/thumb/Desert.jpg'; +import Fishing from '../../../images/Fishing.jpg'; import FishingThumb from '../../../images/thumb/Fishing.jpg'; +import Flower from '../../../images/Flower.jpg'; import FlowerThumb from '../../../images/thumb/Flower.jpg'; +import Kitchen from '../../../images/Kitchen.jpg'; import KitchenThumb from '../../../images/thumb/Kitchen.jpg'; +import ModernHome from '../../../images/ModernHome.jpg'; import ModernHomeThumb from '../../../images/thumb/ModernHome.jpg'; +import Nature from '../../../images/Nature.jpg'; import NatureThumb from '../../../images/thumb/Nature.jpg'; +import Ocean from '../../../images/Ocean.jpg'; import OceanThumb from '../../../images/thumb/Ocean.jpg'; +import Patio from '../../../images/Patio.jpg'; import PatioThumb from '../../../images/thumb/Patio.jpg'; +import Plant from '../../../images/Plant.jpg'; import PlantThumb from '../../../images/thumb/Plant.jpg'; +import SanFrancisco from '../../../images/SanFrancisco.jpg'; import SanFranciscoThumb from '../../../images/thumb/SanFrancisco.jpg'; import { Thumbnail } from '../../BackgroundSelectionDialog/BackgroundThumbnail/BackgroundThumbnail'; @@ -63,6 +79,42 @@ const images = [ SanFranciscoThumb, ]; +const rawImagePaths = [ + Abstract, + BohoHome, + Bookshelf, + CoffeeShop, + Contemporary, + CozyHome, + Desert, + Fishing, + Flower, + Kitchen, + ModernHome, + Nature, + Ocean, + Patio, + Plant, + SanFrancisco, +]; + +let imageElements = new Map(); + +const getImage = (index: number): Promise => { + return new Promise((resolve, reject) => { + if (imageElements.has(index)) { + return resolve(imageElements.get(index)); + } + const img = new Image(); + img.onload = () => { + imageElements.set(index, img); + resolve(img); + }; + img.onerror = reject; + img.src = rawImagePaths[index]; + }); +}; + export const backgroundConfig = { imageNames, images, @@ -70,6 +122,7 @@ export const backgroundConfig = { const virtualBackgroundAssets = '/virtualbackground'; let blurProcessor: GaussianBlurBackgroundProcessor; +let virtualBackgroundProcessor: VirtualBackgroundProcessor; export default function useBackgroundSettings(videoTrack: LocalVideoTrack | undefined, room?: Room | null) { const [backgroundSettings, setBackgroundSettings] = useState(() => { @@ -78,27 +131,43 @@ export default function useBackgroundSettings(videoTrack: LocalVideoTrack | unde }); useEffect(() => { - if (!blurProcessor) { - blurProcessor = new GaussianBlurBackgroundProcessor({ - assetsPath: virtualBackgroundAssets, - }); - blurProcessor.loadModel(); - } + const loadProcessors = async () => { + if (!blurProcessor) { + blurProcessor = new GaussianBlurBackgroundProcessor({ + assetsPath: virtualBackgroundAssets, + }); + await blurProcessor.loadModel(); + } + + if (!virtualBackgroundProcessor) { + virtualBackgroundProcessor = new VirtualBackgroundProcessor({ + assetsPath: virtualBackgroundAssets, + backgroundImage: await getImage(0), + fitType: ImageFit.Cover, + }); + await virtualBackgroundProcessor.loadModel(); + } + }; + loadProcessors(); }, []); useEffect(() => { // make sure localParticipant has joined room before applying video processors // this ensures that the video processors are not applied on the LocalVideoPreview - if (videoTrack && room?.localParticipant) { - if (videoTrack.processor) { - videoTrack.removeProcessor(videoTrack.processor); - } - if (backgroundSettings.type === 'blur') { - videoTrack.addProcessor(blurProcessor); - } else if (backgroundSettings.type === 'image') { - // TODO implement image background replacement logic + const handleProcessorChange = async () => { + if (videoTrack && room?.localParticipant) { + if (videoTrack.processor) { + videoTrack.removeProcessor(videoTrack.processor); + } + if (backgroundSettings.type === 'blur') { + videoTrack.addProcessor(blurProcessor); + } else if (backgroundSettings.type === 'image' && typeof backgroundSettings.index === 'number') { + virtualBackgroundProcessor.backgroundImage = await getImage(backgroundSettings.index); + videoTrack.addProcessor(virtualBackgroundProcessor); + } } - } + }; + handleProcessorChange(); window.localStorage.setItem(SELECTED_BACKGROUND_SETTINGS_KEY, JSON.stringify(backgroundSettings)); }, [backgroundSettings, videoTrack, room]);