Draw face contours

mrousavy · Apr 15, 2024 · ed4c89c · ed4c89c
1 parent d172c97
commit ed4c89c
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 51 deletions.
diff --git a/package/example/ios/Podfile.lock b/package/example/ios/Podfile.lock
@@ -11,13 +11,71 @@ PODS:
     - ReactCommon/turbomodule/core (= 0.72.7)
   - fmt (6.2.1)
   - glog (0.3.5)
+  - GoogleDataTransport (9.4.1):
+    - GoogleUtilities/Environment (~> 7.7)
+    - nanopb (< 2.30911.0, >= 2.30908.0)
+    - PromisesObjC (< 3.0, >= 1.2)
+  - GoogleMLKit/FaceDetection (5.0.0):
+    - GoogleMLKit/MLKitCore
+    - MLKitFaceDetection (~> 4.0.0)
+  - GoogleMLKit/MLKitCore (5.0.0):
+    - MLKitCommon (~> 10.0.0)
+  - GoogleToolboxForMac/DebugUtils (2.3.2):
+    - GoogleToolboxForMac/Defines (= 2.3.2)
+  - GoogleToolboxForMac/Defines (2.3.2)
+  - GoogleToolboxForMac/Logger (2.3.2):
+    - GoogleToolboxForMac/Defines (= 2.3.2)
+  - "GoogleToolboxForMac/NSData+zlib (2.3.2)":
+    - GoogleToolboxForMac/Defines (= 2.3.2)
+  - "GoogleToolboxForMac/NSDictionary+URLArguments (2.3.2)":
+    - GoogleToolboxForMac/DebugUtils (= 2.3.2)
+    - GoogleToolboxForMac/Defines (= 2.3.2)
+    - "GoogleToolboxForMac/NSString+URLArguments (= 2.3.2)"
+  - "GoogleToolboxForMac/NSString+URLArguments (2.3.2)"
+  - GoogleUtilities/Environment (7.13.0):
+    - GoogleUtilities/Privacy
+    - PromisesObjC (< 3.0, >= 1.2)
+  - GoogleUtilities/Logger (7.13.0):
+    - GoogleUtilities/Environment
+    - GoogleUtilities/Privacy
+  - GoogleUtilities/Privacy (7.13.0)
+  - GoogleUtilities/UserDefaults (7.13.0):
+    - GoogleUtilities/Logger
+    - GoogleUtilities/Privacy
+  - GoogleUtilitiesComponents (1.1.0):
+    - GoogleUtilities/Logger
+  - GTMSessionFetcher/Core (3.3.2)
   - hermes-engine (0.72.7):
     - hermes-engine/Pre-built (= 0.72.7)
   - hermes-engine/Pre-built (0.72.7)
   - libevent (2.1.12)
-  - MMKV (1.3.3):
-    - MMKVCore (~> 1.3.3)
-  - MMKVCore (1.3.3)
+  - MLImage (1.0.0-beta5)
+  - MLKitCommon (10.0.0):
+    - GoogleDataTransport (~> 9.0)
+    - GoogleToolboxForMac/Logger (~> 2.1)
+    - "GoogleToolboxForMac/NSData+zlib (~> 2.1)"
+    - "GoogleToolboxForMac/NSDictionary+URLArguments (~> 2.1)"
+    - GoogleUtilities/UserDefaults (~> 7.0)
+    - GoogleUtilitiesComponents (~> 1.0)
+    - GTMSessionFetcher/Core (< 4.0, >= 1.1)
+  - MLKitFaceDetection (4.0.0):
+    - MLKitCommon (~> 10.0)
+    - MLKitVision (~> 6.0)
+  - MLKitVision (6.0.0):
+    - GoogleToolboxForMac/Logger (~> 2.1)
+    - "GoogleToolboxForMac/NSData+zlib (~> 2.1)"
+    - GTMSessionFetcher/Core (< 4.0, >= 1.1)
+    - MLImage (= 1.0.0-beta5)
+    - MLKitCommon (~> 10.0)
+  - MMKV (1.3.4):
+    - MMKVCore (~> 1.3.4)
+  - MMKVCore (1.3.4)
+  - nanopb (2.30910.0):
+    - nanopb/decode (= 2.30910.0)
+    - nanopb/encode (= 2.30910.0)
+  - nanopb/decode (2.30910.0)
+  - nanopb/encode (2.30910.0)
+  - PromisesObjC (2.4.0)
   - RCT-Folly (2021.07.22.00):
     - boost
     - DoubleConversion
@@ -479,6 +537,10 @@ PODS:
     - React-callinvoker
     - React-Core
     - react-native-worklets-core
+  - VisionCameraFaceDetector (1.4.2):
+    - GoogleMLKit/FaceDetection
+    - React-Core
+    - VisionCamera
   - Yoga (1.14.0)
 
 DEPENDENCIES:
@@ -537,14 +599,27 @@ DEPENDENCIES:
   - RNVectorIcons (from `../node_modules/react-native-vector-icons`)
   - vision-camera-resize-plugin (from `../node_modules/vision-camera-resize-plugin`)
   - VisionCamera (from `../..`)
+  - VisionCameraFaceDetector (from `../node_modules/react-native-vision-camera-face-detector`)
   - Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
 
 SPEC REPOS:
   trunk:
     - fmt
+    - GoogleDataTransport
+    - GoogleMLKit
+    - GoogleToolboxForMac
+    - GoogleUtilities
+    - GoogleUtilitiesComponents
+    - GTMSessionFetcher
     - libevent
+    - MLImage
+    - MLKitCommon
+    - MLKitFaceDetection
+    - MLKitVision
     - MMKV
     - MMKVCore
+    - nanopb
+    - PromisesObjC
     - SocketRocket
 
 EXTERNAL SOURCES:
@@ -655,6 +730,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/vision-camera-resize-plugin"
   VisionCamera:
     :path: "../.."
+  VisionCameraFaceDetector:
+    :path: "../node_modules/react-native-vision-camera-face-detector"
   Yoga:
     :path: "../node_modules/react-native/ReactCommon/yoga"
 
@@ -665,10 +742,22 @@ SPEC CHECKSUMS:
   FBReactNativeSpec: 638095fe8a01506634d77b260ef8a322019ac671
   fmt: ff9d55029c625d3757ed641535fd4a75fedc7ce9
   glog: 04b94705f318337d7ead9e6d17c019bd9b1f6b1b
+  GoogleDataTransport: 6c09b596d841063d76d4288cc2d2f42cc36e1e2a
+  GoogleMLKit: 90ba06e028795a50261f29500d238d6061538711
+  GoogleToolboxForMac: 8bef7c7c5cf7291c687cf5354f39f9db6399ad34
+  GoogleUtilities: d053d902a8edaa9904e1bd00c37535385b8ed152
+  GoogleUtilitiesComponents: 679b2c881db3b615a2777504623df6122dd20afe
+  GTMSessionFetcher: 0e876eea9782ec6462e91ab872711c357322c94f
   hermes-engine: 9180d43df05c1ed658a87cc733dc3044cf90c00a
   libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913
-  MMKV: f902fb6719da13c2ab0965233d8963a59416f911
-  MMKVCore: d26e4d3edd5cb8588c2569222cbd8be4231374e9
+  MLImage: 1824212150da33ef225fbd3dc49f184cf611046c
+  MLKitCommon: afcd11b6c0735066a0dde8b4bf2331f6197cbca2
+  MLKitFaceDetection: 96effb0e6407e299fc286034f45d55267a212c26
+  MLKitVision: 90922bca854014a856f8b649d1f1f04f63fd9c79
+  MMKV: ed58ad794b3f88c24d604a5b74f3fba17fcbaf74
+  MMKVCore: a67a1cede26175c413176f404a7cedec43f96a0b
+  nanopb: 438bc412db1928dac798aa6fd75726007be04262
+  PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47
   RCT-Folly: 424b8c9a7a0b9ab2886ffe9c3b041ef628fd4fb1
   RCTRequired: 83bca1c184feb4d2e51c72c8369b83d641443f95
   RCTTypeSafety: 13c4a87a16d7db6cd66006ce9759f073402ef85b
@@ -717,8 +806,9 @@ SPEC CHECKSUMS:
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
   vision-camera-resize-plugin: cef68705394b21ef3ec186a8638f683df7440358
   VisionCamera: 0070916075536a3d2a2beaaa056a711492f379f2
+  VisionCameraFaceDetector: 9cdc4149c540dbb4995761d66c4f591b5952e652
   Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
 
 PODFILE CHECKSUM: 299b350392623e1b01615935e236438d90fd2cff
 
-COCOAPODS: 1.11.3
+COCOAPODS: 1.14.3
diff --git a/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj b/package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj
@@ -235,6 +235,11 @@
 			);
 			inputPaths = (
 				"${PODS_ROOT}/Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample-resources.sh",
+				"${PODS_CONFIGURATION_BUILD_DIR}/GTMSessionFetcher/GTMSessionFetcher_Core_Privacy.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/GoogleDataTransport/GoogleDataTransport_Privacy.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/GoogleUtilities/GoogleUtilities_Privacy.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/MLKitFaceDetection/GoogleMVFaceDetectorResources.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/PromisesObjC/FBLPromises_Privacy.bundle",
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/AntDesign.ttf",
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Entypo.ttf",
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/EvilIcons.ttf",
@@ -255,9 +260,15 @@
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/SimpleLineIcons.ttf",
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Zocial.ttf",
 				"${PODS_CONFIGURATION_BUILD_DIR}/React-Core/AccessibilityResources.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/nanopb/nanopb_Privacy.bundle",
 			);
 			name = "[CP] Copy Pods Resources";
 			outputPaths = (
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GTMSessionFetcher_Core_Privacy.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleDataTransport_Privacy.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleUtilities_Privacy.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVFaceDetectorResources.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/FBLPromises_Privacy.bundle",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AntDesign.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Entypo.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/EvilIcons.ttf",
@@ -278,6 +289,7 @@
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/SimpleLineIcons.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Zocial.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AccessibilityResources.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/nanopb_Privacy.bundle",
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;

diff --git a/package/example/package.json b/package/example/package.json
@@ -21,7 +21,6 @@
     "@shopify/react-native-skia": "^1.2.0",
     "react": "^18.2.0",
     "react-native": "^0.72.3",
-    "react-native-fast-tflite": "^1.2.0",
     "react-native-gesture-handler": "^2.12.1",
     "react-native-mmkv": "^2.10.2",
     "react-native-pressable-opacity": "^1.0.10",
@@ -31,8 +30,8 @@
     "react-native-static-safe-area-insets": "^2.2.0",
     "react-native-vector-icons": "^10.0.0",
     "react-native-video": "^5.2.1",
-    "react-native-worklets-core": "^1.0.0-beta.2",
-    "vision-camera-resize-plugin": "^2.1.1"
+    "react-native-vision-camera-face-detector": "^1.4.2",
+    "react-native-worklets-core": "^1.0.0-beta.2"
   },
   "devDependencies": {
     "@babel/core": "^7.22.10",

diff --git a/package/example/src/CameraPage.tsx b/package/example/src/CameraPage.tsx
@@ -26,8 +26,12 @@ import type { Routes } from './Routes'
 import type { NativeStackScreenProps } from '@react-navigation/native-stack'
 import { useIsFocused } from '@react-navigation/core'
 import { usePreferredCameraDevice } from './hooks/usePreferredCameraDevice'
-import { ClipOp, Skia, TileMode } from '@shopify/react-native-skia'
+import type { SkPoint } from '@shopify/react-native-skia'
+import { ClipOp, PaintStyle, PointMode, Skia, TileMode } from '@shopify/react-native-skia'
 import { useTensorflowModel } from 'react-native-fast-tflite'
+import { useResizePlugin } from 'vision-camera-resize-plugin'
+import type { Face } from 'react-native-vision-camera-face-detector'
+import { detectFaces } from 'react-native-vision-camera-face-detector'
 
 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera)
 Reanimated.addWhitelistedNativeProps({
@@ -177,50 +181,66 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
     location.requestPermission()
   }, [location])
 
-  const plugin = useTensorflowModel(require('./centerface_w640_h480.tflite'))
-  const model = plugin.model
-
+  // idk why but for some reason the face bounding boxes are divided by 2?
+  const MULTIPLIER = 2
   const frameProcessor = useSkiaFrameProcessor((frame) => {
     'worklet'
 
     frame.render()
 
-    if (model == null) {
-      console.log('Model is loading...')
-      return
-    }
-
-    const results = model.runSync([0])
-
-    const time = (performance.now() % 10000) / 1000
-    const radius = 150
-    const centerX = frame.width / 2
-    const centerY = frame.height / 2
-    const width = frame.width * 0.45
-    const speed = 3
-    const x = centerX + radius * Math.cos((time * 2 * Math.PI) / speed) - width / 2
-    const y = centerY + radius * Math.sin((time * 2 * Math.PI) / speed) - width / 2
-
-    const face = { x: x, y: y, width: width, height: width }
+    const result = detectFaces({
+      frame: frame,
+      options: {
+        performanceMode: 'fast',
+        contourMode: 'all',
+        landmarkMode: 'none',
+      },
+    })
 
-    const blurRadius = 10
+    const blurRadius = 20
     const blurFilter = Skia.ImageFilter.MakeBlur(blurRadius, blurRadius, TileMode.Repeat, null)
-
     const paint = Skia.Paint()
     paint.setImageFilter(blurFilter)
 
-    frame.save()
+    const debug = Skia.Paint()
+    debug.setColor(Skia.Color('red'))
+
+    for (const face of result.faces as unknown as Face[]) {
+      // detect faces
+
+      if (face.contours == null) {
+        console.log('no countours for this face!')
+        continue
+      }
+
+      const path = Skia.Path.Make() // Function to add a series of points to the path
+      const addContourToPath = (pointsArray: SkPoint[]) => {
+        if (pointsArray.length > 0) {
+          path.moveTo(pointsArray[0].x * 2, pointsArray[0].y * 2)
+          pointsArray.slice(1).forEach((point) => {
+            path.lineTo(point.x * 2, point.y * 2)
+          })
+          path.close() // Close each contour to complete its loop
+        }
+      }
+
+      // Iterate over each key in the contours object to add to the path
+      Object.keys(face.contours).forEach((key) => {
+        addContourToPath(face.contours[key])
+      })
+
+      // Save the current state of the canvas
+      frame.save()
 
-    // Define a circular clipping path
-    const path = Skia.Path.Make()
-    const margin = 10
-    path.addCircle(face.x + face.width / 2 + margin, face.y + face.height / 2 + margin, face.width / 2 - 2 * margin)
-    frame.clipPath(path, ClipOp.Intersect, true)
+      // Set the path as the clipping region
+      frame.clipPath(path, ClipOp.Intersect, true)
 
-    const faceRect = Skia.XYWHRect(face.x, face.y, face.width, face.height)
+      // Draw the image with the blur filter applied within the clipping region
+      frame.drawImage(frame.__skImage, 0, 0, paint)
 
-    frame.drawImageRect(frame.__skImage, faceRect, faceRect, paint)
-    frame.restore()
+      // Restore the canvas to remove the clip
+      frame.restore()
+    }
   }, [])
 
   const videoHdr = format?.supportsVideoHdr && enableHdr

diff --git a/package/example/src/centerface_w640_h480.tflite b/package/example/src/centerface_w640_h480.tflite
diff --git a/package/example/yarn.lock b/package/example/yarn.lock
@@ -5254,11 +5254,6 @@ react-is@^17.0.1:
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
   integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==
 
-react-native-fast-tflite@^1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/react-native-fast-tflite/-/react-native-fast-tflite-1.2.0.tgz#8813d268402675c58d050adaa407c241aa2b6d41"
-  integrity sha512-1hpDXU+XDJ/fHtI7v0ThJj9/bn/utk7C7FLE2Fa60sJawHcsOAxdOlh92BJzhxpYHBMO9367Jlr5f2bKC1DZmA==
-
 react-native-gesture-handler@^2.12.1:
   version "2.14.0"
   resolved "https://registry.yarnpkg.com/react-native-gesture-handler/-/react-native-gesture-handler-2.14.0.tgz#d6aec0d8b2e55c67557fd6107e828c0a1a248be8"
@@ -5330,6 +5325,11 @@ react-native-video@^5.2.1:
     prop-types "^15.7.2"
     shaka-player "^2.5.9"
 
+react-native-vision-camera-face-detector@^1.4.2:
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/react-native-vision-camera-face-detector/-/react-native-vision-camera-face-detector-1.4.2.tgz#fce9b99b4fc3b41163cc39a4e55df182d67e47ae"
+  integrity sha512-aw7NbHgkavpxBbYOQMBhsz/HYJR6egeGCcram/Hb432jmw4US+9yLvUoTWk+Evv3yvShzvLzI4i9cGHn3I1BkA==
+
 react-native-worklets-core@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/react-native-worklets-core/-/react-native-worklets-core-1.0.0-beta.2.tgz#ee72834557b8585c0400653f0a52427e25fdd57d"
@@ -6281,11 +6281,6 @@ vary@~1.1.2:
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
   integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==
 
-vision-camera-resize-plugin@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/vision-camera-resize-plugin/-/vision-camera-resize-plugin-2.1.1.tgz#e146963f6a0ce7688718031e384999f0f44f7d47"
-  integrity sha512-5Vpn2NyRPrj05n+bcZdBXFFhxdzZt247gDvSz/EA+swmFYZcq+WDuT5UOAeukWEIG2Uk3JLHq6RSVd97HK6vgA==
-
 vlq@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/vlq/-/vlq-1.0.1.tgz#c003f6e7c0b4c1edd623fd6ee50bbc0d6a1de468"