LG-11377 Add Selfie UI Behind Feature Flag

app/javascript/packages/document-capture/components/acuant-camera.tsx

@@ -1,13 +1,16 @@
-import { useContext, useEffect } from 'react';
+import React, { useContext, useEffect } from 'react';
 import type { ReactNode } from 'react';
 import { useI18n } from '@18f/identity-react-i18n';
 import { useImmutableCallback } from '@18f/identity-react-hooks';
 import AcuantContext from '../context/acuant';
 
 declare let AcuantCameraUI: AcuantCameraUIInterface;
+declare let AcuantPassiveLiveness: AcuantPassiveLivenessInterface;
+
 declare global {
   interface Window {
     AcuantCameraUI: AcuantCameraUIInterface;
+    AcuantPassiveLiveness: AcuantPassiveLivenessInterface;
   }
 }
 
@@ -16,6 +19,7 @@ declare global {
  */
 type AcuantGlobals = {
   AcuantCameraUI: AcuantCameraUIInterface;
+  AcuantPassiveLiveness: AcuantPassiveLivenessInterface;
   AcuantCamera: AcuantCameraInterface;
 };
 export type AcuantGlobal = Window & AcuantGlobals;
@@ -138,6 +142,8 @@ type AcuantCameraUIStart = (
   options?: AcuantCameraUIOptions,
 ) => void;
 
+type AcuantPassiveLivenessStart = () => void;
+
 interface AcuantCameraUIInterface {
   /**
    * Start capture
@@ -148,6 +154,16 @@ interface AcuantCameraUIInterface {
    */
   end: () => void;
 }
+interface AcuantPassiveLivenessInterface {
+  /**
+   * Start capture
+   */
+  start: AcuantPassiveLivenessStart;
+  /**
+   * End capture
+   */
+  end: () => void;
+}
 
 type AcuantCameraStart = (
   callback: (response: AcuantImage) => void,
@@ -263,6 +279,10 @@ interface AcuantCameraContextProps {
    * Crop started callback, invoked after capture is made and before image has been evaluated
    */
   onCropStart: () => void;
+  /**
+   * Whether this camera is for selfie mode (other option is captureing an id)
+   */
+  selfieMode: boolean;
   /**
    * React children node
    */
@@ -289,10 +309,22 @@ const getActualAcuantCameraUI = (): AcuantCameraUIInterface => {
   return AcuantCameraUI;
 };
 
+const getActualAcuantPassiveLiveness = (): AcuantPassiveLivenessInterface => {
+  if (window.AcuantPassiveLiveness) {
+    return window.AcuantPassiveLiveness;
+  }
+  if (typeof AcuantPassiveLiveness === 'undefined') {
+    // eslint-disable-next-line no-console
+    console.error('AcuantCameraUI is not defined in the global scope');
+  }
+  return AcuantPassiveLiveness;
+};
+
 function AcuantCamera({
   onImageCaptureSuccess = () => {},
   onImageCaptureFailure = () => {},
   onCropStart = () => {},
+  selfieMode = false,
   children,
 }: AcuantCameraContextProps) {
   const { isReady, setIsActive } = useContext(AcuantContext);
@@ -307,6 +339,45 @@ function AcuantCamera({
     },
     [onImageCaptureSuccess],
   );
+  const faceCaptureCallback = {
+    onDetectorInitialized: () => {
+      console.log('onDetectorInitialized');
+      // This callback is triggered when the face detector is ready.
+      // Until then, no actions are executed and the user sees only the camera stream.
+      // You can opt to display an alert before the callback is triggered.
+    },
+    onDetection: (text) => {
+      console.log('onDetection', text);
+      // Triggered when the face does not pass the scan. The UI element
+      // should be updated here to provide guidence to the user
+    },
+    onOpened: () => {
+      // Camera has opened
+      console.log('onOpened');
+    },
+    onClosed: () => {
+      // Camera has closed
+      console.log('onClosed');
+    },
+    onError: (error) => {
+      // Error occurred. Camera permission not granted will
+      // manifest here with 1 as error code. Unexpected errors will have 2 as error code.
+      console.log('onError', error);
+    },
+    onPhotoTaken: () => {
+      // The photo has been taken and it's showing a preview with a button to accept or retake the image.
+      console.log('onPhotoTaken');
+    },
+    onPhotoRetake: () => {
+      // Triggered when retake button is tapped
+      console.log('onPhotoRetake');
+    },
+    onCaptured: (base64Image) => {
+      // Triggered when accept button is tapped
+      console.log('onCaptured');
+      //onImageCaptureSuccess({image: base64Image});
+    },
+  };
 
   useEffect(() => {
     const textOptions = {
@@ -319,7 +390,24 @@ function AcuantCamera({
         TAP_TO_CAPTURE: t('doc_auth.info.capture_status_tap_to_capture'),
       },
     };
-    if (isReady) {
+    const faceDetectionStates = {
+      FACE_NOT_FOUND: 'FACE NOT FOUND',
+      TOO_MANY_FACES: 'TOO MANY FACES',
+      FACE_ANGLE_TOO_LARGE: 'FACE ANGLE TOO LARGE',
+      PROBABILITY_TOO_SMALL: 'PROBABILITY TOO SMALL',
+      FACE_TOO_SMALL: 'FACE TOO SMALL',
+      FACE_CLOSE_TO_BORDER: 'TOO CLOSE TO THE FRAME',
+    };
+
+    const cleanupCamera = () => {
+      window.AcuantCameraUI.end();
+      setIsActive(false);
+    };
+    const cleanupSelfieCamera = () => {
+      window.AcuantPassiveLiveness.end();
+      setIsActive(false);
+    };
+    const startCamera = () => {
       const onFailureCallbackWithOptions = (...args) => onImageCaptureFailure(...args);
       Object.keys(textOptions).forEach((key) => {
         onFailureCallbackWithOptions[key] = textOptions[key];
@@ -336,12 +424,21 @@ function AcuantCamera({
         textOptions,
       );
       setIsActive(true);
-    }
+    };
+    const startSelfieCamera = () => {
+      window.AcuantPassiveLiveness = getActualAcuantPassiveLiveness();
+      // This opens the native camera, but TODO callbacks
+      //window.AcuantPassiveLiveness.startManualCapture((image) => console.log('image', image));
+      window.AcuantPassiveLiveness.start(faceCaptureCallback, faceDetectionStates);
+      setIsActive(true);
+    };
 
+    if (isReady) {
+      selfieMode ? startSelfieCamera() : startCamera();
+    }
     return () => {
       if (isReady) {
-        window.AcuantCameraUI.end();
-        setIsActive(false);
+        selfieMode ? cleanupSelfieCamera() : cleanupCamera();
       }
     };
   }, [isReady]);

app/javascript/packages/document-capture/components/acuant-capture-canvas.jsx

@@ -67,6 +67,7 @@ function AcuantCaptureCanvas() {
         </p>
       )}
       <div id="acuant-camera" ref={cameraRef} className="acuant-capture-canvas__camera" />
+      <div id="acuant-face-capture-container" ref={cameraRef} />
       <button
         type="button"
         onClick={clickCanvas}

app/javascript/packages/document-capture/components/acuant-capture.tsx

@@ -600,6 +600,7 @@ function AcuantCapture(
     <div className={[className, 'document-capture-acuant-capture'].filter(Boolean).join(' ')}>
       {isCapturingEnvironment && (
         <AcuantCamera
+          selfieMode={name === 'selfie'}
           onCropStart={() => setHasStartedCropping(true)}
           onImageCaptureSuccess={onAcuantImageCaptureSuccess}
           onImageCaptureFailure={onAcuantImageCaptureFailure}

app/javascript/packages/document-capture/components/document-side-acuant-capture.jsx

@@ -10,7 +10,7 @@ import AcuantCapture from './acuant-capture';
 /**
  * @typedef DocumentSideAcuantCaptureProps
  *
- * @prop {'front'|'back'} side
+ * @prop {'front'|'back'|'selfie'} side
  * @prop {RegisterFieldCallback} registerField
  * @prop {Blob|string|null|undefined} value
  * @prop {(nextValues:{[key:string]: Blob|string|null|undefined})=>void} onChange Update values,
@@ -50,9 +50,11 @@ function DocumentSideAcuantCapture({
       ref={registerField(side, { isRequired: true })}
       /* i18n-tasks-use t('doc_auth.headings.document_capture_back') */
       /* i18n-tasks-use t('doc_auth.headings.document_capture_front') */
+      /* i18n-tasks-use t('doc_auth.headings.document_capture_selfie') */
       label={t(`doc_auth.headings.document_capture_${side}`)}
       /* i18n-tasks-use t('doc_auth.headings.back') */
       /* i18n-tasks-use t('doc_auth.headings.front') */
+      /* i18n-tasks-use t('doc_auth.headings.selfie') */
       bannerText={t(`doc_auth.headings.${side}`)}
       value={value}
       onChange={(nextValue, metadata) => {

app/javascript/packages/document-capture/components/documents-step.jsx

@@ -10,7 +10,7 @@ import UploadContext from '../context/upload';
 import TipList from './tip-list';
 
 /**
- * @typedef {'front'|'back'} DocumentSide
+ * @typedef {'front'|'back'|'selfie'} DocumentSide
  */
 
 /**
@@ -27,7 +27,7 @@ import TipList from './tip-list';
  *
  * @type {DocumentSide[]}
  */
-const DOCUMENT_SIDES = ['front', 'back'];
+const DOCUMENT_SIDES = ['front', 'back', 'selfie'];
 
 /**
  * @param {import('@18f/identity-form-steps').FormStepComponentProps<DocumentsStepValue>} props Props object.

config/locales/doc_auth/en.yml

@@ -169,7 +169,9 @@ en:
       document_capture: Add photos of your ID
       document_capture_back: Back of your ID
       document_capture_front: Front of your ID
+      document_capture_selfie: Selfie
       front: Front of your driver’s license or state ID
+      selfie: Photo 
       getting_started: Let’s verify your identity for %{sp_name}
       how_to_verify: Choose how you want to verify your identity
       hybrid_handoff: How would you like to add your ID?

public/acuant/11.9.1/face_landmark_68_tiny_model-weights_manifest.json

@@ -0,0 +1,39 @@
+[
+  {
+      "weights":
+      [
+          {"name":"dense0/conv0/filters","shape":[3,3,3,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008194216092427571,"min":-0.9423348506291708}},
+          {"name":"dense0/conv0/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006839508168837603,"min":-0.8412595047670252}},
+          {"name":"dense0/conv1/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.009194007106855804,"min":-1.2779669878529567}},
+          {"name":"dense0/conv1/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0036026100317637128,"min":-0.3170296827952067}},
+          {"name":"dense0/conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.000740380117706224,"min":-0.06367269012273527}},
+          {"name":"dense0/conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":1,"min":0}},
+          {"name":"dense0/conv2/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":1,"min":0}},
+          {"name":"dense0/conv2/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0037702228508743585,"min":-0.6220867703942692}},
+          {"name":"dense1/conv0/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0033707996209462483,"min":-0.421349952618281}},
+          {"name":"dense1/conv0/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014611541991140328,"min":-1.8556658328748217}},
+          {"name":"dense1/conv0/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002832523046755323,"min":-0.30307996600281956}},
+          {"name":"dense1/conv1/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006593170586754294,"min":-0.6329443763284123}},
+          {"name":"dense1/conv1/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.012215249211180444,"min":-1.6001976466646382}},
+          {"name":"dense1/conv1/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002384825547536214,"min":-0.3028728445370992}},
+          {"name":"dense1/conv2/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005859645441466687,"min":-0.7617539073906693}},
+          {"name":"dense1/conv2/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013121426806730382,"min":-1.7845140457153321}},
+          {"name":"dense1/conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032247188044529336,"min":-0.46435950784122243}},
+          {"name":"dense2/conv0/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002659512618008782,"min":-0.32977956463308894}},
+          {"name":"dense2/conv0/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.015499923743453681,"min":-1.9839902391620712}},
+          {"name":"dense2/conv0/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032450980999890497,"min":-0.522460794098237}},
+          {"name":"dense2/conv1/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005911862382701799,"min":-0.792189559282041}},
+          {"name":"dense2/conv1/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.021025861478319356,"min":-2.2077154552235325}},
+          {"name":"dense2/conv1/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00349616945958605,"min":-0.46149436866535865}},
+          {"name":"dense2/conv2/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008104994250278847,"min":-1.013124281284856}},
+          {"name":"dense2/conv2/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.029337059282789044,"min":-3.5791212325002633}},
+          {"name":"dense2/conv2/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0038808938334969913,"min":-0.4230174278511721}},
+          {"name":"fc/weights","shape":[128,136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014016061670639936,"min":-1.8921683255363912}},
+          {"name":"fc/bias","shape":[136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0029505149698724935,"min":0.088760145008564}}
+      ],
+      "paths":
+      [
+          "face_landmark_68_tiny_model.bin"
+      ]
+  }
+]

public/acuant/11.9.1/tiny_face_detector_model-weights_manifest.json

@@ -0,0 +1 @@
+[{"weights":[{"name":"conv0/filters","shape":[3,3,3,16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.009007044399485869,"min":-1.2069439495311063}},{"name":"conv0/bias","shape":[16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005263455241334205,"min":-0.9211046672334858}},{"name":"conv1/depthwise_filter","shape":[3,3,16,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.004001977630690033,"min":-0.5042491814669441}},{"name":"conv1/pointwise_filter","shape":[1,1,16,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013836609615999109,"min":-1.411334180831909}},{"name":"conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0015159862590771096,"min":-0.30926119685173037}},{"name":"conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002666276225856706,"min":-0.317286870876948}},{"name":"conv2/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.015265831292844286,"min":-1.6792414422128714}},{"name":"conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0020280554598453,"min":-0.37113414915168985}},{"name":"conv3/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006100742489683862,"min":-0.8907084034938438}},{"name":"conv3/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.016276211832083907,"min":-2.0508026908425725}},{"name":"conv3/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003394414279975143,"min":-0.7637432129944072}},{"name":"conv4/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006716050119961009,"min":-0.8059260143953211}},{"name":"conv4/pointwise_filter","shape":[1,1,128,256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.021875603993733724,"min":-2.8875797271728514}},{"name":"conv4/bias","shape":[256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0041141652009066415,"min":-0.8187188749804216}},{"name":"conv5/depthwise_filter","shape":[3,3,256,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008423839597141042,"min":-0.9013508368940915}},{"name":"conv5/pointwise_filter","shape":[1,1,256,512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.030007277283014035,"min":-3.8709387695088107}},{"name":"conv5/bias","shape":[512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008402082966823203,"min":-1.4871686851277068}},{"name":"conv8/filters","shape":[1,1,512,25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.028336129469030042,"min":-4.675461362389957}},{"name":"conv8/bias","shape":[25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002268134028303857,"min":-0.41053225912299807}}],"paths":["tiny_face_detector_model-shard1"]}]