LG-8754 Targeted AB test logging

app/controllers/idv/in_person/ready_to_verify_controller.rb

@@ -10,7 +10,7 @@ class ReadyToVerifyController < ApplicationController
       before_action :confirm_in_person_session
 
       def show
-        analytics.idv_in_person_ready_to_verify_visit
+        analytics.idv_in_person_ready_to_verify_visit(**extra_analytics_attributes)
         @presenter = ReadyToVerifyPresenter.new(enrollment: enrollment)
       end
 
@@ -23,6 +23,13 @@ def confirm_in_person_session
       def enrollment
         current_user.pending_in_person_enrollment
       end
+
+      def extra_analytics_attributes
+        extra = {}
+        bucket = session[:in_person_cta_variant]
+        extra[:in_person_cta_variant] = bucket if bucket
+        extra
+      end
     end
   end
 end

app/javascript/packages/document-capture/components/in-person-call-to-action.tsx

@@ -3,6 +3,7 @@ import { Button } from '@18f/identity-components';
 import { useInstanceId } from '@18f/identity-react-hooks';
 import { t } from '@18f/identity-i18n';
 import AnalyticsContext from '../context/analytics';
+import { InPersonContext } from '../context';
 
 interface InPersonCallToActionProps {
   altHeading?: string;
@@ -13,6 +14,7 @@ interface InPersonCallToActionProps {
 function InPersonCallToAction({ altHeading, altPrompt, altButtonText }: InPersonCallToActionProps) {
   const instanceId = useInstanceId();
   const { trackEvent } = useContext(AnalyticsContext);
+  const { inPersonCtaVariantActive } = useContext(InPersonContext);
 
   return (
     <section
@@ -30,7 +32,11 @@ function InPersonCallToAction({ altHeading, altPrompt, altButtonText }: InPerson
         isWide
         href="#location"
         className="margin-top-3 margin-bottom-1"
-        onClick={() => trackEvent('IdV: verify in person troubleshooting option clicked')}
+        onClick={() =>
+          trackEvent('IdV: verify in person troubleshooting option clicked', {
+            in_person_cta_variant: inPersonCtaVariantActive,
+          })
+        }
       >
         {altButtonText || t('in_person_proofing.body.cta.button')}
       </Button>

app/javascript/packages/document-capture/components/in-person-location-post-office-search-step.tsx

@@ -11,8 +11,10 @@ import AddressSearch, {
   LOCATIONS_URL,
 } from './address-search';
 import InPersonLocations, { FormattedLocation } from './in-person-locations';
+import { InPersonContext } from '../context';
 
 function InPersonLocationPostOfficeSearchStep({ onChange, toPreviousStep, registerField }) {
+  const { inPersonCtaVariantActive } = useContext(InPersonContext);
   const { t } = useI18n();
   const [inProgress, setInProgress] = useState<boolean>(false);
   const [isLoadingLocations, setLoadingLocations] = useState<boolean>(false);
@@ -41,7 +43,10 @@ function InPersonLocationPostOfficeSearchStep({ onChange, toPreviousStep, regist
       const selectedLocation = locationResults![id]!;
       const { streetAddress, formattedCityStateZip } = selectedLocation;
       const selectedLocationAddress = `${streetAddress}, ${formattedCityStateZip}`;
-      setSubmitEventMetadata({ selected_location: selectedLocationAddress });
+      setSubmitEventMetadata({
+        selected_location: selectedLocationAddress,
+        in_person_cta_variant: inPersonCtaVariantActive,
+      });
       onChange({ selectedLocationAddress });
       if (autoSubmit) {
         setDisabledAddressSearch(true);

app/javascript/packages/document-capture/components/in-person-location-step.spec.tsx

@@ -63,6 +63,6 @@ describe('InPersonLocationStep', () => {
 
     await userEvent.click(button);
 
-    await findByText('{"selected_location":"Baltimore"}');
+    await findByText('{"selected_location":"Baltimore","in_person_cta_variant":""}');
   });
 });

app/javascript/packages/document-capture/components/in-person-location-step.tsx

@@ -6,6 +6,7 @@ import BackButton from './back-button';
 import LocationCollection from './location-collection';
 import LocationCollectionItem from './location-collection-item';
 import AnalyticsContext from '../context/analytics';
+import { InPersonContext } from '../context';
 
 interface PostOffice {
   address: string;
@@ -79,6 +80,7 @@ const prepToSend = (location: object) => {
 };
 
 function InPersonLocationStep({ onChange, toPreviousStep }) {
+  const { inPersonCtaVariantActive } = useContext(InPersonContext);
   const { t } = useI18n();
   const [locationData, setLocationData] = useState([] as FormattedLocation[]);
   const [foundAddress] = useState({} as LocationQuery);
@@ -102,7 +104,10 @@ function InPersonLocationStep({ onChange, toPreviousStep }) {
     async (e: any, id: number) => {
       const selectedLocation = locationData[id];
       const { name: selectedLocationName } = selectedLocation;
-      setSubmitEventMetadata({ selected_location: selectedLocationName });
+      setSubmitEventMetadata({
+        selected_location: selectedLocationName,
+        in_person_cta_variant: inPersonCtaVariantActive,
+      });
       onChange({ selectedLocationName });
       if (autoSubmit) {
         return;

app/javascript/packages/document-capture/components/review-issues-step.tsx

@@ -89,7 +89,9 @@ function ReviewIssuesStep({
     setHasDismissed(true);
   }
   function onInPersonSelected() {
-    trackEvent('IdV: verify in person troubleshooting option clicked');
+    trackEvent('IdV: verify in person troubleshooting option clicked', {
+      in_person_cta_variant: inPersonCtaVariantActive,
+    });
   }
 
   // let FormSteps know, via FormStepsContext, whether this page
@@ -102,13 +104,9 @@ function ReviewIssuesStep({
     if (!inPersonURL || isFailedResult) {
       return;
     }
-    if (inPersonCtaVariantActive === 'in_person_variant_a') {
-      trackEvent('IdV: IPP CTA Variant A');
-    } else if (inPersonCtaVariantActive === 'in_person_variant_b') {
-      trackEvent('IdV: IPP CTA Variant B');
-    } else if (inPersonCtaVariantActive === 'in_person_variant_c') {
-      trackEvent('IdV: IPP CTA Variant C');
-    }
+    trackEvent('IdV: IPP CTA Variant Displayed', {
+      in_person_cta_variant: inPersonCtaVariantActive,
+    });
   }, []);
 
   if (!hasDismissed) {

app/javascript/packages/document-capture/context/analytics.spec.tsx

@@ -32,7 +32,9 @@ describe('AnalyticsContextProvider', () => {
 
     result.current.trackVisitEvent(stepName);
 
-    expect(trackEvent).to.have.been.calledWith(`IdV: ${stepName} visited`);
+    expect(trackEvent).to.have.been.calledWith(`IdV: ${stepName} visited`, {
+      in_person_cta_variant: '',
+    });
   });
 
   it('calls trackEvent with submit event', () => {

app/javascript/packages/document-capture/context/analytics.tsx

@@ -1,6 +1,7 @@
-import { createContext, useState } from 'react';
+import { createContext, useContext, useState } from 'react';
 import type { ReactNode } from 'react';
 import type { trackEvent } from '@18f/identity-analytics';
+import InPersonContext from './in-person';
 
 type EventMetadata = Record<string, any>;
 
@@ -66,9 +67,18 @@ export function AnalyticsContextProvider({ children, trackEvent }: AnalyticsCont
 
     setSubmitEventMetadataState(DEFAULT_EVENT_METADATA);
   };
+  const { inPersonCtaVariantActive } = useContext(InPersonContext);
+
+  const extraAnalyticsAttributes = (stepName) => {
+    const extra: EventMetadata = { ...DEFAULT_EVENT_METADATA };
+    if (stepName === 'location') {
+      extra.in_person_cta_variant = inPersonCtaVariantActive;
+    }
+    return extra;
+  };
   const trackVisitEvent: TrackVisitEvent = (stepName) => {
     if (LOGGED_STEPS.includes(stepName)) {
-      trackEvent(`IdV: ${stepName} visited`);
+      trackEvent(`IdV: ${stepName} visited`, extraAnalyticsAttributes(stepName));
     }
   };
 

app/javascript/packs/document-capture.tsx

@@ -128,6 +128,17 @@ const trackEvent: typeof baseTrackEvent = (event, payload) => {
   const App = composeComponents(
     [MarketingSiteContextProvider, { helpCenterRedirectURL, securityAndPrivacyHowItWorksURL }],
     [DeviceContext.Provider, { value: device }],
+    [
+      InPersonContext.Provider,
+      {
+        value: {
+          arcgisSearchEnabled: arcgisSearchEnabled === 'true',
+          inPersonCtaVariantTestingEnabled: inPersonCtaVariantTestingEnabled === true,
+          inPersonCtaVariantActive,
+          inPersonURL,
+        },
+      },
+    ],
     [AnalyticsContextProvider, { trackEvent }],
     [
       AcuantContextProvider,
@@ -172,17 +183,6 @@ const trackEvent: typeof baseTrackEvent = (event, payload) => {
         maxSubmissionAttemptsBeforeNativeCamera: Number(maxSubmissionAttemptsBeforeNativeCamera),
       },
     ],
-    [
-      InPersonContext.Provider,
-      {
-        value: {
-          arcgisSearchEnabled: arcgisSearchEnabled === 'true',
-          inPersonCtaVariantTestingEnabled: inPersonCtaVariantTestingEnabled === true,
-          inPersonCtaVariantActive,
-          inPersonURL,
-        },
-      },
-    ],
     [DocumentCapture, { isAsyncForm, onStepChange: keepAlive }],
   );
 

app/services/analytics_events.rb

@@ -555,11 +555,14 @@ def idv_come_back_later_visit(proofing_components: nil, **extra)
   end
 
   # @param [String] flow_path Document capture path ("hybrid" or "standard")
+  # @param [String] in_person_cta_variant Variant testing bucket label
   # The user clicked the troubleshooting option to start in-person proofing
-  def idv_verify_in_person_troubleshooting_option_clicked(flow_path:, **extra)
+  def idv_verify_in_person_troubleshooting_option_clicked(flow_path:, in_person_cta_variant:,
+                                                          **extra)
     track_event(
       'IdV: verify in person troubleshooting option clicked',
       flow_path: flow_path,
+      in_person_cta_variant: in_person_cta_variant,
       **extra,
     )
   end
@@ -627,9 +630,15 @@ def idv_inherited_proofing_redo_retrieve_user_info_submitted(**extra)
   end
 
   # @param [String] flow_path Document capture path ("hybrid" or "standard")
+  # @param [String] in_person_cta_variant Variant testing bucket label
   # The user visited the in person proofing location step
-  def idv_in_person_location_visited(flow_path:, **extra)
-    track_event('IdV: in person proofing location visited', flow_path: flow_path, **extra)
+  def idv_in_person_location_visited(flow_path:, in_person_cta_variant:, **extra)
+    track_event(
+      'IdV: in person proofing location visited',
+      flow_path: flow_path,
+      in_person_cta_variant: in_person_cta_variant,
+      **extra,
+    )
   end
 
   # @param [Boolean] success
@@ -661,13 +670,16 @@ def idv_in_person_locations_searched(
   end
 
   # @param [String] selected_location Selected in-person location
+  # @param [String] in_person_cta_variant Variant testing bucket label
   # @param [String] flow_path Document capture path ("hybrid" or "standard")
   # The user submitted the in person proofing location step
-  def idv_in_person_location_submitted(selected_location:, flow_path:, **extra)
+  def idv_in_person_location_submitted(selected_location:, in_person_cta_variant:, flow_path:,
+                                       **extra)
     track_event(
       'IdV: in person proofing location submitted',
       selected_location: selected_location,
       flow_path: flow_path,
+      in_person_cta_variant: in_person_cta_variant,
       **extra,
     )
   end
@@ -724,11 +736,14 @@ def idv_in_person_switch_back_submitted(flow_path:, **extra)
     track_event('IdV: in person proofing switch_back submitted', flow_path: flow_path, **extra)
   end
 
+  # @param [String] in_person_cta_variant Variant testing bucket label
   # @param [Idv::ProofingComponentsLogging] proofing_components User's current proofing components
   # The user visited the "ready to verify" page for the in person proofing flow
-  def idv_in_person_ready_to_verify_visit(proofing_components: nil, **extra)
+  def idv_in_person_ready_to_verify_visit(in_person_cta_variant: nil, proofing_components: nil,
+                                          **extra)
     track_event(
       'IdV: in person ready to verify visited',
+      in_person_cta_variant: in_person_cta_variant,
       proofing_components: proofing_components,
       **extra,
     )

app/services/idv/steps/document_capture_step.rb

@@ -71,6 +71,7 @@ def acuant_sdk_upgrade_a_b_testing_variables
 
       def in_person_cta_variant_testing_variables
         bucket = AbTests::IN_PERSON_CTA.bucket(flow_session[:document_capture_session_uuid])
+        session[:in_person_cta_variant] = bucket
         {
           in_person_cta_variant_testing_enabled:
           IdentityConfig.store.in_person_cta_variant_testing_enabled,

config/application.yml.default

@@ -133,7 +133,7 @@ idv_send_link_max_attempts: 5
 ie11_support_end_date: '2022-12-31'
 idv_sp_required: false
 in_person_cta_variant_testing_enabled: true
-in_person_cta_variant_testing_percents: '{"A":50, "B":50}'
+in_person_cta_variant_testing_percents: '{"B":100}'
 in_person_email_reminder_early_benchmark_in_days: 11
 in_person_email_reminder_final_benchmark_in_days: 1
 in_person_email_reminder_late_benchmark_in_days: 4

spec/controllers/frontend_log_controller_spec.rb

@@ -51,6 +51,7 @@
               'IdV: in person proofing location submitted',
               selected_location: selected_location,
               flow_path: flow_path,
+              in_person_cta_variant: nil,
             )
             expect(response).to have_http_status(:ok)
             expect(json[:success]).to eq(true)
@@ -66,6 +67,7 @@
                 'IdV: in person proofing location submitted',
                 flow_path: nil,
                 selected_location: nil,
+                in_person_cta_variant: nil,
               )
             end
           end

spec/features/idv/analytics_spec.rb

@@ -87,9 +87,9 @@
       'Frontend: IdV: back image added' => { 'width' => 284, 'height' => 38, 'mimeType' => 'image/png', 'source' => 'upload', 'size' => 3694, 'attempt' => 1, 'flow_path' => 'standard', 'acuant_sdk_upgrade_a_b_testing_enabled' => 'false', 'use_alternate_sdk' => anything, 'acuant_version' => anything },
       'IdV: doc auth image upload form submitted' => { success: true, errors: {}, attempts: 1, remaining_attempts: 3, user_id: user.uuid, flow_path: 'standard' },
       'IdV: doc auth image upload vendor submitted' => hash_including(success: true, flow_path: 'standard', attention_with_barcode: true, doc_auth_result: 'Attention'),
-      'IdV: verify in person troubleshooting option clicked' => { flow_path: 'standard' },
-      'IdV: in person proofing location visited' => { flow_path: 'standard' },
-      'IdV: in person proofing location submitted' => { flow_path: 'standard', selected_location: '606 E JUNEAU AVE, MILWAUKEE, WI, 53202-9998' },
+      'IdV: verify in person troubleshooting option clicked' => { flow_path: 'standard', in_person_cta_variant: 'in_person_variant_a' },
+      'IdV: in person proofing location visited' => { flow_path: 'standard', in_person_cta_variant: 'in_person_variant_a' },
+      'IdV: in person proofing location submitted' => { flow_path: 'standard', selected_location: '606 E JUNEAU AVE, MILWAUKEE, WI, 53202-9998', in_person_cta_variant: 'in_person_variant_a' },
       'IdV: in person proofing prepare visited' => { flow_path: 'standard' },
       'IdV: in person proofing prepare submitted' => { flow_path: 'standard' },
       'IdV: in person proofing state_id visited' => { step: 'state_id', flow_path: 'standard', step_count: 1, analytics_id: 'In Person Proofing', irs_reproofing: false },
@@ -113,7 +113,7 @@
       'IdV: personal key visited' => { proofing_components: { document_check: 'usps', source_check: 'aamva', resolution_check: 'lexis_nexis', address_check: 'lexis_nexis_address' }, address_verification_method: 'phone' },
       'IdV: personal key acknowledgment toggled' => { checked: true, proofing_components: { document_check: 'usps', source_check: 'aamva', resolution_check: 'lexis_nexis', address_check: 'lexis_nexis_address' } },
       'IdV: personal key submitted' => { proofing_components: { document_check: 'usps', source_check: 'aamva', resolution_check: 'lexis_nexis', address_check: 'lexis_nexis_address' }, address_verification_method: 'phone', fraud_review_pending: false, fraud_rejection: false, deactivation_reason: 'in_person_verification_pending' },
-      'IdV: in person ready to verify visited' => { proofing_components: { document_check: 'usps', source_check: 'aamva', resolution_check: 'lexis_nexis', address_check: 'lexis_nexis_address' } },
+      'IdV: in person ready to verify visited' => { proofing_components: { document_check: 'usps', source_check: 'aamva', resolution_check: 'lexis_nexis', address_check: 'lexis_nexis_address' }, in_person_cta_variant: 'in_person_variant_a' },
       'IdV: user clicked what to bring link on ready to verify page' => {},
       'IdV: user clicked sp link on ready to verify page' => {},
     }
@@ -186,6 +186,9 @@
 
     before do
       allow(IdentityConfig.store).to receive(:in_person_proofing_enabled).and_return(true)
+      allow(AbTests::IN_PERSON_CTA).to receive(:bucket).and_return(:in_person_variant_a)
+      allow(IdentityConfig.store).to receive(:in_person_cta_variant_testing_enabled).
+        and_return(false)
       allow(Idv::InPersonConfig).to receive(:enabled_for_issuer?).and_return(true)
       ServiceProvider.find_by(issuer: sp1_issuer).update(return_to_sp_url: return_sp_url)