Fix flaky redo document capture feature test 100% for sure, no chance of failure

Gemfile

@@ -118,6 +118,7 @@ end
 group :test do
   gem 'axe-core-rspec', '~> 4.2'
   gem 'bundler-audit', require: false
+  gem 'capybara-screenshot'
   gem 'simplecov', '~> 0.21.0', require: false
   gem 'simplecov-cobertura'
   gem 'simplecov_json_formatter'

Gemfile.lock

@@ -222,6 +222,9 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
+    capybara-screenshot (1.0.26)
+      capybara (>= 1.0, < 4)
+      launchy
     cbor (0.5.9.6)
     choice (0.2.0)
     chunky_png (1.4.0)
@@ -748,6 +751,7 @@ DEPENDENCIES
   browser
   bullet (~> 7.0)
   bundler-audit
+  capybara-screenshot
   capybara-webmock!
   connection_pool
   cssbundling-rails

MH-deflakify.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Usage: script RUNS COMMAND ["string signifying a real failure"]
+# Execute COMMAND RUNS times, track outputs and success rates.
+
+set -euo pipefail
+
+LOG_DIR=${LOG_DIR:-.deflakify}
+RUN_ID="run_$(date "+%Y%m%d-%H%M%S")"
+RUN_DIR="${LOG_DIR}/${RUN_ID}"
+
+mkdir -p "$RUN_DIR"
+
+RUNS=$1; shift
+COMMAND=$1; shift
+STRING_THAT_MEANS_FAILURE=${1:-}; shift
+
+ITERATION=0
+SUCCESSES=0
+FAILURES=0
+REAL_FAILURES=0
+
+while (true)
+do
+    ITERATION=$(($ITERATION+1))
+    OUT_FILE="$RUN_DIR/$ITERATION.out.txt"
+    ERR_FILE="$RUN_DIR/$ITERATION.err.txt"
+
+    STARTED_AT=$(date '+%Y-%m-%d %H:%M:%S')
+
+    echo ""
+
+    echo "
+Iteration:   ${ITERATION}
+Command:     $COMMAND
+Started at:  $STARTED_AT" | tee "$OUT_FILE" "$ERR_FILE"
+
+    if sh -c "$COMMAND" >> "$OUT_FILE"  2>> "$ERR_FILE"; then
+        echo "Result:      ✅ Success"
+        SUCCESSES=$(($SUCCESSES + 1))
+    else
+        FAILURES=$(($FAILURES + 1))
+
+        if grep "$STRING_THAT_MEANS_FAILURE" "$OUT_FILE" "$ERR_FILE" > /dev/null 2>&1; then
+            echo "Result:      ❌ Failure"
+            REAL_FAILURES=$(($REAL_FAILURES + 1))
+        else
+            echo "Result:      🤔 Failure, but ignoring"
+        fi
+    fi
+
+    echo "Finished at: $(date '+%Y-%m-%d %H:%M:%S')"
+
+    SUCCESS_PCT=$((($SUCCESSES * 100) / $ITERATION))
+    FAILURE_PCT=$((($FAILURES * 100) / $ITERATION))
+    REAL_FAILURE_PCT=$((($REAL_FAILURES * 100) / $ITERATION))
+
+    echo ""
+    echo "Successes:        $SUCCESSES (${SUCCESS_PCT}%)"
+    echo "Failures (soft):  $FAILURES (${FAILURE_PCT}%)"
+    echo "Failures (hard):  $REAL_FAILURES (${REAL_FAILURE_PCT}%)"
+
+    if [[ "$ITERATION" == "$RUNS" ]]; then
+        if [[ "$REAL_FAILURES" -gt 0 ]]; then
+            exit 1 # Tell git bisect something broke
+        elif [[ "$FAILURES" -g 0 ]]; then
+            exit 125 # Tell git bisect we couldn't figure it out
+        fi
+    fi
+
+done    

app/controllers/idv/document_capture_controller.rb

@@ -27,7 +27,12 @@ def show
     def update
       flow_session['redo_document_capture'] = nil # done with this redo
       result = handle_stored_result
-      analytics.idv_doc_auth_document_capture_submitted(**result.to_h.merge(analytics_arguments))
+      args = result.to_h.merge(analytics_arguments)
+      puts '---------------------------------------------------------------------------------------'
+      puts 'idv_doc_auth_document_capture_submitted'
+      puts args.inspect
+      puts '---------------------------------------------------------------------------------------'
+      analytics.idv_doc_auth_document_capture_submitted(**args)
 
       Funnel::DocAuth::RegisterStep.new(current_user.id, sp_session[:issuer]).
         call('document_capture', :update, true)

spec/features/idv/doc_auth/redo_document_capture_spec.rb

@@ -10,22 +10,26 @@
     allow_any_instance_of(ApplicationController).to receive(:analytics).and_return(fake_analytics)
   end
 
-  context 'when barcode scan returns a warning', allow_browser_log: true do
-    let(:use_bad_ssn) { false }
+  around do |example|
+    example.run
+  rescue
+    # rubocop:disable Rails/Output
+    puts '-----------------------------------------------------------------------------------------'
+    puts 'HERE IS THE HTML'
+    puts page.html
+    puts '-----------------------------------------------------------------------------------------'
+    # rubocop:enable Rails/Output
+    raise
+  end
 
+  context 'when barcode scan returns a warning', allow_browser_log: true do
     before do
       sign_in_and_2fa_user
       complete_doc_auth_steps_before_document_capture_step
       mock_doc_auth_attention_with_barcode
       attach_and_submit_images
       click_idv_continue
-
-      if use_bad_ssn
-        fill_out_ssn_form_with_ssn_that_fails_resolution
-      else
-        fill_out_ssn_form_ok
-      end
-
+      fill_out_ssn_form_with_ssn_that_fails_resolution
       click_idv_continue
     end
 
@@ -37,7 +41,7 @@
         text: t(
           'doc_auth.headings.capture_scan_warning_html',
           link: warning_link_text,
-        ).tr(' ', ' '),
+        ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces
       )
       click_link warning_link_text
 
@@ -56,23 +60,76 @@
 
       expect(current_path).to eq(idv_verify_info_path)
       check t('forms.ssn.show')
-      expect(page).to have_content(DocAuthHelper::GOOD_SSN)
-      expect(page).to have_css('[role="status"]')  # We verified your ID
+      expect(page).to have_content(DocAuthHelper::SSN_THAT_FAILS_RESOLUTION)
+      expect(page).to have_css('[role="status"]') # We verified your ID
+    end
+
+    it 'shows a troubleshooting option to allow the user to cancel and return to SP' do
+      click_idv_continue
+
+      expect(page).to have_link(
+        t('links.cancel'),
+        href: idv_cancel_path,
+      )
+
+      click_link t('links.cancel')
+
+      expect(current_path).to eq(idv_cancel_path)
+    end
+
+    context 'on mobile', driver: :headless_chrome_mobile do
+      it 'shows a warning message to allow the user to return to upload new images' do
+        warning_link_text = t('doc_auth.headings.capture_scan_warning_link')
+
+        expect(page).to have_css(
+          '[role="status"]',
+          text: t(
+            'doc_auth.headings.capture_scan_warning_html',
+            link: warning_link_text,
+          ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces
+        )
+        click_link warning_link_text
+
+        expect(current_path).to eq(idv_document_capture_path)
+        expect(fake_analytics).to have_logged_event(
+          'IdV: doc auth document_capture visited',
+          hash_including(redo_document_capture: true),
+        )
+        DocAuth::Mock::DocAuthMockClient.reset!
+        attach_and_submit_images
+
+        expect(current_path).to eq(idv_verify_info_path)
+        check t('forms.ssn.show')
+        expect(page).to have_content(DocAuthHelper::SSN_THAT_FAILS_RESOLUTION)
+        expect(page).to have_css('[role="status"]') # We verified your ID
+      end
     end
+  end
+
+  context 'after verify info step', allow_browser_log: true do
+    it 'document capture can no longer be reached' do
+      sign_in_and_2fa_user
+      complete_doc_auth_steps_before_document_capture_step
+      mock_doc_auth_attention_with_barcode
+      attach_and_submit_images
+      click_idv_continue
+      fill_out_ssn_form_ok
+      click_idv_continue
 
-    xit 'document capture cannot be reached after submitting verify info step' do
       warning_link_text = t('doc_auth.headings.capture_scan_warning_link')
 
       expect(page).to have_css(
         '[role="status"]',
         text: t(
           'doc_auth.headings.capture_scan_warning_html',
           link: warning_link_text,
-        ).tr(' ', ' '),
+        ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces
       )
+
       click_link warning_link_text
 
       expect(current_path).to eq(idv_hybrid_handoff_path)
+
       complete_hybrid_handoff_step
 
       visit idv_verify_info_url
@@ -102,50 +159,5 @@
       visit idv_verify_info_url
       expect(current_path).to eq(idv_phone_path)
     end
-
-    context 'with a bad SSN' do
-      let(:use_bad_ssn) { true }
-
-      it 'shows a troubleshooting option to allow the user to cancel and return to SP' do
-        click_idv_continue
-
-        expect(page).to have_link(
-          t('links.cancel'),
-          href: idv_cancel_path,
-        )
-
-        click_link t('links.cancel')
-
-        expect(current_path).to eq(idv_cancel_path)
-      end
-    end
-
-    context 'on mobile', driver: :headless_chrome_mobile do
-      it 'shows a warning message to allow the user to return to upload new images' do
-        warning_link_text = t('doc_auth.headings.capture_scan_warning_link')
-
-        expect(page).to have_css(
-          '[role="status"]',
-          text: t(
-            'doc_auth.headings.capture_scan_warning_html',
-            link: warning_link_text,
-          ).tr(' ', ' '),
-        )
-        click_link warning_link_text
-
-        expect(current_path).to eq(idv_document_capture_path)
-        expect(fake_analytics).to have_logged_event(
-          'IdV: doc auth document_capture visited',
-          hash_including(redo_document_capture: true),
-        )
-        DocAuth::Mock::DocAuthMockClient.reset!
-        attach_and_submit_images
-
-        expect(current_path).to eq(idv_verify_info_path)
-        check t('forms.ssn.show')
-        expect(page).to have_content(DocAuthHelper::GOOD_SSN)
-        expect(page).to have_css('[role="status"]') # We verified your ID
-      end
-    end
   end
 end

spec/rails_helper.rb

@@ -16,6 +16,7 @@
 require 'factory_bot'
 require 'view_component/test_helpers'
 require 'capybara/rspec'
+require 'capybara-screenshot/rspec'
 require 'capybara/webmock'
 
 # Checks for pending migrations before tests are run.

spec/support/capybara.rb

@@ -1,4 +1,5 @@
 require 'capybara/rspec'
+require 'capybara-screenshot/rspec'
 require 'rack_session_access/capybara'
 require 'webdrivers/chromedriver'
 require 'selenium/webdriver'
@@ -61,3 +62,7 @@
 end
 
 Capybara.default_driver = :desktop_rack_test
+
+Capybara::Screenshot.register_driver(:headless_chrome) do |driver, path|
+  driver.browser.save_screenshot(path)
+end