diff --git a/Gemfile b/Gemfile index de1eac11a26..7daebdba804 100644 --- a/Gemfile +++ b/Gemfile @@ -118,6 +118,7 @@ end group :test do gem 'axe-core-rspec', '~> 4.2' gem 'bundler-audit', require: false + gem 'capybara-screenshot' gem 'simplecov', '~> 0.21.0', require: false gem 'simplecov-cobertura' gem 'simplecov_json_formatter' diff --git a/Gemfile.lock b/Gemfile.lock index 3b633f9e792..46058b2215f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -222,6 +222,9 @@ GEM rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) + capybara-screenshot (1.0.26) + capybara (>= 1.0, < 4) + launchy cbor (0.5.9.6) choice (0.2.0) chunky_png (1.4.0) @@ -748,6 +751,7 @@ DEPENDENCIES browser bullet (~> 7.0) bundler-audit + capybara-screenshot capybara-webmock! connection_pool cssbundling-rails diff --git a/MH-deflakify.sh b/MH-deflakify.sh new file mode 100755 index 00000000000..70de9b44e50 --- /dev/null +++ b/MH-deflakify.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Usage: script RUNS COMMAND ["string signifying a real failure"] +# Execute COMMAND RUNS times, track outputs and success rates. + +set -euo pipefail + +LOG_DIR=${LOG_DIR:-.deflakify} +RUN_ID="run_$(date "+%Y%m%d-%H%M%S")" +RUN_DIR="${LOG_DIR}/${RUN_ID}" + +mkdir -p "$RUN_DIR" + +RUNS=$1; shift +COMMAND=$1; shift +STRING_THAT_MEANS_FAILURE=${1:-}; shift + +ITERATION=0 +SUCCESSES=0 +FAILURES=0 +REAL_FAILURES=0 + +while (true) +do + ITERATION=$(($ITERATION+1)) + OUT_FILE="$RUN_DIR/$ITERATION.out.txt" + ERR_FILE="$RUN_DIR/$ITERATION.err.txt" + + STARTED_AT=$(date '+%Y-%m-%d %H:%M:%S') + + echo "" + + echo " +Iteration: ${ITERATION} +Command: $COMMAND +Started at: $STARTED_AT" | tee "$OUT_FILE" "$ERR_FILE" + + if sh -c "$COMMAND" >> "$OUT_FILE" 2>> "$ERR_FILE"; then + echo "Result: ✅ Success" + SUCCESSES=$(($SUCCESSES + 1)) + else + FAILURES=$(($FAILURES + 1)) + + if grep "$STRING_THAT_MEANS_FAILURE" "$OUT_FILE" "$ERR_FILE" > /dev/null 2>&1; then + echo "Result: ❌ Failure" + REAL_FAILURES=$(($REAL_FAILURES + 1)) + else + echo "Result: 🤔 Failure, but ignoring" + fi + fi + + echo "Finished at: $(date '+%Y-%m-%d %H:%M:%S')" + + SUCCESS_PCT=$((($SUCCESSES * 100) / $ITERATION)) + FAILURE_PCT=$((($FAILURES * 100) / $ITERATION)) + REAL_FAILURE_PCT=$((($REAL_FAILURES * 100) / $ITERATION)) + + echo "" + echo "Successes: $SUCCESSES (${SUCCESS_PCT}%)" + echo "Failures (soft): $FAILURES (${FAILURE_PCT}%)" + echo "Failures (hard): $REAL_FAILURES (${REAL_FAILURE_PCT}%)" + + if [[ "$ITERATION" == "$RUNS" ]]; then + if [[ "$REAL_FAILURES" -gt 0 ]]; then + exit 1 # Tell git bisect something broke + elif [[ "$FAILURES" -g 0 ]]; then + exit 125 # Tell git bisect we couldn't figure it out + fi + fi + +done \ No newline at end of file diff --git a/app/controllers/idv/document_capture_controller.rb b/app/controllers/idv/document_capture_controller.rb index 22ded80ffc9..168d8a51994 100644 --- a/app/controllers/idv/document_capture_controller.rb +++ b/app/controllers/idv/document_capture_controller.rb @@ -27,7 +27,12 @@ def show def update flow_session['redo_document_capture'] = nil # done with this redo result = handle_stored_result - analytics.idv_doc_auth_document_capture_submitted(**result.to_h.merge(analytics_arguments)) + args = result.to_h.merge(analytics_arguments) + puts '---------------------------------------------------------------------------------------' + puts 'idv_doc_auth_document_capture_submitted' + puts args.inspect + puts '---------------------------------------------------------------------------------------' + analytics.idv_doc_auth_document_capture_submitted(**args) Funnel::DocAuth::RegisterStep.new(current_user.id, sp_session[:issuer]). call('document_capture', :update, true) diff --git a/spec/features/idv/doc_auth/redo_document_capture_spec.rb b/spec/features/idv/doc_auth/redo_document_capture_spec.rb index 4029bc718ec..02a88f9a463 100644 --- a/spec/features/idv/doc_auth/redo_document_capture_spec.rb +++ b/spec/features/idv/doc_auth/redo_document_capture_spec.rb @@ -10,22 +10,26 @@ allow_any_instance_of(ApplicationController).to receive(:analytics).and_return(fake_analytics) end - context 'when barcode scan returns a warning', allow_browser_log: true do - let(:use_bad_ssn) { false } + around do |example| + example.run + rescue + # rubocop:disable Rails/Output + puts '-----------------------------------------------------------------------------------------' + puts 'HERE IS THE HTML' + puts page.html + puts '-----------------------------------------------------------------------------------------' + # rubocop:enable Rails/Output + raise + end + context 'when barcode scan returns a warning', allow_browser_log: true do before do sign_in_and_2fa_user complete_doc_auth_steps_before_document_capture_step mock_doc_auth_attention_with_barcode attach_and_submit_images click_idv_continue - - if use_bad_ssn - fill_out_ssn_form_with_ssn_that_fails_resolution - else - fill_out_ssn_form_ok - end - + fill_out_ssn_form_with_ssn_that_fails_resolution click_idv_continue end @@ -37,7 +41,7 @@ text: t( 'doc_auth.headings.capture_scan_warning_html', link: warning_link_text, - ).tr(' ', ' '), + ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces ) click_link warning_link_text @@ -56,11 +60,62 @@ expect(current_path).to eq(idv_verify_info_path) check t('forms.ssn.show') - expect(page).to have_content(DocAuthHelper::GOOD_SSN) - expect(page).to have_css('[role="status"]') # We verified your ID + expect(page).to have_content(DocAuthHelper::SSN_THAT_FAILS_RESOLUTION) + expect(page).to have_css('[role="status"]') # We verified your ID + end + + it 'shows a troubleshooting option to allow the user to cancel and return to SP' do + click_idv_continue + + expect(page).to have_link( + t('links.cancel'), + href: idv_cancel_path, + ) + + click_link t('links.cancel') + + expect(current_path).to eq(idv_cancel_path) + end + + context 'on mobile', driver: :headless_chrome_mobile do + it 'shows a warning message to allow the user to return to upload new images' do + warning_link_text = t('doc_auth.headings.capture_scan_warning_link') + + expect(page).to have_css( + '[role="status"]', + text: t( + 'doc_auth.headings.capture_scan_warning_html', + link: warning_link_text, + ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces + ) + click_link warning_link_text + + expect(current_path).to eq(idv_document_capture_path) + expect(fake_analytics).to have_logged_event( + 'IdV: doc auth document_capture visited', + hash_including(redo_document_capture: true), + ) + DocAuth::Mock::DocAuthMockClient.reset! + attach_and_submit_images + + expect(current_path).to eq(idv_verify_info_path) + check t('forms.ssn.show') + expect(page).to have_content(DocAuthHelper::SSN_THAT_FAILS_RESOLUTION) + expect(page).to have_css('[role="status"]') # We verified your ID + end end + end + + context 'after verify info step', allow_browser_log: true do + it 'document capture can no longer be reached' do + sign_in_and_2fa_user + complete_doc_auth_steps_before_document_capture_step + mock_doc_auth_attention_with_barcode + attach_and_submit_images + click_idv_continue + fill_out_ssn_form_ok + click_idv_continue - xit 'document capture cannot be reached after submitting verify info step' do warning_link_text = t('doc_auth.headings.capture_scan_warning_link') expect(page).to have_css( @@ -68,11 +123,13 @@ text: t( 'doc_auth.headings.capture_scan_warning_html', link: warning_link_text, - ).tr(' ', ' '), + ).tr(' ', ' '), # Convert non-breaking spaces to regular spaces ) + click_link warning_link_text expect(current_path).to eq(idv_hybrid_handoff_path) + complete_hybrid_handoff_step visit idv_verify_info_url @@ -102,50 +159,5 @@ visit idv_verify_info_url expect(current_path).to eq(idv_phone_path) end - - context 'with a bad SSN' do - let(:use_bad_ssn) { true } - - it 'shows a troubleshooting option to allow the user to cancel and return to SP' do - click_idv_continue - - expect(page).to have_link( - t('links.cancel'), - href: idv_cancel_path, - ) - - click_link t('links.cancel') - - expect(current_path).to eq(idv_cancel_path) - end - end - - context 'on mobile', driver: :headless_chrome_mobile do - it 'shows a warning message to allow the user to return to upload new images' do - warning_link_text = t('doc_auth.headings.capture_scan_warning_link') - - expect(page).to have_css( - '[role="status"]', - text: t( - 'doc_auth.headings.capture_scan_warning_html', - link: warning_link_text, - ).tr(' ', ' '), - ) - click_link warning_link_text - - expect(current_path).to eq(idv_document_capture_path) - expect(fake_analytics).to have_logged_event( - 'IdV: doc auth document_capture visited', - hash_including(redo_document_capture: true), - ) - DocAuth::Mock::DocAuthMockClient.reset! - attach_and_submit_images - - expect(current_path).to eq(idv_verify_info_path) - check t('forms.ssn.show') - expect(page).to have_content(DocAuthHelper::GOOD_SSN) - expect(page).to have_css('[role="status"]') # We verified your ID - end - end end end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 912de11754c..957f696ba6f 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -16,6 +16,7 @@ require 'factory_bot' require 'view_component/test_helpers' require 'capybara/rspec' +require 'capybara-screenshot/rspec' require 'capybara/webmock' # Checks for pending migrations before tests are run. diff --git a/spec/support/capybara.rb b/spec/support/capybara.rb index 783c5c818b5..2d266a1f97b 100644 --- a/spec/support/capybara.rb +++ b/spec/support/capybara.rb @@ -1,4 +1,5 @@ require 'capybara/rspec' +require 'capybara-screenshot/rspec' require 'rack_session_access/capybara' require 'webdrivers/chromedriver' require 'selenium/webdriver' @@ -61,3 +62,7 @@ end Capybara.default_driver = :desktop_rack_test + +Capybara::Screenshot.register_driver(:headless_chrome) do |driver, path| + driver.browser.save_screenshot(path) +end