diff --git a/app/jobs/gpo_expiration_job.rb b/app/jobs/gpo_expiration_job.rb new file mode 100644 index 00000000000..6e9bc06d0c3 --- /dev/null +++ b/app/jobs/gpo_expiration_job.rb @@ -0,0 +1,76 @@ +class GpoExpirationJob < ApplicationJob + queue_as :low + + def initialize(analytics: nil) + @analytics = analytics + end + + def expire_profile(profile:) + gpo_verification_pending_at = profile.gpo_verification_pending_at + + profile.deactivate_due_to_gpo_expiration + + analytics.idv_gpo_expired( + user_id: profile.user.uuid, + user_has_active_profile: profile.user.active_profile.present?, + letters_sent: profile.gpo_confirmation_codes.count, + gpo_verification_pending_at: gpo_verification_pending_at, + ) + end + + def perform(now: Time.zone.now, limit: nil, min_profile_age: nil) + profiles = gpo_profiles_that_should_be_expired(as_of: now, min_profile_age: min_profile_age) + + if limit.present? + profiles = profiles.limit(limit) + end + + profiles.find_each do |profile| + expire_profile(profile: profile) + end + end + + def gpo_profiles_that_should_be_expired(as_of:, min_profile_age: nil) + Profile. + and(are_pending_gpo_verification). + and(user_cant_request_more_letters(as_of: as_of)). + and(most_recent_code_has_expired(as_of: as_of)). + and(are_old_enough(as_of: as_of, min_profile_age: min_profile_age)) + end + + private + + def analytics + @analytics ||= Analytics.new(user: AnonymousUser.new, request: nil, session: {}, sp: nil) + end + + def are_old_enough(as_of:, min_profile_age:) + return Profile.all if min_profile_age.blank? + + max_created_at = as_of - min_profile_age + + return Profile.where(created_at: ..max_created_at) + end + + def are_pending_gpo_verification + Profile.where.not(gpo_verification_pending_at: nil) + end + + def most_recent_code_has_expired(as_of:) + # Any Profile where the most recent code was sent *before* + # usps_confirmation_max_days days ago is now expired + max_code_sent_at = as_of - IdentityConfig.store.usps_confirmation_max_days.days + + Profile.where( + id: GpoConfirmationCode. + select(:profile_id). + group(:profile_id). + having('max(code_sent_at) < ?', max_code_sent_at), + ) + end + + def user_cant_request_more_letters(as_of:) + max_created_at = as_of - IdentityConfig.store.gpo_max_profile_age_to_send_letter_in_days.days + Profile.where(created_at: [..max_created_at]) + end +end diff --git a/app/models/profile.rb b/app/models/profile.rb index b8e59cf7569..d9d8b4edaba 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -173,6 +173,15 @@ def in_person_verification_pending? in_person_verification_pending_at.present? end + def deactivate_due_to_gpo_expiration + raise 'Profile is not pending GPO verification' if gpo_verification_pending_at.nil? + update!( + active: false, + gpo_verification_pending_at: nil, + gpo_verification_expired_at: Time.zone.now, + ) + end + def deactivate_for_in_person_verification update!(active: false, in_person_verification_pending_at: Time.zone.now) end diff --git a/app/services/analytics_events.rb b/app/services/analytics_events.rb index 5b4e285e713..bebf84f9493 100644 --- a/app/services/analytics_events.rb +++ b/app/services/analytics_events.rb @@ -1465,6 +1465,28 @@ def idv_gpo_confirm_start_over_visited(**extra) track_event('IdV: gpo confirm start over visited', **extra) end + # The user ran out of time to complete their address verification by mail. + # @param [String] user_id UUID of the user who expired + # @param [Boolean] user_has_active_profile Whether the user currently has an active profile + # @param [Integer] letters_sent Total # of GPO letters sent for this profile + # @param [Time] gpo_verification_pending_at Date/time when profile originally entered GPO flow + def idv_gpo_expired( + user_id:, + user_has_active_profile:, + letters_sent:, + gpo_verification_pending_at:, + **extra + ) + track_event( + :idv_gpo_expired, + user_id: user_id, + user_has_active_profile: user_has_active_profile, + letters_sent: letters_sent, + gpo_verification_pending_at: gpo_verification_pending_at, + **extra, + ) + end + # A GPO reminder email was sent to the user # @param [String] user_id UUID of user who we sent a reminder to def idv_gpo_reminder_email_sent(user_id:, **extra) diff --git a/db/primary_migrate/20231102211426_add_gpo_verification_expired_at_to_profiles.rb b/db/primary_migrate/20231102211426_add_gpo_verification_expired_at_to_profiles.rb new file mode 100644 index 00000000000..f42fc9c9da3 --- /dev/null +++ b/db/primary_migrate/20231102211426_add_gpo_verification_expired_at_to_profiles.rb @@ -0,0 +1,8 @@ +class AddGpoVerificationExpiredAtToProfiles < ActiveRecord::Migration[7.1] + disable_ddl_transaction! + + def change + add_column :profiles, :gpo_verification_expired_at, :datetime + add_index :profiles, :gpo_verification_expired_at, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index bb0e7352e81..fceaf6e546f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2023_10_24_172229) do +ActiveRecord::Schema[7.1].define(version: 2023_11_02_211426) do # These are extensions that must be enabled in order to support this database enable_extension "citext" enable_extension "pg_stat_statements" @@ -456,9 +456,11 @@ t.datetime "in_person_verification_pending_at" t.text "encrypted_pii_multi_region" t.text "encrypted_pii_recovery_multi_region" + t.datetime "gpo_verification_expired_at" t.index ["fraud_pending_reason"], name: "index_profiles_on_fraud_pending_reason" t.index ["fraud_rejection_at"], name: "index_profiles_on_fraud_rejection_at" t.index ["fraud_review_pending_at"], name: "index_profiles_on_fraud_review_pending_at" + t.index ["gpo_verification_expired_at"], name: "index_profiles_on_gpo_verification_expired_at" t.index ["gpo_verification_pending_at"], name: "index_profiles_on_gpo_verification_pending_at" t.index ["name_zip_birth_year_signature"], name: "index_profiles_on_name_zip_birth_year_signature" t.index ["ssn_signature"], name: "index_profiles_on_ssn_signature" @@ -627,6 +629,8 @@ t.datetime "updated_at", precision: nil, null: false t.datetime "bounced_at", precision: nil t.datetime "reminder_sent_at", precision: nil + t.datetime "expiration_notice_sent_at", precision: nil + t.index ["expiration_notice_sent_at"], name: "index_usps_confirmation_codes_on_expiration_notice_sent_at" t.index ["otp_fingerprint"], name: "index_usps_confirmation_codes_on_otp_fingerprint" t.index ["profile_id"], name: "index_usps_confirmation_codes_on_profile_id" t.index ["reminder_sent_at"], name: "index_usps_confirmation_codes_on_reminder_sent_at" diff --git a/lib/tasks/backfill_gpo_expiration.rake b/lib/tasks/backfill_gpo_expiration.rake new file mode 100644 index 00000000000..63a73505e7f --- /dev/null +++ b/lib/tasks/backfill_gpo_expiration.rake @@ -0,0 +1,69 @@ +namespace :profiles do + desc 'Backfill the gpo_verification_expired_at value' + + ## + # Usage: + # + # Print pending updates + # bundle exec rake profiles:backfill_gpo_expiration > profiles.csv + # + # Commit updates + # bundle exec rake profiles:backfill_gpo_expiration UPDATE_PROFILES=true > profiles.csv + # + task backfill_gpo_expiration: :environment do |_task, _args| + min_profile_age = (ENV['MIN_PROFILE_AGE_IN_DAYS'].to_i || 100).days + update_profiles = ENV['UPDATE_PROFILES'] == 'true' + + job = GpoExpirationJob.new + + profiles = job.gpo_profiles_that_should_be_expired( + as_of: Time.zone.now, + min_profile_age: min_profile_age, + ) + + count = 0 + + profiles.find_each do |profile| + count += 1 + + gpo_verification_pending_at = profile.gpo_verification_pending_at + + if gpo_verification_pending_at.blank? + raise "Profile #{profile.id} does not have gpo_verification_pending_at" + end + + puts "#{profile.id},#{gpo_verification_pending_at.iso8601}" + + if update_profiles + warn "Expired #{count} profiles" if count % 100 == 0 + job.expire_profile(profile: profile) + elsif count % 100 == 0 + warn "Found #{count} profiles" + end + end + end + + ## + # Usage: + # + # Rollback the above: + # + # bundle exec rake profiles:rollback_backfill_gpo_expiration < profiles.csv + # + task rollback_backfill_gpo_expiration: :environment do |_task, _args| + profile_data = STDIN.read.split("\n").map do |profile_row| + profile_row.split(',') + end + + warn "Updating #{profile_data.count} records" + + profile_data.each do |profile_datum| + profile_id, gpo_verification_pending_at = profile_datum + Profile.where(id: profile_id).update!( + gpo_verification_pending_at: Time.zone.parse(gpo_verification_pending_at), + gpo_verification_expired_at: nil, + ) + warn profile_id + end + end +end diff --git a/spec/factories/gpo_confirmation_codes.rb b/spec/factories/gpo_confirmation_codes.rb index 0af40b8b205..d63eeb7f508 100644 --- a/spec/factories/gpo_confirmation_codes.rb +++ b/spec/factories/gpo_confirmation_codes.rb @@ -4,5 +4,6 @@ factory :gpo_confirmation_code do profile otp_fingerprint { Pii::Fingerprinter.fingerprint('ABCDE12345') } + code_sent_at { 1.day.ago } end end diff --git a/spec/factories/users.rb b/spec/factories/users.rb index fa42f7a32c6..2782b012c0c 100644 --- a/spec/factories/users.rb +++ b/spec/factories/users.rb @@ -205,12 +205,30 @@ end trait :with_pending_gpo_profile do - after :build do |user| - profile = create(:profile, :with_pii, gpo_verification_pending_at: 1.day.ago, user: user) - gpo_code = create(:gpo_confirmation_code) - profile.gpo_confirmation_codes << gpo_code - device = create(:device, user: user) - create(:event, user: user, device: device, event_type: :gpo_mail_sent) + transient do + code_sent_at { created_at } + end + + after :create do |user, context| + profile = create( + :profile, + :with_pii, + gpo_verification_pending_at: context.code_sent_at, + user: user, + created_at: context.created_at, + ) + create( + :gpo_confirmation_code, + profile: profile, + code_sent_at: context.code_sent_at, + ) + create( + :event, + user: user, + device: create(:device, user: user), + event_type: :gpo_mail_sent, + created_at: context.code_sent_at, + ) end end diff --git a/spec/jobs/gpo_expiration_job_spec.rb b/spec/jobs/gpo_expiration_job_spec.rb new file mode 100644 index 00000000000..f92c499437a --- /dev/null +++ b/spec/jobs/gpo_expiration_job_spec.rb @@ -0,0 +1,176 @@ +require 'rails_helper' + +RSpec.describe GpoExpirationJob do + include Rails.application.routes.url_helpers + + subject(:job) { described_class.new(analytics: analytics) } + + let(:analytics) { FakeAnalytics.new } + + let(:usps_confirmation_max_days) { 30 } + + let(:gpo_max_profile_age_to_send_letter_in_days) { 30 } + + let(:expired_timestamp) { Time.zone.now.round - usps_confirmation_max_days.days - 1.hour } + + let(:not_expired_timestamp) { Time.zone.now.round - (usps_confirmation_max_days / 2).days } + + let!(:users) do + { + user_with_one_expired_gpo_profile: create( + :user, + :with_pending_gpo_profile, + created_at: expired_timestamp, + ), + user_with_one_unexpired_gpo_profile: create( + :user, + :with_pending_gpo_profile, + created_at: not_expired_timestamp, + ), + user_with_one_expired_code_and_one_unexpired_code: create( + :user, + :with_pending_gpo_profile, + created_at: expired_timestamp, + ).tap do |user| + profile = user.gpo_verification_pending_profile + create(:gpo_confirmation_code, profile: profile, code_sent_at: not_expired_timestamp) + end, + } + end + + before do + allow(IdentityConfig.store).to receive(:gpo_max_profile_age_to_send_letter_in_days).and_return( + gpo_max_profile_age_to_send_letter_in_days, + ) + allow(IdentityConfig.store).to receive(:usps_confirmation_max_days).and_return( + usps_confirmation_max_days, + ) + end + + describe '#gpo_profiles_that_should_be_expired' do + it 'returns the correct profiles' do + profiles = job.gpo_profiles_that_should_be_expired(as_of: Time.zone.now) + + expect( + profiles.map(&:user), + ).to contain_exactly( + users[:user_with_one_expired_gpo_profile], + ) + end + + context 'when users can request letters beyond initial code expiration period' do + let(:gpo_max_profile_age_to_send_letter_in_days) { 45 } + + it 'returns profiles for the correct users' do + profiles = job.gpo_profiles_that_should_be_expired(as_of: Time.zone.now) + expect(profiles.count).to eql(0) + end + end + + context 'when we are enforcing a minimum profile age' do + let!(:really_old_user) do + create( + :user, + :with_pending_gpo_profile, + created_at: expired_timestamp - 5.years, + ) + end + it 'only wants to expire the really old profile' do + profiles = job.gpo_profiles_that_should_be_expired( + as_of: Time.zone.now, + min_profile_age: 2.years, + ) + expect(profiles.map(&:user)).to contain_exactly( + really_old_user, + ) + end + end + end + + describe '#perform' do + it 'expires the profile' do + profile = users[:user_with_one_expired_gpo_profile].reload.gpo_verification_pending_profile + freeze_time do + expect { job.perform }.to change { + profile.reload.gpo_verification_expired_at + }.to eql(Time.zone.now) + end + end + + it 'clears gpo_verification_pending_at' do + profile = users[:user_with_one_expired_gpo_profile].reload.gpo_verification_pending_profile + expect { job.perform }.to change { profile.reload.gpo_verification_pending_at }.to eql(nil) + end + + it 'logged an analytics event' do + job.perform + expect(analytics).to have_logged_event( + :idv_gpo_expired, + user_id: users[:user_with_one_expired_gpo_profile].uuid, + user_has_active_profile: false, + letters_sent: 1, + gpo_verification_pending_at: be_within(1.second).of(expired_timestamp), + ) + end + + context 'when the user has an active profile' do + let!(:active_profile) do + create(:profile, :active, user: users[:user_with_one_expired_gpo_profile]) + end + it 'includes that information in analytics event' do + job.perform + + expect(analytics).to have_logged_event( + :idv_gpo_expired, + user_id: users[:user_with_one_expired_gpo_profile].uuid, + user_has_active_profile: true, + letters_sent: 1, + gpo_verification_pending_at: be_within(1.second).of(expired_timestamp), + ) + end + end + + context 'when the user has multiple codes sent' do + let!(:extra_code) do + create( + :gpo_confirmation_code, + profile: users[:user_with_one_expired_gpo_profile].gpo_verification_pending_profile, + code_sent_at: expired_timestamp, + ) + end + + it 'we note that in the analytics event' do + job.perform + + expect(analytics).to have_logged_event( + :idv_gpo_expired, + user_id: users[:user_with_one_expired_gpo_profile].uuid, + user_has_active_profile: false, + letters_sent: 2, + gpo_verification_pending_at: be_within(1.second).of(expired_timestamp), + ) + end + end + + describe 'limit' do + let(:limit) { 3 } + before do + (0..limit).each do + create( + :user, + :with_pending_gpo_profile, + created_at: expired_timestamp, + ) + end + end + it 'limits the number of records affected' do + initial_count = Profile.where.not(gpo_verification_pending_at: nil).count + + job.perform(limit: limit) + + expect(Profile.where.not(gpo_verification_pending_at: nil).count). + to eql(initial_count - limit) + end + end + end +end diff --git a/spec/models/profile_spec.rb b/spec/models/profile_spec.rb index fbef249cc90..a7c7dd7516d 100644 --- a/spec/models/profile_spec.rb +++ b/spec/models/profile_spec.rb @@ -1054,6 +1054,45 @@ end end + describe '#deactivate_due_to_gpo_expiration' do + let(:profile) { create(:profile, :verify_by_mail_pending, user: user) } + + it 'sets gpo_verification_expired_at' do + freeze_time do + expect do + profile.deactivate_due_to_gpo_expiration + end.to change { profile.gpo_verification_expired_at }.to eql(Time.zone.now) + end + end + + it 'clears gpo_verification_pending_at' do + expect do + profile.deactivate_due_to_gpo_expiration + end.to change { profile.gpo_verification_pending_at }.to eql(nil) + end + + it 'maintains active = false' do + expect do + profile.deactivate_due_to_gpo_expiration + end.not_to change { profile.active }.from(false) + end + + it 'does not set a deactivation_reason' do + expect do + profile.deactivate_due_to_gpo_expiration + end.not_to change { profile.deactivation_reason }.from(nil) + end + + context 'not pending gpo' do + let(:profile) { create(:profile, user: user) } + it 'raises' do + expect do + profile.deactivate_due_to_gpo_expiration + end.to raise_error + end + end + end + describe '#reject_for_fraud' do before do # This is necessary because UserMailer reaches into the