Deploy RC 502 to Production

Gemfile

@@ -70,7 +70,7 @@ gem 'rqrcode'
 gem 'ruby-progressbar'
 gem 'ruby-saml'
 gem 'safe_target_blank', '>= 1.0.2'
-gem 'saml_idp', github: '18F/saml_idp', tag: '0.23.7-18f'
+gem 'saml_idp', github: '18F/saml_idp', tag: '0.23.9-18f'
 gem 'scrypt'
 gem 'simple_form', '>= 5.0.2'
 gem 'stringex', require: false

Gemfile.lock

@@ -49,10 +49,10 @@ GIT
 
 GIT
   remote: https://github.com/18F/saml_idp.git
-  revision: f76c4e093158de5f704e63bcbc09107a717a1c27
-  tag: 0.23.7-18f
+  revision: ddf27d98cde86f80680ab4148653edd1cd745efd
+  tag: 0.23.9-18f
   specs:
-    saml_idp (0.23.7.pre.18f)
+    saml_idp (0.23.9.pre.18f)
       activesupport
       builder
       faraday
@@ -384,7 +384,7 @@ GEM
     jmespath (1.6.2)
     jsbundling-rails (1.1.2)
       railties (>= 6.0.0)
-    json (2.13.0)
+    json (2.13.2)
     jwe (0.4.0)
     jwt (2.7.1)
     knapsack (4.0.0)
@@ -646,7 +646,7 @@ GEM
     rubocop-rspec (3.2.0)
       rubocop (~> 1.61)
     ruby-progressbar (1.13.0)
-    ruby-saml (1.18.0)
+    ruby-saml (1.18.1)
       nokogiri (>= 1.13.10)
       rexml
     ruby-statistics (3.0.2)

app/controllers/concerns/idv/choose_id_type_concern.rb

@@ -29,17 +29,18 @@ def selected_id_type
 
     def dos_passport_api_healthy?(
       analytics:,
+      step:,
       endpoint: IdentityConfig.store.dos_passport_composite_healthcheck_endpoint
     )
       return true if endpoint.blank?
 
       request = DocAuth::Dos::Requests::HealthCheckRequest.new(endpoint:)
-      response = request.fetch(analytics)
+      response = request.fetch(analytics, context_analytics: { step: })
       response.success?
     end
 
     def locals_attrs(analytics:, presenter:, form_submit_url: nil)
-      dos_passport_api_down = !dos_passport_api_healthy?(analytics:)
+      dos_passport_api_down = !dos_passport_api_healthy?(analytics:, step: 'choose_id_type')
       {
         presenter:,
         form_submit_url:,

app/controllers/idv/welcome_controller.rb

@@ -102,7 +102,7 @@ def update_passport_allowed
       end
 
       idv_session.passport_allowed ||= begin
-        if dos_passport_api_healthy?(analytics:)
+        if dos_passport_api_healthy?(analytics:, step: 'welcome')
           (ab_test_bucket(:DOC_AUTH_PASSPORT) == :passport_allowed)
         end
       end

app/models/profile.rb

@@ -16,6 +16,7 @@ class Profile < ApplicationRecord
   # rubocop:enable Rails/InverseOf
   has_many :gpo_confirmation_codes, dependent: :destroy
   has_one :in_person_enrollment, dependent: :destroy
+  has_many :duplicate_profile_confirmations, dependent: :destroy
 
   validates :active, uniqueness: { scope: :user_id, if: :active? }
 

app/services/analytics_events.rb

@@ -6717,14 +6717,23 @@ def otp_phone_validation_failed(error:, message:, context:, country:, **extra)
   # @param [Boolean] success Whether the passport api health check succeeded.
   # @param [Hash] body The health check body, if present.
   # @param [Hash] errors Any additional error information we have
+  # @param [String] step The step in the IdV flow that called the API health check
   # @param [String] exception The Faraday or other exception, if one happened
-  def passport_api_health_check(success:, body: nil, errors: nil, exception: nil, **extra)
+  def passport_api_health_check(
+    success:,
+    body: nil,
+    errors: nil,
+    exception: nil,
+    step: nil,
+    **extra
+  )
     track_event(
       :passport_api_health_check,
       success:,
       body:,
       errors:,
       exception:,
+      step:,
       **extra,
     )
   end

app/services/doc_auth/dos/requests/health_check_request.rb

@@ -17,7 +17,7 @@ def initialize(endpoint:)
           @endpoint = endpoint
         end
 
-        def fetch(analytics)
+        def fetch(analytics, context_analytics: { step: nil })
           begin
             faraday_response = connection.get do |req|
               req.options.context = { service_name: metric_name }
@@ -30,7 +30,8 @@ def fetch(analytics)
           analytics.passport_api_health_check(
             **response.to_h
                 .except(*UNUSED_RESPONSE_KEYS)
-                .merge(response.extra),
+                .merge(response.extra)
+                .merge(context_analytics),
           )
         end
 

app/services/duplicate_profile_checker.rb

@@ -16,8 +16,10 @@ def check_for_duplicate_profiles
 
     pii = cacher.fetch(profile.id)
     duplicate_ssn_finder = Idv::DuplicateSsnFinder.new(user:, ssn: pii[:ssn])
-    associated_profiles = duplicate_ssn_finder.associated_facial_match_profiles_with_ssn
-    if !duplicate_ssn_finder.ial2_profile_ssn_is_unique?
+    associated_profiles = duplicate_ssn_finder.duplicate_facial_match_profiles(
+      service_provider: sp.issuer,
+    )
+    if associated_profiles
       ids = associated_profiles.map(&:id)
       user_session[:duplicate_profile_ids] = ids
     end

app/services/idv/duplicate_ssn_finder.rb

@@ -10,19 +10,19 @@ def initialize(user:, ssn:)
     end
 
     def ssn_is_unique?
-      Profile.where(ssn_signature: ssn_signatures)
-        .where(initiating_service_provider_issuer: sp_eligible_for_one_account)
-        .where.not(user_id: user.id).empty?
+      Profile.where(ssn_signature: ssn_signatures).where.not(user_id: user.id).empty?
     end
 
-    def associated_facial_match_profiles_with_ssn
-      Profile.active.facial_match.where(ssn_signature: ssn_signatures)
-        .where(initiating_service_provider_issuer: sp_eligible_for_one_account)
+    def duplicate_facial_match_profiles(service_provider:)
+      Profile
+        .active
+        .facial_match
+        .where(ssn_signature: ssn_signatures)
+        .joins('INNER JOIN identities ON identities.user_id = profiles.user_id')
+        .where(identities: { service_provider: service_provider })
+        .where(identities: { deleted_at: nil })
         .where.not(user_id: user.id)
-    end
-
-    def ial2_profile_ssn_is_unique?
-      associated_facial_match_profiles_with_ssn.empty?
+        .distinct
     end
 
     # Due to potentially inconsistent normalization of stored SSNs in the past, we must check:

app/services/reporting/irs_credential_tenure_report.rb

@@ -73,40 +73,32 @@ def total_user_count
 
     def average_credential_tenure_months
       end_of_month = report_date.end_of_month
-
-      # Efficiently load only created_at timestamps for IRS users
-      created_ats = User
-        .joins(:identities)
-        .where('users.created_at <= ?', end_of_month)
-        .where(identities: { service_provider: issuers, deleted_at: nil })
-        .distinct
-        .pluck(:created_at)
-
-      return 0 if created_ats.empty?
-
-      total_months = created_ats.sum do |created_at|
-        precise_months_between(created_at.to_date, end_of_month)
+      Reports::BaseReport.transaction_with_timeout do
+        average_months = User
+          .where(
+            '(users.confirmed_at <= :end_of_month AND users.suspended_at IS NULL)
+          OR (users.suspended_at IS NOT NULL AND users.reinstated_at IS NOT NULL)',
+            end_of_month: end_of_month,
+          ).where(
+            'EXISTS (
+            SELECT 1 FROM identities
+            WHERE identities.user_id = users.id
+            AND identities.service_provider IN (:issuers)
+            AND identities.deleted_at IS NULL
+          )',
+            issuers: issuers,
+          ).pick(
+            Arel.sql(
+              'AVG(
+            EXTRACT(YEAR FROM age(?, users.confirmed_at)) * 12 +
+            EXTRACT(MONTH FROM age(?, users.confirmed_at)) +
+            EXTRACT(DAY FROM age(?, users.confirmed_at)) / 30.0
+          )',
+              end_of_month, end_of_month, end_of_month
+            ),
+          ).to_f.round(2)
+        return average_months
       end
-
-      (total_months.to_f / created_ats.size).round(2)
-    end
-
-      private
-
-    def precise_months_between(start_date, end_date)
-      return 0 if end_date < start_date
-
-      # Full months difference
-      months = (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)
-
-      # Adjust for partial month
-      partial_start_day = [start_date.day, Date.new(end_date.year, end_date.month, -1).day].min
-      partial_start_date = Date.new(end_date.year, end_date.month, partial_start_day)
-
-      day_diff = (end_date - partial_start_date).to_f
-      days_in_month = Date.new(end_date.year, end_date.month, -1).day.to_f
-
-      months + (day_diff / days_in_month)
     end
     end
 end

config/locales/es.yml

@@ -2098,8 +2098,8 @@ users.delete.bullet_4: Notificaremos a las agencias a las que acceda con %{app_n
 users.delete.heading: '¿Está seguro de que desea eliminar su cuenta?'
 users.delete.instructions: Ingrese su contraseña para confirmar que desea eliminar su cuenta.
 users.delete.subheading: 'Si elimina su cuenta:'
-users.duplicate_profiles_please_call.error_details_html: <p>If you don’t recognize or can’t delete one of the duplicate accounts you saw on the multiple accounts page, call our contact center at <strong>%{contact_number}</strong> and provide them with the error code <strong>LG33</strong>.</p><p>Having questions about duplicate accounts or how to move forward? Learn more about <a href="%{help_url}">how to resolve duplicate accounts</a>.</p>
-users.duplicate_profiles_please_call.heading: Please give us a call
+users.duplicate_profiles_please_call.error_details_html: <p>Si no reconoce o no puede eliminar una de las cuentas duplicadas que vio en la página que muestra varias cuentas, llame a nuestro centro de contacto al <strong>%{contact_number}</strong> y mencione el código de error <strong>LG33</strong>.</p><p>¿Tiene preguntas acerca de las cuentas duplicadas o de lo que debe hacer? Obtenga más información acerca <a href="%{help_url}">de cómo resolver las cuentas duplicadas</a>.</p>
+users.duplicate_profiles_please_call.heading: Llámenos
 users.password_compromised.warning: Hemos encontrado su contraseña en una filtración de datos en otro sitio o aplicación. %{app_name} requiere que cambie su contraseña para proteger su cuenta.
 users.personal_key.accessible_labels.code_example: Un ejemplo de clave personal con 16 caracteres
 users.personal_key.accessible_labels.preview: Vista previa de la clave personal

config/locales/fr.yml

@@ -2086,8 +2086,8 @@ users.delete.bullet_4: Nous informerons les organismes auxquels vous accédez av
 users.delete.heading: Êtes-vous sûr de vouloir supprimer votre compte ?
 users.delete.instructions: Saisissez votre mot de passe pour confirmer que vous souhaitez supprimer votre compte.
 users.delete.subheading: 'Si vous supprimez votre compte :'
-users.duplicate_profiles_please_call.error_details_html: <p>If you don’t recognize or can’t delete one of the duplicate accounts you saw on the multiple accounts page, call our contact center at <strong>%{contact_number}</strong> and provide them with the error code <strong>LG33</strong>.</p><p>Having questions about duplicate accounts or how to move forward? Learn more about <a href="%{help_url}">how to resolve duplicate accounts</a>.</p>
-users.duplicate_profiles_please_call.heading: Please give us a call
+users.duplicate_profiles_please_call.error_details_html: <p>Si vous ne reconnaissez pas un compte ou ne pouvez pas accéder à l’un des comptes que vous avez identifiés sur la page détaillant vos différents comptes, veuillez appeler notre centre de contact au <strong>%{contact_number}</strong> en fournissant le code d’erreur <strong>LG33</strong>.</p><p>Vous avez des questions sur les comptes en double ou la marche à suivre ? En savoir plus sur <a href="%{help_url}">la manière de résoudre les problèmes de comptes en double</a>.</p>
+users.duplicate_profiles_please_call.heading: Nous appeler
 users.password_compromised.warning: Nous avons trouvé votre mot de passe dans le cadre d’une fuite de données sur un autre site Web ou une application. %{app_name} vous demande de modifier votre mot de passe pour protéger votre compte.
 users.personal_key.accessible_labels.code_example: Un exemple de clé personnelle à 16 caractères
 users.personal_key.accessible_labels.preview: Aperçu de clé personnelle

config/locales/zh.yml

@@ -2099,8 +2099,8 @@ users.delete.bullet_4: 我们将通知你使用 %{app_name} 访问的政府机
 users.delete.heading: 你确定要删除账户吗？
 users.delete.instructions: 输入密码来确认你要删除账户。
 users.delete.subheading: 如果你删除自己的账户：
-users.duplicate_profiles_please_call.error_details_html: <p>If you don’t recognize or can’t delete one of the duplicate accounts you saw on the multiple accounts page, call our contact center at <strong>%{contact_number}</strong> and provide them with the error code <strong>LG33</strong>.</p><p>Having questions about duplicate accounts or how to move forward? Learn more about <a href="%{help_url}">how to resolve duplicate accounts</a>.</p>
-users.duplicate_profiles_please_call.heading: Please give us a call
+users.duplicate_profiles_please_call.error_details_html: <p>如果你不认识或无法删除在列出多个帐户的那一页面上看到的某个重复帐户，请拨打 <strong>%{contact_number}</strong> 致电我们的联系中心并提供出错代码 <strong>LG33</strong>。</p><p>对重复帐户或该怎么办有疑问？了解更多关于<a href="%{help_url}">如何解决重复帐户的信息。</a></p>
+users.duplicate_profiles_please_call.heading: 请给我们打个电话
 users.password_compromised.warning: 另一个网站或应用程序的数据泄露中我们发现有你的密码。%{app_name} 要求你更改密码来保护你的账户。
 users.personal_key.accessible_labels.code_example: 有 16 个字符的个人密钥示例
 users.personal_key.accessible_labels.preview: 个人密钥预览

lib/reporting/fraud_metrics_lg99_report.rb

@@ -17,7 +17,7 @@ class FraudMetricsLg99Report
     attr_reader :time_range
 
     module Events
-      IDV_FINAL_RESOLUTION = 'IdV: Final Resolution'
+      IDV_FINAL_RESOLUTION = 'IdV: final resolution'
       SUSPENDED_USERS = 'User Suspension: Suspended'
       REINSTATED_USERS = 'User Suspension: Reinstated'
 
@@ -158,9 +158,9 @@ def query
         fields
             name
           , properties.user_id as user_id
+         | filter name in %{event_names}
          | filter (name = %{idv_final_resolution} and properties.event_properties.fraud_review_pending = 1)
                  or (name != %{idv_final_resolution})
-        | filter name in %{event_names}
         | limit 10000
       QUERY
     end

lib/reporting/irs_fraud_metrics_lg99_report.rb

@@ -17,7 +17,7 @@ class IrsFraudMetricsLg99Report
     attr_reader :issuers, :time_range
 
     module Events
-      IDV_FINAL_RESOLUTION = 'IdV: Final Resolution'
+      IDV_FINAL_RESOLUTION = 'IdV: final resolution'
       SUSPENDED_USERS = 'User Suspension: Suspended'
       REINSTATED_USERS = 'User Suspension: Reinstated'
 
@@ -69,6 +69,11 @@ def as_emailable_reports
           table: lg99_metrics_table,
           filename: 'lg99_metrics',
         ),
+        Reporting::EmailableReport.new(
+          title: "IRS Credential Tenure Metric #{stats_month}",
+          table: credential_tenure_report_metric,
+          filename: 'Credential_Tenure_Metric',
+        ),
       ]
     end
 
@@ -83,6 +88,7 @@ def definitions_table
         ['Credentials Reinstated', 'Count',
          'The count of unique suspended accounts ' + '
          that are reinstated within the reporting month.'],
+        ['Credential Tenure', 'Count', 'The average age, in months, of all accounts'],
       ]
     end
 
@@ -120,6 +126,13 @@ def lg99_metrics_table
       ]
     end
 
+    def credential_tenure_report_metric
+      Reporting::IrsCredentialTenureReport.new(
+        time_range.end,
+        issuers: issuers,
+      ).irs_credential_tenure_report
+    end
+
     def stats_month
       time_range.begin.strftime('%b-%Y')
     end
@@ -156,9 +169,9 @@ def query
             name
           , properties.user_id as user_id
         | filter properties.service_provider IN %{issuers}
-        | filter (name = %{idv_final_resolution} and properties.event_properties.fraud_review_pending = 1)
-                 or (name != %{idv_final_resolution})
         | filter name in %{event_names}
+        | filter (name = %{idv_final_resolution} and properties.event_properties.fraud_review_pending = 1)
+                 or (name != %{idv_final_resolution})        
         | limit 10000
       QUERY
     end

spec/controllers/concerns/idv/choose_id_type_concern_spec.rb

@@ -8,6 +8,8 @@
   subject { controller }
 
   let(:analytics) { FakeAnalytics.new }
+  let(:step) { 'choose_id_type' }
+  let(:context_analytics) { { step: step } }
   let(:document_capture_session) { double(DocumentCaptureSession) }
   let(:parameters) do
     ActionController::Parameters.new(
@@ -145,7 +147,8 @@
           :dos_passport_composite_healthcheck_endpoint,
         ).and_return('http://dostest.com/status')
         allow(DocAuth::Dos::Requests::HealthCheckRequest).to receive(:new).and_return(request)
-        allow(request).to receive(:fetch).with(analytics).and_return(response)
+        allow(request).to receive(:fetch).with(analytics, context_analytics: context_analytics)
+          .and_return(response)
       end
 
       context 'when the dos response is successful' do
@@ -154,7 +157,7 @@
         end
 
         it 'returns true' do
-          expect(subject.dos_passport_api_healthy?(analytics:)).to be(true)
+          expect(subject.dos_passport_api_healthy?(analytics:, step:)).to be(true)
         end
       end
 
@@ -164,14 +167,14 @@
         end
 
         it 'returns false' do
-          expect(subject.dos_passport_api_healthy?(analytics:)).to be(false)
+          expect(subject.dos_passport_api_healthy?(analytics:, step:)).to be(false)
         end
       end
     end
 
     context 'when the endpoint is an empty string' do
       it 'returns true' do
-        expect(subject.dos_passport_api_healthy?(analytics:, endpoint: '')).to be(true)
+        expect(subject.dos_passport_api_healthy?(analytics:, step:, endpoint: '')).to be(true)
       end
     end
   end
@@ -187,7 +190,8 @@
         :dos_passport_composite_healthcheck_endpoint,
       ).and_return('http://dostest.com/status')
       allow(DocAuth::Dos::Requests::HealthCheckRequest).to receive(:new).and_return(request)
-      allow(request).to receive(:fetch).with(analytics).and_return(response)
+      allow(request).to receive(:fetch).with(analytics, context_analytics: context_analytics)
+        .and_return(response)
     end
 
     context 'when the dos passport api is healthy' do