LG-7306 Redacted logging of ThreatMetrix response

app/jobs/resolution_proofing_job.rb

@@ -85,13 +85,16 @@ def logger_info_hash(hash)
   def add_threatmetrix_result_to_callback_result(callback_log_data:, threatmetrix_result:)
     exception = threatmetrix_result.exception.inspect if threatmetrix_result.exception
 
+    response_h = Proofing::LexisNexis::Ddp::ResponseRedacter.
+      redact(threatmetrix_result.response_body)
     callback_log_data.result[:context][:stages][:threatmetrix] = {
       client: lexisnexis_ddp_proofer.class.vendor_name,
       errors: threatmetrix_result.errors,
       exception: exception,
       success: threatmetrix_result.success?,
       timed_out: threatmetrix_result.timed_out?,
       transaction_id: threatmetrix_result.transaction_id,
+      response_body: response_h,
     }
   end
 

app/services/proofing/lexis_nexis/ddp/proofer.rb

@@ -37,6 +37,7 @@ def proof_applicant(applicant, result)
 
         def process_response(response, result)
           body = response.response_body
+          result.response_body = body
           result.transaction_id = body['request_id']
           request_result = body['request_result']
           review_status = body['review_status']

app/services/proofing/lexis_nexis/ddp/response_redacter.rb

@@ -0,0 +1,139 @@
+module Proofing
+  module LexisNexis
+    module Ddp
+      class ResponseRedacter
+        ALLOWED_RESPONSE_FIELDS = %w[
+          account_email_assert_history
+          account_email_first_seen
+          account_email_last_event
+          account_email_last_update
+          account_email_result
+          account_email_score
+          account_email_worst_score
+          account_address_state
+          account_lex_id
+          account_lex_id_first_seen
+          account_lex_id_last_event
+          account_lex_id_last_update
+          account_lex_id_region
+          account_lex_id_result
+          account_lex_id_score
+          account_lex_id_worst_score
+          account_name_assert_history
+          account_name_first_seen
+          account_name_last_event
+          account_name_last_update
+          account_name_result
+          account_name_score
+          account_name_worst_score
+          api_call_datetime
+          api_caller_ip
+          api_type
+          api_version
+          bb_assessment
+          bb_assessment_rating
+          bb_bot_rating
+          bb_bot_score
+          bb_fraud_rating
+          bb_fraud_score
+          champion_request_duration
+          digital_id
+          digital_id_confidence
+          digital_id_confidence_rating
+          digital_id_first_seen
+          digital_id_last_event
+          digital_id_last_update
+          digital_id_result
+          digital_id_trust_score
+          digital_id_trust_score_rating
+          digital_id_trust_score_reason_code
+          digital_id_trust_score_summary_reason_code
+          emailage.emailriskscore.billriskcountry
+          emailage.emailriskscore.correlationid
+          emailage.emailriskscore.domainexists
+          emailage.emailriskscore.domainrelevantinfo
+          emailage.emailriskscore.domainrelevantinfoid
+          emailage.emailriskscore.domainriskcountry
+          emailage.emailriskscore.domainrisklevel
+          emailage.emailriskscore.domainrisklevelid
+          emailage.emailriskscore.eaadvice
+          emailage.emailriskscore.eaadviceid
+          emailage.emailriskscore.eareason
+          emailage.emailriskscore.eareasonid
+          emailage.emailriskscore.eariskband
+          emailage.emailriskscore.eariskbandid
+          emailage.emailriskscore.eascore
+          emailage.emailriskscore.eastatusid
+          emailage.emailriskscore.emailexists
+          emailage.emailriskscore.first_seen_days
+          emailage.emailriskscore.firstverificationdate
+          emailage.emailriskscore.fraudrisk
+          emailage.emailriskscore.namematch
+          emailage.emailriskscore.phone_status
+          emailage.emailriskscore.responsestatus.errorcode
+          emailage.emailriskscore.responsestatus.status
+          emailage.emailriskscore.shipforward
+          emailage.emailriskscore.status
+          emailage.emailriskscore.totalhits
+          emailage.emailriskscore.uniquehits
+          enabled_services
+          event_datetime
+          event_type
+          fraudpoint.conversation_id
+          fraudpoint.friendly_fraud_index
+          fraudpoint.manipulated_identity_index
+          fraudpoint.product_status
+          fraudpoint.transaction_reason_code
+          fraudpoint.risk_indicators_codes
+          fraudpoint.risk_indicators_descriptions
+          fraudpoint.score
+          fraudpoint.stolen_identity_index
+          fraudpoint.suspicious_activity_index
+          fraudpoint.synthetic_identity_index
+          fraudpoint.transaction_status
+          fraudpoint.vulnerable_victim_index
+          integration_hub_results
+          org_id
+          policy
+          policy_details_api
+          policy_engine_version
+          policy_score
+          primary_industry
+          reason_code
+          request_duration
+          request_id
+          request_result
+          review_status
+          risk_rating
+          secondary_industry
+          service_type
+          session_id
+          session_id_query_count
+          summary_risk_score
+          tmx_reason_code
+          tmx_risk_rating
+          tmx_summary_reason_code
+          tmx_variables
+          tps_datetime
+          tps_duration
+          tps_error
+          tps_result
+          tps_type
+          tps_vendor
+          tps_was_timeout
+          unknown_session
+        ]
+
+        # @param [Hash] body
+        def self.redact(hash)
+          filtered_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS)
+          unfiltered_keys = hash.keys - filtered_response_h.keys
+          unfiltered_keys.each do |key|
+            filtered_response_h[key] = '[redacted]'
+          end
+          filtered_response_h
+        end
+      end
+    end
+  end
+end

app/services/proofing/mock/ddp_mock_client.rb

@@ -32,6 +32,11 @@ class DdpMockClient < Proofing::Base
 
       proof do |applicant, result|
         result.transaction_id = TRANSACTION_ID
+        response_body = File.read(
+          Rails.root.join(
+            'spec', 'fixtures', 'proofing', 'lexis_nexis', 'ddp', 'successful_response.json'
+          ),
+        )
         result.review_status = case SsnFormatter.format(applicant[:ssn])
         when REJECT_STATUS_SSN
           'reject'
@@ -42,6 +47,9 @@ class DdpMockClient < Proofing::Base
         else
           'pass'
         end
+        result.response_body = JSON.parse(
+          response_body.gsub('REVIEW_STATUS', result.review_status.to_s),
+        )
       end
     end
   end

app/services/proofing/result.rb

@@ -1,7 +1,7 @@
 module Proofing
   class Result
     attr_reader :exception
-    attr_accessor :context, :transaction_id, :reference, :review_status
+    attr_accessor :context, :transaction_id, :reference, :review_status, :response_body
 
     def initialize(
       errors: {},

spec/fixtures/proofing/lexis_nexis/ddp/error_response.json

@@ -1,5 +1,6 @@
 {
   "error_detail": "service_type",
   "request_id":"1234-abcd",
-  "request_result":"fail_invalid_parameter"
+  "request_result":"fail_invalid_parameter",
+  "review_status":"REVIEW_STATUS"
 }

spec/fixtures/proofing/lexis_nexis/ddp/successful_redacted_response.json

@@ -0,0 +1,10 @@
+{
+  "request_id": "1234",
+  "request_result": "success",
+  "review_status": "pass",
+  "risk_rating": "trusted",
+  "summary_risk_score": "-6",
+  "tmx_risk_rating": "neutral",
+  "fraudpoint.score": "500",
+  "first_name": "[redacted]"
+}

spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json

@@ -5,5 +5,6 @@
   "risk_rating": "trusted",
   "summary_risk_score": "-6",
   "tmx_risk_rating": "neutral",
-  "fraudpoint.score": "500"
+  "fraudpoint.score": "500",
+  "first_name": "WARNING! YOU SHOULD NEVER SEE THIS PII FIELD IN THE LOGS"
 }

spec/jobs/resolution_proofing_job_spec.rb

@@ -42,6 +42,9 @@
   let(:threatmetrix_session_id) { SecureRandom.uuid }
   let(:threatmetrix_request_id) { Proofing::Mock::DdpMockClient::TRANSACTION_ID }
   let(:request_ip) { Faker::Internet.ip_v4_address }
+  let(:ddp_response_body) do
+    JSON.parse(LexisNexisFixtures.ddp_success_redacted_response_json, symbolize_names: true)
+  end
 
   describe '.perform_later' do
     it 'stores results' do
@@ -153,6 +156,7 @@
                 success: true,
                 timed_out: false,
                 transaction_id: threatmetrix_request_id,
+                response_body: ddp_response_body,
               },
             },
           },
@@ -230,6 +234,7 @@
                   success: true,
                   timed_out: false,
                   transaction_id: threatmetrix_request_id,
+                  response_body: ddp_response_body,
                 },
               },
             },

spec/support/lexis_nexis_fixtures.rb

@@ -32,6 +32,11 @@ def ddp_success_response_json
       JSON.parse(raw).to_json
     end
 
+    def ddp_success_redacted_response_json
+      raw = read_fixture_file_at_path('ddp/successful_redacted_response.json')
+      JSON.parse(raw).to_json
+    end
+
     def ddp_failure_response_json
       raw = read_fixture_file_at_path('ddp/failed_response.json')
       JSON.parse(raw).to_json