From 6dfd29de5a8bca75392d3cc9f6b3f79eed137702 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Sun, 28 Aug 2022 17:03:35 -0500 Subject: [PATCH 01/21] LG-7306 Redacted logging of ThreatMetrix response --- app/jobs/resolution_proofing_job.rb | 3 ++ app/services/idv/steps/verify_base_step.rb | 21 +++++++------ .../proofing/lexis_nexis/ddp/proofer.rb | 2 +- .../lexis_nexis/ddp/response_whitelister.rb | 31 +++++++++++++++++++ app/services/proofing/mock/ddp_mock_client.rb | 5 +++ app/services/proofing/result.rb | 2 +- .../lexis_nexis/ddp/successful_response.json | 3 +- 7 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 app/services/proofing/lexis_nexis/ddp/response_whitelister.rb diff --git a/app/jobs/resolution_proofing_job.rb b/app/jobs/resolution_proofing_job.rb index b763f3de590..0e746a8f256 100644 --- a/app/jobs/resolution_proofing_job.rb +++ b/app/jobs/resolution_proofing_job.rb @@ -85,6 +85,8 @@ def logger_info_hash(hash) def add_threatmetrix_result_to_callback_result(callback_log_data:, threatmetrix_result:) exception = threatmetrix_result.exception.inspect if threatmetrix_result.exception + response_h = Proofing::LexisNexis::Ddp::ResponseWhitelister. + whilelist_response_and_redact_unwhitelisted_fields(threatmetrix_result.response_body) callback_log_data.result[:context][:stages][:threatmetrix] = { client: lexisnexis_ddp_proofer.class.vendor_name, errors: threatmetrix_result.errors, @@ -92,6 +94,7 @@ def add_threatmetrix_result_to_callback_result(callback_log_data:, threatmetrix_ success: threatmetrix_result.success?, timed_out: threatmetrix_result.timed_out?, transaction_id: threatmetrix_result.transaction_id, + response_body: response_h, } end diff --git a/app/services/idv/steps/verify_base_step.rb b/app/services/idv/steps/verify_base_step.rb index 90cbfb54591..94bbca3b505 100644 --- a/app/services/idv/steps/verify_base_step.rb +++ b/app/services/idv/steps/verify_base_step.rb @@ -49,10 +49,11 @@ def add_proofing_costs(results) if stage == :resolution # transaction_id comes from ConversationId add_cost(:lexis_nexis_resolution, transaction_id: hash[:transaction_id]) - tmx_id = hash[:threatmetrix_request_id] - add_cost(:threatmetrix, transaction_id: tmx_id) if tmx_id elsif stage == :state_id process_aamva(hash[:transaction_id]) + elsif stage == :threatmetrix + tmx_id = hash[:transaction_id] + add_cost(:threatmetrix, transaction_id: tmx_id) if tmx_id end end end @@ -158,14 +159,14 @@ def enqueue_job throttle_type: :proof_ssn, ) - if throttle.throttled_else_increment? - @flow.analytics.throttler_rate_limit_triggered( - throttle_type: :proof_ssn, - step_name: self.class, - ) - redirect_to idv_session_errors_ssn_failure_url - return - end + # if throttle.throttled_else_increment? + # @flow.analytics.throttler_rate_limit_triggered( + # throttle_type: :proof_ssn, + # step_name: self.class, + # ) + # redirect_to idv_session_errors_ssn_failure_url + # return + # end end document_capture_session = create_document_capture_session( diff --git a/app/services/proofing/lexis_nexis/ddp/proofer.rb b/app/services/proofing/lexis_nexis/ddp/proofer.rb index fc880b667ce..648fb1eb046 100644 --- a/app/services/proofing/lexis_nexis/ddp/proofer.rb +++ b/app/services/proofing/lexis_nexis/ddp/proofer.rb @@ -36,7 +36,7 @@ def proof_applicant(applicant, result) private def process_response(response, result) - body = response.response_body + result.response_body = response.response_body result.transaction_id = body['request_id'] request_result = body['request_result'] review_status = body['review_status'] diff --git a/app/services/proofing/lexis_nexis/ddp/response_whitelister.rb b/app/services/proofing/lexis_nexis/ddp/response_whitelister.rb new file mode 100644 index 00000000000..d45b2aa2023 --- /dev/null +++ b/app/services/proofing/lexis_nexis/ddp/response_whitelister.rb @@ -0,0 +1,31 @@ +module Proofing + module LexisNexis + module Ddp + class ResponseWhitelister + WHITELISTED_RESPONSE_FIELDS = %w[ + fraudpoint.score + request_id + request_result + review_status + risk_rating + summary_risk_score + tmx_risk_rating + ] + + def self.whilelist_response_and_redact_unwhitelisted_fields(response_json) + begin + hash = JSON.parse(response_json) + whielisted_response_h = hash.slice(*WHITELISTED_RESPONSE_FIELDS) + unwhitelisted_fields = hash.keys - whielisted_response_h.keys + unwhitelisted_fields.each do |key| + whielisted_response_h[key] = '[redacted]' + end + whielisted_response_h + rescue JSON::ParserError + {} + end + end + end + end + end +end diff --git a/app/services/proofing/mock/ddp_mock_client.rb b/app/services/proofing/mock/ddp_mock_client.rb index 4a5e80113ee..c9d147d1fac 100644 --- a/app/services/proofing/mock/ddp_mock_client.rb +++ b/app/services/proofing/mock/ddp_mock_client.rb @@ -32,6 +32,11 @@ class DdpMockClient < Proofing::Base proof do |applicant, result| result.transaction_id = TRANSACTION_ID + result.response_body = File.read( + Rails.root.join( + 'spec', 'fixtures', 'proofing', 'lexis_nexis', 'ddp', 'successful_response.json' + ), + ) result.review_status = case SsnFormatter.format(applicant[:ssn]) when REJECT_STATUS_SSN 'reject' diff --git a/app/services/proofing/result.rb b/app/services/proofing/result.rb index 644083c5e4b..9adbcc3f662 100644 --- a/app/services/proofing/result.rb +++ b/app/services/proofing/result.rb @@ -1,7 +1,7 @@ module Proofing class Result attr_reader :exception - attr_accessor :context, :transaction_id, :reference, :review_status + attr_accessor :context, :transaction_id, :reference, :review_status, :response_body def initialize( errors: {}, diff --git a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json index d514e487563..5b41f4911e5 100644 --- a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json +++ b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json @@ -5,5 +5,6 @@ "risk_rating": "trusted", "summary_risk_score": "-6", "tmx_risk_rating": "neutral", - "fraudpoint.score": "500" + "fraudpoint.score": "500", + "first_name": "WARNING! YOU SHOULD NEVER SEE THIS PII FIELD IN LOG FILES" } From 6c17eb1d674a8dea06f1887ff1efcfcdf690377f Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Sun, 28 Aug 2022 17:06:12 -0500 Subject: [PATCH 02/21] Rm testing --- app/services/idv/steps/verify_base_step.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/services/idv/steps/verify_base_step.rb b/app/services/idv/steps/verify_base_step.rb index 94bbca3b505..cc9836ae337 100644 --- a/app/services/idv/steps/verify_base_step.rb +++ b/app/services/idv/steps/verify_base_step.rb @@ -159,14 +159,14 @@ def enqueue_job throttle_type: :proof_ssn, ) - # if throttle.throttled_else_increment? - # @flow.analytics.throttler_rate_limit_triggered( - # throttle_type: :proof_ssn, - # step_name: self.class, - # ) - # redirect_to idv_session_errors_ssn_failure_url - # return - # end + if throttle.throttled_else_increment? + @flow.analytics.throttler_rate_limit_triggered( + throttle_type: :proof_ssn, + step_name: self.class, + ) + redirect_to idv_session_errors_ssn_failure_url + return + end end document_capture_session = create_document_capture_session( From e703a53692da9c5cdda01f1f42adf170ad18c878 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Sun, 28 Aug 2022 21:12:00 -0500 Subject: [PATCH 03/21] Fix --- app/services/proofing/lexis_nexis/ddp/proofer.rb | 3 ++- .../lexis_nexis/ddp/successful_redacted_response.json | 10 ++++++++++ .../proofing/lexis_nexis/ddp/successful_response.json | 3 +-- spec/jobs/resolution_proofing_job_spec.rb | 5 +++++ 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 spec/fixtures/proofing/lexis_nexis/ddp/successful_redacted_response.json diff --git a/app/services/proofing/lexis_nexis/ddp/proofer.rb b/app/services/proofing/lexis_nexis/ddp/proofer.rb index 648fb1eb046..7a78f1164a7 100644 --- a/app/services/proofing/lexis_nexis/ddp/proofer.rb +++ b/app/services/proofing/lexis_nexis/ddp/proofer.rb @@ -36,7 +36,8 @@ def proof_applicant(applicant, result) private def process_response(response, result) - result.response_body = response.response_body + body = response.response_body + result.response_body = body result.transaction_id = body['request_id'] request_result = body['request_result'] review_status = body['review_status'] diff --git a/spec/fixtures/proofing/lexis_nexis/ddp/successful_redacted_response.json b/spec/fixtures/proofing/lexis_nexis/ddp/successful_redacted_response.json new file mode 100644 index 00000000000..85b43947a59 --- /dev/null +++ b/spec/fixtures/proofing/lexis_nexis/ddp/successful_redacted_response.json @@ -0,0 +1,10 @@ +{ + "request_id": "1234", + "request_result": "success", + "review_status": "pass", + "risk_rating": "trusted", + "summary_risk_score": "-6", + "tmx_risk_rating": "neutral", + "fraudpoint.score": "500", + "first_name": "[redacted]" +} diff --git a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json index 5b41f4911e5..d514e487563 100644 --- a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json +++ b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json @@ -5,6 +5,5 @@ "risk_rating": "trusted", "summary_risk_score": "-6", "tmx_risk_rating": "neutral", - "fraudpoint.score": "500", - "first_name": "WARNING! YOU SHOULD NEVER SEE THIS PII FIELD IN LOG FILES" + "fraudpoint.score": "500" } diff --git a/spec/jobs/resolution_proofing_job_spec.rb b/spec/jobs/resolution_proofing_job_spec.rb index 2ff8246c433..86e582ff443 100644 --- a/spec/jobs/resolution_proofing_job_spec.rb +++ b/spec/jobs/resolution_proofing_job_spec.rb @@ -42,6 +42,9 @@ let(:threatmetrix_session_id) { SecureRandom.uuid } let(:threatmetrix_request_id) { Proofing::Mock::DdpMockClient::TRANSACTION_ID } let(:request_ip) { Faker::Internet.ip_v4_address } + let(:ddp_response_body) do + JSON.parse(LexisNexisFixtures.ddp_success_response_json, symbolize_names: true) + end describe '.perform_later' do it 'stores results' do @@ -153,6 +156,7 @@ success: true, timed_out: false, transaction_id: threatmetrix_request_id, + response_body: ddp_response_body, }, }, }, @@ -230,6 +234,7 @@ success: true, timed_out: false, transaction_id: threatmetrix_request_id, + response_body: ddp_response_body, }, }, }, From 334e96a6ce9a6e031d6ed81f43c0bb2d89773481 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Sun, 28 Aug 2022 21:18:46 -0500 Subject: [PATCH 04/21] Specs --- .../proofing/lexis_nexis/ddp/successful_response.json | 3 ++- spec/jobs/resolution_proofing_job_spec.rb | 2 +- spec/support/lexis_nexis_fixtures.rb | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json index d514e487563..4f63de45a75 100644 --- a/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json +++ b/spec/fixtures/proofing/lexis_nexis/ddp/successful_response.json @@ -5,5 +5,6 @@ "risk_rating": "trusted", "summary_risk_score": "-6", "tmx_risk_rating": "neutral", - "fraudpoint.score": "500" + "fraudpoint.score": "500", + "first_name": "WARNING! YOU SHOULD NEVER SEE THIS PII FIELD IN THE LOGS" } diff --git a/spec/jobs/resolution_proofing_job_spec.rb b/spec/jobs/resolution_proofing_job_spec.rb index 86e582ff443..ad4c9b88e0b 100644 --- a/spec/jobs/resolution_proofing_job_spec.rb +++ b/spec/jobs/resolution_proofing_job_spec.rb @@ -43,7 +43,7 @@ let(:threatmetrix_request_id) { Proofing::Mock::DdpMockClient::TRANSACTION_ID } let(:request_ip) { Faker::Internet.ip_v4_address } let(:ddp_response_body) do - JSON.parse(LexisNexisFixtures.ddp_success_response_json, symbolize_names: true) + JSON.parse(LexisNexisFixtures.ddp_success_redacted_response_json, symbolize_names: true) end describe '.perform_later' do diff --git a/spec/support/lexis_nexis_fixtures.rb b/spec/support/lexis_nexis_fixtures.rb index ab1c91b78f4..30e83cb5264 100644 --- a/spec/support/lexis_nexis_fixtures.rb +++ b/spec/support/lexis_nexis_fixtures.rb @@ -32,6 +32,11 @@ def ddp_success_response_json JSON.parse(raw).to_json end + def ddp_success_redacted_response_json + raw = read_fixture_file_at_path('ddp/successful_redacted_response.json') + JSON.parse(raw).to_json + end + def ddp_failure_response_json raw = read_fixture_file_at_path('ddp/failed_response.json') JSON.parse(raw).to_json From f418d06dfd1595a1c97b5a61aa062d120d802f57 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 10:18:51 -0500 Subject: [PATCH 05/21] ResponseRedacter --- app/jobs/resolution_proofing_job.rb | 2 +- .../ddp/{response_whitelister.rb => response_redacter.rb} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename app/services/proofing/lexis_nexis/ddp/{response_whitelister.rb => response_redacter.rb} (82%) diff --git a/app/jobs/resolution_proofing_job.rb b/app/jobs/resolution_proofing_job.rb index 0e746a8f256..b8ad0c68975 100644 --- a/app/jobs/resolution_proofing_job.rb +++ b/app/jobs/resolution_proofing_job.rb @@ -85,7 +85,7 @@ def logger_info_hash(hash) def add_threatmetrix_result_to_callback_result(callback_log_data:, threatmetrix_result:) exception = threatmetrix_result.exception.inspect if threatmetrix_result.exception - response_h = Proofing::LexisNexis::Ddp::ResponseWhitelister. + response_h = Proofing::LexisNexis::Ddp::ResponseRedacter. whilelist_response_and_redact_unwhitelisted_fields(threatmetrix_result.response_body) callback_log_data.result[:context][:stages][:threatmetrix] = { client: lexisnexis_ddp_proofer.class.vendor_name, diff --git a/app/services/proofing/lexis_nexis/ddp/response_whitelister.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb similarity index 82% rename from app/services/proofing/lexis_nexis/ddp/response_whitelister.rb rename to app/services/proofing/lexis_nexis/ddp/response_redacter.rb index d45b2aa2023..9f96afd7ef5 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_whitelister.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -1,8 +1,8 @@ module Proofing module LexisNexis module Ddp - class ResponseWhitelister - WHITELISTED_RESPONSE_FIELDS = %w[ + class ResponseRedacter + ALLOWED_RESPONSE_FIELDS = %w[ fraudpoint.score request_id request_result @@ -15,7 +15,7 @@ class ResponseWhitelister def self.whilelist_response_and_redact_unwhitelisted_fields(response_json) begin hash = JSON.parse(response_json) - whielisted_response_h = hash.slice(*WHITELISTED_RESPONSE_FIELDS) + whielisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) unwhitelisted_fields = hash.keys - whielisted_response_h.keys unwhitelisted_fields.each do |key| whielisted_response_h[key] = '[redacted]' From 448c0670e859f95d4afe34edbb6de429664cbd65 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 10:38:22 -0500 Subject: [PATCH 06/21] whilelist_response_and_redact_unwhitelisted_fields --- app/jobs/resolution_proofing_job.rb | 2 +- app/services/proofing/lexis_nexis/ddp/response_redacter.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/jobs/resolution_proofing_job.rb b/app/jobs/resolution_proofing_job.rb index b8ad0c68975..7b6c305ebe1 100644 --- a/app/jobs/resolution_proofing_job.rb +++ b/app/jobs/resolution_proofing_job.rb @@ -86,7 +86,7 @@ def add_threatmetrix_result_to_callback_result(callback_log_data:, threatmetrix_ exception = threatmetrix_result.exception.inspect if threatmetrix_result.exception response_h = Proofing::LexisNexis::Ddp::ResponseRedacter. - whilelist_response_and_redact_unwhitelisted_fields(threatmetrix_result.response_body) + redact(threatmetrix_result.response_body) callback_log_data.result[:context][:stages][:threatmetrix] = { client: lexisnexis_ddp_proofer.class.vendor_name, errors: threatmetrix_result.errors, diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index 9f96afd7ef5..4d1fa2fdfe6 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -12,7 +12,8 @@ class ResponseRedacter tmx_risk_rating ] - def self.whilelist_response_and_redact_unwhitelisted_fields(response_json) + # @param [String] body + def self.redact(response_json) begin hash = JSON.parse(response_json) whielisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) From 1e9f2d1bb957833d60a601914f8a2e26b4e64bf6 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 10:48:02 -0500 Subject: [PATCH 07/21] Set review_status on forced testing fail states for the mock --- app/services/proofing/mock/ddp_mock_client.rb | 3 ++- spec/fixtures/proofing/lexis_nexis/ddp/error_response.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/services/proofing/mock/ddp_mock_client.rb b/app/services/proofing/mock/ddp_mock_client.rb index c9d147d1fac..54a14835eac 100644 --- a/app/services/proofing/mock/ddp_mock_client.rb +++ b/app/services/proofing/mock/ddp_mock_client.rb @@ -32,7 +32,7 @@ class DdpMockClient < Proofing::Base proof do |applicant, result| result.transaction_id = TRANSACTION_ID - result.response_body = File.read( + response_body = File.read( Rails.root.join( 'spec', 'fixtures', 'proofing', 'lexis_nexis', 'ddp', 'successful_response.json' ), @@ -47,6 +47,7 @@ class DdpMockClient < Proofing::Base else 'pass' end + result.response_body = response_body.gsub('REVIEW_STATUS', result.review_status.to_s) end end end diff --git a/spec/fixtures/proofing/lexis_nexis/ddp/error_response.json b/spec/fixtures/proofing/lexis_nexis/ddp/error_response.json index 4aa2c84f848..d36ca4063bf 100644 --- a/spec/fixtures/proofing/lexis_nexis/ddp/error_response.json +++ b/spec/fixtures/proofing/lexis_nexis/ddp/error_response.json @@ -1,5 +1,6 @@ { "error_detail": "service_type", "request_id":"1234-abcd", - "request_result":"fail_invalid_parameter" + "request_result":"fail_invalid_parameter", + "review_status":"REVIEW_STATUS" } From d3ce41c73d4f6477358a75deb1295c4b7f701c88 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 10:51:31 -0500 Subject: [PATCH 08/21] transaction_id comes from request_id (comment) --- app/services/idv/steps/verify_base_step.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/services/idv/steps/verify_base_step.rb b/app/services/idv/steps/verify_base_step.rb index cc9836ae337..919d8b70bf7 100644 --- a/app/services/idv/steps/verify_base_step.rb +++ b/app/services/idv/steps/verify_base_step.rb @@ -52,6 +52,7 @@ def add_proofing_costs(results) elsif stage == :state_id process_aamva(hash[:transaction_id]) elsif stage == :threatmetrix + # transaction_id comes from request_id tmx_id = hash[:transaction_id] add_cost(:threatmetrix, transaction_id: tmx_id) if tmx_id end From f3ae6b54fb9ba683021e43d2a521e92a03c3f853 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 11:17:33 -0500 Subject: [PATCH 09/21] changelog: Upcoming Features, ThreatMetrix, Redacted response logging From ba70cf0ac3b6346ac1f8a7c8833d3145e629d74a Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 12:28:23 -0500 Subject: [PATCH 10/21] redact(hash) --- app/services/proofing/lexis_nexis/ddp/response_redacter.rb | 5 ++--- app/services/proofing/mock/ddp_mock_client.rb | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index 4d1fa2fdfe6..9036dcc1d67 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -12,10 +12,9 @@ class ResponseRedacter tmx_risk_rating ] - # @param [String] body - def self.redact(response_json) + # @param [Hash] body + def self.redact(hash) begin - hash = JSON.parse(response_json) whielisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) unwhitelisted_fields = hash.keys - whielisted_response_h.keys unwhitelisted_fields.each do |key| diff --git a/app/services/proofing/mock/ddp_mock_client.rb b/app/services/proofing/mock/ddp_mock_client.rb index 54a14835eac..d6553fed8ef 100644 --- a/app/services/proofing/mock/ddp_mock_client.rb +++ b/app/services/proofing/mock/ddp_mock_client.rb @@ -47,7 +47,9 @@ class DdpMockClient < Proofing::Base else 'pass' end - result.response_body = response_body.gsub('REVIEW_STATUS', result.review_status.to_s) + result.response_body = JSON.parse( + response_body.gsub('REVIEW_STATUS', result.review_status.to_s), + ) end end end From ba79fff979265f0eb153046506a96e1d8f1a4191 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 12:33:52 -0500 Subject: [PATCH 11/21] Full ALLOWED_RESPONSE_FIELDS --- .../lexis_nexis/ddp/response_redacter.rb | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index 9036dcc1d67..688b791fc95 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -3,13 +3,134 @@ module LexisNexis module Ddp class ResponseRedacter ALLOWED_RESPONSE_FIELDS = %w[ + account_email_assert_history + account_email_first_seen + account_email_last_event + account_email_last_update + account_email_result + account_email_score + account_email_worst_score + account_address_state + account_lex_id + account_lex_id_first_seen + account_lex_id_last_event + account_lex_id_last_update + account_lex_id_region + account_lex_id_result + account_lex_id_score + account_lex_id_worst_score + account_name_assert_history + account_name_first_seen + account_name_last_event + account_name_last_update + account_name_result + account_name_score + account_name_worst_score + api_call_datetime + api_caller_ip + api_type + api_version + bb_assessment + bb_assessment_rating + bb_bot_rating + bb_bot_score + bb_fraud_rating + bb_fraud_score + champion_request_duration + digital_id + digital_id_confidence + digital_id_confidence_rating + digital_id_first_seen + digital_id_last_event + digital_id_last_update + digital_id_result + digital_id_trust_score + digital_id_trust_score_rating + digital_id_trust_score_reason_code + digital_id_trust_score_summary_reason_code + emailage.emailriskscore.billriskcountry + emailage.emailriskscore.correlationid + emailage.emailriskscore.domainexists + emailage.emailriskscore.domainrelevantinfo + emailage.emailriskscore.domainrelevantinfoid + emailage.emailriskscore.domainriskcountry + emailage.emailriskscore.domainrisklevel + emailage.emailriskscore.domainrisklevelid + emailage.emailriskscore.eaadvice + emailage.emailriskscore.eaadviceid + emailage.emailriskscore.eareason + emailage.emailriskscore.eareasonid + emailage.emailriskscore.eariskband + emailage.emailriskscore.eariskbandid + emailage.emailriskscore.eascore + emailage.emailriskscore.eastatusid + emailage.emailriskscore.emailexists + emailage.emailriskscore.first_seen_days + emailage.emailriskscore.firstverificationdate + emailage.emailriskscore.fraudrisk + emailage.emailriskscore.namematch + emailage.emailriskscore.phone_status + emailage.emailriskscore.responsestatus.errorcode + emailage.emailriskscore.responsestatus.status + emailage.emailriskscore.shipforward + emailage.emailriskscore.status + emailage.emailriskscore.totalhits + emailage.emailriskscore.uniquehits + enabled_services + event_datetime + event_type + fraudpoint.conversation_id + fraudpoint.friendly_fraud_index + fraudpoint.manipulated_identity_index + fraudpoint.product_status + fraudpoint.transaction_reason_code + fraudpoint.risk_indicators_codes + fraudpoint.risk_indicators_descriptions fraudpoint.score + fraudpoint.stolen_identity_index + fraudpoint.suspicious_activity_index + fraudpoint.synthetic_identity_index + fraudpoint.transaction_status + fraudpoint.vulnerable_victim_index + integration_hub_results + org_id + policy + policy_details_api + policy_engine_version + policy_score + primary_industry + reason_code + request_duration request_id request_result review_status risk_rating + secondary_industry + service_type + session_id + session_id_query_count + ssn_hash + ssn_hash_assert_history + ssn_hash_attributes + ssn_hash_first_seen + ssn_hash_last_event + ssn_hash_last_update + ssn_hash_result + ssn_hash_score + ssn_hash_worst_score summary_risk_score + tmx_reason_code tmx_risk_rating + tmx_summary_reason_code + tmx_variables + tps_datetime + tps_duration + tps_error + tps_result + tps_type + tps_vendor + tps_was_timeout + unknown_session ] # @param [Hash] body From 74f01bb1bb1e1048827a15b9016ab041cc82cb01 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 15:37:48 -0500 Subject: [PATCH 12/21] rm ssn_hash fields --- .../proofing/lexis_nexis/ddp/response_redacter.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index 688b791fc95..a243aa3f06a 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -109,15 +109,6 @@ class ResponseRedacter service_type session_id session_id_query_count - ssn_hash - ssn_hash_assert_history - ssn_hash_attributes - ssn_hash_first_seen - ssn_hash_last_event - ssn_hash_last_update - ssn_hash_result - ssn_hash_score - ssn_hash_worst_score summary_risk_score tmx_reason_code tmx_risk_rating From ad8f8fd415229c0bae2b7a3e3452b1c699393499 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 18:09:36 -0500 Subject: [PATCH 13/21] whielisted_response_h typo and rm ParserError --- .../proofing/lexis_nexis/ddp/response_redacter.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index a243aa3f06a..e70c973749f 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -126,16 +126,12 @@ class ResponseRedacter # @param [Hash] body def self.redact(hash) - begin - whielisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) - unwhitelisted_fields = hash.keys - whielisted_response_h.keys - unwhitelisted_fields.each do |key| - whielisted_response_h[key] = '[redacted]' - end - whielisted_response_h - rescue JSON::ParserError - {} + whitelisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) + unwhitelisted_fields = hash.keys - whitelisted_response_h.keys + unwhitelisted_fields.each do |key| + whitelisted_response_h[key] = '[redacted]' end + whitelisted_response_h end end end From 69addb89a97256e0c04f52832a3e4f7ac47d855b Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Mon, 29 Aug 2022 18:59:10 -0500 Subject: [PATCH 14/21] rm whitelist --- .../proofing/lexis_nexis/ddp/response_redacter.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb index e70c973749f..fa584f1e399 100644 --- a/app/services/proofing/lexis_nexis/ddp/response_redacter.rb +++ b/app/services/proofing/lexis_nexis/ddp/response_redacter.rb @@ -126,12 +126,12 @@ class ResponseRedacter # @param [Hash] body def self.redact(hash) - whitelisted_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) - unwhitelisted_fields = hash.keys - whitelisted_response_h.keys - unwhitelisted_fields.each do |key| - whitelisted_response_h[key] = '[redacted]' + filtered_response_h = hash.slice(*ALLOWED_RESPONSE_FIELDS) + unfiltered_keys = hash.keys - filtered_response_h.keys + unfiltered_keys.each do |key| + filtered_response_h[key] = '[redacted]' end - whitelisted_response_h + filtered_response_h end end end From 42efb4d251c3ec7d278a7b9e6fc877142e5a180c Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 13:14:11 -0500 Subject: [PATCH 15/21] Fix costing --- .../db/proofing_cost/add_user_proofing_cost.rb | 1 + app/services/db/sp_cost/add_sp_cost.rb | 1 + ...0220830121822_add_threatmetrix_to_proofing_cost.rb | 10 ++++++++++ db/schema.rb | 3 ++- spec/features/idv/doc_auth/verify_step_spec.rb | 11 ++++++++--- 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb diff --git a/app/services/db/proofing_cost/add_user_proofing_cost.rb b/app/services/db/proofing_cost/add_user_proofing_cost.rb index fa7d00d35f3..e818ac6b535 100644 --- a/app/services/db/proofing_cost/add_user_proofing_cost.rb +++ b/app/services/db/proofing_cost/add_user_proofing_cost.rb @@ -13,6 +13,7 @@ class ProofingCostTypeError < StandardError; end lexis_nexis_address gpo_letter phone_otp + threatmetrix ].freeze def self.call(user_id, token) diff --git a/app/services/db/sp_cost/add_sp_cost.rb b/app/services/db/sp_cost/add_sp_cost.rb index 6c4ef015e18..fe798e4e130 100644 --- a/app/services/db/sp_cost/add_sp_cost.rb +++ b/app/services/db/sp_cost/add_sp_cost.rb @@ -12,6 +12,7 @@ class SpCostTypeError < StandardError; end lexis_nexis_resolution lexis_nexis_address gpo_letter + threatmetrix ].freeze def self.call(service_provider, ial, token, transaction_id: nil, user: nil) diff --git a/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb b/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb new file mode 100644 index 00000000000..9a1839e3208 --- /dev/null +++ b/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb @@ -0,0 +1,10 @@ +class AddThreatmetrixToProofingCost < ActiveRecord::Migration[7.0] + def up + add_column :proofing_costs, :threatmetrix_count, :integer + change_column_default :proofing_costs, :threatmetrix_count, 0 + end + + def down + remove_column :proofing_costs, :threatmetrix_count + end +end diff --git a/db/schema.rb b/db/schema.rb index 0a150f2319e..9113fdd76f2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_08_08_140030) do +ActiveRecord::Schema[7.0].define(version: 2022_08_30_121822) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" enable_extension "pgcrypto" @@ -476,6 +476,7 @@ t.datetime "updated_at", precision: nil, null: false t.integer "acuant_result_count", default: 0 t.integer "acuant_selfie_count", default: 0 + t.integer "threatmetrix_count", default: 0 t.index ["user_id"], name: "index_proofing_costs_on_user_id", unique: true end diff --git a/spec/features/idv/doc_auth/verify_step_spec.rb b/spec/features/idv/doc_auth/verify_step_spec.rb index 1760d3aa540..8217cb92889 100644 --- a/spec/features/idv/doc_auth/verify_step_spec.rb +++ b/spec/features/idv/doc_auth/verify_step_spec.rb @@ -9,6 +9,9 @@ let(:fake_analytics) { FakeAnalytics.new } before do allow_any_instance_of(ApplicationController).to receive(:analytics).and_return(fake_analytics) + allow(IdentityConfig.store).to receive(:proofing_device_profiling_collecting_enabled). + and_return(true) + allow(IdentityConfig.store).to receive(:lexisnexis_threatmetrix_enabled).and_return(true) end it 'displays the expected content' do @@ -177,7 +180,7 @@ anything, should_proof_state_id: true, trace_id: anything, - threatmetrix_session_id: nil, + threatmetrix_session_id: anything, user_id: user.id, request_ip: kind_of(String), ). @@ -188,6 +191,8 @@ click_idv_continue expect(DocAuthLog.find_by(user_id: user.id).aamva).not_to be_nil + expect(SpCost.find_by(cost_type: 'threatmetrix').transaction_id). + to eq(Proofing::Mock::DdpMockClient::TRANSACTION_ID) end end @@ -204,7 +209,7 @@ anything, should_proof_state_id: false, trace_id: anything, - threatmetrix_session_id: nil, + threatmetrix_session_id: anything, user_id: user.id, request_ip: kind_of(String), ). @@ -229,7 +234,7 @@ anything, should_proof_state_id: false, trace_id: anything, - threatmetrix_session_id: nil, + threatmetrix_session_id: anything, user_id: user.id, request_ip: kind_of(String), ). From b0848e25779a142cfa8ce9f19a838d8249e6f7f1 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 14:01:14 -0500 Subject: [PATCH 16/21] Update costing report --- spec/features/reports/proofing_costs_report_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/features/reports/proofing_costs_report_spec.rb b/spec/features/reports/proofing_costs_report_spec.rb index 1e920612e36..14978fd2129 100644 --- a/spec/features/reports/proofing_costs_report_spec.rb +++ b/spec/features/reports/proofing_costs_report_spec.rb @@ -28,6 +28,7 @@ 'gpo_letter_count_average' => 0.0, 'lexis_nexis_address_count_average' => 0.0, 'phone_otp_count_average' => 0.0, + 'threatmetrix_count_average' => 0.0, } end From f4266dbd4c7c58fad44d6e0261038bf09ca70baa Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 14:38:40 -0500 Subject: [PATCH 17/21] Revert --- spec/features/idv/doc_auth/verify_step_spec.rb | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/spec/features/idv/doc_auth/verify_step_spec.rb b/spec/features/idv/doc_auth/verify_step_spec.rb index 8217cb92889..1760d3aa540 100644 --- a/spec/features/idv/doc_auth/verify_step_spec.rb +++ b/spec/features/idv/doc_auth/verify_step_spec.rb @@ -9,9 +9,6 @@ let(:fake_analytics) { FakeAnalytics.new } before do allow_any_instance_of(ApplicationController).to receive(:analytics).and_return(fake_analytics) - allow(IdentityConfig.store).to receive(:proofing_device_profiling_collecting_enabled). - and_return(true) - allow(IdentityConfig.store).to receive(:lexisnexis_threatmetrix_enabled).and_return(true) end it 'displays the expected content' do @@ -180,7 +177,7 @@ anything, should_proof_state_id: true, trace_id: anything, - threatmetrix_session_id: anything, + threatmetrix_session_id: nil, user_id: user.id, request_ip: kind_of(String), ). @@ -191,8 +188,6 @@ click_idv_continue expect(DocAuthLog.find_by(user_id: user.id).aamva).not_to be_nil - expect(SpCost.find_by(cost_type: 'threatmetrix').transaction_id). - to eq(Proofing::Mock::DdpMockClient::TRANSACTION_ID) end end @@ -209,7 +204,7 @@ anything, should_proof_state_id: false, trace_id: anything, - threatmetrix_session_id: anything, + threatmetrix_session_id: nil, user_id: user.id, request_ip: kind_of(String), ). @@ -234,7 +229,7 @@ anything, should_proof_state_id: false, trace_id: anything, - threatmetrix_session_id: anything, + threatmetrix_session_id: nil, user_id: user.id, request_ip: kind_of(String), ). From 8c10b11562cf157b10629e718d6a5c2feebc1003 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 15:49:27 -0500 Subject: [PATCH 18/21] Revert costing --- .../db/proofing_cost/add_user_proofing_cost.rb | 1 - ...20220830121822_add_threatmetrix_to_proofing_cost.rb | 10 ---------- db/schema.rb | 3 +-- 3 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb diff --git a/app/services/db/proofing_cost/add_user_proofing_cost.rb b/app/services/db/proofing_cost/add_user_proofing_cost.rb index e818ac6b535..fa7d00d35f3 100644 --- a/app/services/db/proofing_cost/add_user_proofing_cost.rb +++ b/app/services/db/proofing_cost/add_user_proofing_cost.rb @@ -13,7 +13,6 @@ class ProofingCostTypeError < StandardError; end lexis_nexis_address gpo_letter phone_otp - threatmetrix ].freeze def self.call(user_id, token) diff --git a/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb b/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb deleted file mode 100644 index 9a1839e3208..00000000000 --- a/db/primary_migrate/20220830121822_add_threatmetrix_to_proofing_cost.rb +++ /dev/null @@ -1,10 +0,0 @@ -class AddThreatmetrixToProofingCost < ActiveRecord::Migration[7.0] - def up - add_column :proofing_costs, :threatmetrix_count, :integer - change_column_default :proofing_costs, :threatmetrix_count, 0 - end - - def down - remove_column :proofing_costs, :threatmetrix_count - end -end diff --git a/db/schema.rb b/db/schema.rb index 9113fdd76f2..0a150f2319e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_08_30_121822) do +ActiveRecord::Schema[7.0].define(version: 2022_08_08_140030) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" enable_extension "pgcrypto" @@ -476,7 +476,6 @@ t.datetime "updated_at", precision: nil, null: false t.integer "acuant_result_count", default: 0 t.integer "acuant_selfie_count", default: 0 - t.integer "threatmetrix_count", default: 0 t.index ["user_id"], name: "index_proofing_costs_on_user_id", unique: true end From f7cdb211ae9f1bc0a0f4ddf7c97b8cbf0eb072d5 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 15:51:30 -0500 Subject: [PATCH 19/21] One more revert --- app/services/db/sp_cost/add_sp_cost.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/services/db/sp_cost/add_sp_cost.rb b/app/services/db/sp_cost/add_sp_cost.rb index fe798e4e130..6c4ef015e18 100644 --- a/app/services/db/sp_cost/add_sp_cost.rb +++ b/app/services/db/sp_cost/add_sp_cost.rb @@ -12,7 +12,6 @@ class SpCostTypeError < StandardError; end lexis_nexis_resolution lexis_nexis_address gpo_letter - threatmetrix ].freeze def self.call(service_provider, ial, token, transaction_id: nil, user: nil) From 6e4572a3ddc765d714b4c848e50613687450f5f3 Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 15:52:29 -0500 Subject: [PATCH 20/21] Revert --- spec/features/reports/proofing_costs_report_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/features/reports/proofing_costs_report_spec.rb b/spec/features/reports/proofing_costs_report_spec.rb index 14978fd2129..1e920612e36 100644 --- a/spec/features/reports/proofing_costs_report_spec.rb +++ b/spec/features/reports/proofing_costs_report_spec.rb @@ -28,7 +28,6 @@ 'gpo_letter_count_average' => 0.0, 'lexis_nexis_address_count_average' => 0.0, 'phone_otp_count_average' => 0.0, - 'threatmetrix_count_average' => 0.0, } end From ab1b64e0e993c5968315521563e8b2d9cd0df3fe Mon Sep 17 00:00:00 2001 From: Steve Urciuoli Date: Tue, 30 Aug 2022 16:05:24 -0500 Subject: [PATCH 21/21] One more --- app/services/idv/steps/verify_base_step.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/services/idv/steps/verify_base_step.rb b/app/services/idv/steps/verify_base_step.rb index 919d8b70bf7..90cbfb54591 100644 --- a/app/services/idv/steps/verify_base_step.rb +++ b/app/services/idv/steps/verify_base_step.rb @@ -49,12 +49,10 @@ def add_proofing_costs(results) if stage == :resolution # transaction_id comes from ConversationId add_cost(:lexis_nexis_resolution, transaction_id: hash[:transaction_id]) + tmx_id = hash[:threatmetrix_request_id] + add_cost(:threatmetrix, transaction_id: tmx_id) if tmx_id elsif stage == :state_id process_aamva(hash[:transaction_id]) - elsif stage == :threatmetrix - # transaction_id comes from request_id - tmx_id = hash[:transaction_id] - add_cost(:threatmetrix, transaction_id: tmx_id) if tmx_id end end end