LG-8413: Background job should remove events from Redis

app/jobs/irs_attempts_events_batch_job.rb

@@ -15,7 +15,7 @@ def perform(timestamp = Time.zone.now - 1.hour)
       data: event_values, timestamp: timestamp, public_key_str: public_key,
     )
 
-    create_and_upload_to_attempts_s3_resource(
+    upload_to_s3_response = create_and_upload_to_attempts_s3_resource(
       bucket_name: s3_helper.attempts_bucket_name, filename: result.filename,
       encrypted_data: result.encrypted_data
     )
@@ -31,6 +31,7 @@ def perform(timestamp = Time.zone.now - 1.hour)
     )
 
     log_irs_attempts_events_job_info(result, events, start_time)
+    redis_client.remove_events(timestamp: timestamp) if upload_to_s3_response&.etag
     irs_attempts_api_log_file
   end
 

app/services/irs_attempts_api/redis_client.rb

@@ -25,6 +25,15 @@ def read_events(timestamp:, batch_size: 5000)
       events
     end
 
+    def remove_events(timestamp:)
+      return unless IdentityConfig.store.irs_attempt_api_delete_events_after_s3_upload
+
+      key = key(timestamp)
+      redis_pool.with do |client|
+        client.del(key)
+      end
+    end
+
     def key(timestamp)
       'irs-attempt-api:' + timestamp.in_time_zone('UTC').change(min: 0, sec: 0).iso8601
     end

config/application.yml.default

@@ -153,6 +153,7 @@ irs_attempt_api_event_count_default: 1000
 irs_attempt_api_event_count_max: 10000
 irs_attempt_api_payload_size_logging_enabled: true
 irs_attempt_api_track_tmx_fraud_check_event: false
+irs_attempt_api_delete_events_after_s3_upload: false
 key_pair_generation_percent: 0
 logins_per_ip_track_only_mode: false
 # LexisNexis #####################################################

lib/identity_config.rb

@@ -233,6 +233,7 @@ def self.build_store(config_map)
     config.add(:irs_attempt_api_event_count_max, type: :integer)
     config.add(:irs_attempt_api_payload_size_logging_enabled, type: :boolean)
     config.add(:irs_attempt_api_track_tmx_fraud_check_event, type: :boolean)
+    config.add(:irs_attempt_api_delete_events_after_s3_upload, type: :boolean)
     config.add(:irs_attempt_api_public_key)
     config.add(:irs_attempt_api_public_key_id)
     config.add(:lexisnexis_base_url, type: :string)

spec/jobs/irs_attempts_events_batch_job_spec.rb

@@ -31,6 +31,7 @@
         )
       end
 
+      let!(:s3_put_object_response) { double(etag: true) }
       before do
         allow(IdentityConfig.store).to receive(:irs_attempt_api_enabled).and_return(true)
         allow(IdentityConfig.store).to receive(:irs_attempt_api_aws_s3_enabled).and_return(true)
@@ -45,7 +46,7 @@
 
         allow_any_instance_of(described_class).to receive(
           :create_and_upload_to_attempts_s3_resource,
-        )
+        ).and_return(s3_put_object_response)
 
         allow_any_instance_of(described_class).to receive(:duration_ms).and_return(0.1234)
 
@@ -104,6 +105,46 @@
           IrsAttemptsApi::EnvelopeEncryptor.formatted_timestamp(start_time),
         )
       end
+
+      context 'When irs delete events feature flag and s3 put_object response is true' do
+        before do
+          allow(IdentityConfig.store).to receive(:irs_attempt_api_delete_events_after_s3_upload).
+            and_return(true)
+        end
+
+        it 'delete the events from redis' do
+          IrsAttemptsEventsBatchJob.perform_now
+          events = IrsAttemptsApi::RedisClient.new.read_events(timestamp: start_time)
+          expect(events.count).to eq 0
+        end
+      end
+
+      context 'When irs delete events feature flag is false' do
+        before do
+          allow(IdentityConfig.store).to receive(:irs_attempt_api_delete_events_after_s3_upload).
+            and_return(false)
+        end
+
+        it 'does not delete the events from redis' do
+          IrsAttemptsEventsBatchJob.perform_now
+          events = IrsAttemptsApi::RedisClient.new.read_events(timestamp: start_time)
+          expect(events.count).to eq 2
+        end
+      end
+
+      context 'When irs delete events feature flag is true and s3 put_object response is false' do
+        let!(:s3_put_object_response) { double(etag: false) }
+        before do
+          allow(IdentityConfig.store).to receive(:irs_attempt_api_delete_events_after_s3_upload).
+            and_return(true)
+        end
+
+        it 'does not delete the events from redis' do
+          IrsAttemptsEventsBatchJob.perform_now
+          events = IrsAttemptsApi::RedisClient.new.read_events(timestamp: start_time)
+          expect(events.count).to eq 2
+        end
+      end
     end
 
     context 'IRS attempts API is not enabled' do