LG-8056 Encrypt document submissions and write them to S3

app/controllers/idv/image_uploads_controller.rb

@@ -24,7 +24,13 @@ def image_upload_form
         analytics: analytics,
         uuid_prefix: current_sp&.app_id,
         irs_attempts_api_tracker: irs_attempts_api_tracker,
+        store_encrypted_images: store_encrypted_images?,
       )
     end
+
+    def store_encrypted_images?
+      IdentityConfig.store.encrypted_document_storage_enabled &&
+        irs_attempt_api_enabled_for_session?
+    end
   end
 end

app/forms/idv/api_image_upload_form.rb

@@ -11,13 +11,14 @@ class ApiImageUploadForm
     validate :throttle_if_rate_limited
 
     def initialize(params, service_provider:, analytics: nil,
-                   uuid_prefix: nil, irs_attempts_api_tracker: nil)
+                   uuid_prefix: nil, irs_attempts_api_tracker: nil, store_encrypted_images: false)
       @params = params
       @service_provider = service_provider
       @analytics = analytics
       @readable = {}
       @uuid_prefix = uuid_prefix
       @irs_attempts_api_tracker = irs_attempts_api_tracker
+      @store_encrypted_images = store_encrypted_images
     end
 
     def submit
@@ -64,8 +65,8 @@ def validate_form
 
     def post_images_to_client
       response = doc_auth_client.post_images(
-        front_image: front.read,
-        back_image: back.read,
+        front_image: front_image_bytes,
+        back_image: back_image_bytes,
         image_source: image_source,
         user_uuid: user_uuid,
         uuid_prefix: uuid_prefix,
@@ -79,6 +80,14 @@ def post_images_to_client
       response
     end
 
+    def front_image_bytes
+      @front_image_bytes ||= front.read
+    end
+
+    def back_image_bytes
+      @back_image_bytes ||= back.read
+    end
+
     def validate_pii_from_doc(client_response)
       response = Idv::DocPiiForm.new(
         pii: client_response.pii_from_doc,
@@ -210,6 +219,7 @@ def update_analytics(client_response)
         ).merge(native_camera_ab_test_data),
       )
       pii_from_doc = client_response.pii_from_doc || {}
+      store_encrypted_images_if_required
       irs_attempts_api_tracker.idv_document_upload_submitted(
         success: client_response.success?,
         document_state: pii_from_doc[:state],
@@ -224,6 +234,23 @@ def update_analytics(client_response)
       )
     end
 
+    def store_encrypted_images_if_required
+      return unless store_encrypted_images?
+
+      encrypted_document_storage_writer.encrypt_and_write_document(
+        front_image: front_image_bytes,
+        back_image: back_image_bytes,
+      )
+    end
+
+    def store_encrypted_images?
+      @store_encrypted_images
+    end
+
+    def encrypted_document_storage_writer
+      @encrypted_document_storage_writer ||= EncryptedDocumentStorage::DocumentWriter.new
+    end
+
     def native_camera_ab_test_data
       return {} unless IdentityConfig.store.idv_native_camera_a_b_testing_enabled
 

app/services/encrypted_document_storage/document_writer.rb

@@ -0,0 +1,36 @@
+module EncryptedDocumentStorage
+  class DocumentWriter
+    def encrypt_and_write_document(front_image:, back_image:)
+      key = SecureRandom.bytes(32)
+      encrypted_front_image = aes_cipher.encrypt(front_image, key)
+      encrypted_back_image = aes_cipher.encrypt(back_image, key)
+
+      front_image_uuid = SecureRandom.uuid
+      back_image_uiid = SecureRandom.uuid
+
+      storage.write_image(encrypted_image: encrypted_front_image, name: front_image_uuid)
+      storage.write_image(encrypted_image: encrypted_back_image, name: back_image_uiid)
+
+      WriteDocumentResult.new(
+        front_uuid: front_image_uuid,
+        back_uuid: back_image_uiid,
+        front_encryption_key: Base64.strict_encode64(key),
+        back_encryption_key: Base64.strict_encode64(key),
+      )
+    end
+
+    def storage
+      @storage ||= begin
+        if Rails.env.production?
+          S3Storage.new
+        else
+          LocalStorage.new
+        end
+      end
+    end
+
+    def aes_cipher
+      @aes_cipher ||= Encryption::AesCipher.new
+    end
+  end
+end

app/services/encrypted_document_storage/local_storage.rb

@@ -0,0 +1,13 @@
+module EncryptedDocumentStorage
+  class LocalStorage
+    def write_image(encrypted_image:, name:)
+      FileUtils.mkdir_p(tmp_document_storage_dir)
+      filepath = tmp_document_storage_dir.join(name)
+      File.write(filepath, encrypted_image)
+    end
+
+    def tmp_document_storage_dir
+      Rails.root.join('tmp', 'encrypted_doc_storage')
+    end
+  end
+end

app/services/encrypted_document_storage/s3_storage.rb

@@ -0,0 +1,21 @@
+module EncryptedDocumentStorage
+  class S3Storage
+    def write_image(encrypted_image:, name:)
+      s3_client.put_object(
+        bucket: IdentityConfig.store.encrypted_document_storage_s3_bucket,
+        body: encrypted_image,
+        key: name,
+      )
+    end
+
+    private
+
+    def s3_client
+      Aws::S3::Client.new(
+        http_open_timeout: 5,
+        http_read_timeout: 5,
+        compute_checksums: false,
+      )
+    end
+  end
+end

app/services/encrypted_document_storage/write_document_result.rb

@@ -0,0 +1,9 @@
+module EncryptedDocumentStorage
+  WriteDocumentResult = Struct.new(
+    :front_uuid,
+    :back_uuid,
+    :front_encryption_key,
+    :back_encryption_key,
+    keyword_init: true,
+  )
+end

bin/setup

@@ -73,6 +73,7 @@ Dir.chdir APP_ROOT do
   puts "\n== Removing old logs and tempfiles =="
   run "rm -f log/*"
   run "rm -rf tmp/cache"
+  run "rm -rf tmp/encrypted_doc_storage"
 
   puts "\n== Restarting application server =="
   run "mkdir -p tmp"

config/application.yml.default

@@ -100,6 +100,8 @@ enable_partner_api: false
 enable_rate_limiting: true
 enable_test_routes: true
 enable_usps_verification: true
+encrypted_document_storage_enabled: true
+encrypted_document_storage_s3_bucket: 'test-bucket-changeme'
 event_disavowal_expiration_hours: 240
 geo_data_file_path: 'geo_data/GeoLite2-City.mmdb'
 good_job_max_threads: 5
@@ -425,6 +427,8 @@ production:
   email_registrations_per_ip_track_only_mode: true
   enable_test_routes: false
   enable_usps_verification: false
+  encrypted_document_storage_enabled: false
+  encrypted_document_storage_s3_bucket: ''
   hmac_fingerprinter_key:
   hmac_fingerprinter_key_queue: '[]'
   idv_sp_required: true

lib/identity_config.rb

@@ -177,6 +177,8 @@ def self.build_store(config_map)
     config.add(:enable_rate_limiting, type: :boolean)
     config.add(:enable_test_routes, type: :boolean)
     config.add(:enable_usps_verification, type: :boolean)
+    config.add(:encrypted_document_storage_enabled, type: :boolean)
+    config.add(:encrypted_document_storage_s3_bucket, type: :string)
     config.add(:event_disavowal_expiration_hours, type: :integer)
     config.add(:geo_data_file_path, type: :string)
     config.add(:good_job_max_threads, type: :integer)

spec/forms/idv/api_image_upload_form_spec.rb

@@ -15,6 +15,7 @@
       service_provider: build(:service_provider, issuer: 'test_issuer'),
       analytics: fake_analytics,
       irs_attempts_api_tracker: irs_attempts_api_tracker,
+      store_encrypted_images: store_encrypted_images,
     )
   end
 
@@ -30,6 +31,7 @@
   let(:document_capture_session_uuid) { document_capture_session.uuid }
   let(:fake_analytics) { FakeAnalytics.new }
   let(:irs_attempts_api_tracker) { IrsAttemptsApiTrackingHelper::FakeAttemptsTracker.new }
+  let(:store_encrypted_images) { false }
 
   describe '#valid?' do
     context 'with all valid images' do
@@ -245,6 +247,39 @@
       end
     end
 
+    describe 'encrypted document storage' do
+      context 'when encrypted image storage is enabled' do
+        let(:store_encrypted_images) { true }
+
+        it 'writes encrypted documents' do
+          # This is not a _great_ way to test this. Once we start writing these events to the
+          # attempts API we should use the fake attempts API to grab the 'reference` value for the
+          # front and back image and check that those files are written.
+          document_writer = form.send(:encrypted_document_storage_writer)
+
+          expect(document_writer).to receive(:encrypt_and_write_document).with(
+            front_image: DocAuthImageFixtures.document_front_image_multipart.read,
+            back_image: DocAuthImageFixtures.document_back_image_multipart.read,
+          ).and_call_original
+
+          form.submit
+        end
+      end
+
+      context 'when the attempts API is not enabled' do
+        let(:store_encrypted_images) { false }
+
+        it 'when encrypted image storage is disabled' do
+          document_writer = instance_double(EncryptedDocumentStorage::DocumentWriter)
+          allow(form).to receive(:encrypted_document_storage_writer).and_return(document_writer)
+
+          expect(document_writer).to_not receive(:encrypt_and_write_document)
+
+          form.submit
+        end
+      end
+    end
+
     describe 'image source' do
       let(:source) { nil }
       let(:front_image_metadata) do

spec/services/encrypted_document_storage/document_writer_spec.rb

@@ -0,0 +1,54 @@
+require 'rails_helper'
+
+RSpec.describe EncryptedDocumentStorage::DocumentWriter do
+  describe '#encrypt_and_write_document' do
+    it 'encrypts the document and writes it to storage' do
+      front_image = 'hello, i am the front image'
+      back_image = 'hello, i am the back image'
+
+      result = EncryptedDocumentStorage::DocumentWriter.new.encrypt_and_write_document(
+        front_image: front_image,
+        back_image: back_image,
+      )
+
+      front_file_path = Rails.root.join('tmp', 'encrypted_doc_storage', result.front_uuid)
+      back_file_path = Rails.root.join('tmp', 'encrypted_doc_storage', result.back_uuid)
+      front_key = Base64.strict_decode64(result.front_encryption_key)
+      back_key = Base64.strict_decode64(result.back_encryption_key)
+
+      aes_cipher = Encryption::AesCipher.new
+
+      written_front_image = aes_cipher.decrypt(
+        File.read(front_file_path),
+        front_key,
+      )
+      written_back_image = aes_cipher.decrypt(
+        File.read(back_file_path),
+        back_key,
+      )
+
+      expect(written_front_image).to eq(front_image)
+      expect(written_back_image).to eq(back_image)
+    end
+  end
+
+  describe '#storage' do
+    subject { EncryptedDocumentStorage::DocumentWriter.new }
+
+    context 'in production' do
+      it 'is uses S3' do
+        allow(Rails.env).to receive(:production?).and_return(true)
+
+        expect(subject.storage).to be_a(EncryptedDocumentStorage::S3Storage)
+      end
+    end
+
+    context 'outside production' do
+      it 'it uses the disk' do
+        allow(Rails.env).to receive(:production?).and_return(false)
+
+        expect(subject.storage).to be_a(EncryptedDocumentStorage::LocalStorage)
+      end
+    end
+  end
+end

spec/services/encrypted_document_storage/local_storage_spec.rb

@@ -0,0 +1,20 @@
+require 'rails_helper'
+
+RSpec.describe EncryptedDocumentStorage::LocalStorage do
+  describe '#write_image' do
+    it 'writes the document to the disk' do
+      encrypted_image = "hello, i'm the encrypted document."
+      name = SecureRandom.uuid
+
+      EncryptedDocumentStorage::LocalStorage.new.write_image(
+        encrypted_image: encrypted_image,
+        name: name,
+      )
+
+      result = File.read(
+        Rails.root.join('tmp', 'encrypted_doc_storage', name),
+      )
+      expect(result).to eq(encrypted_image)
+    end
+  end
+end

spec/services/encrypted_document_storage/s3_storage_spec.rb

@@ -0,0 +1,28 @@
+require 'rails_helper'
+
+RSpec.describe EncryptedDocumentStorage::S3Storage do
+  describe '#write_image' do
+    it 'writes the document to S3' do
+      encrypted_image = 'hello, i am the encrypted document.'
+      name = '123abc'
+
+      storage = EncryptedDocumentStorage::S3Storage.new
+
+      stubbed_s3_client = Aws::S3::Client.new(stub_responses: true)
+      allow(storage).to receive(:s3_client).and_return(stubbed_s3_client)
+
+      expect(stubbed_s3_client).to receive(:put_object).and_call_original
+      stubbed_s3_client.stub_responses(
+        :put_object,
+        ->(context) {
+          params = context.params
+          expect(params[:bucket]).to eq(IdentityConfig.store.encrypted_document_storage_s3_bucket)
+          expect(params[:key]).to eq(name)
+          expect(params[:body]).to eq(encrypted_image)
+        },
+      )
+
+      storage.write_image(encrypted_image: encrypted_image, name: name)
+    end
+  end
+end