diff --git a/Gemfile b/Gemfile index 826c9ef550f..d8611d53d61 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,7 @@ gem 'jwt' gem 'lograge', '>= 0.11.2' gem 'lookbook', '~> 1.4.5', require: false gem 'lru_redux' +gem 'msgpack', '~> 1.6' gem 'maxminddb' gem 'multiset' gem 'net-sftp' diff --git a/Gemfile.lock b/Gemfile.lock index 6a57fd7e791..7d456fc816e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -765,6 +765,7 @@ DEPENDENCIES lookbook (~> 1.4.5) lru_redux maxminddb + msgpack (~> 1.6) multiset net-sftp newrelic_rpm (~> 8.0) diff --git a/app/services/encryption/aes_cipher.rb b/app/services/encryption/aes_cipher.rb index 70c8cc321e6..18b049f9993 100644 --- a/app/services/encryption/aes_cipher.rb +++ b/app/services/encryption/aes_cipher.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This module is still needed by existing functionality, but any new AES encryption +# should prefer using AesEncryptorV2 and AesCipherV2. module Encryption class AesCipher include Encodable @@ -32,7 +34,7 @@ def self.encryption_cipher def encipher(plaintext) iv = cipher.random_iv cipher.auth_data = 'PII' - ciphertext = cipher.update(plaintext) + cipher.final + ciphertext = cipher.update(plaintext) << cipher.final tag = cipher.auth_tag { iv: encode(iv), ciphertext: encode(ciphertext), tag: encode(tag) }.to_json end @@ -46,9 +48,9 @@ def decipher(payload) end def try_decipher(unpacked_payload) - cipher.update(ciphertext(unpacked_payload)) + cipher.final + cipher.update(ciphertext(unpacked_payload)) << cipher.final rescue OpenSSL::Cipher::CipherError => err - raise EncryptionError, 'failed to decipher payload: ' + err.to_s + raise EncryptionError, "failed to decipher payload: #{err}" end def unpack_payload(payload) diff --git a/app/services/encryption/aes_cipher_v2.rb b/app/services/encryption/aes_cipher_v2.rb new file mode 100644 index 00000000000..fe178087a1a --- /dev/null +++ b/app/services/encryption/aes_cipher_v2.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Encryption + class AesCipherV2 + def encrypt(plaintext, cek) + self.cipher = self.class.encryption_cipher + # The key length for the AES-256-GCM cipher is fixed at 128 bits, or 32 + # characters. Starting with Ruby 2.4, an expection is thrown if you try to + # set a key longer than 32 characters, which is what we have been doing + # all along. In prior versions of Ruby, the key was silently truncated. + cipher.key = cek[0..31] + encipher(plaintext) + end + + def decrypt(payload, cek) + self.cipher = OpenSSL::Cipher.new 'aes-256-gcm' + cipher.decrypt + cipher.key = cek[0..31] + decipher(payload) + end + + def self.encryption_cipher + OpenSSL::Cipher.new('aes-256-gcm').encrypt + end + + private + + attr_accessor :cipher + + def encipher(plaintext) + iv = cipher.random_iv + cipher.auth_data = 'PII' + ciphertext = cipher.update(plaintext) << cipher.final + tag = cipher.auth_tag + + { iv: iv, ciphertext: ciphertext, tag: tag }.to_msgpack + end + + def decipher(payload) + unpacked_payload = unpack_payload(payload) + cipher.iv = iv(unpacked_payload) + cipher.auth_tag = tag(unpacked_payload) + cipher.auth_data = 'PII' + try_decipher(unpacked_payload) + end + + def try_decipher(unpacked_payload) + cipher.update(ciphertext(unpacked_payload)) << cipher.final + rescue OpenSSL::Cipher::CipherError => err + raise EncryptionError, "failed to decipher payload: #{err}" + end + + def unpack_payload(payload) + MessagePack.unpack(payload) + rescue StandardError + raise EncryptionError, 'Unable to parse encrypted payload' + end + + def iv(unpacked_payload) + unpacked_payload['iv'] + end + + def tag(unpacked_payload) + unpacked_payload['tag'] + end + + def ciphertext(unpacked_payload) + unpacked_payload['ciphertext'] + end + end +end diff --git a/app/services/encryption/encryptors/aes_encryptor.rb b/app/services/encryption/encryptors/aes_encryptor.rb index 62ebe2a125a..72f95130eed 100644 --- a/app/services/encryption/encryptors/aes_encryptor.rb +++ b/app/services/encryption/encryptors/aes_encryptor.rb @@ -1,3 +1,5 @@ +# This module is still needed by existing functionality, but any new AES encryption +# should prefer using AesEncryptorV2 and AesCipherV2. module Encryption module Encryptors class AesEncryptor diff --git a/app/services/encryption/encryptors/aes_encryptor_v2.rb b/app/services/encryption/encryptors/aes_encryptor_v2.rb new file mode 100644 index 00000000000..8c5a0d04325 --- /dev/null +++ b/app/services/encryption/encryptors/aes_encryptor_v2.rb @@ -0,0 +1,36 @@ +module Encryption + module Encryptors + class AesEncryptorV2 + def initialize + self.cipher = AesCipherV2.new + end + + def encrypt(plaintext, cek) + payload = fingerprint_and_concat(plaintext) + cipher.encrypt(payload, cek) + end + + def decrypt(ciphertext, cek) + decrypt_and_test_payload(ciphertext, cek) + end + + private + + attr_accessor :cipher + + def fingerprint_and_concat(plaintext) + fingerprint = Pii::Fingerprinter.fingerprint(plaintext) + [plaintext, fingerprint].to_msgpack + end + + def decrypt_and_test_payload(payload, cek) + begin + plaintext, fingerprint = MessagePack.unpack(cipher.decrypt(payload, cek)) + rescue OpenSSL::Cipher::CipherError, MessagePack::MalformedFormatError => err + raise EncryptionError, err.inspect + end + return plaintext if Pii::Fingerprinter.verify(plaintext, fingerprint) + end + end + end +end diff --git a/config/application.yml.default b/config/application.yml.default index 463c41ffc72..eab057dcf3d 100644 --- a/config/application.yml.default +++ b/config/application.yml.default @@ -288,7 +288,7 @@ service_provider_request_ttl_hours: 24 session_check_delay: 30 session_check_frequency: 30 session_encryptor_alert_enabled: false -session_encryptor_v2_enabled: true +session_encryptor_v3_enabled: false session_timeout_in_minutes: 15 session_timeout_warning_seconds: 150 session_total_duration_timeout_in_minutes: 720 diff --git a/lib/identity_config.rb b/lib/identity_config.rb index b554938209f..65f6d4f0ecf 100644 --- a/lib/identity_config.rb +++ b/lib/identity_config.rb @@ -396,7 +396,7 @@ def self.build_store(config_map) config.add(:session_check_frequency, type: :integer) config.add(:session_encryption_key, type: :string) config.add(:session_encryptor_alert_enabled, type: :boolean) - config.add(:session_encryptor_v2_enabled, type: :boolean) + config.add(:session_encryptor_v3_enabled, type: :boolean) config.add(:session_timeout_in_minutes, type: :integer) config.add(:session_timeout_warning_seconds, type: :integer) config.add(:session_total_duration_timeout_in_minutes, type: :integer) diff --git a/lib/legacy_session_encryptor.rb b/lib/legacy_session_encryptor.rb index 660c787577d..ce54b980219 100644 --- a/lib/legacy_session_encryptor.rb +++ b/lib/legacy_session_encryptor.rb @@ -1,18 +1,150 @@ +# frozen_string_literal: true + class LegacySessionEncryptor + CIPHERTEXT_HEADER = 'v2' def load(value) - decrypted = encryptor.decrypt(value) + _v2, ciphertext = value.split(':') + decrypted = outer_decrypt(ciphertext) + + session = JSON.parse(decrypted, quirks_mode: true).with_indifferent_access + kms_decrypt_sensitive_paths!(session) - JSON.parse(decrypted, quirks_mode: true).with_indifferent_access + session end def dump(value) + value.deep_stringify_keys! + + kms_encrypt_pii!(value) + kms_encrypt_sensitive_paths!(value, SessionEncryptor::SENSITIVE_PATHS) + alert_or_raise_if_contains_sensitive_keys!(value) plain = JSON.generate(value, quirks_mode: true) - encryptor.encrypt(plain) + alert_or_raise_if_contains_sensitive_value!(plain, value) + CIPHERTEXT_HEADER + ':' + outer_encrypt(plain) + end + + def kms_encrypt(text) + Base64.encode64(Encryption::KmsClient.new.encrypt(text, 'context' => 'session-encryption')) + end + + def kms_decrypt(text) + Encryption::KmsClient.new.decrypt( + Base64.decode64(text), 'context' => 'session-encryption' + ) + end + + def outer_encrypt(plaintext) + Encryption::Encryptors::AesEncryptor.new.encrypt(plaintext, session_encryption_key) + end + + def outer_decrypt(ciphertext) + Encryption::Encryptors::AesEncryptor.new.decrypt(ciphertext, session_encryption_key) end private - def encryptor - Encryption::Encryptors::SessionEncryptor.new + # The PII bundle is stored in the user session in the 'decrypted_pii' key. + # The PII is decrypted with the user's password when they successfully submit it and then + # stored in the session. Before saving the session, this method encrypts the PII with KMS and + # stores it in the 'encrypted_pii' key. + # + # The PII is not frequently needed in its KMS-decrypted state. To reduce the + # risks around holding plaintext PII in memory during requests, this PII is KMS-decrypted + # on-demand by the Pii::Cacher. + def kms_encrypt_pii!(session) + return unless session.dig('warden.user.user.session', 'decrypted_pii') + decrypted_pii = session['warden.user.user.session'].delete('decrypted_pii') + session['warden.user.user.session']['encrypted_pii'] = + kms_encrypt(decrypted_pii) + nil + end + + # This method extracts all of the sensitive paths that exist into a + # separate hash. This separate hash is then encrypted and placed in the session. + # We use #reduce to build the nested empty hash if needed. If Hash#bury + # (https://bugs.ruby-lang.org/issues/11747) existed, we could use that instead. + def kms_encrypt_sensitive_paths!(session, sensitive_paths) + sensitive_data = {} + + sensitive_paths.each do |path| + *all_but_last_key, last_key = path + + if all_but_last_key.blank? + value = session.delete(last_key) + else + value = session.dig(*all_but_last_key)&.delete(last_key) + end + + if value + all_but_last_key.reduce(sensitive_data) do |hash, key| + hash[key] ||= {} + + hash[key] + end + + if all_but_last_key.blank? + sensitive_data[last_key] = value + else + sensitive_data.dig(*all_but_last_key).store(last_key, value) + end + end + end + + raise "invalid session, 'sensitive_data' is reserved key" if session['sensitive_data'].present? + return if sensitive_data.blank? + session['sensitive_data'] = kms_encrypt(JSON.generate(sensitive_data)) + end + + # This method reverses the steps taken in #kms_encrypt_sensitive_paths! + # The encrypted hash is decrypted and then deep merged into the session hash. + # The merge must be a deep merge to avoid collisions with existing hashes in the + # session. + def kms_decrypt_sensitive_paths!(session) + sensitive_data = session.delete('sensitive_data') + return if sensitive_data.blank? + + sensitive_data = JSON.parse( + kms_decrypt(sensitive_data), quirks_mode: true + ) + + session.deep_merge!(sensitive_data) + end + + def alert_or_raise_if_contains_sensitive_value!(string, hash) + if SessionEncryptor::SENSITIVE_REGEX.match?(string) + exception = SessionEncryptor::SensitiveValueError.new + if IdentityConfig.store.session_encryptor_alert_enabled + NewRelic::Agent.notice_error( + exception, custom_params: { + session_structure: hash.deep_transform_values { |v| nil }, + } + ) + else + raise exception + end + end + end + + def alert_or_raise_if_contains_sensitive_keys!(hash) + hash.deep_transform_keys do |key| + if SessionEncryptor::SENSITIVE_KEYS.include?(key.to_s) + exception = SessionEncryptor::SensitiveKeyError.new( + "#{key} unexpectedly appeared in session", + ) + if IdentityConfig.store.session_encryptor_alert_enabled + NewRelic::Agent.notice_error( + exception, custom_params: { + session_structure: hash.deep_transform_values { |_v| '' }, + } + ) + else + raise exception + end + end + end + end + + def session_encryption_key + IdentityConfig.store.session_encryption_key end end diff --git a/lib/session_encryptor.rb b/lib/session_encryptor.rb index 57aae4a88cc..113a1b496b1 100644 --- a/lib/session_encryptor.rb +++ b/lib/session_encryptor.rb @@ -4,13 +4,17 @@ class SessionEncryptor class SensitiveKeyError < StandardError; end class SensitiveValueError < StandardError; end - NEW_CIPHERTEXT_HEADER = 'v2' + CIPHERTEXT_HEADER = 'v3' + MINIMUM_COMPRESS_LIMIT = 300 SENSITIVE_KEYS = [ 'first_name', 'middle_name', 'last_name', 'address1', 'address2', 'city', 'state', 'zipcode', 'zip_code', 'same_address_as_id', 'dob', 'phone_number', 'phone', 'ssn', 'prev_address1', 'prev_address2', 'prev_city', 'prev_state', 'prev_zipcode', 'pii', 'pii_from_doc', 'pii_from_user', 'password', 'personal_key', 'email', 'email_address', 'unconfirmed_phone' ].to_set.freeze + CIPHERTEXT_KEY = 't' + COMPRESSED_KEY = 'c' + VERSION_KEY = 'v' # 'idv/doc_auth' and 'idv' are used during the proofing process and can contain PII # personal keys are generated and stored in the session between requests, but are used @@ -39,10 +43,17 @@ class SensitiveValueError < StandardError; end def load(value) return LegacySessionEncryptor.new.load(value) if should_use_legacy_encryptor_for_read?(value) - _v2, ciphertext = value.split(':') + payload = MessagePack.unpack(value) + ciphertext = payload[CIPHERTEXT_KEY] + compressed = payload[COMPRESSED_KEY] decrypted = outer_decrypt(ciphertext) + decrypted = if compressed == 1 + Zlib.gunzip(decrypted) + else + decrypted + end - session = JSON.parse(decrypted, quirks_mode: true).with_indifferent_access + session = JSON.parse(decrypted).with_indifferent_access kms_decrypt_sensitive_paths!(session) session @@ -55,9 +66,22 @@ def dump(value) kms_encrypt_pii!(value) kms_encrypt_sensitive_paths!(value, SENSITIVE_PATHS) alert_or_raise_if_contains_sensitive_keys!(value) - plain = JSON.generate(value, quirks_mode: true) + plain = JSON.generate(value) alert_or_raise_if_contains_sensitive_value!(plain, value) - NEW_CIPHERTEXT_HEADER + ':' + outer_encrypt(plain) + + if should_compress?(plain) + { + VERSION_KEY => CIPHERTEXT_HEADER, + CIPHERTEXT_KEY => outer_encrypt(Zlib.gzip(plain)), + COMPRESSED_KEY => 1, + }.to_msgpack + else + { + VERSION_KEY => CIPHERTEXT_HEADER, + CIPHERTEXT_KEY => outer_encrypt(plain), + COMPRESSED_KEY => 0, + }.to_msgpack + end end def kms_encrypt(text) @@ -71,11 +95,11 @@ def kms_decrypt(text) end def outer_encrypt(plaintext) - Encryption::Encryptors::AesEncryptor.new.encrypt(plaintext, session_encryption_key) + Encryption::Encryptors::AesEncryptorV2.new.encrypt(plaintext, session_encryption_key) end def outer_decrypt(ciphertext) - Encryption::Encryptors::AesEncryptor.new.decrypt(ciphertext, session_encryption_key) + Encryption::Encryptors::AesEncryptorV2.new.decrypt(ciphertext, session_encryption_key) end private @@ -142,7 +166,7 @@ def kms_decrypt_sensitive_paths!(session) return if sensitive_data.blank? sensitive_data = JSON.parse( - kms_decrypt(sensitive_data), quirks_mode: true + kms_decrypt(sensitive_data), ) session.deep_merge!(sensitive_data) @@ -181,13 +205,15 @@ def alert_or_raise_if_contains_sensitive_keys!(hash) end def should_use_legacy_encryptor_for_read?(value) - ## Legacy ciphertexts will not include a colon and thus will have no header - header = value.split(':').first - header != NEW_CIPHERTEXT_HEADER + value.start_with?(LegacySessionEncryptor::CIPHERTEXT_HEADER) + end + + def should_compress?(value) + value.bytesize >= MINIMUM_COMPRESS_LIMIT end def should_use_legacy_encryptor_for_write? - !IdentityConfig.store.session_encryptor_v2_enabled + !IdentityConfig.store.session_encryptor_v3_enabled end def session_encryption_key diff --git a/spec/lib/session_encryptor_spec.rb b/spec/lib/session_encryptor_spec.rb index 0bb522dc94c..62592372afa 100644 --- a/spec/lib/session_encryptor_spec.rb +++ b/spec/lib/session_encryptor_spec.rb @@ -15,9 +15,9 @@ end end - context 'with version 2 encryption enabled' do + context 'with version 3 encryption enabled' do before do - allow(IdentityConfig.store).to receive(:session_encryptor_v2_enabled).and_return(true) + allow(IdentityConfig.store).to receive(:session_encryptor_v3_enabled).and_return(true) end it 'decrypts the new version of the session' do @@ -33,9 +33,9 @@ end describe '#dump' do - context 'with version 2 encryption enabled' do + context 'with version 3 encryption enabled' do before do - allow(IdentityConfig.store).to receive(:session_encryptor_v2_enabled).and_return(true) + allow(IdentityConfig.store).to receive(:session_encryptor_v3_enabled).and_return(true) end it 'transparently encrypts/decrypts sensitive elements of the session' do @@ -97,10 +97,11 @@ 'idv/doc_auth' => { 'ssn' => '666-66-6666' }, 'other_value' => 42, } } - ciphertext = subject.dump(session) - partially_decrypted = subject.outer_decrypt(ciphertext.split(':').last) + partially_decrypted = Zlib.gunzip( + subject.outer_decrypt(MessagePack.unpack(ciphertext)[SessionEncryptor::CIPHERTEXT_KEY]), + ) partially_decrypted_json = JSON.parse(partially_decrypted) expect(partially_decrypted_json.fetch('warden.user.user.session')['idv']).to eq nil @@ -114,6 +115,18 @@ ).to eq 42 end + it 'does not compress when payload is small' do + session = { 'a' => 0 } + ciphertext = subject.dump(session) + + session_payload = MessagePack.unpack(ciphertext) + expect(session_payload[SessionEncryptor::COMPRESSED_KEY]).to eq 0 + session_decrypted = JSON.parse( + subject.outer_decrypt(session_payload[SessionEncryptor::CIPHERTEXT_KEY]), + ) + expect(session_decrypted).to eq session + end + it 'raises if reserved key is used' do session = { 'sensitive_data' => 'test', @@ -182,9 +195,9 @@ end end - context 'without version 2 encryption enabled' do + context 'without version 3 encryption enabled' do before do - allow(IdentityConfig.store).to receive(:session_encryptor_v2_enabled).and_return(false) + allow(IdentityConfig.store).to receive(:session_encryptor_v3_enabled).and_return(false) end it 'encrypts the session with the legacy encryptor' do diff --git a/spec/services/encryption/aes_cipher_v2_spec.rb b/spec/services/encryption/aes_cipher_v2_spec.rb new file mode 100644 index 00000000000..7efb7317d96 --- /dev/null +++ b/spec/services/encryption/aes_cipher_v2_spec.rb @@ -0,0 +1,42 @@ +require 'rails_helper' + +describe Encryption::AesCipherV2 do + let(:plaintext) { 'some long secret' } + let(:cek) { SecureRandom.random_bytes(32) } + + describe '#encrypt' do + it 'returns MessagePack string containing AES-encrypted ciphertext' do + ciphertext = subject.encrypt(plaintext, cek) + + expect(ciphertext).to_not match plaintext + expect(ciphertext).to be_a String + expect { MessagePack.unpack(ciphertext) }.to_not raise_error + end + end + + describe '#decrypt' do + it 'returns plaintext' do + ciphertext = subject.encrypt(plaintext, cek) + + expect(subject.decrypt(ciphertext, cek)).to eq plaintext + end + + it 'raises error on invalid input' do + ciphertext = subject.encrypt(plaintext, cek) + ciphertext += 'foo' + + expect { subject.decrypt(ciphertext, cek) }.to raise_error Encryption::EncryptionError + end + end + + describe '.encryption_cipher' do + it 'returns an AES cipher for encryption operation' do + expect_any_instance_of(OpenSSL::Cipher).to receive(:encrypt).and_call_original + + cipher = subject.class.encryption_cipher + + expect(cipher).to be_kind_of(OpenSSL::Cipher) + expect(cipher.name).to eq OpenSSL::Cipher.new('aes-256-gcm').name + end + end +end diff --git a/spec/services/encryption/encryptors/aes_encryptor_v2_spec.rb b/spec/services/encryption/encryptors/aes_encryptor_v2_spec.rb new file mode 100644 index 00000000000..91dbdc4c674 --- /dev/null +++ b/spec/services/encryption/encryptors/aes_encryptor_v2_spec.rb @@ -0,0 +1,29 @@ +require 'rails_helper' + +describe Encryption::Encryptors::AesEncryptorV2 do + let(:aes_cek) { SecureRandom.random_bytes(32) } + let(:plaintext) { 'four score and seven years ago' } + + describe '#encrypt' do + it 'returns encrypted text' do + encrypted = subject.encrypt(plaintext, aes_cek) + + expect(encrypted).to_not match plaintext + end + end + + describe '#decrypt' do + it 'returns original text' do + encrypted = subject.encrypt(plaintext, aes_cek) + + expect(subject.decrypt(encrypted, aes_cek)).to eq plaintext + end + + it 'requires same password used for encrypt' do + encrypted = subject.encrypt(plaintext, aes_cek) + diff_cek = SecureRandom.random_bytes(32) + + expect { subject.decrypt(encrypted, diff_cek) }.to raise_error Encryption::EncryptionError + end + end +end