Reduce session size with encoding and compression improvements

Gemfile

@@ -35,6 +35,7 @@ gem 'jwt'
 gem 'lograge', '>= 0.11.2'
 gem 'lookbook', '~> 1.4.5', require: false
 gem 'lru_redux'
+gem 'msgpack', '~> 1.6'
 gem 'maxminddb'
 gem 'multiset'
 gem 'net-sftp'

Gemfile.lock

@@ -765,6 +765,7 @@ DEPENDENCIES
   lookbook (~> 1.4.5)
   lru_redux
   maxminddb
+  msgpack (~> 1.6)
   multiset
   net-sftp
   newrelic_rpm (~> 8.0)

app/services/encryption/aes_cipher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# This module is still needed by existing functionality, but any new AES encryption
+# should prefer using AesEncryptorV2 and AesCipherV2.
 module Encryption
   class AesCipher
     include Encodable
@@ -32,7 +34,7 @@ def self.encryption_cipher
     def encipher(plaintext)
       iv = cipher.random_iv
       cipher.auth_data = 'PII'
-      ciphertext = cipher.update(plaintext) + cipher.final
+      ciphertext = cipher.update(plaintext) << cipher.final
       tag = cipher.auth_tag
       { iv: encode(iv), ciphertext: encode(ciphertext), tag: encode(tag) }.to_json
     end
@@ -46,9 +48,9 @@ def decipher(payload)
     end
 
     def try_decipher(unpacked_payload)
-      cipher.update(ciphertext(unpacked_payload)) + cipher.final
+      cipher.update(ciphertext(unpacked_payload)) << cipher.final
     rescue OpenSSL::Cipher::CipherError => err
-      raise EncryptionError, 'failed to decipher payload: ' + err.to_s
+      raise EncryptionError, "failed to decipher payload: #{err}"
     end
 
     def unpack_payload(payload)

app/services/encryption/aes_cipher_v2.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Encryption
+  class AesCipherV2
+    def encrypt(plaintext, cek)
+      self.cipher = self.class.encryption_cipher
+      # The key length for the AES-256-GCM cipher is fixed at 128 bits, or 32
+      # characters. Starting with Ruby 2.4, an expection is thrown if you try to
+      # set a key longer than 32 characters, which is what we have been doing
+      # all along. In prior versions of Ruby, the key was silently truncated.
+      cipher.key = cek[0..31]
+      encipher(plaintext)
+    end
+
+    def decrypt(payload, cek)
+      self.cipher = OpenSSL::Cipher.new 'aes-256-gcm'
+      cipher.decrypt
+      cipher.key = cek[0..31]
+      decipher(payload)
+    end
+
+    def self.encryption_cipher
+      OpenSSL::Cipher.new('aes-256-gcm').encrypt
+    end
+
+    private
+
+    attr_accessor :cipher
+
+    def encipher(plaintext)
+      iv = cipher.random_iv
+      cipher.auth_data = 'PII'
+      ciphertext = cipher.update(plaintext) << cipher.final
+      tag = cipher.auth_tag
+
+      { iv: iv, ciphertext: ciphertext, tag: tag }.to_msgpack
+    end
+
+    def decipher(payload)
+      unpacked_payload = unpack_payload(payload)
+      cipher.iv = iv(unpacked_payload)
+      cipher.auth_tag = tag(unpacked_payload)
+      cipher.auth_data = 'PII'
+      try_decipher(unpacked_payload)
+    end
+
+    def try_decipher(unpacked_payload)
+      cipher.update(ciphertext(unpacked_payload)) << cipher.final
+    rescue OpenSSL::Cipher::CipherError => err
+      raise EncryptionError, "failed to decipher payload: #{err}"
+    end
+
+    def unpack_payload(payload)
+      MessagePack.unpack(payload)
+    rescue StandardError
+      raise EncryptionError, 'Unable to parse encrypted payload'
+    end
+
+    def iv(unpacked_payload)
+      unpacked_payload['iv']
+    end
+
+    def tag(unpacked_payload)
+      unpacked_payload['tag']
+    end
+
+    def ciphertext(unpacked_payload)
+      unpacked_payload['ciphertext']
+    end
+  end
+end

app/services/encryption/encryptors/aes_encryptor.rb

@@ -1,3 +1,5 @@
+# This module is still needed by existing functionality, but any new AES encryption
+# should prefer using AesEncryptorV2 and AesCipherV2.
 module Encryption
   module Encryptors
     class AesEncryptor

app/services/encryption/encryptors/aes_encryptor_v2.rb

@@ -0,0 +1,36 @@
+module Encryption
+  module Encryptors
+    class AesEncryptorV2
+      def initialize
+        self.cipher = AesCipherV2.new
+      end
+
+      def encrypt(plaintext, cek)
+        payload = fingerprint_and_concat(plaintext)
+        cipher.encrypt(payload, cek)
+      end
+
+      def decrypt(ciphertext, cek)
+        decrypt_and_test_payload(ciphertext, cek)
+      end
+
+      private
+
+      attr_accessor :cipher
+
+      def fingerprint_and_concat(plaintext)
+        fingerprint = Pii::Fingerprinter.fingerprint(plaintext)
+        [plaintext, fingerprint].to_msgpack
+      end
+
+      def decrypt_and_test_payload(payload, cek)
+        begin
+          plaintext, fingerprint = MessagePack.unpack(cipher.decrypt(payload, cek))
+        rescue OpenSSL::Cipher::CipherError, MessagePack::MalformedFormatError => err
+          raise EncryptionError, err.inspect
+        end
+        return plaintext if Pii::Fingerprinter.verify(plaintext, fingerprint)
+      end
+    end
+  end
+end

config/application.yml.default

@@ -288,7 +288,7 @@ service_provider_request_ttl_hours: 24
 session_check_delay: 30
 session_check_frequency: 30
 session_encryptor_alert_enabled: false
-session_encryptor_v2_enabled: true
+session_encryptor_v3_enabled: false
 session_timeout_in_minutes: 15
 session_timeout_warning_seconds: 150
 session_total_duration_timeout_in_minutes: 720

lib/identity_config.rb

@@ -396,7 +396,7 @@ def self.build_store(config_map)
     config.add(:session_check_frequency, type: :integer)
     config.add(:session_encryption_key, type: :string)
     config.add(:session_encryptor_alert_enabled, type: :boolean)
-    config.add(:session_encryptor_v2_enabled, type: :boolean)
+    config.add(:session_encryptor_v3_enabled, type: :boolean)
     config.add(:session_timeout_in_minutes, type: :integer)
     config.add(:session_timeout_warning_seconds, type: :integer)
     config.add(:session_total_duration_timeout_in_minutes, type: :integer)

lib/legacy_session_encryptor.rb

@@ -1,18 +1,150 @@
+# frozen_string_literal: true
+
 class LegacySessionEncryptor
+  CIPHERTEXT_HEADER = 'v2'
   def load(value)
-    decrypted = encryptor.decrypt(value)
+    _v2, ciphertext = value.split(':')
+    decrypted = outer_decrypt(ciphertext)
+
+    session = JSON.parse(decrypted, quirks_mode: true).with_indifferent_access
+    kms_decrypt_sensitive_paths!(session)
 
-    JSON.parse(decrypted, quirks_mode: true).with_indifferent_access
+    session
   end
 
   def dump(value)
+    value.deep_stringify_keys!
+
+    kms_encrypt_pii!(value)
+    kms_encrypt_sensitive_paths!(value, SessionEncryptor::SENSITIVE_PATHS)
+    alert_or_raise_if_contains_sensitive_keys!(value)
     plain = JSON.generate(value, quirks_mode: true)
-    encryptor.encrypt(plain)
+    alert_or_raise_if_contains_sensitive_value!(plain, value)
+    CIPHERTEXT_HEADER + ':' + outer_encrypt(plain)
+  end
+
+  def kms_encrypt(text)
+    Base64.encode64(Encryption::KmsClient.new.encrypt(text, 'context' => 'session-encryption'))
+  end
+
+  def kms_decrypt(text)
+    Encryption::KmsClient.new.decrypt(
+      Base64.decode64(text), 'context' => 'session-encryption'
+    )
+  end
+
+  def outer_encrypt(plaintext)
+    Encryption::Encryptors::AesEncryptor.new.encrypt(plaintext, session_encryption_key)
+  end
+
+  def outer_decrypt(ciphertext)
+    Encryption::Encryptors::AesEncryptor.new.decrypt(ciphertext, session_encryption_key)
   end
 
   private
 
-  def encryptor
-    Encryption::Encryptors::SessionEncryptor.new
+  # The PII bundle is stored in the user session in the 'decrypted_pii' key.
+  # The PII is decrypted with the user's password when they successfully submit it and then
+  # stored in the session.  Before saving the session, this method encrypts the PII with KMS and
+  # stores it in the 'encrypted_pii' key.
+  #
+  # The PII is not frequently needed in its KMS-decrypted state. To reduce the
+  # risks around holding plaintext PII in memory during requests, this PII is KMS-decrypted
+  # on-demand by the Pii::Cacher.
+  def kms_encrypt_pii!(session)
+    return unless session.dig('warden.user.user.session', 'decrypted_pii')
+    decrypted_pii = session['warden.user.user.session'].delete('decrypted_pii')
+    session['warden.user.user.session']['encrypted_pii'] =
+      kms_encrypt(decrypted_pii)
+    nil
+  end
+
+  # This method extracts all of the sensitive paths that exist into a
+  # separate hash.  This separate hash is then encrypted and placed in the session.
+  # We use #reduce to build the nested empty hash if needed. If Hash#bury
+  # (https://bugs.ruby-lang.org/issues/11747) existed, we could use that instead.
+  def kms_encrypt_sensitive_paths!(session, sensitive_paths)
+    sensitive_data = {}
+
+    sensitive_paths.each do |path|
+      *all_but_last_key, last_key = path
+
+      if all_but_last_key.blank?
+        value = session.delete(last_key)
+      else
+        value = session.dig(*all_but_last_key)&.delete(last_key)
+      end
+
+      if value
+        all_but_last_key.reduce(sensitive_data) do |hash, key|
+          hash[key] ||= {}
+
+          hash[key]
+        end
+
+        if all_but_last_key.blank?
+          sensitive_data[last_key] = value
+        else
+          sensitive_data.dig(*all_but_last_key).store(last_key, value)
+        end
+      end
+    end
+
+    raise "invalid session, 'sensitive_data' is reserved key" if session['sensitive_data'].present?
+    return if sensitive_data.blank?
+    session['sensitive_data'] = kms_encrypt(JSON.generate(sensitive_data))
+  end
+
+  # This method reverses the steps taken in #kms_encrypt_sensitive_paths!
+  # The encrypted hash is decrypted and then deep merged into the session hash.
+  # The merge must be a deep merge to avoid collisions with existing hashes in the
+  # session.
+  def kms_decrypt_sensitive_paths!(session)
+    sensitive_data = session.delete('sensitive_data')
+    return if sensitive_data.blank?
+
+    sensitive_data = JSON.parse(
+      kms_decrypt(sensitive_data), quirks_mode: true
+    )
+
+    session.deep_merge!(sensitive_data)
+  end
+
+  def alert_or_raise_if_contains_sensitive_value!(string, hash)
+    if SessionEncryptor::SENSITIVE_REGEX.match?(string)
+      exception = SessionEncryptor::SensitiveValueError.new
+      if IdentityConfig.store.session_encryptor_alert_enabled
+        NewRelic::Agent.notice_error(
+          exception, custom_params: {
+            session_structure: hash.deep_transform_values { |v| nil },
+          }
+        )
+      else
+        raise exception
+      end
+    end
+  end
+
+  def alert_or_raise_if_contains_sensitive_keys!(hash)
+    hash.deep_transform_keys do |key|
+      if SessionEncryptor::SENSITIVE_KEYS.include?(key.to_s)
+        exception = SessionEncryptor::SensitiveKeyError.new(
+          "#{key} unexpectedly appeared in session",
+        )
+        if IdentityConfig.store.session_encryptor_alert_enabled
+          NewRelic::Agent.notice_error(
+            exception, custom_params: {
+              session_structure: hash.deep_transform_values { |_v| '' },
+            }
+          )
+        else
+          raise exception
+        end
+      end
+    end
+  end
+
+  def session_encryption_key
+    IdentityConfig.store.session_encryption_key
   end
 end