Update OIG report format

lib/data_pull.rb

@@ -152,13 +152,36 @@ def run(args:, config:)
       ActiveRecord::Base.connection.execute('SET statement_timeout = 0')
       uuids = args
 
-      users = uuids.map { |uuid| DataRequests::Deployed::LookupUserByUuid.new(uuid).call }.compact
+      users, missing_uuids = uuids.map do |uuid|
+        DataRequests::Deployed::LookupUserByUuid.new(uuid).call || uuid
+      end.partition { |u| u.is_a?(User) }
+
       shared_device_users = DataRequests::Deployed::LookupSharedDeviceUsers.new(users).call
 
       output = shared_device_users.map do |user|
         DataRequests::Deployed::CreateUserReport.new(user, config.requesting_issuers).call
       end
 
+      if config.include_missing?
+        output += missing_uuids.map do |uuid|
+          {
+            user_id: nil,
+            login_uuid: nil,
+            requesting_issuer_uuid: uuid,
+            email_addresses: [],
+            mfa_configurations: {
+              phone_configurations: [],
+              auth_app_configurations: [],
+              webauthn_configurations: [],
+              piv_cac_configurations: [],
+              backup_code_configurations: [],
+            },
+            user_events: [],
+            not_found: true,
+          }
+        end
+      end
+
       ScriptBase::Result.new(
         subtask: 'ig-request',
         uuids: users.map(&:uuid),

lib/data_requests/deployed/create_mfa_configurations_report.rb

@@ -40,6 +40,7 @@ def backup_code_configurations_report
       def phone_configurations_report
         user.phone_configurations.map do |phone_configuration|
           {
+            id: phone_configuration.id,
             phone: phone_configuration.phone,
             created_at: phone_configuration.created_at,
             confirmed_at: phone_configuration.confirmed_at,

lib/data_requests/local/write_cloudwatch_logs.rb

@@ -4,6 +4,7 @@ module DataRequests
   module Local
     class WriteCloudwatchLogs
       HEADERS = %w[
+        uuid
         timestamp
         event_name
         success
@@ -14,19 +15,24 @@ class WriteCloudwatchLogs
         user_agent
       ].freeze
 
-      attr_reader :cloudwatch_results, :output_dir
+      attr_reader :cloudwatch_results, :requesting_issuer_uuid, :csv
 
-      def initialize(cloudwatch_results, output_dir)
+      def initialize(cloudwatch_results:, requesting_issuer_uuid:, csv:, include_header: false)
         @cloudwatch_results = cloudwatch_results
-        @output_dir = output_dir
+        @requesting_issuer_uuid = requesting_issuer_uuid
+        @csv = csv
+        @include_header = include_header
+      end
+
+      def include_header?
+        !!@include_header
       end
 
       def call
-        CSV.open(File.join(output_dir, 'logs.csv'), 'w') do |csv|
-          csv << HEADERS
-          cloudwatch_results.each do |row|
-            csv << build_row(row)
-          end
+        csv << HEADERS if include_header?
+
+        cloudwatch_results.each do |row|
+          csv << build_row(row)
         end
       end
 
@@ -60,6 +66,7 @@ def build_row(row)
         user_agent = data.dig('properties', 'user_agent')
 
         [
+          requesting_issuer_uuid,
           timestamp,
           event_name,
           success,

lib/data_requests/local/write_user_events.rb

@@ -3,16 +3,21 @@
 module DataRequests
   module Local
     class WriteUserEvents
-      attr_reader :user_report, :output_dir, :requesting_issuer_uuid
+      attr_reader :user_report, :requesting_issuer_uuid, :csv
 
-      def initialize(user_report, output_dir, requesting_issuer_uuid)
+      def initialize(user_report:, requesting_issuer_uuid:, csv:, include_header: false)
         @user_report = user_report
-        @output_dir = output_dir
+        @csv = csv
         @requesting_issuer_uuid = requesting_issuer_uuid
+        @include_header = include_header
+      end
+
+      def include_header?
+        !!@include_header
       end
 
       def call
-        CSV.open(File.join(output_dir, 'events.csv'), 'w') do |csv|
+        if include_header?
           csv << %w[
             uuid
             event_name
@@ -22,17 +27,17 @@ def call
             user_agent
             device_cookie
           ]
+        end
 
-          user_report[:user_events].each do |row|
-            csv << [requesting_issuer_uuid] + row.values_at(
-              :event_name,
-              :date_time,
-              :ip,
-              :disavowed_at,
-              :user_agent,
-              :device_cookie,
-            )
-          end
+        user_report[:user_events].each do |row|
+          csv << [requesting_issuer_uuid] + row.values_at(
+            :event_name,
+            :date_time,
+            :ip,
+            :disavowed_at,
+            :user_agent,
+            :device_cookie,
+          )
         end
       end
     end

lib/data_requests/local/write_user_info.rb

@@ -3,86 +3,122 @@
 module DataRequests
   module Local
     class WriteUserInfo
-      attr_reader :user_report, :output_dir
+      attr_reader :user_report, :csv
 
-      def initialize(user_report, output_dir)
+      def initialize(user_report:, csv:, include_header: false)
         @user_report = user_report
-        @output_dir = output_dir
+        @csv = csv
+        @include_header = include_header
+      end
+
+      def include_header?
+        !!@include_header
       end
 
       def call
+        if include_header?
+          csv << %w[
+            uuid
+            type
+            value
+            created_at
+            confirmed_at
+            internal_id
+          ]
+        end
+
+        write_not_found
         write_emails
         write_phone_configurations
         write_auth_app_configurations
         write_webauthn_configurations
         write_piv_cac_configurations
         write_backup_code_configurations
-        output_file.close
       end
 
       private
 
-      def output_file
-        @output_file ||= begin
-          output_path = File.join(output_dir, 'user.csv')
-          File.open(output_path, 'w')
-        end
+      def uuid
+        user_report[:requesting_issuer_uuid]
       end
 
-      def write_rows_to_csv(rows, *columns)
-        output_file.puts(columns.join(','))
-
-        return output_file.puts("No data\n\n") if rows.empty?
-
-        rows.each do |row|
-          output_file.puts CSV.generate_line(row.values_at(*columns))
+      def write_not_found
+        if user_report[:not_found]
+          csv << [
+            uuid,
+            'not found',
+          ]
         end
-        output_file.puts("\n")
       end
 
       def write_auth_app_configurations
-        output_file.puts('Auth app configurations:')
-        write_rows_to_csv(
-          user_report[:mfa_configurations][:auth_app_configurations],
-          :name, :created_at
-        )
+        user_report[:mfa_configurations][:auth_app_configurations].each do |auth_app_config|
+          csv << [
+            uuid,
+            'Auth app configuration',
+            auth_app_config[:name],
+            auth_app_config[:created_at],
+          ]
+        end
       end
 
       def write_backup_code_configurations
-        output_file.puts('Backup code configurations:')
-        write_rows_to_csv(
-          user_report[:mfa_configurations][:backup_code_configurations],
-          :created_at, :used_at
-        )
+        user_report[:mfa_configurations][:backup_code_configurations].each do |backup_code_config|
+          csv << [
+            uuid,
+            'Backup code configuration',
+            nil,
+            backup_code_config[:created_at],
+            backup_code_config[:used_at],
+          ]
+        end
       end
 
       def write_emails
-        output_file.puts('Emails:')
-        write_rows_to_csv(user_report[:email_addresses], :email, :created_at, :confirmed_at)
+        user_report[:email_addresses].each do |email|
+          csv << [
+            uuid,
+            'Email',
+            email[:email],
+            email[:created_at],
+            email[:confirmed_at],
+          ]
+        end
       end
 
       def write_phone_configurations
-        output_file.puts('Phone configurations:')
-        write_rows_to_csv(
-          user_report[:mfa_configurations][:phone_configurations],
-          :phone, :created_at, :confirmed_at
-        )
+        user_report[:mfa_configurations][:phone_configurations].each do |phone_config|
+          csv << [
+            uuid,
+            'Phone configuration',
+            phone_config[:phone],
+            phone_config[:created_at],
+            phone_config[:confirmed_at],
+            phone_config[:id],
+          ]
+        end
       end
 
       def write_piv_cac_configurations
-        output_file.puts('PIV/CAC configurations:')
-        write_rows_to_csv(
-          user_report[:mfa_configurations][:piv_cac_configurations],
-          :name, :created_at
-        )
+        user_report[:mfa_configurations][:piv_cac_configurations].each do |piv_cac_config|
+          csv << [
+            uuid,
+            'PIV/CAC configuration',
+            piv_cac_config[:name],
+            piv_cac_config[:created_at],
+          ]
+        end
       end
 
       def write_webauthn_configurations
-        output_file.puts('WebAuthn configurations:')
-        write_rows_to_csv(
-          user_report[:mfa_configurations][:webauthn_configurations],
-          :name, :created_at
-        )
+        user_report[:mfa_configurations][:webauthn_configurations].each do |webauthn_config|
+          csv << [
+            uuid,
+            'WebAuthn configuration',
+            webauthn_config[:name],
+            webauthn_config[:created_at],
+          ]
+        end
       end
     end
   end

lib/tasks/data_requests.rake

@@ -39,25 +39,49 @@ namespace :data_requests do
     users_report = JSON.parse(File.read(ENV['USERS_REPORT']), symbolize_names: true)
     output_dir = ENV['OUTPUT_DIR']
 
-    users_report.each do |user_report|
+    users_csv = CSV.open(File.join(output_dir, 'users.csv'), 'w')
+    user_events_csv = CSV.open(File.join(output_dir, 'user_events.csv'), 'w')
+    logs_csv = CSV.open(File.join(output_dir, 'logs.csv'), 'w')
+
+    users_report.each_with_index do |user_report, idx|
       puts "Processing user: #{user_report[:requesting_issuer_uuid]}"
-      user_output_dir = File.join(output_dir, user_report[:requesting_issuer_uuid])
-      FileUtils.mkdir_p(user_output_dir)
 
-      DataRequests::Local::WriteUserInfo.new(user_report, user_output_dir).call
+      DataRequests::Local::WriteUserInfo.new(
+        user_report:,
+        csv: users_csv,
+        include_header: idx == 0,
+      ).call
+
       DataRequests::Local::WriteUserEvents.new(
-        user_report, user_output_dir, user_report[:requesting_issuer_uuid]
+        user_report:,
+        requesting_issuer_uuid: user_report[:requesting_issuer_uuid],
+        csv: user_events_csv,
+        include_header: idx == 0,
       ).call
 
       cloudwatch_dates = user_report[:user_events].pluck(:date_time).map do |date_time|
         Time.zone.parse(date_time).to_date
       end.uniq
+
       cloudwatch_results = DataRequests::Local::FetchCloudwatchLogs.new(
         user_report[:login_uuid],
         cloudwatch_dates,
       ).call
 
-      DataRequests::Local::WriteCloudwatchLogs.new(cloudwatch_results, user_output_dir).call
+      DataRequests::Local::WriteCloudwatchLogs.new(
+        cloudwatch_results:,
+        requesting_issuer_uuid: user_report[:requesting_issuer_uuid],
+        csv: logs_csv,
+        include_header: idx == 0,
+      ).call
+
+      users_csv.flush
+      user_events_csv.flush
+      logs_csv.flush
     end
+  ensure
+    users_csv&.close
+    user_events_csv&.close
+    logs_csv&.close
   end
 end

spec/fixtures/data_request.json

@@ -13,6 +13,7 @@
     "mfa_configurations": {
       "phone_configurations": [
         {
+          "id": 123456,
           "phone": "+1 555-555-5555",
           "created_at": "2021-10-21T14:53:08.803Z",
           "confirmed_at": "2021-10-21T14:53:08.790Z"