LG-11116 Refactor Report Building

app/jobs/reports/base_report.rb

@@ -62,10 +62,11 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension)
       url
     end
 
-    def generate_s3_paths(name, extension, now: Time.zone.now)
+    def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now)
       host_data_env = Identity::Hostdata.env
-      latest = "#{host_data_env}/#{name}/latest.#{name}.#{extension}"
-      [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name}.#{extension}"]
+      name_subdir_ext = "#{name}#{subname ? '/' : ''}#{subname}.#{extension}"
+      latest = "#{host_data_env}/#{name}/latest.#{name_subdir_ext}"
+      [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name_subdir_ext}"]
     end
 
     def logger

app/jobs/reports/monthly_key_metrics_report.rb

@@ -6,9 +6,30 @@ class MonthlyKeyMetricsReport < BaseReport
 
     attr_reader :report_date
 
-    def perform(date)
+    def perform(date = Time.zone.today)
       @report_date = date
-      csv_for_email = monthly_key_metrics_report_array
+
+      account_reuse_table = account_reuse_report.account_reuse_report
+      total_profiles_table = account_reuse_report.total_identities_report
+
+      upload_to_s3(account_reuse_table, report_name: 'account_reuse')
+      upload_to_s3(total_profiles_table, report_name: 'total_profiles')
+
+      email_tables = [
+        [
+          {
+            title: "IDV app reuse rate #{account_reuse_report.stats_month}",
+            float_as_percent: true,
+            precision: 4,
+          },
+          *account_reuse_table,
+        ],
+        [
+          { title: 'Total proofed identities' },
+          *total_profiles_table,
+        ],
+      ]
+
       email_message = "Report: #{REPORT_NAME} #{date}"
       email_addresses = emails.select(&:present?)
 
@@ -17,7 +38,7 @@ def perform(date)
           email: email_addresses,
           subject: "Monthly Key Metrics Report - #{date}",
           message: email_message,
-          tables: csv_for_email,
+          tables: email_tables,
         ).deliver_now
       else
         Rails.logger.warn 'No email addresses received - Monthly Key Metrics Report NOT SENT'
@@ -32,19 +53,29 @@ def emails
       emails
     end
 
-    def monthly_key_metrics_report_array
-      csv_array = []
+    def account_reuse_report
+      @account_reuse_report ||= Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date)
+    end
 
-      account_reuse_report_csv.each do |row|
-        csv_array << row
-      end
+    def upload_to_s3(report_body, report_name: nil)
+      _latest, path = generate_s3_paths(REPORT_NAME, 'csv', subname: report_name, now: report_date)
 
-      csv_array
+      if bucket_name.present?
+        upload_file_to_s3_bucket(
+          path: path,
+          body: csv_file(report_body),
+          content_type: 'text/csv',
+          bucket: bucket_name,
+        )
+      end
     end
 
-    # Individual Key Metric Report
-    def account_reuse_report_csv
-      Reports::MonthlyAccountReuseReport.new(report_date).report_csv
+    def csv_file(report_array)
+      CSV.generate do |csv|
+        report_array.each do |row|
+          csv << row
+        end
+      end
     end
   end
 end

app/jobs/reports/monthly_account_reuse_report.rb → app/services/reporting/account_reuse_and_total_identities_report.rb

@@ -1,38 +1,76 @@
-require 'csv'
-
-module Reports
-  class MonthlyAccountReuseReport < BaseReport
-    REPORT_NAME = 'monthly-account-reuse-report'
-
+module Reporting
+  class AccountReuseAndTotalIdentitiesReport
     attr_reader :report_date
 
     def initialize(report_date = Time.zone.today)
       @report_date = report_date
     end
 
-    def perform(report_date = Time.zone.today)
-      @report_date = report_date
-
-      _latest, path = generate_s3_paths(REPORT_NAME, 'json', now: report_date)
+    # Return array of arrays
+    def account_reuse_report
+      account_reuse_table = []
+      account_reuse_table << ['Num. SPs', 'Num. users', 'Percentage']
 
-      if bucket_name.present?
-        upload_file_to_s3_bucket(
-          path: path,
-          body: report_body,
-          content_type: 'text/csv',
-          bucket: bucket_name,
-        )
+      total_reuse_report[:reuse_stats].each do |result_entry|
+        account_reuse_table << [
+          result_entry['num_agencies'],
+          result_entry['num_users'],
+          result_entry['percentage'],
+        ]
       end
+
+      account_reuse_table << [
+        'Total (all >1)',
+        total_reuse_report[:total_users],
+        total_reuse_report[:total_percentage],
+      ]
+
+      account_reuse_table
     end
 
-    def first_day_of_report_month
-      report_date.beginning_of_month.strftime('%Y-%m-%d')
+    def total_identities_report
+      total_identities_table = []
+      total_identities_table << ["Total proofed identities (#{stats_month})"]
+      total_identities_table << [total_reuse_report[:total_proofed]]
+      total_identities_table
     end
 
-    def params
-      {
-        query_date: first_day_of_report_month,
-      }.transform_values { |v| ActiveRecord::Base.connection.quote(v) }
+    def stats_month
+      report_date.prev_month(1).strftime('%b-%Y')
+    end
+
+    private
+
+    def total_reuse_report
+      return @total_reuse_report if defined?(@total_reuse_report)
+      reuse_stats = agency_reuse_results
+
+      reuse_total_users = 0
+      reuse_total_percentage = 0
+
+      total_proofed = num_active_profiles
+
+      if !reuse_stats.empty?
+        reuse_stats.each do |result_entry|
+          reuse_total_users += result_entry['num_users']
+        end
+
+        if total_proofed > 0
+          reuse_stats.each_with_index do |result_entry, index|
+            reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f
+
+            reuse_total_percentage += reuse_stats[index]['percentage']
+          end
+        end
+      end
+
+      # reuse_stats and total_stats
+      @total_reuse_report = {
+        reuse_stats: reuse_stats,
+        total_users: reuse_total_users,
+        total_percentage: reuse_total_percentage,
+        total_proofed: total_proofed,
+      }
     end
 
     def agency_reuse_results
@@ -64,7 +102,7 @@ def agency_reuse_results
               num_agencies ASC
       SQL
 
-      agency_results = transaction_with_timeout do
+      agency_results = Reports::BaseReport.transaction_with_timeout do
         ActiveRecord::Base.connection.execute(agency_sql)
       end
 
@@ -83,87 +121,21 @@ def num_active_profiles
             profiles.activated_at < %{query_date}
       SQL
 
-      proofed_results = transaction_with_timeout do
+      proofed_results = Reports::BaseReport.transaction_with_timeout do
         ActiveRecord::Base.connection.execute(proofed_sql)
       end
 
       proofed_results.first['num_proofed']
     end
 
-    def stats_month
-      report_date.prev_month(1).strftime('%b-%Y')
-    end
-
-    def total_reuse_report
-      reuse_stats = agency_reuse_results
-
-      reuse_total_users = 0
-      reuse_total_percentage = 0
-
-      total_proofed = num_active_profiles
-
-      if !reuse_stats.empty?
-        reuse_stats.each do |result_entry|
-          reuse_total_users += result_entry['num_users']
-        end
-
-        if total_proofed > 0
-          reuse_stats.each_with_index do |result_entry, index|
-            reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f
-
-            reuse_total_percentage += reuse_stats[index]['percentage']
-          end
-        end
-      end
-
-      # reuse_stats and total_stats
-      { reuse_stats: reuse_stats,
-        total_users: reuse_total_users,
-        total_percentage: reuse_total_percentage,
-        total_proofed: total_proofed }
-    end
-
-    def report_csv
-      monthly_reuse_report = total_reuse_report
-
-      tables_array = []
-      reuse_rate_table = []
-      reuse_rate_table << {
-        title: "IDV app reuse rate #{stats_month}",
-        float_as_percent: true,
-        precision: 4,
-      }
-      reuse_rate_table << ['Num. SPs', 'Num. users', 'Percentage']
-
-      monthly_reuse_report[:reuse_stats].each do |result_entry|
-        reuse_rate_table << [
-          result_entry['num_agencies'],
-          result_entry['num_users'],
-          result_entry['percentage'],
-        ]
-      end
-      reuse_rate_table << [
-        'Total (all >1)',
-        monthly_reuse_report[:total_users],
-        monthly_reuse_report[:total_percentage],
-      ]
-      tables_array << reuse_rate_table
-
-      total_proofed_identities_table = []
-      total_proofed_identities_table << { title: 'Total proofed identities' }
-      total_proofed_identities_table << ["Total proofed identities (#{stats_month})"]
-      total_proofed_identities_table << [monthly_reuse_report[:total_proofed]]
-      tables_array << total_proofed_identities_table
-
-      tables_array
+    def params
+      {
+        query_date: first_day_of_report_month,
+      }.transform_values { |v| ActiveRecord::Base.connection.quote(v) }
     end
 
-    def report_body
-      CSV.generate do |csv|
-        report_csv.each do |row|
-          csv << row
-        end
-      end
+    def first_day_of_report_month
+      report_date.beginning_of_month.strftime('%Y-%m-%d')
     end
   end
 end

config/initializers/job_configurations.rb

@@ -4,7 +4,6 @@
 cron_24h = '0 0 * * *'
 gpo_cron_24h = '0 10 * * *' # 10am UTC is 5am EST/6am EDT
 cron_1w = '0 0 * * 0'
-cron_1st_of_mo = '0 0 1 * *'
 
 if defined?(Rails::Console)
   Rails.logger.info 'job_configurations: console detected, skipping schedule'
@@ -189,12 +188,6 @@
         cron: cron_24h,
         args: -> { [14.days.ago] },
       },
-      # Monthly report describing account reuse
-      monthly_account_reuse_report: {
-        class: 'Reports::MonthlyAccountReuseReport',
-        cron: cron_1st_of_mo,
-        args: -> { [Time.zone.today] },
-      },
       # Monthly report checking in on key metrics
       monthly_key_metrics_report: {
         class: 'Reports::MonthlyKeyMetricsReport',

spec/jobs/reports/monthly_key_metrics_report_spec.rb

@@ -7,12 +7,31 @@
   let(:name) { 'monthly-key-metrics-report' }
   let(:agnes_email) { 'fake@agnes_email.com' }
   let(:feds_email) { 'fake@feds_email.com' }
+  let(:s3_report_bucket_prefix) { 'reports-bucket' }
+  let(:account_reuse_s3_path) do
+    'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/account_reuse.csv'
+  end
+  let(:total_profiles_s3_path) do
+    'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/total_profiles.csv'
+  end
 
   before do
     allow(IdentityConfig.store).to receive(:team_agnes_email).
       and_return(agnes_email)
     allow(IdentityConfig.store).to receive(:team_all_feds_email).
       and_return(feds_email)
+
+    allow(Identity::Hostdata).to receive(:env).and_return('int')
+    allow(Identity::Hostdata).to receive(:aws_account_id).and_return('1234')
+    allow(Identity::Hostdata).to receive(:aws_region).and_return('us-west-1')
+    allow(IdentityConfig.store).to receive(:s3_report_bucket_prefix).
+      and_return(s3_report_bucket_prefix)
+
+    Aws.config[:s3] = {
+      stub_responses: {
+        put_object: {},
+      },
+    }
   end
 
   it 'sends out a report to the email listed with one total user' do
@@ -51,4 +70,22 @@
 
     subject.perform(report_date)
   end
+
+  it 'uploads a file to S3 based on the report date' do
+    expect(subject).to receive(:upload_file_to_s3_bucket).with(
+      path: account_reuse_s3_path,
+      body: anything,
+      content_type: 'text/csv',
+      bucket: 'reports-bucket.1234-us-west-1',
+    ).exactly(1).time.and_call_original
+
+    expect(subject).to receive(:upload_file_to_s3_bucket).with(
+      path: total_profiles_s3_path,
+      body: anything,
+      content_type: 'text/csv',
+      bucket: 'reports-bucket.1234-us-west-1',
+    ).exactly(1).time.and_call_original
+
+    subject.perform(report_date)
+  end
 end