diff --git a/app/jobs/reports/base_report.rb b/app/jobs/reports/base_report.rb index e424ffc8f49..e2148679d4d 100644 --- a/app/jobs/reports/base_report.rb +++ b/app/jobs/reports/base_report.rb @@ -62,10 +62,11 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension) url end - def generate_s3_paths(name, extension, now: Time.zone.now) + def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now) host_data_env = Identity::Hostdata.env - latest = "#{host_data_env}/#{name}/latest.#{name}.#{extension}" - [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name}.#{extension}"] + name_subdir_ext = "#{name}#{subname ? '/' : ''}#{subname}.#{extension}" + latest = "#{host_data_env}/#{name}/latest.#{name_subdir_ext}" + [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name_subdir_ext}"] end def logger diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index 7ba755a7862..8438089d1b6 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -6,9 +6,30 @@ class MonthlyKeyMetricsReport < BaseReport attr_reader :report_date - def perform(date) + def perform(date = Time.zone.today) @report_date = date - csv_for_email = monthly_key_metrics_report_array + + account_reuse_table = account_reuse_report.account_reuse_report + total_profiles_table = account_reuse_report.total_identities_report + + upload_to_s3(account_reuse_table, report_name: 'account_reuse') + upload_to_s3(total_profiles_table, report_name: 'total_profiles') + + email_tables = [ + [ + { + title: "IDV app reuse rate #{account_reuse_report.stats_month}", + float_as_percent: true, + precision: 4, + }, + *account_reuse_table, + ], + [ + { title: 'Total proofed identities' }, + *total_profiles_table, + ], + ] + email_message = "Report: #{REPORT_NAME} #{date}" email_addresses = emails.select(&:present?) @@ -17,7 +38,7 @@ def perform(date) email: email_addresses, subject: "Monthly Key Metrics Report - #{date}", message: email_message, - tables: csv_for_email, + tables: email_tables, ).deliver_now else Rails.logger.warn 'No email addresses received - Monthly Key Metrics Report NOT SENT' @@ -32,19 +53,29 @@ def emails emails end - def monthly_key_metrics_report_array - csv_array = [] + def account_reuse_report + @account_reuse_report ||= Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) + end - account_reuse_report_csv.each do |row| - csv_array << row - end + def upload_to_s3(report_body, report_name: nil) + _latest, path = generate_s3_paths(REPORT_NAME, 'csv', subname: report_name, now: report_date) - csv_array + if bucket_name.present? + upload_file_to_s3_bucket( + path: path, + body: csv_file(report_body), + content_type: 'text/csv', + bucket: bucket_name, + ) + end end - # Individual Key Metric Report - def account_reuse_report_csv - Reports::MonthlyAccountReuseReport.new(report_date).report_csv + def csv_file(report_array) + CSV.generate do |csv| + report_array.each do |row| + csv << row + end + end end end end diff --git a/app/jobs/reports/monthly_account_reuse_report.rb b/app/services/reporting/account_reuse_and_total_identities_report.rb similarity index 62% rename from app/jobs/reports/monthly_account_reuse_report.rb rename to app/services/reporting/account_reuse_and_total_identities_report.rb index ee02545ce3d..ca4966e6608 100644 --- a/app/jobs/reports/monthly_account_reuse_report.rb +++ b/app/services/reporting/account_reuse_and_total_identities_report.rb @@ -1,38 +1,76 @@ -require 'csv' - -module Reports - class MonthlyAccountReuseReport < BaseReport - REPORT_NAME = 'monthly-account-reuse-report' - +module Reporting + class AccountReuseAndTotalIdentitiesReport attr_reader :report_date def initialize(report_date = Time.zone.today) @report_date = report_date end - def perform(report_date = Time.zone.today) - @report_date = report_date - - _latest, path = generate_s3_paths(REPORT_NAME, 'json', now: report_date) + # Return array of arrays + def account_reuse_report + account_reuse_table = [] + account_reuse_table << ['Num. SPs', 'Num. users', 'Percentage'] - if bucket_name.present? - upload_file_to_s3_bucket( - path: path, - body: report_body, - content_type: 'text/csv', - bucket: bucket_name, - ) + total_reuse_report[:reuse_stats].each do |result_entry| + account_reuse_table << [ + result_entry['num_agencies'], + result_entry['num_users'], + result_entry['percentage'], + ] end + + account_reuse_table << [ + 'Total (all >1)', + total_reuse_report[:total_users], + total_reuse_report[:total_percentage], + ] + + account_reuse_table end - def first_day_of_report_month - report_date.beginning_of_month.strftime('%Y-%m-%d') + def total_identities_report + total_identities_table = [] + total_identities_table << ["Total proofed identities (#{stats_month})"] + total_identities_table << [total_reuse_report[:total_proofed]] + total_identities_table end - def params - { - query_date: first_day_of_report_month, - }.transform_values { |v| ActiveRecord::Base.connection.quote(v) } + def stats_month + report_date.prev_month(1).strftime('%b-%Y') + end + + private + + def total_reuse_report + return @total_reuse_report if defined?(@total_reuse_report) + reuse_stats = agency_reuse_results + + reuse_total_users = 0 + reuse_total_percentage = 0 + + total_proofed = num_active_profiles + + if !reuse_stats.empty? + reuse_stats.each do |result_entry| + reuse_total_users += result_entry['num_users'] + end + + if total_proofed > 0 + reuse_stats.each_with_index do |result_entry, index| + reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f + + reuse_total_percentage += reuse_stats[index]['percentage'] + end + end + end + + # reuse_stats and total_stats + @total_reuse_report = { + reuse_stats: reuse_stats, + total_users: reuse_total_users, + total_percentage: reuse_total_percentage, + total_proofed: total_proofed, + } end def agency_reuse_results @@ -64,7 +102,7 @@ def agency_reuse_results num_agencies ASC SQL - agency_results = transaction_with_timeout do + agency_results = Reports::BaseReport.transaction_with_timeout do ActiveRecord::Base.connection.execute(agency_sql) end @@ -83,87 +121,21 @@ def num_active_profiles profiles.activated_at < %{query_date} SQL - proofed_results = transaction_with_timeout do + proofed_results = Reports::BaseReport.transaction_with_timeout do ActiveRecord::Base.connection.execute(proofed_sql) end proofed_results.first['num_proofed'] end - def stats_month - report_date.prev_month(1).strftime('%b-%Y') - end - - def total_reuse_report - reuse_stats = agency_reuse_results - - reuse_total_users = 0 - reuse_total_percentage = 0 - - total_proofed = num_active_profiles - - if !reuse_stats.empty? - reuse_stats.each do |result_entry| - reuse_total_users += result_entry['num_users'] - end - - if total_proofed > 0 - reuse_stats.each_with_index do |result_entry, index| - reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f - - reuse_total_percentage += reuse_stats[index]['percentage'] - end - end - end - - # reuse_stats and total_stats - { reuse_stats: reuse_stats, - total_users: reuse_total_users, - total_percentage: reuse_total_percentage, - total_proofed: total_proofed } - end - - def report_csv - monthly_reuse_report = total_reuse_report - - tables_array = [] - reuse_rate_table = [] - reuse_rate_table << { - title: "IDV app reuse rate #{stats_month}", - float_as_percent: true, - precision: 4, - } - reuse_rate_table << ['Num. SPs', 'Num. users', 'Percentage'] - - monthly_reuse_report[:reuse_stats].each do |result_entry| - reuse_rate_table << [ - result_entry['num_agencies'], - result_entry['num_users'], - result_entry['percentage'], - ] - end - reuse_rate_table << [ - 'Total (all >1)', - monthly_reuse_report[:total_users], - monthly_reuse_report[:total_percentage], - ] - tables_array << reuse_rate_table - - total_proofed_identities_table = [] - total_proofed_identities_table << { title: 'Total proofed identities' } - total_proofed_identities_table << ["Total proofed identities (#{stats_month})"] - total_proofed_identities_table << [monthly_reuse_report[:total_proofed]] - tables_array << total_proofed_identities_table - - tables_array + def params + { + query_date: first_day_of_report_month, + }.transform_values { |v| ActiveRecord::Base.connection.quote(v) } end - def report_body - CSV.generate do |csv| - report_csv.each do |row| - csv << row - end - end + def first_day_of_report_month + report_date.beginning_of_month.strftime('%Y-%m-%d') end end end diff --git a/config/initializers/job_configurations.rb b/config/initializers/job_configurations.rb index f1832446985..e39a398ac04 100644 --- a/config/initializers/job_configurations.rb +++ b/config/initializers/job_configurations.rb @@ -4,7 +4,6 @@ cron_24h = '0 0 * * *' gpo_cron_24h = '0 10 * * *' # 10am UTC is 5am EST/6am EDT cron_1w = '0 0 * * 0' -cron_1st_of_mo = '0 0 1 * *' if defined?(Rails::Console) Rails.logger.info 'job_configurations: console detected, skipping schedule' @@ -189,12 +188,6 @@ cron: cron_24h, args: -> { [14.days.ago] }, }, - # Monthly report describing account reuse - monthly_account_reuse_report: { - class: 'Reports::MonthlyAccountReuseReport', - cron: cron_1st_of_mo, - args: -> { [Time.zone.today] }, - }, # Monthly report checking in on key metrics monthly_key_metrics_report: { class: 'Reports::MonthlyKeyMetricsReport', diff --git a/spec/jobs/reports/monthly_key_metrics_report_spec.rb b/spec/jobs/reports/monthly_key_metrics_report_spec.rb index f6cbefc7309..bc0298d6ccd 100644 --- a/spec/jobs/reports/monthly_key_metrics_report_spec.rb +++ b/spec/jobs/reports/monthly_key_metrics_report_spec.rb @@ -7,12 +7,31 @@ let(:name) { 'monthly-key-metrics-report' } let(:agnes_email) { 'fake@agnes_email.com' } let(:feds_email) { 'fake@feds_email.com' } + let(:s3_report_bucket_prefix) { 'reports-bucket' } + let(:account_reuse_s3_path) do + 'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/account_reuse.csv' + end + let(:total_profiles_s3_path) do + 'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/total_profiles.csv' + end before do allow(IdentityConfig.store).to receive(:team_agnes_email). and_return(agnes_email) allow(IdentityConfig.store).to receive(:team_all_feds_email). and_return(feds_email) + + allow(Identity::Hostdata).to receive(:env).and_return('int') + allow(Identity::Hostdata).to receive(:aws_account_id).and_return('1234') + allow(Identity::Hostdata).to receive(:aws_region).and_return('us-west-1') + allow(IdentityConfig.store).to receive(:s3_report_bucket_prefix). + and_return(s3_report_bucket_prefix) + + Aws.config[:s3] = { + stub_responses: { + put_object: {}, + }, + } end it 'sends out a report to the email listed with one total user' do @@ -51,4 +70,22 @@ subject.perform(report_date) end + + it 'uploads a file to S3 based on the report date' do + expect(subject).to receive(:upload_file_to_s3_bucket).with( + path: account_reuse_s3_path, + body: anything, + content_type: 'text/csv', + bucket: 'reports-bucket.1234-us-west-1', + ).exactly(1).time.and_call_original + + expect(subject).to receive(:upload_file_to_s3_bucket).with( + path: total_profiles_s3_path, + body: anything, + content_type: 'text/csv', + bucket: 'reports-bucket.1234-us-west-1', + ).exactly(1).time.and_call_original + + subject.perform(report_date) + end end diff --git a/spec/jobs/reports/monthly_account_reuse_report_spec.rb b/spec/services/reporting/account_reuse_and_total_identities_report_spec.rb similarity index 60% rename from spec/jobs/reports/monthly_account_reuse_report_spec.rb rename to spec/services/reporting/account_reuse_and_total_identities_report_spec.rb index e16f933d5eb..fad3350bc5e 100644 --- a/spec/jobs/reports/monthly_account_reuse_report_spec.rb +++ b/spec/services/reporting/account_reuse_and_total_identities_report_spec.rb @@ -1,45 +1,16 @@ require 'rails_helper' require 'csv' -RSpec.describe Reports::MonthlyAccountReuseReport do +RSpec.describe Reporting::AccountReuseAndTotalIdentitiesReport do let(:report_date) { Date.new(2021, 3, 1) } - subject(:report) { Reports::MonthlyAccountReuseReport.new } - - let(:s3_report_bucket_prefix) { 'reports-bucket' } - let(:s3_report_path) do - 'int/monthly-account-reuse-report/2021/2021-03-01.monthly-account-reuse-report.json' - end + subject(:report) { Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) } before do travel_to report_date - allow(Identity::Hostdata).to receive(:env).and_return('int') - allow(Identity::Hostdata).to receive(:aws_account_id).and_return('1234') - allow(Identity::Hostdata).to receive(:aws_region).and_return('us-west-1') - allow(IdentityConfig.store).to receive(:s3_report_bucket_prefix). - and_return(s3_report_bucket_prefix) - - Aws.config[:s3] = { - stub_responses: { - put_object: {}, - }, - } end describe '#perform' do - it 'uploads a file to S3 based on the report date' do - expect(report).to receive(:upload_file_to_s3_bucket).with( - path: s3_report_path, - body: anything, - content_type: 'text/csv', - bucket: 'reports-bucket.1234-us-west-1', - ).exactly(1).time.and_call_original - - expect(report).to receive(:report_body).and_call_original.once - - report.perform - end - context 'with data' do let(:in_query) { report_date - 12.days } let(:out_of_query) { report_date + 12.days } @@ -121,32 +92,16 @@ def create_identity(id, provider, verified_time) end end - it 'aggregates by issuer' do - expect(report).to receive(:upload_file_to_s3_bucket). - exactly(1).times do |path:, body:, content_type:, bucket:| - actual_csv = body - expected_csv = CSV.generate do |csv| - [ - [ - { title: 'IDV app reuse rate Feb-2021', float_as_percent: true, precision: 4 }, - ['Num. SPs', 'Num. users', 'Percentage'], - [2, 3, 0.3], - [3, 2, 0.2], - ['Total (all >1)', 5, 0.5], - ], - [ - { title: 'Total proofed identities' }, - ['Total proofed identities (Feb-2021)'], - [10], - ], - ].each do |row| - csv << row - end - end - expect(actual_csv).to eq(expected_csv) - end + it 'returns correct queries' do + actual_account_reuse_table = report.account_reuse_report + actual_total_profiles_table = report.total_identities_report + + expected_account_reuse_table = [['Num. SPs', 'Num. users', 'Percentage'], [2, 3, 0.3], + [3, 2, 0.2], ['Total (all >1)', 5, 0.5]] + expected_total_profiles_table = [['Total proofed identities (Feb-2021)'], [10]] - report.perform + expect(actual_account_reuse_table).to eq(expected_account_reuse_table) + expect(actual_total_profiles_table).to eq(expected_total_profiles_table) end end end