From 5279fbd0d4f3e9478ab8ef9c7cd4341b0d5b78d1 Mon Sep 17 00:00:00 2001 From: Luis Date: Mon, 2 Oct 2023 15:31:55 -0500 Subject: [PATCH 1/9] LG-11116 Refactor Report Building [skip changelog] --- app/jobs/reports/base_report.rb | 6 +- .../reports/monthly_key_metrics_report.rb | 50 ++++-- ...ount_reuse_and_total_identities_report.rb} | 158 ++++++++---------- 3 files changed, 107 insertions(+), 107 deletions(-) rename app/{jobs/reports/monthly_account_reuse_report.rb => services/reporting/account_reuse_and_total_identities_report.rb} (65%) diff --git a/app/jobs/reports/base_report.rb b/app/jobs/reports/base_report.rb index e424ffc8f49..ae413959943 100644 --- a/app/jobs/reports/base_report.rb +++ b/app/jobs/reports/base_report.rb @@ -62,10 +62,10 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension) url end - def generate_s3_paths(name, extension, now: Time.zone.now) + def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now) host_data_env = Identity::Hostdata.env - latest = "#{host_data_env}/#{name}/latest.#{name}.#{extension}" - [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name}.#{extension}"] + latest = "#{host_data_env}/#{name}/latest.#{name}#{subname ? '/' + subname : '' }.#{extension}" + [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name}#{subname ? '/' + subname : '' }.#{extension}"] end def logger diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index 7ba755a7862..986f95ec9e7 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -6,9 +6,30 @@ class MonthlyKeyMetricsReport < BaseReport attr_reader :report_date - def perform(date) + def perform(date = Time.zone.today) @report_date = date - csv_for_email = monthly_key_metrics_report_array + + account_reuse_table = account_reuse_queries.account_reuse_report + total_profiles_table = account_reuse_queries.total_identities_report + + upload_to_s3(account_reuse_data, report_name: 'account_reuse') + upload_to_s3(total_profiles_data, report_name: 'total_profiles', ) + + email_tables = [ + [ + { + title: "IDV app reuse rate #{account_reuse_queries.stats_month}", + float_as_percent: true, + precision: 4, + }, + account_reuse_table + ], + [ + { title: 'Total proofed identities' }, + total_profiles_table, + ] + ] + email_message = "Report: #{REPORT_NAME} #{date}" email_addresses = emails.select(&:present?) @@ -17,7 +38,7 @@ def perform(date) email: email_addresses, subject: "Monthly Key Metrics Report - #{date}", message: email_message, - tables: csv_for_email, + tables: email_tables, ).deliver_now else Rails.logger.warn 'No email addresses received - Monthly Key Metrics Report NOT SENT' @@ -32,19 +53,22 @@ def emails emails end - def monthly_key_metrics_report_array - csv_array = [] + def account_reuse_queries + @account_reuse_queries ||= AccountReuseAndTotalIdentities.new(report_date) + end - account_reuse_report_csv.each do |row| - csv_array << row - end + def upload_to_s3(report_body, report_name: nil) - csv_array - end + _latest, path = generate_s3_paths(REPORT_NAME, 'csv', subname: report_name, now: report_date) - # Individual Key Metric Report - def account_reuse_report_csv - Reports::MonthlyAccountReuseReport.new(report_date).report_csv + if bucket_name.present? + upload_file_to_s3_bucket( + path: path, + body: report_body, + content_type: 'text/csv', + bucket: bucket_name, + ) + end end end end diff --git a/app/jobs/reports/monthly_account_reuse_report.rb b/app/services/reporting/account_reuse_and_total_identities_report.rb similarity index 65% rename from app/jobs/reports/monthly_account_reuse_report.rb rename to app/services/reporting/account_reuse_and_total_identities_report.rb index ee02545ce3d..996aa3af68c 100644 --- a/app/jobs/reports/monthly_account_reuse_report.rb +++ b/app/services/reporting/account_reuse_and_total_identities_report.rb @@ -1,8 +1,7 @@ require 'csv' -module Reports - class MonthlyAccountReuseReport < BaseReport - REPORT_NAME = 'monthly-account-reuse-report' +module Reporting + class AccountReuseAndTotalIdentitiesReport attr_reader :report_date @@ -10,29 +9,71 @@ def initialize(report_date = Time.zone.today) @report_date = report_date end - def perform(report_date = Time.zone.today) - @report_date = report_date - - _latest, path = generate_s3_paths(REPORT_NAME, 'json', now: report_date) + # Return array of arrays + def account_reuse_report + account_reuse_table = [] + account_reuse_table << ['Num. SPs', 'Num. users', 'Percentage'] - if bucket_name.present? - upload_file_to_s3_bucket( - path: path, - body: report_body, - content_type: 'text/csv', - bucket: bucket_name, - ) + total_reuse_report[:reuse_stats].each do |result_entry| + account_reuse_table << [ + result_entry['num_agencies'], + result_entry['num_users'], + result_entry['percentage'], + ] end + + account_reuse_table << [ + 'Total (all >1)', + total_reuse_report[:total_users], + total_reuse_report[:total_percentage], + ] + + account_reuse_table end - def first_day_of_report_month - report_date.beginning_of_month.strftime('%Y-%m-%d') + def total_identities_report + total_identities_table = [] + total_identities_table << ["Total proofed identities (#{stats_month})"] + total_identities_table << [total_reuse_report[:total_proofed]] + total_identities_table end - def params - { - query_date: first_day_of_report_month, - }.transform_values { |v| ActiveRecord::Base.connection.quote(v) } + def stats_month + report_date.prev_month(1).strftime('%b-%Y') + end + + private + + def total_reuse_report + return @total_reuse_report if defined?(@total_reuse_report) + reuse_stats = agency_reuse_results + + reuse_total_users = 0 + reuse_total_percentage = 0 + + total_proofed = num_active_profiles + + if !reuse_stats.empty? + reuse_stats.each do |result_entry| + reuse_total_users += result_entry['num_users'] + end + + if total_proofed > 0 + reuse_stats.each_with_index do |result_entry, index| + reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f + + reuse_total_percentage += reuse_stats[index]['percentage'] + end + end + end + + # reuse_stats and total_stats + @total_reuse_report = { + reuse_stats: reuse_stats, + total_users: reuse_total_users, + total_percentage: reuse_total_percentage, + total_proofed: total_proofed, + } end def agency_reuse_results @@ -90,80 +131,15 @@ def num_active_profiles proofed_results.first['num_proofed'] end - def stats_month - report_date.prev_month(1).strftime('%b-%Y') - end - - def total_reuse_report - reuse_stats = agency_reuse_results - - reuse_total_users = 0 - reuse_total_percentage = 0 - - total_proofed = num_active_profiles - - if !reuse_stats.empty? - reuse_stats.each do |result_entry| - reuse_total_users += result_entry['num_users'] - end - - if total_proofed > 0 - reuse_stats.each_with_index do |result_entry, index| - reuse_stats[index]['percentage'] = result_entry['num_users'] / total_proofed.to_f - - reuse_total_percentage += reuse_stats[index]['percentage'] - end - end - end - - # reuse_stats and total_stats - { reuse_stats: reuse_stats, - total_users: reuse_total_users, - total_percentage: reuse_total_percentage, - total_proofed: total_proofed } + def params + { + query_date: first_day_of_report_month, + }.transform_values { |v| ActiveRecord::Base.connection.quote(v) } end - def report_csv - monthly_reuse_report = total_reuse_report - - tables_array = [] - reuse_rate_table = [] - reuse_rate_table << { - title: "IDV app reuse rate #{stats_month}", - float_as_percent: true, - precision: 4, - } - reuse_rate_table << ['Num. SPs', 'Num. users', 'Percentage'] - - monthly_reuse_report[:reuse_stats].each do |result_entry| - reuse_rate_table << [ - result_entry['num_agencies'], - result_entry['num_users'], - result_entry['percentage'], - ] - end - reuse_rate_table << [ - 'Total (all >1)', - monthly_reuse_report[:total_users], - monthly_reuse_report[:total_percentage], - ] - tables_array << reuse_rate_table - - total_proofed_identities_table = [] - total_proofed_identities_table << { title: 'Total proofed identities' } - total_proofed_identities_table << ["Total proofed identities (#{stats_month})"] - total_proofed_identities_table << [monthly_reuse_report[:total_proofed]] - tables_array << total_proofed_identities_table - - tables_array + def first_day_of_report_month + report_date.beginning_of_month.strftime('%Y-%m-%d') end - def report_body - CSV.generate do |csv| - report_csv.each do |row| - csv << row - end - end - end end end From fdd910fad5ca8d8b9ab2d291bba80e5c04047344 Mon Sep 17 00:00:00 2001 From: "Luis H. Matos" Date: Tue, 3 Oct 2023 12:30:58 -0500 Subject: [PATCH 2/9] Update app/jobs/reports/base_report.rb Co-authored-by: Zach Margolis --- app/jobs/reports/base_report.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/jobs/reports/base_report.rb b/app/jobs/reports/base_report.rb index ae413959943..4ab38c20df1 100644 --- a/app/jobs/reports/base_report.rb +++ b/app/jobs/reports/base_report.rb @@ -64,8 +64,9 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension) def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now) host_data_env = Identity::Hostdata.env - latest = "#{host_data_env}/#{name}/latest.#{name}#{subname ? '/' + subname : '' }.#{extension}" - [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name}#{subname ? '/' + subname : '' }.#{extension}"] + name_subdir_ext = "#{name}#{subdir}.#{extension}" + latest = "#{host_data_env}/#{name}/latest.#{name_subdir_ext}" + [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name_subdir_ext}"] end def logger From a8b83cd67274834fa94bbb7a57200153fa4653f9 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 3 Oct 2023 14:21:26 -0500 Subject: [PATCH 3/9] changelog: Internal, Reporting, Reorganize Individual Metric Reports --- app/jobs/reports/base_report.rb | 2 +- .../reports/monthly_key_metrics_report.rb | 29 +++++--- ...count_reuse_and_total_identities_report.rb | 6 +- .../monthly_key_metrics_report_spec.rb | 37 ++++++++++ ...reuse_and_total_identities_report_spec.rb} | 67 +++---------------- 5 files changed, 69 insertions(+), 72 deletions(-) rename spec/{jobs/reports/monthly_account_reuse_report_spec.rb => services/reporting/account_reuse_and_total_identities_report_spec.rb} (60%) diff --git a/app/jobs/reports/base_report.rb b/app/jobs/reports/base_report.rb index 4ab38c20df1..cb9af509a16 100644 --- a/app/jobs/reports/base_report.rb +++ b/app/jobs/reports/base_report.rb @@ -64,7 +64,7 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension) def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now) host_data_env = Identity::Hostdata.env - name_subdir_ext = "#{name}#{subdir}.#{extension}" + name_subdir_ext = "#{name}/#{subname}.#{extension}" latest = "#{host_data_env}/#{name}/latest.#{name_subdir_ext}" [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name_subdir_ext}"] end diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index 986f95ec9e7..2ea56c71e6e 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -12,22 +12,22 @@ def perform(date = Time.zone.today) account_reuse_table = account_reuse_queries.account_reuse_report total_profiles_table = account_reuse_queries.total_identities_report - upload_to_s3(account_reuse_data, report_name: 'account_reuse') - upload_to_s3(total_profiles_data, report_name: 'total_profiles', ) + upload_to_s3(account_reuse_table, report_name: 'account_reuse') + upload_to_s3(total_profiles_table, report_name: 'total_profiles') email_tables = [ [ { - title: "IDV app reuse rate #{account_reuse_queries.stats_month}", - float_as_percent: true, - precision: 4, + title: "IDV app reuse rate #{account_reuse_queries.stats_month}", + float_as_percent: true, + precision: 4, }, - account_reuse_table - ], + account_reuse_table, + ], [ { title: 'Total proofed identities' }, total_profiles_table, - ] + ], ] email_message = "Report: #{REPORT_NAME} #{date}" @@ -54,21 +54,28 @@ def emails end def account_reuse_queries - @account_reuse_queries ||= AccountReuseAndTotalIdentities.new(report_date) + @account_reuse_queries ||= Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) end def upload_to_s3(report_body, report_name: nil) - _latest, path = generate_s3_paths(REPORT_NAME, 'csv', subname: report_name, now: report_date) if bucket_name.present? upload_file_to_s3_bucket( path: path, - body: report_body, + body: csv_file(report_body), content_type: 'text/csv', bucket: bucket_name, ) end end + + def csv_file(report_array) + CSV.generate do |csv| + report_array.each do |row| + csv << row + end + end + end end end diff --git a/app/services/reporting/account_reuse_and_total_identities_report.rb b/app/services/reporting/account_reuse_and_total_identities_report.rb index 996aa3af68c..59ac781a380 100644 --- a/app/services/reporting/account_reuse_and_total_identities_report.rb +++ b/app/services/reporting/account_reuse_and_total_identities_report.rb @@ -2,7 +2,6 @@ module Reporting class AccountReuseAndTotalIdentitiesReport - attr_reader :report_date def initialize(report_date = Time.zone.today) @@ -105,7 +104,7 @@ def agency_reuse_results num_agencies ASC SQL - agency_results = transaction_with_timeout do + agency_results = Reports::BaseReport.transaction_with_timeout do ActiveRecord::Base.connection.execute(agency_sql) end @@ -124,7 +123,7 @@ def num_active_profiles profiles.activated_at < %{query_date} SQL - proofed_results = transaction_with_timeout do + proofed_results = Reports::BaseReport.transaction_with_timeout do ActiveRecord::Base.connection.execute(proofed_sql) end @@ -140,6 +139,5 @@ def params def first_day_of_report_month report_date.beginning_of_month.strftime('%Y-%m-%d') end - end end diff --git a/spec/jobs/reports/monthly_key_metrics_report_spec.rb b/spec/jobs/reports/monthly_key_metrics_report_spec.rb index f6cbefc7309..bc0298d6ccd 100644 --- a/spec/jobs/reports/monthly_key_metrics_report_spec.rb +++ b/spec/jobs/reports/monthly_key_metrics_report_spec.rb @@ -7,12 +7,31 @@ let(:name) { 'monthly-key-metrics-report' } let(:agnes_email) { 'fake@agnes_email.com' } let(:feds_email) { 'fake@feds_email.com' } + let(:s3_report_bucket_prefix) { 'reports-bucket' } + let(:account_reuse_s3_path) do + 'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/account_reuse.csv' + end + let(:total_profiles_s3_path) do + 'int/monthly-key-metrics-report/2021/2021-03-02.monthly-key-metrics-report/total_profiles.csv' + end before do allow(IdentityConfig.store).to receive(:team_agnes_email). and_return(agnes_email) allow(IdentityConfig.store).to receive(:team_all_feds_email). and_return(feds_email) + + allow(Identity::Hostdata).to receive(:env).and_return('int') + allow(Identity::Hostdata).to receive(:aws_account_id).and_return('1234') + allow(Identity::Hostdata).to receive(:aws_region).and_return('us-west-1') + allow(IdentityConfig.store).to receive(:s3_report_bucket_prefix). + and_return(s3_report_bucket_prefix) + + Aws.config[:s3] = { + stub_responses: { + put_object: {}, + }, + } end it 'sends out a report to the email listed with one total user' do @@ -51,4 +70,22 @@ subject.perform(report_date) end + + it 'uploads a file to S3 based on the report date' do + expect(subject).to receive(:upload_file_to_s3_bucket).with( + path: account_reuse_s3_path, + body: anything, + content_type: 'text/csv', + bucket: 'reports-bucket.1234-us-west-1', + ).exactly(1).time.and_call_original + + expect(subject).to receive(:upload_file_to_s3_bucket).with( + path: total_profiles_s3_path, + body: anything, + content_type: 'text/csv', + bucket: 'reports-bucket.1234-us-west-1', + ).exactly(1).time.and_call_original + + subject.perform(report_date) + end end diff --git a/spec/jobs/reports/monthly_account_reuse_report_spec.rb b/spec/services/reporting/account_reuse_and_total_identities_report_spec.rb similarity index 60% rename from spec/jobs/reports/monthly_account_reuse_report_spec.rb rename to spec/services/reporting/account_reuse_and_total_identities_report_spec.rb index e16f933d5eb..fad3350bc5e 100644 --- a/spec/jobs/reports/monthly_account_reuse_report_spec.rb +++ b/spec/services/reporting/account_reuse_and_total_identities_report_spec.rb @@ -1,45 +1,16 @@ require 'rails_helper' require 'csv' -RSpec.describe Reports::MonthlyAccountReuseReport do +RSpec.describe Reporting::AccountReuseAndTotalIdentitiesReport do let(:report_date) { Date.new(2021, 3, 1) } - subject(:report) { Reports::MonthlyAccountReuseReport.new } - - let(:s3_report_bucket_prefix) { 'reports-bucket' } - let(:s3_report_path) do - 'int/monthly-account-reuse-report/2021/2021-03-01.monthly-account-reuse-report.json' - end + subject(:report) { Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) } before do travel_to report_date - allow(Identity::Hostdata).to receive(:env).and_return('int') - allow(Identity::Hostdata).to receive(:aws_account_id).and_return('1234') - allow(Identity::Hostdata).to receive(:aws_region).and_return('us-west-1') - allow(IdentityConfig.store).to receive(:s3_report_bucket_prefix). - and_return(s3_report_bucket_prefix) - - Aws.config[:s3] = { - stub_responses: { - put_object: {}, - }, - } end describe '#perform' do - it 'uploads a file to S3 based on the report date' do - expect(report).to receive(:upload_file_to_s3_bucket).with( - path: s3_report_path, - body: anything, - content_type: 'text/csv', - bucket: 'reports-bucket.1234-us-west-1', - ).exactly(1).time.and_call_original - - expect(report).to receive(:report_body).and_call_original.once - - report.perform - end - context 'with data' do let(:in_query) { report_date - 12.days } let(:out_of_query) { report_date + 12.days } @@ -121,32 +92,16 @@ def create_identity(id, provider, verified_time) end end - it 'aggregates by issuer' do - expect(report).to receive(:upload_file_to_s3_bucket). - exactly(1).times do |path:, body:, content_type:, bucket:| - actual_csv = body - expected_csv = CSV.generate do |csv| - [ - [ - { title: 'IDV app reuse rate Feb-2021', float_as_percent: true, precision: 4 }, - ['Num. SPs', 'Num. users', 'Percentage'], - [2, 3, 0.3], - [3, 2, 0.2], - ['Total (all >1)', 5, 0.5], - ], - [ - { title: 'Total proofed identities' }, - ['Total proofed identities (Feb-2021)'], - [10], - ], - ].each do |row| - csv << row - end - end - expect(actual_csv).to eq(expected_csv) - end + it 'returns correct queries' do + actual_account_reuse_table = report.account_reuse_report + actual_total_profiles_table = report.total_identities_report + + expected_account_reuse_table = [['Num. SPs', 'Num. users', 'Percentage'], [2, 3, 0.3], + [3, 2, 0.2], ['Total (all >1)', 5, 0.5]] + expected_total_profiles_table = [['Total proofed identities (Feb-2021)'], [10]] - report.perform + expect(actual_account_reuse_table).to eq(expected_account_reuse_table) + expect(actual_total_profiles_table).to eq(expected_total_profiles_table) end end end From 908bdb57b7ef11f80330d41393f4ecba17298483 Mon Sep 17 00:00:00 2001 From: "Luis H. Matos" Date: Tue, 3 Oct 2023 15:35:21 -0500 Subject: [PATCH 4/9] Update app/jobs/reports/monthly_key_metrics_report.rb Co-authored-by: Zach Margolis --- app/jobs/reports/monthly_key_metrics_report.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index 2ea56c71e6e..d23e881da2a 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -22,11 +22,11 @@ def perform(date = Time.zone.today) float_as_percent: true, precision: 4, }, - account_reuse_table, + *account_reuse_table, ], [ { title: 'Total proofed identities' }, - total_profiles_table, + *total_profiles_table, ], ] From a0fbfc0286fd62f59fb6397cde8d93ed30723505 Mon Sep 17 00:00:00 2001 From: "Luis H. Matos" Date: Tue, 3 Oct 2023 15:35:42 -0500 Subject: [PATCH 5/9] Update app/jobs/reports/monthly_key_metrics_report.rb Co-authored-by: Zach Margolis --- app/jobs/reports/monthly_key_metrics_report.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index d23e881da2a..f3adf4dfc69 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -53,8 +53,8 @@ def emails emails end - def account_reuse_queries - @account_reuse_queries ||= Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) + def account_reuse_report + @account_reuse_report ||= Reporting::AccountReuseAndTotalIdentitiesReport.new(report_date) end def upload_to_s3(report_body, report_name: nil) From 581fbc89f76c12870fb2f8e1fbf1b7dd15ecbc65 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 3 Oct 2023 15:53:32 -0500 Subject: [PATCH 6/9] PR comments --- .../reporting/account_reuse_and_total_identities_report.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/services/reporting/account_reuse_and_total_identities_report.rb b/app/services/reporting/account_reuse_and_total_identities_report.rb index 59ac781a380..ca4966e6608 100644 --- a/app/services/reporting/account_reuse_and_total_identities_report.rb +++ b/app/services/reporting/account_reuse_and_total_identities_report.rb @@ -1,5 +1,3 @@ -require 'csv' - module Reporting class AccountReuseAndTotalIdentitiesReport attr_reader :report_date From b3b296072921473675077b8b160e0bb9d155f123 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 3 Oct 2023 16:17:20 -0500 Subject: [PATCH 7/9] fix specs --- app/jobs/reports/base_report.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/reports/base_report.rb b/app/jobs/reports/base_report.rb index cb9af509a16..e2148679d4d 100644 --- a/app/jobs/reports/base_report.rb +++ b/app/jobs/reports/base_report.rb @@ -64,7 +64,7 @@ def upload_file_to_s3_timestamped_and_latest(report_name, body, extension) def generate_s3_paths(name, extension, subname: nil, now: Time.zone.now) host_data_env = Identity::Hostdata.env - name_subdir_ext = "#{name}/#{subname}.#{extension}" + name_subdir_ext = "#{name}#{subname ? '/' : ''}#{subname}.#{extension}" latest = "#{host_data_env}/#{name}/latest.#{name_subdir_ext}" [latest, "#{host_data_env}/#{name}/#{now.year}/#{now.strftime('%F')}.#{name_subdir_ext}"] end From 087cbf0554205415c649a8a6589604fd23c3fc6d Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 3 Oct 2023 16:32:04 -0500 Subject: [PATCH 8/9] spec fixes --- config/initializers/job_configurations.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config/initializers/job_configurations.rb b/config/initializers/job_configurations.rb index f1832446985..e39a398ac04 100644 --- a/config/initializers/job_configurations.rb +++ b/config/initializers/job_configurations.rb @@ -4,7 +4,6 @@ cron_24h = '0 0 * * *' gpo_cron_24h = '0 10 * * *' # 10am UTC is 5am EST/6am EDT cron_1w = '0 0 * * 0' -cron_1st_of_mo = '0 0 1 * *' if defined?(Rails::Console) Rails.logger.info 'job_configurations: console detected, skipping schedule' @@ -189,12 +188,6 @@ cron: cron_24h, args: -> { [14.days.ago] }, }, - # Monthly report describing account reuse - monthly_account_reuse_report: { - class: 'Reports::MonthlyAccountReuseReport', - cron: cron_1st_of_mo, - args: -> { [Time.zone.today] }, - }, # Monthly report checking in on key metrics monthly_key_metrics_report: { class: 'Reports::MonthlyKeyMetricsReport', From 86a5a04bf6089589466a335566595d2e83c0f6d0 Mon Sep 17 00:00:00 2001 From: Luis Date: Tue, 3 Oct 2023 16:46:16 -0500 Subject: [PATCH 9/9] spec fix --- app/jobs/reports/monthly_key_metrics_report.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/jobs/reports/monthly_key_metrics_report.rb b/app/jobs/reports/monthly_key_metrics_report.rb index f3adf4dfc69..8438089d1b6 100644 --- a/app/jobs/reports/monthly_key_metrics_report.rb +++ b/app/jobs/reports/monthly_key_metrics_report.rb @@ -9,8 +9,8 @@ class MonthlyKeyMetricsReport < BaseReport def perform(date = Time.zone.today) @report_date = date - account_reuse_table = account_reuse_queries.account_reuse_report - total_profiles_table = account_reuse_queries.total_identities_report + account_reuse_table = account_reuse_report.account_reuse_report + total_profiles_table = account_reuse_report.total_identities_report upload_to_s3(account_reuse_table, report_name: 'account_reuse') upload_to_s3(total_profiles_table, report_name: 'total_profiles') @@ -18,7 +18,7 @@ def perform(date = Time.zone.today) email_tables = [ [ { - title: "IDV app reuse rate #{account_reuse_queries.stats_month}", + title: "IDV app reuse rate #{account_reuse_report.stats_month}", float_as_percent: true, precision: 4, },