diff --git a/app/jobs/reports/month_helper.rb b/app/jobs/reports/month_helper.rb index e7a41fb2b04..992d2fb1c6d 100644 --- a/app/jobs/reports/month_helper.rb +++ b/app/jobs/reports/month_helper.rb @@ -33,10 +33,5 @@ def months(date_range) results end - - def full_month?(date_range) - date_range.begin == date_range.begin.beginning_of_month && - date_range.end == date_range.end.end_of_month - end end end diff --git a/app/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window.rb b/app/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window.rb index e7c090d1507..be97f93686c 100644 --- a/app/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window.rb +++ b/app/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window.rb @@ -19,32 +19,31 @@ def call(issuer:, iaa:, iaa_start_date:, iaa_end_date:) iaa_range = (iaa_start_date..iaa_end_date) - full_months, partial_months = Reports::MonthHelper.months(iaa_range). - partition do |month_range| - Reports::MonthHelper.full_month?(month_range) - end - - # The subqueries create a uniform representation of data: - # - full months from monthly_sp_auth_counts - # - partial months by aggregating sp_return_logs + # Query a month at a time, to keep query time/result size fairly reasonable # The results are rows with [user_id, ial, auth_count, year_month] - queries = [ - *full_month_subquery(issuer: issuer, full_months: full_months), - *partial_month_subqueries(issuer: issuer, partial_months: partial_months), - ] + months = Reports::MonthHelper.months(iaa_range) + queries = build_queries(issuer: issuer, months: months) ial_to_year_month_to_users = Hash.new do |ial_h, ial_k| ial_h[ial_k] = Hash.new { |ym_h, ym_k| ym_h[ym_k] = Multiset.new } end queries.each do |query| - stream_query(query) do |row| - user_id = row['user_id'] - year_month = row['year_month'] - auth_count = row['auth_count'] - ial = row['ial'] - - ial_to_year_month_to_users[ial][year_month].add(user_id, auth_count) + temp_copy = ial_to_year_month_to_users.deep_dup + + with_retries( + max_tries: 3, + rescue: PG::TRSerializationFailure, + handler: proc { ial_to_year_month_to_users = temp_copy }, + ) do + stream_query(query) do |row| + user_id = row['user_id'] + year_month = row['year_month'] + auth_count = row['auth_count'] + ial = row['ial'] + + ial_to_year_month_to_users[ial][year_month].add(user_id, auth_count) + end end end @@ -81,35 +80,12 @@ def call(issuer:, iaa:, iaa_start_date:, iaa_end_date:) rows end - # @return [String] - def full_month_subquery(issuer:, full_months:) - return if full_months.blank? - params = { - issuer: issuer, - year_months: full_months.map { |r| r.begin.strftime('%Y%m') }, - }.transform_values { |value| quote(value) } - - format(<<~SQL, params) - SELECT - monthly_sp_auth_counts.user_id - , monthly_sp_auth_counts.year_month - , monthly_sp_auth_counts.ial - , SUM(monthly_sp_auth_counts.auth_count)::bigint AS auth_count - FROM - monthly_sp_auth_counts - WHERE - monthly_sp_auth_counts.issuer = %{issuer} - AND monthly_sp_auth_counts.year_month IN %{year_months} - GROUP BY - monthly_sp_auth_counts.year_month - , monthly_sp_auth_counts.ial - , monthly_sp_auth_counts.user_id - SQL - end - + # @param [String] issuer + # @param [Array>] months ranges of dates by month that are included in this iaa, + # the first and last may be partial months # @return [Array] - def partial_month_subqueries(issuer:, partial_months:) - partial_months.map do |month_range| + def build_queries(issuer:, months:) + months.map do |month_range| params = { range_start: month_range.begin, range_end: month_range.end, diff --git a/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa.rb b/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa.rb index df505c918e8..968474d40e1 100644 --- a/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa.rb +++ b/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa.rb @@ -15,32 +15,31 @@ def call(key:, issuers:, start_date:, end_date:) return [] if !date_range || issuers.blank? - full_months, partial_months = Reports::MonthHelper.months(date_range). - partition do |month_range| - Reports::MonthHelper.full_month?(month_range) - end - - # The subqueries create a uniform representation of data: - # - full months from monthly_sp_auth_counts - # - partial months by aggregating sp_return_logs + # Query a month at a time, to keep query time/result size fairly reasonable # The results are rows with [user_id, ial, auth_count, year_month] - queries = [ - *full_month_subquery(issuers: issuers, full_months: full_months), - *partial_month_subqueries(issuers: issuers, partial_months: partial_months), - ] + months = Reports::MonthHelper.months(date_range) + queries = build_queries(issuers: issuers, months: months) ial_to_year_month_to_users = Hash.new do |ial_h, ial_k| ial_h[ial_k] = Hash.new { |ym_h, ym_k| ym_h[ym_k] = Multiset.new } end queries.each do |query| - stream_query(query) do |row| - user_id = row['user_id'] - year_month = row['year_month'] - auth_count = row['auth_count'] - ial = row['ial'] - - ial_to_year_month_to_users[ial][year_month].add(user_id, auth_count) + temp_copy = ial_to_year_month_to_users.deep_dup + + with_retries( + max_tries: 3, + rescue: PG::TRSerializationFailure, + handler: proc { ial_to_year_month_to_users = temp_copy }, + ) do + stream_query(query) do |row| + user_id = row['user_id'] + year_month = row['year_month'] + auth_count = row['auth_count'] + ial = row['ial'] + + ial_to_year_month_to_users[ial][year_month].add(user_id, auth_count) + end end end @@ -76,31 +75,12 @@ def call(key:, issuers:, start_date:, end_date:) rows end - # @return [String] - def full_month_subquery(issuers:, full_months:) - return if full_months.blank? - params = { - issuers: issuers, - year_months: full_months.map { |r| r.begin.strftime('%Y%m') }, - }.transform_values { |value| quote(value) } - - format(<<~SQL, params) - SELECT - monthly_sp_auth_counts.user_id - , monthly_sp_auth_counts.year_month - , monthly_sp_auth_counts.auth_count - , monthly_sp_auth_counts.ial - FROM - monthly_sp_auth_counts - WHERE - monthly_sp_auth_counts.issuer IN %{issuers} - AND monthly_sp_auth_counts.year_month IN %{year_months} - SQL - end - + # @param [Array] issuers all the issuers for this iaa + # @param [Array>] months ranges of dates by month that are included in this iaa, + # the first and last may be partial months # @return [Array] - def partial_month_subqueries(issuers:, partial_months:) - partial_months.map do |month_range| + def build_queries(issuers:, months:) + months.map do |month_range| params = { range_start: month_range.begin, range_end: month_range.end, diff --git a/spec/jobs/reports/agency_invoice_iaa_supplement_report_spec.rb b/spec/jobs/reports/agency_invoice_iaa_supplement_report_spec.rb index 0af4b89e442..8feac5fac06 100644 --- a/spec/jobs/reports/agency_invoice_iaa_supplement_report_spec.rb +++ b/spec/jobs/reports/agency_invoice_iaa_supplement_report_spec.rb @@ -102,24 +102,28 @@ # 1 unique user in month 1 at IAA 2 @ IAL 2 create( - :monthly_sp_auth_count, + :sp_return_log, user_id: user1.id, - auth_count: 1, ial: 2, issuer: iaa2_sp.issuer, - year_month: inside_iaa2.strftime('%Y%m'), + requested_at: inside_iaa2, + returned_at: inside_iaa2, + billable: true, ) # 2 users, each 2 auths (1 unique) in month 2 at IAA 2 @ IAL 2 [user1, user2].each do |user| - create( - :monthly_sp_auth_count, - user_id: user.id, - auth_count: 2, - ial: 2, - issuer: iaa2_sp.issuer, - year_month: (inside_iaa2 + 1.month).strftime('%Y%m'), - ) + 2.times do + create( + :sp_return_log, + user_id: user.id, + ial: 2, + issuer: iaa2_sp.issuer, + requested_at: inside_iaa2 + 1.month, + returned_at: inside_iaa2 + 1.month, + billable: true, + ) + end end end @@ -188,7 +192,7 @@ let(:iaa2_key) { "#{gtc1.gtc_number}-#{format('%04d', iaa_order2.order_number)}" } let(:iaa1_range) { Date.new(2020, 4, 15)..Date.new(2021, 4, 14) } - let(:iaa2_range) { Date.new(2021, 4, 14)..Date.new(2022, 4, 13) } + let(:iaa2_range) { Date.new(2021, 4, 15)..Date.new(2022, 4, 14) } it 'counts up costs by issuer + ial, and includes iaa and app_id' do results = JSON.parse(report.perform(Time.zone.today), symbolize_names: true) @@ -205,6 +209,18 @@ iaa_start_date: iaa1_range.begin.to_s, iaa_end_date: iaa1_range.end.to_s, }, + { + iaa: iaa2_key, + ial1_total_auth_count: 0, + ial2_total_auth_count: 1, + ial1_unique_users: 0, + ial2_unique_users: 1, + ial1_new_unique_users: 0, + ial2_new_unique_users: 1, + year_month: inside_iaa2.strftime('%Y%m'), + iaa_start_date: iaa2_range.begin.to_s, + iaa_end_date: iaa2_range.end.to_s, + }, { iaa: iaa2_key, ial1_total_auth_count: 0, @@ -212,7 +228,7 @@ ial1_unique_users: 0, ial2_unique_users: 2, ial1_new_unique_users: 0, - ial2_new_unique_users: 2, + ial2_new_unique_users: 1, year_month: (inside_iaa2 + 1.month).strftime('%Y%m'), iaa_start_date: iaa2_range.begin.to_s, iaa_end_date: iaa2_range.end.to_s, diff --git a/spec/jobs/reports/agency_invoice_issuer_supplement_report_spec.rb b/spec/jobs/reports/agency_invoice_issuer_supplement_report_spec.rb index 3c3408ab553..ff602ace3c4 100644 --- a/spec/jobs/reports/agency_invoice_issuer_supplement_report_spec.rb +++ b/spec/jobs/reports/agency_invoice_issuer_supplement_report_spec.rb @@ -21,22 +21,28 @@ end before do - create( - :monthly_sp_auth_count, - user: user, - service_provider: sp, - ial: 1, - auth_count: 11, - year_month: iaa_range.begin.strftime('%Y%m'), - ) - create( - :monthly_sp_auth_count, - user: user, - service_provider: sp, - ial: 2, - auth_count: 22, - year_month: iaa_range.begin.strftime('%Y%m'), - ) + 3.times do + create( + :sp_return_log, + user: user, + service_provider: sp, + ial: 1, + requested_at: iaa_range.begin + 1.day, + returned_at: iaa_range.begin + 1.day, + billable: true, + ) + end + 4.times do + create( + :sp_return_log, + user: user, + service_provider: sp, + ial: 2, + requested_at: iaa_range.begin + 1.day, + returned_at: iaa_range.begin + 1.day, + billable: true, + ) + end end it 'totals up auth counts within IAA window by month' do @@ -50,8 +56,8 @@ iaa_start_date: '2021-01-01', iaa_end_date: '2021-12-31', year_month: '202101', - ial1_total_auth_count: 11, - ial2_total_auth_count: 22, + ial1_total_auth_count: 3, + ial2_total_auth_count: 4, ial1_unique_users: 1, ial2_unique_users: 1, }, diff --git a/spec/jobs/reports/combined_invoice_supplement_report_spec.rb b/spec/jobs/reports/combined_invoice_supplement_report_spec.rb index e95582d36b7..9f7ab9ea9e7 100644 --- a/spec/jobs/reports/combined_invoice_supplement_report_spec.rb +++ b/spec/jobs/reports/combined_invoice_supplement_report_spec.rb @@ -112,22 +112,24 @@ # 1 unique user in month 1 at IAA 2 sp 1 @ IAL 2 create( - :monthly_sp_auth_count, + :sp_return_log, user_id: user1.id, - auth_count: 1, ial: 2, issuer: iaa2_sp1.issuer, - year_month: inside_iaa2.strftime('%Y%m'), + requested_at: inside_iaa2, + returned_at: inside_iaa2, + billable: true, ) # 1 unique user in month 1 at IAA 2 sp 2 @ IAL 2 create( - :monthly_sp_auth_count, + :sp_return_log, user_id: user2.id, - auth_count: 1, ial: 2, issuer: iaa2_sp2.issuer, - year_month: inside_iaa2.strftime('%Y%m'), + requested_at: inside_iaa2, + returned_at: inside_iaa2, + billable: true, ) end diff --git a/spec/jobs/reports/month_helper_spec.rb b/spec/jobs/reports/month_helper_spec.rb index c3c46f24eb0..c8c12571451 100644 --- a/spec/jobs/reports/month_helper_spec.rb +++ b/spec/jobs/reports/month_helper_spec.rb @@ -14,14 +14,4 @@ ) end end - - describe '.full_month?' do - it 'is true when the range is a full month' do - expect(full_month?(Date.new(2021, 1, 1)..Date.new(2021, 1, 31))).to eq(true) - end - - it 'is false when the range is not a full month' do - expect(full_month?(Date.new(2021, 1, 2)..Date.new(2021, 1, 30))).to eq(false) - end - end end diff --git a/spec/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window_spec.rb b/spec/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window_spec.rb index e73cdde3894..572c060f9e2 100644 --- a/spec/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window_spec.rb +++ b/spec/services/db/monthly_sp_auth_count/total_monthly_auth_counts_within_iaa_window_spec.rb @@ -36,7 +36,7 @@ context 'with data' do let(:partial_month_date) { iaa_range.begin + 1.day } - let(:full_month_date) { (iaa_range.begin + 1.month).beginning_of_month } + let(:full_month_date) { iaa_range.begin + 1.month } let(:user) { create(:user) } before do @@ -65,17 +65,20 @@ ) # 11 IAL 1 auths during full month - create( - :monthly_sp_auth_count, - user: user, - service_provider: service_provider, - ial: 1, - year_month: full_month_date.strftime('%Y%m'), - auth_count: 11, - ) + 11.times do + create( + :sp_return_log, + user: user, + service_provider: service_provider, + ial: 1, + requested_at: full_month_date, + returned_at: full_month_date, + billable: true, + ) + end end - it 'counts and uniqes auths across sp_return_logs and monthly_sp_auth_counts' do + it 'counts and uniques auths' do rows = [ { year_month: partial_month_date.strftime('%Y%m'), @@ -108,7 +111,7 @@ context 'with only partial months' do let(:iaa_range) { Date.new(2021, 1, 15)..Date.new(2021, 1, 17) } - it 'counts auths across sp_return_logs and monthly_sp_auth_counts' do + it 'counts and uniques auths' do expect(result).to match_array([]) end end diff --git a/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa_spec.rb b/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa_spec.rb index cb20ee443a0..adcbc987ff7 100644 --- a/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa_spec.rb +++ b/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_iaa_spec.rb @@ -90,26 +90,32 @@ # 1 old user + 1 new user in whole month @ IAL 1 [user1, user2].each do |user| - create( - :monthly_sp_auth_count, - user_id: user.id, - auth_count: 10, - ial: 1, - issuer: issuer1, - year_month: inside_whole_month.strftime('%Y%m'), - ) + 10.times do + create( + :sp_return_log, + user_id: user.id, + ial: 1, + issuer: issuer1, + requested_at: inside_whole_month, + returned_at: inside_whole_month, + billable: true, + ) + end end # 2 old user + 1 new user in whole month @ IAL 2 [user1, user2, user3].each do |user| - create( - :monthly_sp_auth_count, - user_id: user.id, - auth_count: 100, - ial: 2, - issuer: issuer2, - year_month: inside_whole_month.strftime('%Y%m'), - ) + 7.times do + create( + :sp_return_log, + user_id: user.id, + ial: 2, + issuer: issuer2, + requested_at: inside_whole_month, + returned_at: inside_whole_month, + billable: true, + ) + end end end @@ -151,7 +157,7 @@ year_month: '202010', iaa_start_date: iaa_range.begin.to_s, iaa_end_date: iaa_range.end.to_s, - total_auth_count: 300, + total_auth_count: 21, unique_users: 3, new_unique_users: 1, },