diff --git a/app/jobs/reports/combined_invoice_supplement_report_v2.rb b/app/jobs/reports/combined_invoice_supplement_report_v2.rb new file mode 100644 index 00000000000..915d23fa2de --- /dev/null +++ b/app/jobs/reports/combined_invoice_supplement_report_v2.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +require 'csv' + +module Reports + class CombinedInvoiceSupplementReportV2 < BaseReport + REPORT_NAME = 'combined-invoice-supplement-report-v2' + + def perform(_date) + csv = build_csv(IaaReportingHelper.iaas, IaaReportingHelper.partner_accounts) + save_report(REPORT_NAME, csv, extension: 'csv') + end + + # @param [Array] iaas + # @param [Array] partner_accounts + # @return [String] CSV report + def build_csv(iaas, partner_accounts) + by_iaa_results = iaas.flat_map do |iaa| + Db::MonthlySpAuthCount::UniqueMonthlyAuthCountsByIaa.call( + key: iaa.key, + issuers: iaa.issuers, + start_date: iaa.start_date, + end_date: iaa.end_date, + ) + end + + by_issuer_results = iaas.flat_map do |iaa| + iaa.issuers.flat_map do |issuer| + Db::MonthlySpAuthCount::TotalMonthlyAuthCountsWithinIaaWindow.call( + issuer: issuer, + iaa_start_date: iaa.start_date, + iaa_end_date: iaa.end_date, + iaa: iaa.key, + ) + end + end + + by_partner_results = partner_accounts.flat_map do |partner_account| + Db::MonthlySpAuthCount::NewUniqueMonthlyUserCountsByPartner.call( + partner: partner_account.partner, + issuers: partner_account.issuers, + start_date: partner_account.start_date, + end_date: partner_account.end_date, + ) + end + + combine_by_iaa_month( + by_iaa_results: by_iaa_results, + by_issuer_results: by_issuer_results, + by_partner_results: by_partner_results, + ) + end + + def combine_by_iaa_month(by_iaa_results:, by_issuer_results:, by_partner_results:) + by_iaa_and_year_month = by_iaa_results.group_by do |result| + [result[:key], result[:year_month]] + end + + by_issuer_iaa_issuer_year_months = by_issuer_results. + group_by { |r| r[:iaa] }. + transform_values do |iaa| + iaa.group_by { |r| r[:issuer] }. + transform_values { |issuer| issuer.group_by { |r| r[:year_month] } } + end + + # rubocop:disable Metrics/BlockLength + CSV.generate do |csv| + csv << [ + 'iaa_order_number', + 'partner', + 'iaa_start_date', + 'iaa_end_date', + + 'issuer', + 'friendly_name', + + 'year_month', + 'year_month_readable', + + 'iaa_ial1_unique_users', + 'iaa_ial2_unique_users', + 'iaa_ial1_plus_2_unique_users', + 'partner_ial2_new_unique_users_year1', + 'partner_ial2_new_unique_users_year2', + 'partner_ial2_new_unique_users_year3', + 'partner_ial2_new_unique_users_year4', + 'partner_ial2_new_unique_users_year5', + 'partner_ial2_new_unique_users_year_greater_than_5', + 'partner_ial2_new_unique_users_year_unknown', + + 'issuer_ial1_total_auth_count', + 'issuer_ial2_total_auth_count', + 'issuer_ial1_plus_2_total_auth_count', + + 'issuer_ial1_unique_users', + 'issuer_ial2_unique_users', + 'issuer_ial1_plus_2_unique_users', + 'issuer_ial2_new_unique_users', + ] + by_issuer_iaa_issuer_year_months.each do |iaa_key, issuer_year_months| + issuer_year_months.each do |issuer, year_months_data| + friendly_name = ServiceProvider.find_by(issuer: issuer).friendly_name + year_months = year_months_data.keys.sort + + year_months.each do |year_month| + iaa_results = by_iaa_and_year_month[ [iaa_key, year_month] ] + issuer_results = year_months_data[year_month] + year_month_start = Date.strptime(year_month, '%Y%m') + iaa_start_date = Date.parse(iaa_results.first[:iaa_start_date]) + iaa_end_date = Date.parse(iaa_results.first[:iaa_end_date]) + + partner_results = by_partner_results.find do |result| + result[:year_month] == year_month && result[:issuers]&.include?(issuer) + end || {} + csv << [ + iaa_key, + partner_results[:partner], + iaa_start_date, + iaa_end_date, + + issuer, + friendly_name, + + year_month, + year_month_start.strftime('%B %Y'), + + (iaa_ial1_unique_users = extract(iaa_results, :unique_users, ial: 1)), + (iaa_ial2_unique_users = extract(iaa_results, :unique_users, ial: 2)), + iaa_ial1_unique_users + iaa_ial2_unique_users, + partner_results[:partner_ial2_new_unique_users_year1] || 0, + partner_results[:partner_ial2_new_unique_users_year2] || 0, + partner_results[:partner_ial2_new_unique_users_year3] || 0, + partner_results[:partner_ial2_new_unique_users_year4] || 0, + partner_results[:partner_ial2_new_unique_users_year5] || 0, + partner_results[:partner_ial2_new_unique_users_year_greater_than_5] || 0, + partner_results[:partner_ial2_new_unique_users_year_unknown] || 0, + + (ial1_total_auth_count = extract(issuer_results, :total_auth_count, ial: 1)), + (ial2_total_auth_count = extract(issuer_results, :total_auth_count, ial: 2)), + ial1_total_auth_count + ial2_total_auth_count, + + (issuer_ial1_unique_users = extract(issuer_results, :unique_users, ial: 1)), + (issuer_ial2_unique_users = extract(issuer_results, :unique_users, ial: 2)), + issuer_ial1_unique_users + issuer_ial2_unique_users, + extract(issuer_results, :new_unique_users, ial: 2), + ] + end + end + end + end + # rubocop:enable Metrics/BlockLength + end + + def extract(arr, key, ial:) + arr.find { |elem| elem[:ial] == ial && elem[key] }&.dig(key) || 0 + end + end +end diff --git a/app/models/agreements/integration.rb b/app/models/agreements/integration.rb index cb32e11c5e6..067219a5ac8 100644 --- a/app/models/agreements/integration.rb +++ b/app/models/agreements/integration.rb @@ -5,7 +5,7 @@ class Agreements::Integration < ApplicationRecord belongs_to :partner_account belongs_to :integration_status - belongs_to :service_provider + belongs_to :service_provider, foreign_key: :issuer, primary_key: :issuer, inverse_of: :integration has_many :integration_usages, dependent: :restrict_with_exception has_many :iaa_orders, through: :integration_usages diff --git a/app/models/service_provider.rb b/app/models/service_provider.rb index fa1ba8cb174..d0c518dbf68 100644 --- a/app/models/service_provider.rb +++ b/app/models/service_provider.rb @@ -20,7 +20,14 @@ class ServiceProvider < ApplicationRecord primary_key: 'issuer', dependent: :destroy - # Do not define validations in this model. + has_one :integration, + inverse_of: :service_provider, + foreign_key: 'issuer', + primary_key: 'issuer', + class_name: 'Agreements::Integration', + dependent: nil + + # Do not define validations in this model # See https://github.com/18F/identity_validations include IdentityValidations::ServiceProviderValidation diff --git a/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner.rb b/app/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner.rb similarity index 94% rename from app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner.rb rename to app/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner.rb index 6e88e72513e..ff251c1a88d 100644 --- a/app/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner.rb +++ b/app/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner.rb @@ -2,17 +2,17 @@ module Db module MonthlySpAuthCount - module UniqueMonthlyAuthCountsByPartner + module NewUniqueMonthlyUserCountsByPartner extend Reports::QueryHelpers module_function - # @param [String] key label for billing (Partner requesting agency) + # @param [String] partner label for billing (Partner requesting agency) # @param [Array] issuers issuers for the iaa # @param [Date] start_date iaa start date # @param [Date] end_date iaa end date # @return [PG::Result, Array] - def call(key:, issuers:, start_date:, end_date:) + def call(partner:, issuers:, start_date:, end_date:) date_range = start_date...end_date return [] if !date_range || issuers.blank? @@ -56,6 +56,7 @@ def call(key:, issuers:, start_date:, end_date:) rows = [] prev_seen_users = Set.new + issuers_set = issuers.to_set year_months = year_month_to_users_to_profile_age.keys.sort # rubocop:disable Metrics/BlockLength @@ -79,7 +80,8 @@ def call(key:, issuers:, start_date:, end_date:) prev_seen_users |= this_month_users rows << { - key: key, + partner: partner, + issuers: issuers_set, year_month: year_month, iaa_start_date: date_range.begin.to_s, iaa_end_date: date_range.end.to_s, diff --git a/config/initializers/job_configurations.rb b/config/initializers/job_configurations.rb index 0d0d2368bf5..ea87a1a2fba 100644 --- a/config/initializers/job_configurations.rb +++ b/config/initializers/job_configurations.rb @@ -4,6 +4,7 @@ cron_12m = '0/12 * * * *' cron_1h = '0 * * * *' cron_24h = '0 0 * * *' +cron_24h_1am = '0 1 * * *' # 1am UTC is 8pm EST/9pm EDT gpo_cron_24h = '0 10 * * *' # 10am UTC is 5am EST/6am EDT cron_1w = '0 0 * * 0' @@ -59,6 +60,11 @@ cron: cron_24h, args: -> { [Time.zone.today] }, }, + combined_invoice_supplement_report_v2: { + class: 'Reports::CombinedInvoiceSupplementReportV2', + cron: cron_24h_1am, + args: -> { [Time.zone.today] }, + }, agreement_summary_report: { class: 'Reports::AgreementSummaryReport', cron: cron_24h, diff --git a/spec/jobs/reports/combined_invoice_supplement_report_v2_spec.rb b/spec/jobs/reports/combined_invoice_supplement_report_v2_spec.rb new file mode 100644 index 00000000000..f42fe8a4e99 --- /dev/null +++ b/spec/jobs/reports/combined_invoice_supplement_report_v2_spec.rb @@ -0,0 +1,240 @@ +require 'rails_helper' + +RSpec.describe Reports::CombinedInvoiceSupplementReportV2 do + subject(:report) { Reports::CombinedInvoiceSupplementReportV2.new } + + before do + clear_agreements_data + ServiceProvider.delete_all + end + + let(:partner_account1) { create(:partner_account) } + let(:partner_account2) { create(:partner_account) } + let(:gtc1) do + create( + :iaa_gtc, + gtc_number: 'gtc1234', + partner_account: partner_account1, + start_date: iaa1_range.begin, + end_date: iaa1_range.end, + ) + end + + let(:gtc2) do + create( + :iaa_gtc, + gtc_number: 'gtc5678', + partner_account: partner_account2, + start_date: iaa2_range.begin, + end_date: iaa2_range.end, + ) + end + + let(:iaa_order1) do + build_iaa_order(order_number: 1, date_range: iaa1_range, iaa_gtc: gtc1) + end + let(:iaa_order2) do + build_iaa_order(order_number: 2, date_range: iaa2_range, iaa_gtc: gtc2) + end + + # Do this because of invalid check when building integration usages + let!(:iaa_orders) do + [ + iaa_order1, + iaa_order2, + ] + end + + let!(:iaa1_sp) do + create( + :service_provider, + iaa: iaa1, + iaa_start_date: iaa1_range.begin, + iaa_end_date: iaa1_range.end, + ) + end + + let!(:iaa2_sp1) do + create( + :service_provider, + iaa: iaa2, + iaa_start_date: iaa2_range.begin, + iaa_end_date: iaa2_range.end, + ) + end + let!(:iaa2_sp2) do + create( + :service_provider, + iaa: iaa2, + iaa_start_date: iaa2_range.begin, + iaa_end_date: iaa2_range.end, + ) + end + + let(:iaa1) { 'iaa1' } + let(:iaa1_range) { Date.new(2020, 4, 15)..Date.new(2021, 4, 14) } + let(:inside_iaa1) { iaa1_range.begin + 1.day } + + let(:iaa2) { 'iaa2' } + let(:iaa2_range) { Date.new(2020, 9, 1)..Date.new(2021, 8, 30) } + let(:inside_iaa2) { iaa2_range.begin + 1.day } + + describe '#perform' do + it 'is empty with no data' do + csv = CSV.parse(report.perform(Time.zone.today), headers: true) + expect(csv).to be_empty + end + + context 'with data' do + let(:user1) { create(:user) } + let(:user2) { create(:user) } + + before do + iaa_order1.integrations << build_integration( + issuer: iaa1_sp.issuer, + partner_account: partner_account1, + ) + iaa_order2.integrations << build_integration( + issuer: iaa2_sp1.issuer, + partner_account: partner_account2, + ) + iaa_order2.integrations << build_integration( + issuer: iaa2_sp2.issuer, + partner_account: partner_account2, + ) + iaa_order1.save + iaa_order2.save + + create( + :sp_return_log, + user_id: user1.id, + issuer: iaa1_sp.issuer, + ial: 1, + requested_at: inside_iaa1, + returned_at: inside_iaa1, + profile_verified_at: nil, + billable: true, + ) + + # 1 unique user in month 1 at IAA 2 sp 1 @ IAL 2 with profile age 1 + create( + :sp_return_log, + user_id: user1.id, + ial: 2, + issuer: iaa2_sp1.issuer, + requested_at: inside_iaa2, + returned_at: inside_iaa2, + profile_verified_at: '2019-01-01 00:00:00', + billable: true, + ) + + # 1 unique user in month 1 at IAA 2 sp 2 @ IAL 2 with profile age 2 + create( + :sp_return_log, + user_id: user2.id, + ial: 2, + issuer: iaa2_sp2.issuer, + requested_at: inside_iaa2, + returned_at: inside_iaa2, + profile_verified_at: '2020-01-01 00:00:00', + billable: true, + ) + end + + it 'generates a report by iaa + order number and issuer and year month' do + csv = CSV.parse(report.perform(Time.zone.today), headers: true) + + expect(csv.length).to eq(3) + + aggregate_failures do + row = csv.find { |r| r['issuer'] == iaa1_sp.issuer } + expect(row['iaa_order_number']).to eq('gtc1234-0001') + expect(row['partner']).to eq(nil) + expect(row['iaa_start_date']).to eq('2020-04-15') + expect(row['iaa_end_date']).to eq('2021-04-14') + + expect(row['issuer']).to eq(iaa1_sp.issuer) + expect(row['friendly_name']).to eq(iaa1_sp.friendly_name) + + expect(row['year_month']).to eq('202004') + expect(row['year_month_readable']).to eq('April 2020') + + expect(row['iaa_ial1_unique_users'].to_i).to eq(1) + expect(row['iaa_ial2_unique_users'].to_i).to eq(0) + expect(row['iaa_ial1_plus_2_unique_users'].to_i).to eq(1) + expect(row['partner_ial2_new_unique_users_year1'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year2'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year3'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year4'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year5'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year_greater_than_5'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year_unknown'].to_i).to eq(0) + + expect(row['issuer_ial1_total_auth_count'].to_i).to eq(1) + expect(row['issuer_ial2_total_auth_count'].to_i).to eq(0) + expect(row['issuer_ial1_plus_2_total_auth_count'].to_i).to eq(1) + + expect(row['issuer_ial1_unique_users'].to_i).to eq(1) + expect(row['issuer_ial2_unique_users'].to_i).to eq(0) + expect(row['issuer_ial1_plus_2_unique_users'].to_i).to eq(1) + expect(row['issuer_ial2_new_unique_users'].to_i).to eq(0) + end + + [iaa2_sp1, iaa2_sp2].each do |sp| + aggregate_failures do + row = csv.find { |r| r['issuer'] == sp.issuer } + + expect(row['iaa_order_number']).to eq('gtc5678-0002') + expect(row['partner']).to eq(partner_account2.requesting_agency) + expect(row['iaa_start_date']).to eq('2020-09-01') + expect(row['iaa_end_date']).to eq('2021-08-30') + + expect(row['issuer']).to eq(sp.issuer) + expect(row['friendly_name']).to eq(sp.friendly_name) + + expect(row['year_month']).to eq('202009') + expect(row['year_month_readable']).to eq('September 2020') + + expect(row['iaa_ial1_unique_users'].to_i).to eq(0) + expect(row['iaa_ial2_unique_users'].to_i).to eq(2) + expect(row['iaa_ial1_plus_2_unique_users'].to_i).to eq(2) + expect(row['partner_ial2_new_unique_users_year1'].to_i).to eq(1) + expect(row['partner_ial2_new_unique_users_year2'].to_i).to eq(1) + expect(row['partner_ial2_new_unique_users_year3'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year4'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year5'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year_greater_than_5'].to_i).to eq(0) + expect(row['partner_ial2_new_unique_users_year_unknown'].to_i).to eq(0) + + expect(row['issuer_ial1_total_auth_count'].to_i).to eq(0) + expect(row['issuer_ial2_total_auth_count'].to_i).to eq(1) + expect(row['issuer_ial1_plus_2_total_auth_count'].to_i).to eq(1) + + expect(row['issuer_ial1_unique_users'].to_i).to eq(0) + expect(row['issuer_ial2_unique_users'].to_i).to eq(1) + expect(row['issuer_ial1_plus_2_unique_users'].to_i).to eq(1) + expect(row['issuer_ial2_new_unique_users'].to_i).to eq(1) + end + end + end + end + end + + def build_iaa_order(order_number:, date_range:, iaa_gtc:) + create( + :iaa_order, + order_number: order_number, + start_date: date_range.begin, + end_date: date_range.end, + iaa_gtc: iaa_gtc, + ) + end + + def build_integration(issuer:, partner_account:) + create( + :integration, + issuer: issuer, + partner_account: partner_account, + ) + end +end diff --git a/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner_spec.rb b/spec/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner_spec.rb similarity index 94% rename from spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner_spec.rb rename to spec/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner_spec.rb index b82d6787eda..02c1c32f6ff 100644 --- a/spec/services/db/monthly_sp_auth_count/unique_monthly_auth_counts_by_partner_spec.rb +++ b/spec/services/db/monthly_sp_auth_count/new_unique_monthly_user_counts_by_partner_spec.rb @@ -1,10 +1,10 @@ require 'rails_helper' -RSpec.describe Db::MonthlySpAuthCount::UniqueMonthlyAuthCountsByPartner do +RSpec.describe Db::MonthlySpAuthCount::NewUniqueMonthlyUserCountsByPartner do describe '.call' do subject(:results) do - Db::MonthlySpAuthCount::UniqueMonthlyAuthCountsByPartner.call( - key: key, + Db::MonthlySpAuthCount::NewUniqueMonthlyUserCountsByPartner.call( + partner: partner_key, start_date: partner_range.begin, end_date: partner_range.end, issuers: issuers, @@ -12,8 +12,7 @@ end context 'with no data' do - let(:key) { partner } - let(:partner) { 'DHS' } + let(:partner_key) { 'DHS' } let(:partner_range) { 1.year.ago..Time.zone.now } let(:issuers) { [] } @@ -23,7 +22,7 @@ end context 'with data' do - let(:key) { 'DHS' } + let(:partner_key) { 'DHS' } let(:issuers) { [issuer1, issuer2, issuer3] } let(:partner_range) { Date.new(2020, 9, 15)..Date.new(2021, 9, 14) } @@ -67,6 +66,8 @@ let(:issuer1) { 'issuer1' } let(:issuer2) { 'issuer2' } let(:issuer3) { 'issuer3' } + let(:issuer4) { 'issuer4' } + let(:issuer5) { 'issuer5' } before do # Inside partial month @@ -198,7 +199,8 @@ it 'adds up new unique users from sp_return_log instances and splits based on profile age' do rows = [ { - key: key, + partner: partner_key, + issuers: issuers, year_month: '202009', iaa_start_date: partner_range.begin.to_s, iaa_end_date: partner_range.end.to_s, @@ -213,7 +215,8 @@ partner_ial2_new_unique_users_unknown: 0, }, { - key: key, + partner: partner_key, + issuers: issuers, year_month: '202010', iaa_start_date: partner_range.begin.to_s, iaa_end_date: partner_range.end.to_s, @@ -228,13 +231,12 @@ partner_ial2_new_unique_users_unknown: 1, }, ] - expect(results).to match_array(rows) end end context 'with only partial month data' do - let(:key) { 'DHS' } + let(:partner_key) { 'DHS' } let(:partner_range) { Date.new(2020, 9, 15)..Date.new(2020, 9, 17) } let(:issuers) { ['issuer1'] } let(:rows) { [] }