LG-15559: Add support for A/B tests daily reporting

app/jobs/reports/ab_tests_report.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'reporting/ab_tests_report'
+
+module Reports
+  class AbTestsReport < BaseReport
+    attr_reader :report_date
+
+    def initialize(report_date = nil, *args, **kwargs)
+      @report_date = report_date
+      super(*args, **kwargs)
+    end
+
+    # @param [DateTime]
+    def perform(report_date)
+      @report_date = report_date
+
+      report_configs.each do |config|
+        tables_report(config).deliver_now
+      end
+    end
+
+    def tables_report(config)
+      experiment_name = config.experiment_name
+      subject = "A/B Tests Report - #{experiment_name} - #{report_date}"
+      reports = ab_tests_report(config).as_emailable_reports
+
+      ReportMailer.tables_report(
+        email: config.email,
+        subject:,
+        message: subject,
+        reports:,
+        attachment_format: :csv,
+      )
+    end
+
+    def ab_tests_report(config)
+      Reporting::AbTestsReport.new(
+        queries: config.queries,
+        time_range: report_date.yesterday..report_date,
+      )
+    end
+
+    private
+
+    def report_configs
+      AbTests
+        .all
+        .values
+        .select { |ab_test| ab_test.report&.email&.present? && ab_test.active? }
+        .map(&:report)
+    end
+  end
+end

app/views/report_mailer/tables_report.html.erb

@@ -35,7 +35,7 @@
         <tr>
           <% row.each do |cell| %>
             <td <%= cell.is_a?(Numeric) ? 'class=table-number' : nil %> >
-              <% if cell.is_a?(Float) && report.float_as_percent? %>
+              <% if cell.is_a?(Float) && report.float_as_percent? && cell.finite? %>
                 <%= number_to_percentage(cell * 100, precision: report.precision || 2) %>
               <% else %>
                 <%= number_with_delimiter(cell) %>

config/initializers/job_configurations.rb

@@ -236,6 +236,12 @@
         cron: cron_every_monday,
         args: -> { [Time.zone.yesterday.end_of_day] },
       },
+      # Send A/B test reports
+      ab_tests_report: {
+        class: 'Reports::AbTestsReport',
+        cron: cron_24h,
+        args: -> { [Time.zone.yesterday.end_of_day] },
+      },
       # Send fraud metrics to Team Judy
       fraud_metrics_report: {
         class: 'Reports::FraudMetricsReport',

lib/ab_test.rb

@@ -3,10 +3,19 @@
 class AbTest
   include ::NewRelic::Agent::MethodTracer
 
-  attr_reader :buckets, :experiment_name, :default_bucket, :should_log
+  attr_reader :buckets, :experiment_name, :default_bucket, :should_log, :report
 
   MAX_SHA = (16 ** 64) - 1
 
+  ReportQueryConfig = Struct.new(:title, :query, :row_labels, keyword_init: true).freeze
+
+  ReportConfig = Struct.new(:experiment_name, :email, :queries, keyword_init: true) do
+    def initialize(queries: [], **)
+      super
+      self.queries.map!(&ReportQueryConfig.method(:new))
+    end
+  end.freeze
+
   # @param [Proc<String>,Regexp,string,Boolean,nil] should_log Controls whether bucket data for this
   #                                                            A/B test is logged with specific
   #                                                            events.
@@ -20,16 +29,19 @@ def initialize(
     buckets: {},
     should_log: nil,
     default_bucket: :default,
+    report: nil,
     &discriminator
   )
     @buckets = buckets
     @discriminator = discriminator
     @experiment_name = experiment_name
     @default_bucket = default_bucket
     @should_log = should_log
+    @report = ReportConfig.new(experiment_name:, **report.to_h) if report
     raise 'invalid bucket data structure' unless valid_bucket_data_structure?
     ensure_numeric_percentages
     raise 'bucket percentages exceed 100' unless within_100_percent?
+    @active = buckets.present? && buckets.values.any?(&:positive?)
   end
 
   # @param [ActionDispatch::Request] request
@@ -39,7 +51,7 @@ def initialize(
   # @param [User] user
   # @param [Hash] user_session
   def bucket(request:, service_provider:, session:, user:, user_session:)
-    return nil if no_percentages?
+    return nil if !active?
 
     discriminator = resolve_discriminator(
       request:, service_provider:, session:, user:,
@@ -71,6 +83,10 @@ def include_in_analytics_event?(event_name)
     end
   end
 
+  def active?
+    @active
+  end
+
   private
 
   def resolve_discriminator(user:, **)
@@ -81,10 +97,6 @@ def resolve_discriminator(user:, **)
     end
   end
 
-  def no_percentages?
-    buckets.empty? || buckets.values.all? { |pct| pct == 0 }
-  end
-
   def percent(discriminator)
     Digest::SHA256.hexdigest("#{discriminator}:#{experiment_name}").to_i(16).to_f / MAX_SHA * 100
   end

lib/reporting/ab_tests_report.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'reporting/cloudwatch_client'
+require 'reporting/cloudwatch_query_quoting'
+
+module Reporting
+  class AbTestsReport
+    attr_reader :queries, :time_range
+
+    # @param [Array<AbTest::ReportQueryConfig>] queries
+    # @param [Range<Time>] time_range
+    def initialize(
+      queries:,
+      time_range:
+    )
+      @queries = queries
+      @time_range = time_range
+    end
+
+    def as_tables
+      queries.map(&method(:table_for_query))
+    end
+
+    def as_emailable_reports
+      queries.map do |query|
+        Reporting::EmailableReport.new(
+          title: query.title,
+          table: table_for_query(query),
+        )
+      end
+    end
+
+    private
+
+    def table_for_query(query)
+      query_data = fetch_results(query: query.query)
+      headers = column_labels(query_data.first)
+      rows = query_data.map(&:values)
+
+      [
+        headers,
+        *format_rows(rows, headers, query.row_labels),
+      ]
+    end
+
+    def format_rows(rows, headers, row_labels)
+      if row_labels
+        rows = rows.each_with_index.map do |value, index|
+          [row_labels[index], *value.slice(1)]
+        end
+      end
+
+      headers.each_index.select { |i| headers[i] =~ /percent/i }.each do |percent_index|
+        rows.each { |row| row[percent_index] = format_as_percent(row[percent_index]) }
+      end
+
+      rows
+    end
+
+    def fetch_results(query:)
+      cloudwatch_client.fetch(query:, from: time_range.begin, to: time_range.end)
+    end
+
+    def column_labels(row)
+      row.keys
+    end
+
+    def cloudwatch_client
+      @cloudwatch_client ||= Reporting::CloudwatchClient.new(
+        progress: false,
+        ensure_complete_logs: false,
+      )
+    end
+
+    # @return [String]
+    def format_as_percent(number)
+      format('%.2f%%', number)
+    end
+  end
+end

spec/jobs/reports/ab_tests_report_spec.rb

@@ -0,0 +1,94 @@
+require 'rails_helper'
+
+RSpec.describe Reports::AbTestsReport do
+  let(:report_date) { Date.new(2023, 12, 25) }
+  let(:email) { 'email@example.com' }
+  let(:tested_percent) { 1 }
+  let(:queries) do
+    [
+      {
+        title: 'Sign in success rate by CAPTCHA validation performed',
+        query: <<~QUERY,
+          fields properties.event_properties.captcha_validation_performed as `Captcha Validation Performed`
+          | filter name = 'Email and Password Authentication'
+          | stats avg(properties.event_properties.success)*100 as `Success Percent` by `Captcha Validation Performed`
+          | sort `Captcha Validation Performed` asc
+        QUERY
+        row_labels: ['Validation Not Performed', 'Validation Performed'],
+      },
+    ]
+  end
+  let(:ab_tests) do
+    {
+      RECAPTCHA_SIGN_IN: AbTest.new(
+        experiment_name: 'reCAPTCHA at Sign-In',
+        buckets: { sign_in_recaptcha: tested_percent },
+        report: { email:, queries: },
+      ),
+    }
+  end
+
+  before do
+    allow(AbTests).to receive(:all).and_return(ab_tests)
+  end
+
+  describe '#perform' do
+    let(:emailable_reports) do
+      [
+        Reporting::EmailableReport.new(
+          title: 'Sign in success rate by CAPTCHA validation performed',
+          table: [
+            ['Captcha Validation Performed', 'Success Percent'],
+            ['Validation Not Performed', '90.19%'],
+            ['Validation Performed', '85.68%'],
+          ],
+        ),
+      ]
+    end
+
+    let(:report) do
+      double(Reporting::AbTestsReport, as_emailable_reports: emailable_reports)
+    end
+
+    before do
+      allow(Reporting::AbTestsReport).to receive(:new).with(
+        queries:,
+        time_range: report_date.yesterday..report_date,
+      ).and_return(report)
+
+      allow(ReportMailer).to receive(:tables_report).and_call_original
+    end
+
+    it 'emails the table report with csv' do
+      expect(ReportMailer).to receive(:tables_report).with(
+        email:,
+        subject: "A/B Tests Report - reCAPTCHA at Sign-In - #{report_date}",
+        message: "A/B Tests Report - reCAPTCHA at Sign-In - #{report_date}",
+        reports: emailable_reports,
+        attachment_format: :csv,
+      )
+
+      subject.perform(report_date)
+    end
+
+    context 'when associated report email is nil' do
+      let(:email) { nil }
+
+      it 'does not email the table report' do
+        expect(ReportMailer).not_to receive(:tables_report)
+
+        subject.perform(report_date)
+      end
+    end
+
+    context 'when a/b test buckets are zero (test is inactive)' do
+      let(:tested_percent) { 0 }
+
+      it 'does not email the table report' do
+        expect(ReportMailer).not_to receive(:tables_report)
+
+        subject.perform(report_date)
+      end
+    end
+  end
+end