Add script to get OTP delivery logs for SMS, Voice

bin/oncall/email-deliveries

@@ -10,6 +10,7 @@ require 'terminal-table'
 $LOAD_PATH.unshift(File.expand_path(File.join(__dir__, '../../lib')))
 require 'reporting/cloudwatch_client'
 require 'reporting/cloudwatch_query_quoting'
+require 'reporting/unknown_progress_bar'
 
 class EmailDeliveries
   include Reporting::CloudwatchQueryQuoting
@@ -82,70 +83,53 @@ class EmailDeliveries
 
   # @return [Array<Result>]
   def query_data(uuids)
-    if progress_bar?
-      bar = ProgressBar.create(
-        title: 'Querying logs',
-        total: nil,
-        format: '[ %t ] %B %a',
-        output: STDERR,
+    Reporting::UnknownProgressBar.wrap(show_bar: progress_bar?, title: 'Querying logs') do
+      event_log = cloudwatch_client('prod_/srv/idp/shared/log/events.log').fetch(
+        query: <<~EOS,
+          fields
+            @timestamp
+          , properties.user_id AS user_id
+          , properties.event_properties.ses_message_id AS ses_message_id
+          | filter name = 'Email Sent'
+          | filter properties.user_id IN #{quote(uuids)}
+          | limit 10000
+        EOS
+        from: 1.week.ago,
+        to: Time.now,
       )
-      thread = Thread.new do
-        loop do
-          sleep 0.1
-          bar.increment
-        end
-      end
-    end
 
-    event_log = cloudwatch_client('prod_/srv/idp/shared/log/events.log').fetch(
-      query: <<~EOS,
-        fields
-          @timestamp
-        , properties.user_id AS user_id
-        , properties.event_properties.ses_message_id AS ses_message_id
-        | filter name = 'Email Sent'
-        | filter properties.user_id IN #{quote(uuids)}
-        | limit 10000
-      EOS
-      from: 1.week.ago,
-      to: Time.now,
-    )
-
-    events_by_message_id = event_log.index_by { |event| event['ses_message_id'] }
-
-    message_id_filters = events_by_message_id.keys.map do |message_id|
-      "@message LIKE /#{message_id}/"
-    end.join(' OR ')
-
-    email_events = cloudwatch_client('/aws/lambda/SESAllEvents_Lambda').fetch(
-      query: <<~EOS,
-        fields
-          eventType AS event_type
-        | filter #{message_id_filters}
-        | parse '"messageId": "*"' as ses_message_id
-        | display @timestamp, event_type, ses_message_id
-        | limit 10000
-      EOS
-      from: 1.week.ago,
-      to: Time.now,
-    )
+      events_by_message_id = event_log.index_by { |event| event['ses_message_id'] }
+
+      message_id_filters = events_by_message_id.keys.map do |message_id|
+        "@message LIKE /#{message_id}/"
+      end.join(' OR ')
+
+      email_events = cloudwatch_client('/aws/lambda/SESAllEvents_Lambda').fetch(
+        query: <<~EOS,
+          fields
+            eventType AS event_type
+          | filter #{message_id_filters}
+          | parse '"messageId": "*"' as ses_message_id
+          | display @timestamp, event_type, ses_message_id
+          | limit 10000
+        EOS
+        from: 1.week.ago,
+        to: Time.now,
+      )
 
-    email_events.
-      group_by { |event| event['ses_message_id'] }.
-      map do |message_id, events|
-        Result.new(
-          user_id: events_by_message_id[message_id]['user_id'],
-          timestamp: events_by_message_id[message_id]['@timestamp'],
-          message_id: message_id,
-          events: events.sort_by { |e| e['@timestamp'] }.map { |e| e['event_type'] },
-        )
-      end
-  ensure
-    thread&.kill
-    bar&.stop
+      email_events.
+        group_by { |event| event['ses_message_id'] }.
+        map do |message_id, events|
+          Result.new(
+            user_id: events_by_message_id[message_id]['user_id'],
+            timestamp: events_by_message_id[message_id]['@timestamp'],
+            message_id: message_id,
+            events: events.sort_by { |e| e['@timestamp'] }.map { |e| e['event_type'] },
+          )
+        end
+    end
   end
 
-
   def cloudwatch_client(log_group_name)
     Reporting::CloudwatchClient.new(
       ensure_complete_logs: false,

bin/oncall/otp-deliveries

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/integer/time'
+require 'optparse'
+
+$LOAD_PATH.unshift(File.expand_path(File.join(__dir__, '../../lib')))
+require 'script_base'
+require 'reporting/cloudwatch_client'
+require 'reporting/cloudwatch_query_quoting'
+require 'reporting/unknown_progress_bar'
+
+class OtpDeliveries
+  include Reporting::CloudwatchQueryQuoting
+
+  # @return [OtpDeliveries]
+  def self.parse!(argv: ARGV, out: STDOUT)
+    show_help = false
+    output_format = :table
+    filter = nil
+
+    parser = OptionParser.new do |opts|
+      opts.banner = <<~EOM
+        Usage: #{$PROGRAM_NAME} uuid1 [uuid2...]
+
+        Looks up OTP delivery attempts for SMS/Voice by user UUID, within the last 72 hours
+
+        Options:
+      EOM
+
+      opts.on('--csv') do
+        output_format = :csv
+      end
+
+      opts.on('--table', 'Output format as an ASCII table (default)') do
+        output_format = :table
+      end
+
+      opts.on('--json') do
+        output_format = :json
+      end
+
+      opts.on('--filter=FILTER', 'Filters output to be only SMS or VOICE') do |filter_v|
+        filter = filter_v
+      end
+
+      opts.on('--help', 'Show this help message') do
+        show_help = true
+      end
+    end
+
+    uuids = parser.parse!(argv)
+
+    if uuids.empty? || show_help
+      out.puts parser
+      exit 1
+    end
+
+    new(uuids: uuids, filter: filter, output_format: output_format)
+  end
+
+  attr_reader :uuids, :output_format, :filter
+
+  def initialize(uuids:, output_format:, filter: nil, progress_bar: true)
+    @uuids = uuids
+    @output_format = output_format
+    @filter = filter
+    @progress_bar = progress_bar
+  end
+
+  def progress_bar?
+    @progress_bar
+  end
+
+  def run(out: STDOUT)
+    results = query_data(uuids)
+
+    table = []
+    table << %w[user_id timestamp message_id delivery_preference country_code]
+    results.each do |result|
+      table << [
+        result.user_id,
+        result.timestamp,
+        result.message_id,
+        result.delivery_preference,
+        result.country_code,
+      ]
+    end
+
+    ScriptBase.render_output(table, format: output_format, stdout: out)
+  end
+
+  Result = Struct.new(
+    :user_id,
+    :timestamp,
+    :message_id,
+    :delivery_preference,
+    :country_code,
+    keyword_init: true,
+  )
+
+  # @return [Array<Result>]
+  def query_data(uuids)
+    Reporting::UnknownProgressBar.wrap(show_bar: progress_bar?, title: 'Querying logs') do
+      cloudwatch_client.fetch(
+        query: <<~EOS,
+          fields
+            @timestamp
+          , properties.user_id
+          , properties.event_properties.telephony_response.message_id
+          , properties.event_properties.otp_delivery_preference
+          , properties.event_properties.telephony_response.delivery_status
+          , properties.event_properties.country_code
+          | filter name = 'Telephony: OTP sent'
+          #{filter ?
+            "| filter properties.event_properties.otp_delivery_preference = '#{filter.downcase}'" :
+            nil}
+          | filter properties.user_id IN #{quote(uuids)}
+          | limit 10000
+        EOS
+        from: 72.hours.ago,
+        to: Time.now,
+      ).map do |row|
+        Result.new(
+          user_id: row['properties.user_id'],
+          timestamp: row['@timestamp'],
+          message_id: row['properties.event_properties.telephony_response.message_id'],
+          delivery_preference: row['properties.event_properties.otp_delivery_preference'],
+          country_code: row['properties.event_properties.country_code'],
+        )
+      end
+    end
+  end
+
+  def cloudwatch_client
+    @cloudwatch_client ||= Reporting::CloudwatchClient.new(
+      ensure_complete_logs: false,
+      slice_interval: nil,
+      progress: false,
+    )
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  OtpDeliveries.parse!.run
+end

lib/reporting/unknown_progress_bar.rb

@@ -0,0 +1,30 @@
+require 'ruby-progressbar'
+
+module Reporting
+  # Small wrapper around ruby-progressbar
+  class UnknownProgressBar
+    # Wraps a block and displays a progress bar while the block executes
+    # Returns the value from the block
+    def self.wrap(show_bar:, title: 'Waiting', output: STDERR)
+      if show_bar
+        bar = ProgressBar.create(
+          title: title,
+          total: nil,
+          format: '[ %t ] %B %a',
+          output: output,
+        )
+        thread = Thread.fork do
+          loop do
+            sleep 0.1
+            bar.increment
+          end
+        end
+      end
+
+      yield
+    ensure
+      thread&.kill
+      bar&.stop
+    end
+  end
+end

lib/script_base.rb

@@ -62,7 +62,7 @@ def run
     if result.json
       stdout.puts result.json.to_json
     else
-      render_output(result.table)
+      self.class.render_output(result.table, format: config.format, stdout: stdout)
     end
   end
 
@@ -101,10 +101,10 @@ def option_parser
   end
 
   # @param [Array<Array<String>>] rows
-  def render_output(rows)
+  def self.render_output(rows, format:, stdout: STDOUT)
     return if rows.blank?
 
-    case config.format
+    case format
     when :table
       require 'terminal-table'
       table = Terminal::Table.new
@@ -131,7 +131,7 @@ def render_output(rows)
 
       stdout.puts JSON.pretty_generate(objects)
     else
-      raise "Unknown format=#{config.format}"
+      raise "Unknown format=#{format}"
     end
   end
 end