Data Pull Script Script (LG-9705)

Gemfile

@@ -65,6 +65,7 @@ gem 'sprockets-rails'
 gem 'stringex', require: false
 gem 'strong_migrations', '>= 0.4.2'
 gem 'subprocess', require: false
+gem 'terminal-table', require: false
 gem 'uglifier', '~> 4.2'
 gem 'valid_email', '>= 0.1.3'
 gem 'view_component', '~> 2.82.0'
@@ -124,6 +125,7 @@ group :test do
   gem 'rspec-retry'
   gem 'rspec_junit_formatter'
   gem 'shoulda-matchers', '~> 4.0', require: false
+  gem 'tableparser', require: false
   gem 'webdrivers', '~> 5.2.0'
   gem 'webmock'
   gem 'zonebie'

Gemfile.lock

@@ -645,6 +645,7 @@ GEM
     strong_migrations (0.8.0)
       activerecord (>= 5.2)
     subprocess (1.5.5)
+    tableparser (1.0.1)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.2.1)
@@ -821,6 +822,8 @@ DEPENDENCIES
   stringex
   strong_migrations (>= 0.4.2)
   subprocess
+  tableparser
+  terminal-table
   uglifier (~> 4.2)
   valid_email (>= 0.1.3)
   view_component (~> 2.82.0)

bin/data-pull

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../config/environment.rb'
+require 'data_pull'
+DataPull.new(argv: ARGV.dup, stdout: STDOUT, stderr: STDERR).run

lib/data_pull.rb

@@ -0,0 +1,224 @@
+require 'optparse'
+
+class DataPull
+  attr_reader :argv, :stdout, :stderr
+
+  def initialize(argv:, stdout:, stderr:)
+    @argv = argv
+    @stdout = stdout
+    @stderr = stderr
+  end
+
+  Result = Struct.new(
+    :table, # tabular output, rendered as an ASCII table or as CSV
+    :log_message, # summary message used for audit logging, DO NOT PUT PII HERE
+    keyword_init: true,
+  )
+
+  Config = Struct.new(
+    :include_missing,
+    :format,
+    :show_help,
+    keyword_init: true,
+  ) do
+    alias_method :include_missing?, :include_missing
+    alias_method :show_help?, :show_help
+  end
+
+  def config
+    @config ||= Config.new(
+      include_missing: true,
+      format: :table,
+      show_help: false,
+    )
+  end
+
+  def run
+    option_parser.parse!(argv)
+    subtask_class = subtask(argv.shift)
+
+    if config.show_help? || !subtask_class
+      stdout.puts option_parser
+      return
+    end
+
+    result = subtask_class.new.run(args: argv, include_missing: config.include_missing?)
+
+    stderr.puts result.log_message
+
+    render_output(result.table)
+  end
+
+  # @param [Array<Array<String>>] rows
+  def render_output(rows)
+    return if rows.blank?
+
+    case config.format
+    when :table
+      require 'terminal-table'
+      table = Terminal::Table.new
+      header, *body = rows
+      table << header
+      table << :separator
+      body.each do |row|
+        table << row
+      end
+      stdout.puts table
+    when :csv
+      require 'csv'
+      CSV.instance(stdout) do |csv|
+        rows.each do |row|
+          csv << row
+        end
+      end
+    when :json
+      headers, *body = rows
+
+      objects = body.map do |values|
+        headers.zip(values).to_h
+      end
+
+      stdout.puts JSON.pretty_generate(objects)
+    else
+      raise "Unknown format=#{config.format}"
+    end
+  end
+
+  # @api private
+  # A subtask is a class that has a run method, the type signature should look like:
+  # +#run(args: Array<String>, include_missing: Boolean) -> Result+
+  # @return [Class,nil]
+  def subtask(name)
+    {
+      'uuid-lookup' => UuidLookup,
+      'uuid-convert' => UuidConvert,
+      'email-lookup' => EmailLookup,
+      'profile-status' => ProfileStatus,
+    }[name]
+  end
+
+  def option_parser
+    @option_parser ||= OptionParser.new do |opts|
+      opts.banner = <<~EOS
+        #{$PROGRAM_NAME} [subcommand] [arguments] [options]
+
+        Example usage:
+
+          * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com
+
+          * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2
+
+          * #{$PROGRAM_NAME} email-lookup uuid1 uuid2...
+
+          * #{$PROGRAM_NAME} profile-status uuid1 uuid2...
+
+        Options:
+      EOS
+
+      opts.on('--help') do
+        config.show_help = true
+      end
+
+      opts.on('--csv') do
+        config.format = :csv
+      end
+
+      opts.on('--table', 'Output format as an ASCII table (default)') do\
+        config.format = :table
+      end
+
+      opts.on('--json') do
+        config.format = :json
+      end
+
+      opts.on('--[no-]include-missing', <<~STR) do |include_missing|
+        Whether or not to add rows in the output for missing inputs, defaults to off
+      STR
+        config.include_missing = include_missing
+      end
+    end
+  end
+
+  class UuidLookup
+    def run(args:, include_missing:)
+      emails = args
+
+      table = []
+      table << %w[email uuid]
+
+      uuids = []
+
+      emails.each do |email|
+        user = User.find_with_email(email)
+        if user
+          table << [email, user.uuid]
+          uuids << user.uuid
+        elsif include_missing
+          table << [email, '[NOT FOUND]']
+        end
+      end
+
+      Result.new(
+        log_message: "uuid-lookup, uuids: #{uuids.join(', ')}",
+        table:,
+      )
+    end
+  end
+
+  class UuidConvert
+    def run(args:, include_missing:)
+      partner_uuids = args
+
+      table = []
+      table << %w[partner_uuid source internal_uuid]
+      identities = AgencyIdentity.includes(:user, :agency).where(uuid: partner_uuids)
+
+      identities.each do |identity|
+        table << [identity.uuid, identity.agency.name, identity.user.uuid]
+      end
+
+      if include_missing
+        (partner_uuids - identities.map(&:uuid)).each do |missing_uuid|
+          table << [missing_uuid, '[NOT FOUND]', '[NOT FOUND]']
+        end
+      end
+
+      Result.new(
+        log_message: "uuid-convert, uuids: #{identities.map { |u| u.user.uuid }.join(', ')}",
+        table:,
+      )
+    end
+  end
+
+  class EmailLookup
+    def run(args:, include_missing:)
+      uuids = args
+
+      users = User.includes(:email_addresses).where(uuid: uuids)
+
+      table = []
+      table << %w[uuid email]
+
+      users.each do |user|
+        table << [user.uuid, *user.email_addresses.map(&:email)]
+      end
+
+      if include_missing
+        (uuids - users.map(&:uuid)).each do |missing_uuid|
+          table << [missing_uuid, '[NOT FOUND]']
+        end
+      end
+
+      Result.new(
+        log_message: "email-lookup, uuids: #{users.map(&:uuid).join(', ')}",
+        table:,
+      )
+    end
+  end
+
+  class ProfileStatus
+    def run(*)
+      Result.new
+    end
+  end
+end

spec/lib/data_pull_spec.rb

@@ -0,0 +1,125 @@
+require 'rails_helper'
+require 'tableparser'
+require 'data_pull'
+
+RSpec.describe DataPull do
+  let(:stdout) { StringIO.new }
+  let(:stderr) { StringIO.new }
+  let(:argv) { [] }
+
+  subject(:data_pull) { DataPull.new(argv:, stdout:, stderr:) }
+
+  describe 'command line flags' do
+    let(:argv) { ['uuid-lookup', user.email_addresses.first.email] }
+    let(:user) { create(:user) }
+
+    describe '--help' do
+      before { argv << '--help' }
+      it 'prints a help message' do
+        data_pull.run
+
+        expect(stdout.string).to include('Options:')
+      end
+    end
+
+    describe '--csv' do
+      before { argv << '--csv' }
+      it 'formats output as CSV' do
+        data_pull.run
+
+        expect(CSV.parse(stdout.string)).to eq(
+          [
+            ['email', 'uuid'],
+            [user.email_addresses.first.email, user.uuid],
+          ],
+        )
+      end
+    end
+
+    describe '--table' do
+      before { argv << '--table' }
+      it 'formats output as an ASCII table' do
+        data_pull.run
+
+        expect(Tableparser.parse(stdout.string)).to eq(
+          [
+            ['email', 'uuid'],
+            [user.email_addresses.first.email, user.uuid],
+          ],
+        )
+      end
+    end
+
+    describe '--json' do
+      before { argv << '--json' }
+      it 'formats output as JSON' do
+        data_pull.run
+
+        expect(JSON.parse(stdout.string)).to eq(
+          [
+            {
+              'email' => user.email_addresses.first.email,
+              'uuid' => user.uuid,
+            },
+          ],
+        )
+      end
+    end
+
+    describe '--include-missing' do
+      let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', '--json'] }
+      it 'adds rows for missing values' do
+        data_pull.run
+
+        expect(JSON.parse(stdout.string)).to eq(
+          [
+            {
+              'email' => 'does_not_exist@example.com',
+              'uuid' => '[NOT FOUND]',
+            },
+          ],
+        )
+      end
+    end
+
+    describe '--no-include-missing' do
+      let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', '--json'] }
+      it 'does not add rows for missing values' do
+        data_pull.run
+
+        expect(JSON.parse(stdout.string)).to be_empty
+      end
+    end
+  end
+
+  describe DataPull::UuidLookup do
+    subject(:subtask) { DataPull::UuidLookup.new }
+
+    describe '#run' do
+      let(:include_missing) { true }
+    end
+  end
+
+  describe DataPull::UuidConvert do
+    subject(:subtask) { DataPull::UuidConvert.new }
+
+    describe '#run' do
+      let(:include_missing) { true }
+    end
+  end
+
+  describe DataPull::EmailLookup do
+    subject(:subtask) { DataPull::EmailLookup.new }
+
+    describe '#run' do
+      let(:include_missing) { true }
+    end
+  end
+
+  describe DataPull::ProfileStatus do
+    subject(:subtask) { DataPull::ProfileStatus.new }
+
+    describe '#run' do
+    end
+  end
+end