From 323ae60c6f85fb64a0115dbf823bb42e9719be15 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Thu, 4 May 2023 16:59:22 -0700 Subject: [PATCH 01/11] Add first pass at extendable data-pull script --- Gemfile | 1 + Gemfile.lock | 1 + bin/data-pull | 5 + lib/data_pull.rb | 212 +++++++++++++++++++++++++++++++++++++ spec/lib/data_pull_spec.rb | 55 ++++++++++ 5 files changed, 274 insertions(+) create mode 100755 bin/data-pull create mode 100644 lib/data_pull.rb create mode 100644 spec/lib/data_pull_spec.rb diff --git a/Gemfile b/Gemfile index a1ac6a19fbf..92aa362fd04 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,7 @@ gem 'sprockets-rails' gem 'stringex', require: false gem 'strong_migrations', '>= 0.4.2' gem 'subprocess', require: false +gem 'terminal-table', require: false gem 'uglifier', '~> 4.2' gem 'valid_email', '>= 0.1.3' gem 'view_component', '~> 2.82.0' diff --git a/Gemfile.lock b/Gemfile.lock index 09c160a7175..c5ee90bfd35 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -821,6 +821,7 @@ DEPENDENCIES stringex strong_migrations (>= 0.4.2) subprocess + terminal-table uglifier (~> 4.2) valid_email (>= 0.1.3) view_component (~> 2.82.0) diff --git a/bin/data-pull b/bin/data-pull new file mode 100755 index 00000000000..d36486de8fe --- /dev/null +++ b/bin/data-pull @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require_relative '../config/environment.rb' +require 'data_pull' +DataPull.new(argv: ARGV.dup, stdout: STDOUT, stderr: STDERR).run diff --git a/lib/data_pull.rb b/lib/data_pull.rb new file mode 100644 index 00000000000..68b5ab5c3b4 --- /dev/null +++ b/lib/data_pull.rb @@ -0,0 +1,212 @@ +require 'optparse' + +class DataPull + attr_reader :argv, :stdout, :stderr + + def initialize(argv:, stdout:, stderr:) + @argv = argv + @stdout = stdout + @stderr = stderr + end + + Result = Struct.new( + :table, # tabular output, rendered as an ASCII table or as CSV + :log_message, # summary message used for audit logging, DO NOT PUT PII HERE + keyword_init: true + ) + + Config = Struct.new( + :include_missing, + :format, + :show_help, + keyword_init: true, + ) do + alias_method :include_missing?, :include_missing + alias_method :show_help?, :show_help + end + + def config + @config ||= Config.new( + include_missing: true, + format: :table, + show_help: false, + ) + end + + def run + option_parser.parse!(argv) + subtask_class = subtask(argv.shift) + + if config.show_help? || !subtask_class + stdout.puts option_parser + return + end + + result = subtask_class.new.run(args: argv, include_missing: config.include_missing?) + + stderr.puts result.log_message + + render_output(result.table) + end + + # @param [Array>] rows + def render_output(rows) + return if rows.blank? + + case config.format + when :table + require 'terminal-table' + table = Terminal::Table.new + header, *body = rows + table << header + table << :separator + body.each do |row| + table << row + end + stdout.puts table + when :csv + require 'csv' + CSV.instance(stdout) do |csv| + rows.each do |row| + csv << row + end + end + else + raise "Unknown format=#{config.format}" + end + end + + # @api private + # A subtask is a class that has a run method, the type signature should look like: + # +#run(args: Array, include_missing: Boolean) -> Result+ + # @return [Class,nil] + def subtask(name) + { + 'uuid-lookup' => UuidLookup, + 'uuid-convert' => UuidConvert, + 'email-lookup' => EmailLookup, + 'profile-status' => ProfileStatus, + }[name] + end + + def option_parser + @option_parser ||= OptionParser.new do |opts| + opts.banner = <<~EOS + #{$PROGRAM_NAME} [subcommand] [arguments] [options] + + Example usage: + + * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com + + * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2 + + * #{$PROGRAM_NAME} email-lookup uuid1 uuid2... + + * #{$PROGRAM_NAME} profile-status uuid1 uuid2... + + Options: + EOS + + opts.on('--help') do + config.show_help = true + end + + opts.on('--csv') do + config.format = :csv + end + + opts.on('--table', 'Output format as an ASCII table (default)') do\ + config.format = :table + end + + opts.on('--[no-]include-missing', <<~STR) do |include_missing| + Whether or not to add rows in the output for missing inputs, defaults to off + STR + config.include_missing = include_missing + end + end + end + + class UuidLookup + def run(args:, include_missing:) + emails = args + + table = [] + table << %w[email uuid] + + uuids = [] + + emails.each do |email| + user = User.find_with_email(email) + if user + table << [email, user.uuid] + uuids << user.uuid + elsif include_missing + table << [email, '[NOT FOUND]'] + end + end + + Result.new( + log_message: "uuid-lookup, uuids: #{uuids.join(', ')}", + table:, + ) + end + end + + class UuidConvert + def run(args:, include_missing:) + partner_uuids = args + + table = [] + table << %w[partner_uuid source internal_uuid] + identities = AgencyIdentity.includes(:user, :agency).where(uuid: partner_uuids) + + identities.each do |identity| + table << [identity.uuid, identity.agency.name, identity.user.uuid] + end + + if include_missing + (partner_uuids - identities.map(&:uuid)).each do |missing_uuid| + table << [missing_uuid, '[NOT FOUND]', '[NOT FOUND]'] + end + end + + Result.new( + log_message: "uuid-convert, uuids: #{identities.map { |u| u.user.uuid }.join(', ')}", + table:, + ) + end + end + + class EmailLookup + def run(args:, include_missing:) + uuids = args + + users = User.includes(:email_addresses).where(uuid: uuids) + + table = [] + table << %w[uuid email] + + users.each do |user| + table << [user.uuid, *user.email_addresses.map(&:email)] + end + + if include_missing + (uuids - users.map(&:uuid)).each do |missing_uuid| + table << [missing_uuid, '[NOT FOUND]'] + end + end + + Result.new( + log_message: "email-lookup, uuids: #{users.map(&:uuid).join(', ')}", + table:, + ) + end + end + + class ProfileStatus + def run(args:, include_missing:) + Result.new + end + end +end \ No newline at end of file diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb new file mode 100644 index 00000000000..6cf2dd02595 --- /dev/null +++ b/spec/lib/data_pull_spec.rb @@ -0,0 +1,55 @@ +require 'rails_helper' + +RSpec.describe DataPull do + let(:stdout) { StringIO.new } + let(:stderr) { StringIO.new } + let(:argv) { [] } + + subject(:data_pull) { DataPull.new(argv:, stdout:, stderr:) } + + describe 'command line flags' do + describe '--help' do + end + + describe '--csv' do + end + + describe '--table' do + end + + describe '--include-missing' do + end + + describe '--no-include-missing' do + end + end + + + describe DataPull::UuidLookup do + subject(:subtask) { DataPull::UuidLookup.new } + + describe '#run' do + end + end + + describe DataPull::UuidConvert do + subject(:subtask) { DataPull::UuidConvert.new } + + describe '#run' do + end + end + + describe DataPull::EmailLookup do + subject(:subtask) { DataPull::EmailLookup.new } + + describe '#run' do + end + end + + describe DataPull::ProfileStatus do + subject(:subtask) { DataPull::ProfileStatus.new } + + describe '#run' do + end + end +end From c93ddaead07e9788aa60e45a6a830cadfe70d734 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Thu, 4 May 2023 17:01:20 -0700 Subject: [PATCH 02/11] lints --- lib/data_pull.rb | 6 +++--- spec/lib/data_pull_spec.rb | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index 68b5ab5c3b4..ccadee38b08 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -12,7 +12,7 @@ def initialize(argv:, stdout:, stderr:) Result = Struct.new( :table, # tabular output, rendered as an ASCII table or as CSV :log_message, # summary message used for audit logging, DO NOT PUT PII HERE - keyword_init: true + keyword_init: true, ) Config = Struct.new( @@ -205,8 +205,8 @@ def run(args:, include_missing:) end class ProfileStatus - def run(args:, include_missing:) + def run(*) Result.new end end -end \ No newline at end of file +end diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index 6cf2dd02595..5b1f51cbd6d 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -24,7 +24,6 @@ end end - describe DataPull::UuidLookup do subject(:subtask) { DataPull::UuidLookup.new } From 62bb4657149cfbc392feda2dea6ee0f72db728e9 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Thu, 4 May 2023 17:17:21 -0700 Subject: [PATCH 03/11] Add some tests, JSON output --- Gemfile | 1 + Gemfile.lock | 2 ++ lib/data_pull.rb | 12 +++++++ spec/lib/data_pull_spec.rb | 71 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+) diff --git a/Gemfile b/Gemfile index 92aa362fd04..6b671ec27e0 100644 --- a/Gemfile +++ b/Gemfile @@ -125,6 +125,7 @@ group :test do gem 'rspec-retry' gem 'rspec_junit_formatter' gem 'shoulda-matchers', '~> 4.0', require: false + gem 'tableparser', require: false gem 'webdrivers', '~> 5.2.0' gem 'webmock' gem 'zonebie' diff --git a/Gemfile.lock b/Gemfile.lock index c5ee90bfd35..4e7644ded2b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -645,6 +645,7 @@ GEM strong_migrations (0.8.0) activerecord (>= 5.2) subprocess (1.5.5) + tableparser (1.0.1) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) thor (1.2.1) @@ -821,6 +822,7 @@ DEPENDENCIES stringex strong_migrations (>= 0.4.2) subprocess + tableparser terminal-table uglifier (~> 4.2) valid_email (>= 0.1.3) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index ccadee38b08..ee62727f7c8 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -71,6 +71,14 @@ def render_output(rows) csv << row end end + when :json + headers, *body = rows + + objects = body.map do |values| + headers.zip(values).to_h + end + + stdout.puts JSON.pretty_generate(objects) else raise "Unknown format=#{config.format}" end @@ -119,6 +127,10 @@ def option_parser config.format = :table end + opts.on('--json') do + config.format = :json + end + opts.on('--[no-]include-missing', <<~STR) do |include_missing| Whether or not to add rows in the output for missing inputs, defaults to off STR diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index 5b1f51cbd6d..8556b149a1e 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -1,4 +1,6 @@ require 'rails_helper' +require 'tableparser' +require 'data_pull' RSpec.describe DataPull do let(:stdout) { StringIO.new } @@ -8,19 +10,85 @@ subject(:data_pull) { DataPull.new(argv:, stdout:, stderr:) } describe 'command line flags' do + let(:argv) { ['uuid-lookup', user.email_addresses.first.email] } + let(:user) { create(:user) } + describe '--help' do + before { argv << '--help' } + it 'prints a help message' do + data_pull.run + + expect(stdout.string).to include('Options:') + end end describe '--csv' do + before { argv << '--csv' } + it 'formats output as CSV' do + data_pull.run + + expect(CSV.parse(stdout.string)).to eq( + [ + ['email', 'uuid'], + [user.email_addresses.first.email, user.uuid], + ], + ) + end end describe '--table' do + before { argv << '--table' } + it 'formats output as an ASCII table' do + data_pull.run + + expect(Tableparser.parse(stdout.string)).to eq( + [ + ['email', 'uuid'], + [user.email_addresses.first.email, user.uuid], + ], + ) + end + end + + describe '--json' do + before { argv << '--json' } + it 'formats output as JSON' do + data_pull.run + + expect(JSON.parse(stdout.string)).to eq( + [ + { + 'email' => user.email_addresses.first.email, + 'uuid' => user.uuid, + }, + ], + ) + end end describe '--include-missing' do + let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', '--json'] } + it 'adds rows for missing values' do + data_pull.run + + expect(JSON.parse(stdout.string)).to eq( + [ + { + 'email' => 'does_not_exist@example.com', + 'uuid' => '[NOT FOUND]', + }, + ], + ) + end end describe '--no-include-missing' do + let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', '--json'] } + it 'does not add rows for missing values' do + data_pull.run + + expect(JSON.parse(stdout.string)).to be_empty + end end end @@ -28,6 +96,7 @@ subject(:subtask) { DataPull::UuidLookup.new } describe '#run' do + let(:include_missing) { true } end end @@ -35,6 +104,7 @@ subject(:subtask) { DataPull::UuidConvert.new } describe '#run' do + let(:include_missing) { true } end end @@ -42,6 +112,7 @@ subject(:subtask) { DataPull::EmailLookup.new } describe '#run' do + let(:include_missing) { true } end end From 54279b2e48bac618b4915901f33dd20fefa71c8a Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 5 May 2023 08:01:46 -0700 Subject: [PATCH 04/11] Update email-lookup to have one row per email, confirmed_at column --- lib/data_pull.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index ee62727f7c8..2e5bb34413c 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -197,15 +197,17 @@ def run(args:, include_missing:) users = User.includes(:email_addresses).where(uuid: uuids) table = [] - table << %w[uuid email] + table << %w[uuid email confirmed_at] users.each do |user| - table << [user.uuid, *user.email_addresses.map(&:email)] + user.email_addresses.each do |email_address| + table << [user.uuid, email_address.email, email_address.confirmed_at] + end end if include_missing (uuids - users.map(&:uuid)).each do |missing_uuid| - table << [missing_uuid, '[NOT FOUND]'] + table << [missing_uuid, '[NOT FOUND]', nil] end end From a8b3ac01c0da120b79f9dda64c2341e042d47924 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 5 May 2023 08:16:47 -0700 Subject: [PATCH 05/11] Add changelog changelog: Internal, Tooling, Add script to streamline data pulls From 4d816716769f33fff1b8490cb51aaa0e5eb178d3 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 5 May 2023 08:47:34 -0700 Subject: [PATCH 06/11] - Remove profile-lookup subtask - Add required --reason arg - Add subtask specs --- lib/data_pull.rb | 30 +++++++------- spec/factories/agency_identities.rb | 7 ++++ spec/lib/data_pull_spec.rb | 61 ++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 spec/factories/agency_identities.rb diff --git a/lib/data_pull.rb b/lib/data_pull.rb index 2e5bb34413c..650b230ca35 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -19,6 +19,7 @@ def initialize(argv:, stdout:, stderr:) :include_missing, :format, :show_help, + :reason, keyword_init: true, ) do alias_method :include_missing?, :include_missing @@ -37,14 +38,14 @@ def run option_parser.parse!(argv) subtask_class = subtask(argv.shift) - if config.show_help? || !subtask_class + if config.reason.blank? || config.show_help? || !subtask_class stdout.puts option_parser return end result = subtask_class.new.run(args: argv, include_missing: config.include_missing?) - stderr.puts result.log_message + stderr.puts [result.log_message, "reason: #{config.reason}"].join("\n") render_output(result.table) end @@ -93,10 +94,10 @@ def subtask(name) 'uuid-lookup' => UuidLookup, 'uuid-convert' => UuidConvert, 'email-lookup' => EmailLookup, - 'profile-status' => ProfileStatus, }[name] end + # rubocop:disable Metrics/BlockLength def option_parser @option_parser ||= OptionParser.new do |opts| opts.banner = <<~EOS @@ -104,13 +105,11 @@ def option_parser Example usage: - * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com + * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com --reason "support case 123" - * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2 + * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2 --reason "investigation" - * #{$PROGRAM_NAME} email-lookup uuid1 uuid2... - - * #{$PROGRAM_NAME} profile-status uuid1 uuid2... + * #{$PROGRAM_NAME} email-lookup uuid1 uuid2 --reason "investigation" Options: EOS @@ -123,7 +122,7 @@ def option_parser config.format = :csv end - opts.on('--table', 'Output format as an ASCII table (default)') do\ + opts.on('--table', 'Output format as an ASCII table (default)') do config.format = :table end @@ -136,8 +135,13 @@ def option_parser STR config.include_missing = include_missing end + + opts.on('--reason=REASON', 'reason for this data pull (required, will be logged)') do |reason| + config.reason = reason + end end end + # rubocop:enable Metrics/BlockLength class UuidLookup def run(args:, include_missing:) @@ -171,7 +175,7 @@ def run(args:, include_missing:) table = [] table << %w[partner_uuid source internal_uuid] - identities = AgencyIdentity.includes(:user, :agency).where(uuid: partner_uuids) + identities = AgencyIdentity.includes(:user, :agency).where(uuid: partner_uuids).order(:uuid) identities.each do |identity| table << [identity.uuid, identity.agency.name, identity.user.uuid] @@ -217,10 +221,4 @@ def run(args:, include_missing:) ) end end - - class ProfileStatus - def run(*) - Result.new - end - end end diff --git a/spec/factories/agency_identities.rb b/spec/factories/agency_identities.rb new file mode 100644 index 00000000000..c0d26ef90cf --- /dev/null +++ b/spec/factories/agency_identities.rb @@ -0,0 +1,7 @@ +FactoryBot.define do + factory :agency_identity do + association :user, factory: %i[user fully_registered] + association :agency + uuid { SecureRandom.uuid } + end +end diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index 8556b149a1e..836339f17d6 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -10,7 +10,8 @@ subject(:data_pull) { DataPull.new(argv:, stdout:, stderr:) } describe 'command line flags' do - let(:argv) { ['uuid-lookup', user.email_addresses.first.email] } + let(:argv) { ['uuid-lookup', user.email_addresses.first.email, *reason_args] } + let(:reason_args) { ['--reason', 'test'] } let(:user) { create(:user) } describe '--help' do @@ -67,7 +68,9 @@ end describe '--include-missing' do - let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', '--json'] } + let(:argv) do + ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', *reason_args, '--json'] + end it 'adds rows for missing values' do data_pull.run @@ -83,28 +86,72 @@ end describe '--no-include-missing' do - let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', '--json'] } + let(:argv) do + ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', *reason_args, + '--json'] + end it 'does not add rows for missing values' do data_pull.run expect(JSON.parse(stdout.string)).to be_empty end end + + describe 'missing --reason' do + before do + reason_args.each { |arg| argv.delete(arg) } + end + + it 'prints the help message' do + data_pull.run + + expect(stdout.string).to include('Options:') + end + end end describe DataPull::UuidLookup do subject(:subtask) { DataPull::UuidLookup.new } describe '#run' do + let(:users) { create_list(:user, 2) } + + let(:args) { [*users.map { |u| u.email_addresses.first.email }, 'missing@example.com'] } let(:include_missing) { true } + + subject(:result) { subtask.run(args:, include_missing:) } + + it 'looks up the UUIDs for the given email addresses' do + expect(result.table).to eq( + [ + ['email', 'uuid'], + *users.map { |u| [u.email_addresses.first.email, u.uuid] }, + ['missing@example.com', '[NOT FOUND]'], + ], + ) + end end end describe DataPull::UuidConvert do subject(:subtask) { DataPull::UuidConvert.new } + let(:agency_identities) { create_list(:agency_identity, 2).sort_by(&:uuid) } + describe '#run' do let(:include_missing) { true } + let(:args) { [*agency_identities.map(&:uuid), 'does-not-exist'] } + subject(:result) { subtask.run(args:, include_missing:) } + + it 'converts the agency agency identities to internal UUIDs' do + expect(result.table).to eq( + [ + ['partner_uuid', 'source', 'internal_uuid'], + *agency_identities.map { |a| [a.uuid, a.agency.name, a.user.uuid] }, + ['does-not-exist', '[NOT FOUND]', '[NOT FOUND]'], + ], + ) + end end end @@ -113,13 +160,7 @@ describe '#run' do let(:include_missing) { true } - end - end - - describe DataPull::ProfileStatus do - subject(:subtask) { DataPull::ProfileStatus.new } - - describe '#run' do + subject(:result) { subtask.run(args:, include_missing:) } end end end From 6a44eb76fb56fe2aa03946b7f029421322b81478 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 5 May 2023 08:54:49 -0700 Subject: [PATCH 07/11] Specs for email-lookup --- lib/data_pull.rb | 4 ++-- spec/lib/data_pull_spec.rb | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index 650b230ca35..cdcde9a278a 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -198,13 +198,13 @@ class EmailLookup def run(args:, include_missing:) uuids = args - users = User.includes(:email_addresses).where(uuid: uuids) + users = User.includes(:email_addresses).where(uuid: uuids).order(:uuid) table = [] table << %w[uuid email confirmed_at] users.each do |user| - user.email_addresses.each do |email_address| + user.email_addresses.sort_by(&:id).each do |email_address| table << [user.uuid, email_address.email, email_address.confirmed_at] end end diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index 836339f17d6..b66d9c0a97b 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -136,11 +136,11 @@ describe DataPull::UuidConvert do subject(:subtask) { DataPull::UuidConvert.new } - let(:agency_identities) { create_list(:agency_identity, 2).sort_by(&:uuid) } - describe '#run' do - let(:include_missing) { true } + let(:agency_identities) { create_list(:agency_identity, 2).sort_by(&:uuid) } + let(:args) { [*agency_identities.map(&:uuid), 'does-not-exist'] } + let(:include_missing) { true } subject(:result) { subtask.run(args:, include_missing:) } it 'converts the agency agency identities to internal UUIDs' do @@ -159,8 +159,23 @@ subject(:subtask) { DataPull::EmailLookup.new } describe '#run' do + let(:user) { create(:user, :with_multiple_emails) } + + let(:args) { [user.uuid, 'does-not-exist'] } let(:include_missing) { true } subject(:result) { subtask.run(args:, include_missing:) } + + it 'loads email addresses for the user' do + expect(result.table).to eq( + [ + ['uuid', 'email', 'confirmed_at'], + *user.email_addresses.sort_by(&:id).map do |e| + [e.user.uuid, e.email, e.confirmed_at] + end, + ['does-not-exist', '[NOT FOUND]', nil], + ], + ) + end end end end From c96a9b023173cdec2e9377a418cbb474fee57b91 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Fri, 5 May 2023 13:52:13 -0700 Subject: [PATCH 08/11] Remove --reason, will be handled by devops script --- lib/data_pull.rb | 15 +++++---------- spec/lib/data_pull_spec.rb | 24 +++--------------------- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index cdcde9a278a..9bbb71c59dc 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -19,7 +19,6 @@ def initialize(argv:, stdout:, stderr:) :include_missing, :format, :show_help, - :reason, keyword_init: true, ) do alias_method :include_missing?, :include_missing @@ -38,14 +37,14 @@ def run option_parser.parse!(argv) subtask_class = subtask(argv.shift) - if config.reason.blank? || config.show_help? || !subtask_class + if config.show_help? || !subtask_class stdout.puts option_parser return end result = subtask_class.new.run(args: argv, include_missing: config.include_missing?) - stderr.puts [result.log_message, "reason: #{config.reason}"].join("\n") + stderr.puts result.log_message render_output(result.table) end @@ -105,11 +104,11 @@ def option_parser Example usage: - * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com --reason "support case 123" + * #{$PROGRAM_NAME} uuid-lookup email1@example.com email2@example.com - * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2 --reason "investigation" + * #{$PROGRAM_NAME} uuid-convert partner-uuid1 partner-uuid2 - * #{$PROGRAM_NAME} email-lookup uuid1 uuid2 --reason "investigation" + * #{$PROGRAM_NAME} email-lookup uuid1 uuid2 Options: EOS @@ -135,10 +134,6 @@ def option_parser STR config.include_missing = include_missing end - - opts.on('--reason=REASON', 'reason for this data pull (required, will be logged)') do |reason| - config.reason = reason - end end end # rubocop:enable Metrics/BlockLength diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index b66d9c0a97b..9d12397d3ac 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -10,8 +10,7 @@ subject(:data_pull) { DataPull.new(argv:, stdout:, stderr:) } describe 'command line flags' do - let(:argv) { ['uuid-lookup', user.email_addresses.first.email, *reason_args] } - let(:reason_args) { ['--reason', 'test'] } + let(:argv) { ['uuid-lookup', user.email_addresses.first.email] } let(:user) { create(:user) } describe '--help' do @@ -68,9 +67,7 @@ end describe '--include-missing' do - let(:argv) do - ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', *reason_args, '--json'] - end + let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--include-missing', '--json'] } it 'adds rows for missing values' do data_pull.run @@ -86,28 +83,13 @@ end describe '--no-include-missing' do - let(:argv) do - ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', *reason_args, - '--json'] - end + let(:argv) { ['uuid-lookup', 'does_not_exist@example.com', '--no-include-missing', '--json'] } it 'does not add rows for missing values' do data_pull.run expect(JSON.parse(stdout.string)).to be_empty end end - - describe 'missing --reason' do - before do - reason_args.each { |arg| argv.delete(arg) } - end - - it 'prints the help message' do - data_pull.run - - expect(stdout.string).to include('Options:') - end - end end describe DataPull::UuidLookup do From 16cc9c31ecb141e0a355acae90349c02cf03029b Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Mon, 8 May 2023 10:55:36 -0700 Subject: [PATCH 09/11] Set default LOGIN_TASK_LOG_LEVEL to minimize extra output --- bin/data-pull | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/data-pull b/bin/data-pull index d36486de8fe..f4b64f46bce 100755 --- a/bin/data-pull +++ b/bin/data-pull @@ -1,5 +1,6 @@ #!/usr/bin/env ruby +ENV['LOGIN_TASK_LOG_LEVEL'] ||= 'warn' require_relative '../config/environment.rb' require 'data_pull' DataPull.new(argv: ARGV.dup, stdout: STDOUT, stderr: STDERR).run From f5cf0674fe77a97f3ce33b59f191527d973dd8b4 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Mon, 8 May 2023 11:36:30 -0700 Subject: [PATCH 10/11] remove redundant rubocop disable --- lib/data_pull.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index 9bbb71c59dc..89b99d65110 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -96,7 +96,6 @@ def subtask(name) }[name] end - # rubocop:disable Metrics/BlockLength def option_parser @option_parser ||= OptionParser.new do |opts| opts.banner = <<~EOS @@ -136,7 +135,6 @@ def option_parser end end end - # rubocop:enable Metrics/BlockLength class UuidLookup def run(args:, include_missing:) From 90c2598b46a4a0e9d9294d71fa714b7906006e86 Mon Sep 17 00:00:00 2001 From: Zach Margolis Date: Mon, 8 May 2023 11:38:35 -0700 Subject: [PATCH 11/11] Round timestamps for consistency in CI --- lib/data_pull.rb | 2 +- spec/lib/data_pull_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/data_pull.rb b/lib/data_pull.rb index 89b99d65110..512eba6394d 100644 --- a/lib/data_pull.rb +++ b/lib/data_pull.rb @@ -198,7 +198,7 @@ def run(args:, include_missing:) users.each do |user| user.email_addresses.sort_by(&:id).each do |email_address| - table << [user.uuid, email_address.email, email_address.confirmed_at] + table << [user.uuid, email_address.email, email_address.confirmed_at.round(6)] end end diff --git a/spec/lib/data_pull_spec.rb b/spec/lib/data_pull_spec.rb index 9d12397d3ac..38d2cb58e38 100644 --- a/spec/lib/data_pull_spec.rb +++ b/spec/lib/data_pull_spec.rb @@ -152,7 +152,7 @@ [ ['uuid', 'email', 'confirmed_at'], *user.email_addresses.sort_by(&:id).map do |e| - [e.user.uuid, e.email, e.confirmed_at] + [e.user.uuid, e.email, e.confirmed_at.round(6)] end, ['does-not-exist', '[NOT FOUND]', nil], ],