From fed2fbbe29032579497181cb95fa59e7decc8b47 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Tue, 24 Oct 2023 13:34:37 -0400 Subject: [PATCH 01/13] changelog: Internal, Authentication, Disposable emails database loaded --- Gemfile | 1 + Gemfile.lock | 2 ++ app/models/disposable_domain.rb | 3 +++ .../20231024170146_create_email_data_table.rb | 11 +++++++++++ db/primary_migrate/20231024172229_load_email_data.rb | 9 +++++++++ db/schema.rb | 8 +++++++- 6 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 app/models/disposable_domain.rb create mode 100644 db/primary_migrate/20231024170146_create_email_data_table.rb create mode 100644 db/primary_migrate/20231024172229_load_email_data.rb diff --git a/Gemfile b/Gemfile index f13db8b387a..fb7f18410eb 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,7 @@ gem 'concurrent-ruby' gem 'connection_pool' gem 'cssbundling-rails' gem 'devise', '~> 4.8' +gem "email_data", require: "email_data/source/active_record" gem 'dotiw', '>= 4.0.1' gem 'faraday', '~> 2' gem 'faker' diff --git a/Gemfile.lock b/Gemfile.lock index 4b04e5cc7e1..f3c847a29ce 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -271,6 +271,7 @@ GEM activesupport i18n dumb_delegator (1.0.0) + email_data (1697934462.0.0) email_spec (2.2.2) htmlentities (~> 4.3.3) launchy (~> 2.1) @@ -736,6 +737,7 @@ DEPENDENCIES derailed_benchmarks devise (~> 4.8) dotiw (>= 4.0.1) + email_data email_spec erb_lint (~> 0.4.0) factory_bot_rails (>= 6.2.0) diff --git a/app/models/disposable_domain.rb b/app/models/disposable_domain.rb new file mode 100644 index 00000000000..830c9da8a7c --- /dev/null +++ b/app/models/disposable_domain.rb @@ -0,0 +1,3 @@ +class DisposableDomain < ApplicationRecord +end + \ No newline at end of file diff --git a/db/primary_migrate/20231024170146_create_email_data_table.rb b/db/primary_migrate/20231024170146_create_email_data_table.rb new file mode 100644 index 00000000000..92dfa8ff017 --- /dev/null +++ b/db/primary_migrate/20231024170146_create_email_data_table.rb @@ -0,0 +1,11 @@ +class CreateEmailDataTable < ActiveRecord::Migration[7.0] + def change + enable_extension "citext" + + create_table :disposable_domains do |t| + t.citext :name, null: false + end + + add_index :disposable_domains, :name, unique: true + end +end diff --git a/db/primary_migrate/20231024172229_load_email_data.rb b/db/primary_migrate/20231024172229_load_email_data.rb new file mode 100644 index 00000000000..cc1ea665d05 --- /dev/null +++ b/db/primary_migrate/20231024172229_load_email_data.rb @@ -0,0 +1,9 @@ +class LoadEmailData < ActiveRecord::Migration[7.0] + def change + ActiveRecord::Base.connection.execute <<~PG + COPY disposable_domains (name) + FROM '#{EmailData.data_dir.join('disposable_domains')}.txt' + (FORMAT CSV) + PG + end +end diff --git a/db/schema.rb b/db/schema.rb index 2e1b1e1b8ac..66ce355c079 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,9 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_08_31_124437) do +ActiveRecord::Schema[7.0].define(version: 2023_10_24_172229) do # These are extensions that must be enabled in order to support this database + enable_extension "citext" enable_extension "pg_stat_statements" enable_extension "pgcrypto" enable_extension "plpgsql" @@ -94,6 +95,11 @@ t.index ["user_id", "last_used_at"], name: "index_device_user_id_last_used_at" end + create_table "disposable_domains", force: :cascade do |t| + t.citext "name", null: false + t.index ["name"], name: "index_disposable_domains_on_name", unique: true + end + create_table "doc_auth_logs", force: :cascade do |t| t.integer "user_id", null: false t.datetime "welcome_view_at", precision: nil From 7d7076dc5a8c125147ee66592d5387c32c12c09d Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Wed, 25 Oct 2023 10:31:23 -0400 Subject: [PATCH 02/13] fix lint --- Gemfile | 2 +- app/models/disposable_domain.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index fb7f18410eb..15e7bec699d 100644 --- a/Gemfile +++ b/Gemfile @@ -23,7 +23,7 @@ gem 'concurrent-ruby' gem 'connection_pool' gem 'cssbundling-rails' gem 'devise', '~> 4.8' -gem "email_data", require: "email_data/source/active_record" +gem 'email_data', require: 'email_data/source/active_record' gem 'dotiw', '>= 4.0.1' gem 'faraday', '~> 2' gem 'faker' diff --git a/app/models/disposable_domain.rb b/app/models/disposable_domain.rb index 830c9da8a7c..f32d48d914d 100644 --- a/app/models/disposable_domain.rb +++ b/app/models/disposable_domain.rb @@ -1,3 +1,2 @@ class DisposableDomain < ApplicationRecord end - \ No newline at end of file From ad3074106f2695c85450a2891a010c7077fd7d78 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Thu, 26 Oct 2023 11:04:17 -0400 Subject: [PATCH 03/13] delete load --- db/primary_migrate/20231024172229_load_email_data.rb | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 db/primary_migrate/20231024172229_load_email_data.rb diff --git a/db/primary_migrate/20231024172229_load_email_data.rb b/db/primary_migrate/20231024172229_load_email_data.rb deleted file mode 100644 index cc1ea665d05..00000000000 --- a/db/primary_migrate/20231024172229_load_email_data.rb +++ /dev/null @@ -1,9 +0,0 @@ -class LoadEmailData < ActiveRecord::Migration[7.0] - def change - ActiveRecord::Base.connection.execute <<~PG - COPY disposable_domains (name) - FROM '#{EmailData.data_dir.join('disposable_domains')}.txt' - (FORMAT CSV) - PG - end -end From d285f1dcb18dee8f78ed8210825f40d1e9dde4d1 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Mon, 30 Oct 2023 10:27:07 -0400 Subject: [PATCH 04/13] email db script --- disposable_domains/disposable-domains.txt | 0 scripts/load_email_data.rb | 6 ++++++ 2 files changed, 6 insertions(+) create mode 100644 disposable_domains/disposable-domains.txt create mode 100644 scripts/load_email_data.rb diff --git a/disposable_domains/disposable-domains.txt b/disposable_domains/disposable-domains.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/load_email_data.rb b/scripts/load_email_data.rb new file mode 100644 index 00000000000..632065de034 --- /dev/null +++ b/scripts/load_email_data.rb @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +require 'faraday' + +response = Faraday.get("https://raw.githubusercontent.com/fnando/email_data/46d7d9aab2cd7f5c26709f27c02932f7c19f59d0/data/disposable_domains.txt") + +disposable_domains_file="disposable_domains/disposable-domains.txt" From 97a79f223969e4703c36cdd91dcdca37d9fb4144 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Mon, 30 Oct 2023 16:33:17 -0400 Subject: [PATCH 05/13] move loading email data to class --- Gemfile | 1 - Gemfile.lock | 2 -- config/application.rb | 1 + disposable_domains/disposable-domains.txt | 0 lib/load_disposable_domain.rb | 7 +++++++ scripts/load_email_data.rb | 6 ------ 6 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 disposable_domains/disposable-domains.txt create mode 100644 lib/load_disposable_domain.rb delete mode 100644 scripts/load_email_data.rb diff --git a/Gemfile b/Gemfile index 15e7bec699d..f13db8b387a 100644 --- a/Gemfile +++ b/Gemfile @@ -23,7 +23,6 @@ gem 'concurrent-ruby' gem 'connection_pool' gem 'cssbundling-rails' gem 'devise', '~> 4.8' -gem 'email_data', require: 'email_data/source/active_record' gem 'dotiw', '>= 4.0.1' gem 'faraday', '~> 2' gem 'faker' diff --git a/Gemfile.lock b/Gemfile.lock index f3c847a29ce..4b04e5cc7e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -271,7 +271,6 @@ GEM activesupport i18n dumb_delegator (1.0.0) - email_data (1697934462.0.0) email_spec (2.2.2) htmlentities (~> 4.3.3) launchy (~> 2.1) @@ -737,7 +736,6 @@ DEPENDENCIES derailed_benchmarks devise (~> 4.8) dotiw (>= 4.0.1) - email_data email_spec erb_lint (~> 0.4.0) factory_bot_rails (>= 6.2.0) diff --git a/config/application.rb b/config/application.rb index 00ac6b48093..9db407209a3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -9,6 +9,7 @@ require_relative '../lib/asset_sources' require_relative '../lib/identity_config' +require_relative '../lib/load_disposable_domain' require_relative '../lib/fingerprinter' require_relative '../lib/identity_job_log_subscriber' require_relative '../lib/email_delivery_observer' diff --git a/disposable_domains/disposable-domains.txt b/disposable_domains/disposable-domains.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/load_disposable_domain.rb b/lib/load_disposable_domain.rb new file mode 100644 index 00000000000..17041297bd5 --- /dev/null +++ b/lib/load_disposable_domain.rb @@ -0,0 +1,7 @@ +class LoadDisposableDomain + def self.load_disposable_domains(url) + Faraday.get(url).body.each_line do |line| + DisposableDomain.find_or_create_by(name: line) + end + end +end diff --git a/scripts/load_email_data.rb b/scripts/load_email_data.rb deleted file mode 100644 index 632065de034..00000000000 --- a/scripts/load_email_data.rb +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env ruby -require 'faraday' - -response = Faraday.get("https://raw.githubusercontent.com/fnando/email_data/46d7d9aab2cd7f5c26709f27c02932f7c19f59d0/data/disposable_domains.txt") - -disposable_domains_file="disposable_domains/disposable-domains.txt" From 04eaf69326d721721a92d8b3152fd295b247932d Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Tue, 31 Oct 2023 09:44:26 -0400 Subject: [PATCH 06/13] rubocop --- lib/load_disposable_domain.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/load_disposable_domain.rb b/lib/load_disposable_domain.rb index 17041297bd5..8aca1cf9c66 100644 --- a/lib/load_disposable_domain.rb +++ b/lib/load_disposable_domain.rb @@ -2,6 +2,6 @@ class LoadDisposableDomain def self.load_disposable_domains(url) Faraday.get(url).body.each_line do |line| DisposableDomain.find_or_create_by(name: line) - end + end end end From 84c05034a0390a419ca8a1cfa877d026c322c8d3 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Thu, 2 Nov 2023 09:01:48 -0400 Subject: [PATCH 07/13] domain pull --- lib/load_disposable_domain.rb | 7 ------- lib/tasks/disposable_domains.rake | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 lib/load_disposable_domain.rb create mode 100644 lib/tasks/disposable_domains.rake diff --git a/lib/load_disposable_domain.rb b/lib/load_disposable_domain.rb deleted file mode 100644 index 8aca1cf9c66..00000000000 --- a/lib/load_disposable_domain.rb +++ /dev/null @@ -1,7 +0,0 @@ -class LoadDisposableDomain - def self.load_disposable_domains(url) - Faraday.get(url).body.each_line do |line| - DisposableDomain.find_or_create_by(name: line) - end - end -end diff --git a/lib/tasks/disposable_domains.rake b/lib/tasks/disposable_domains.rake new file mode 100644 index 00000000000..f3fee44fa83 --- /dev/null +++ b/lib/tasks/disposable_domains.rake @@ -0,0 +1,7 @@ +namespace :disposable_domains do + task :load, %i[s3_url] => [:environment] do |_task, args| + file = Identity::Hostdata.secrets_s3.read_file(args[:s3_url]) + DisposableDomain.insert_all(file) + end +end +# rake "disposable_domains:load" From 5b4e733c824361a3311a39db546e29378cb9b11f Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Thu, 2 Nov 2023 10:39:42 -0400 Subject: [PATCH 08/13] fix rake and application.rb --- config/application.rb | 1 - lib/tasks/disposable_domains.rake | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config/application.rb b/config/application.rb index 9db407209a3..00ac6b48093 100644 --- a/config/application.rb +++ b/config/application.rb @@ -9,7 +9,6 @@ require_relative '../lib/asset_sources' require_relative '../lib/identity_config' -require_relative '../lib/load_disposable_domain' require_relative '../lib/fingerprinter' require_relative '../lib/identity_job_log_subscriber' require_relative '../lib/email_delivery_observer' diff --git a/lib/tasks/disposable_domains.rake b/lib/tasks/disposable_domains.rake index f3fee44fa83..6f7c2f3a45d 100644 --- a/lib/tasks/disposable_domains.rake +++ b/lib/tasks/disposable_domains.rake @@ -4,4 +4,4 @@ namespace :disposable_domains do DisposableDomain.insert_all(file) end end -# rake "disposable_domains:load" +# rake "disposable_domains:load['URL_HERE']" From 3c9baed23cd97c32c55a9568628c4c0ab105103d Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Thu, 2 Nov 2023 14:59:33 -0400 Subject: [PATCH 09/13] use script in csv format --- lib/tasks/disposable_domains.rake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/tasks/disposable_domains.rake b/lib/tasks/disposable_domains.rake index 6f7c2f3a45d..de4411fd964 100644 --- a/lib/tasks/disposable_domains.rake +++ b/lib/tasks/disposable_domains.rake @@ -1,7 +1,11 @@ +# rubocop:disable Rails/SkipsModelValidations +require 'csv' namespace :disposable_domains do task :load, %i[s3_url] => [:environment] do |_task, args| file = Identity::Hostdata.secrets_s3.read_file(args[:s3_url]) - DisposableDomain.insert_all(file) + csv_of_file = CSV.parse(file) + DisposableDomain.insert_all(csv_of_file.map(&:to_h)) end end # rake "disposable_domains:load['URL_HERE']" +# rubocop:enable Rails/SkipsModelValidations From 9ca06e73811c29b7a12b5dc3f17b6a8ca0fc4e0a Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Thu, 2 Nov 2023 15:33:56 -0400 Subject: [PATCH 10/13] make sure header is true --- lib/tasks/disposable_domains.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/disposable_domains.rake b/lib/tasks/disposable_domains.rake index de4411fd964..3142c7996b2 100644 --- a/lib/tasks/disposable_domains.rake +++ b/lib/tasks/disposable_domains.rake @@ -3,7 +3,7 @@ require 'csv' namespace :disposable_domains do task :load, %i[s3_url] => [:environment] do |_task, args| file = Identity::Hostdata.secrets_s3.read_file(args[:s3_url]) - csv_of_file = CSV.parse(file) + csv_of_file = CSV.parse(file, headers: true) DisposableDomain.insert_all(csv_of_file.map(&:to_h)) end end From 636b3320f60a7dc7b3dac5698743ffa501fdaeb9 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Fri, 3 Nov 2023 09:56:00 -0400 Subject: [PATCH 11/13] update schema --- db/schema.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 66ce355c079..2e6af469c17 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_10_24_172229) do +ActiveRecord::Schema[7.1].define(version: 2023_10_24_172229) do # These are extensions that must be enabled in order to support this database enable_extension "citext" enable_extension "pg_stat_statements" @@ -656,7 +656,7 @@ add_foreign_key "iaa_gtcs", "partner_accounts" add_foreign_key "iaa_orders", "iaa_gtcs" add_foreign_key "in_person_enrollments", "profiles" - add_foreign_key "in_person_enrollments", "service_providers", column: "issuer", primary_key: "issuer" + add_foreign_key "in_person_enrollments", "service_providers", column: "issuer", primary_key: "issuer", validate: false add_foreign_key "in_person_enrollments", "users" add_foreign_key "integration_usages", "iaa_orders" add_foreign_key "integration_usages", "integrations" From 0134d1bcd387c94031bbbfbc8e67ebeaf427d7ef Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Mon, 6 Nov 2023 10:27:41 -0500 Subject: [PATCH 12/13] remove schema change that wasnt needed --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 2e6af469c17..bb0e7352e81 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -656,7 +656,7 @@ add_foreign_key "iaa_gtcs", "partner_accounts" add_foreign_key "iaa_orders", "iaa_gtcs" add_foreign_key "in_person_enrollments", "profiles" - add_foreign_key "in_person_enrollments", "service_providers", column: "issuer", primary_key: "issuer", validate: false + add_foreign_key "in_person_enrollments", "service_providers", column: "issuer", primary_key: "issuer" add_foreign_key "in_person_enrollments", "users" add_foreign_key "integration_usages", "iaa_orders" add_foreign_key "integration_usages", "integrations" From a3ad00eb450cbefd05b24b208b430905843601a2 Mon Sep 17 00:00:00 2001 From: Malick Diarra Date: Tue, 7 Nov 2023 09:54:47 -0500 Subject: [PATCH 13/13] update rake to deal with text file --- lib/tasks/disposable_domains.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/disposable_domains.rake b/lib/tasks/disposable_domains.rake index 3142c7996b2..665f40cc163 100644 --- a/lib/tasks/disposable_domains.rake +++ b/lib/tasks/disposable_domains.rake @@ -3,8 +3,8 @@ require 'csv' namespace :disposable_domains do task :load, %i[s3_url] => [:environment] do |_task, args| file = Identity::Hostdata.secrets_s3.read_file(args[:s3_url]) - csv_of_file = CSV.parse(file, headers: true) - DisposableDomain.insert_all(csv_of_file.map(&:to_h)) + names = file.split("\n") + DisposableDomain.insert_all(names.map { |name| { name: } }) end end # rake "disposable_domains:load['URL_HERE']"