Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .rubocop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ require:
AllCops:
Exclude:
- 'bin/**/*'
- 'db/migrate/*'
- 'db/primary_migrate/*'
- 'db/schema.rb'
- 'db/worker_jobs_migrate/*'
- 'db/worker_jobs_schema.rb'
- 'lib/tasks/create_test_accounts.rb'
- 'node_modules/**/*'
- 'tmp/**/*'
Expand Down
7 changes: 7 additions & 0 deletions app/models/worker_job_application_record.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# rubocop:disable Rails/ApplicationRecord
class WorkerJobApplicationRecord < ActiveRecord::Base
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need coverage here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this class is actually used, the GoodJob::Job dynamically subclasses it, however that may be set up before the code coverage is set up

If this class was not used, GoodJob would look in the wrong DB for its table

self.abstract_class = true

connects_to database: { writing: :worker_jobs, reading: :worker_jobs }
end
# rubocop:enable Rails/ApplicationRecord
8 changes: 8 additions & 0 deletions config/application.yml.default
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,10 @@ development:
database_readonly_password: ''
database_readonly_username: ''
database_username: ''
database_worker_jobs_name: ''
database_worker_jobs_username: ''
database_worker_jobs_host: ''
database_worker_jobs_password: ''
doc_auth_vendor: 'mock'
domain_name: localhost:3000
enable_rate_limiting: 'false'
Expand Down Expand Up @@ -341,6 +345,10 @@ test:
database_readonly_password: ''
database_readonly_username: ''
database_username: ''
database_worker_jobs_name: ''
database_worker_jobs_username: ''
database_worker_jobs_host: ''
database_worker_jobs_password: ''
doc_auth_enable_presigned_s3_urls: 'true'
doc_auth_vendor: 'mock'
doc_capture_polling_enabled: 'false'
Expand Down
21 changes: 21 additions & 0 deletions config/database.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,14 @@ defaults: &defaults
development:
primary:
<<: *defaults
migrations_paths: db/primary_migrate
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the default is db/migrate but specifying this makes sure we don't run the old migrations in the jobs-only DB, complete separation of DBs

read_replica:
<<: *defaults
replica: true
worker_jobs:
<<: *defaults
database: upaya_worker_jobs_<%= Rails.env %>
migrations_paths: db/worker_jobs_migrate

test:
primary: &test
Expand All @@ -36,9 +41,14 @@ test:
database: <%= ENV['POSTGRES_DB'] || "upaya_test#{ENV['TEST_ENV_NUMBER']}" %>
user: <%= ENV['POSTGRES_USER'] %>
password: <%= ENV['POSTGRES_PASSWORD'] %>
migrations_paths: db/primary_migrate
read_replica:
<<: *test
replica: true
worker_jobs:
<<: *test
database: <%= "upaya_worker_jobs_test#{ENV['TEST_ENV_NUMBER']}" %>
migrations_paths: db/worker_jobs_migrate

<%
pool = if Identity::Hostdata.instance_role == 'worker'
Expand All @@ -58,6 +68,7 @@ production:
pool: <%= pool %>
sslmode: 'verify-full'
sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem'
migrations_paths: db/primary_migrate
read_replica:
<<: *defaults
database: <%= IdentityConfig.store.database_name %>
Expand All @@ -68,3 +79,13 @@ production:
sslmode: 'verify-full'
sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem'
replica: true
worker_jobs:
<<: *defaults
database: <%= IdentityConfig.store.database_worker_jobs_name %>
username: <%= IdentityConfig.store.database_worker_jobs_username %>
host: <%= IdentityConfig.store.database_worker_jobs_host %>
password: <%= IdentityConfig.store.database_worker_jobs_password %>
pool: <%= pool %>
sslmode: 'verify-full'
sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem'
migrations_paths: db/worker_jobs_migrate
1 change: 1 addition & 0 deletions config/initializers/good_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
# see config/initializers/job_configurations.rb for cron schedule
end

GoodJob.active_record_parent_class = 'WorkerJobApplicationRecord'
GoodJob.retry_on_unhandled_error = false
GoodJob.on_thread_error = ->(exception) { NewRelic::Agent.notice_error(exception) }
end
24 changes: 12 additions & 12 deletions db/schema.rb
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@

create_table "backup_code_configurations", force: :cascade do |t|
t.integer "user_id", null: false
t.string "encrypted_code", default: "", null: false
t.string "code_fingerprint", default: "", null: false
t.string "encrypted_code", default: "", null: false
t.datetime "used_at"
Comment on lines 74 to 76
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is just from doing a full drop/recreat, basically a true-up of this file

t.datetime "created_at", null: false
t.datetime "updated_at", null: false
Expand Down Expand Up @@ -178,12 +178,12 @@
t.integer "choose_method_view_count", default: 0
t.datetime "present_cac_view_at"
t.integer "present_cac_view_count", default: 0
t.integer "present_cac_submit_count", default: 0
t.integer "present_cac_error_count", default: 0
t.datetime "enter_info_view_at"
t.integer "enter_info_view_count", default: 0
t.datetime "success_view_at"
t.integer "success_view_count", default: 0
t.integer "present_cac_submit_count", default: 0
t.integer "present_cac_error_count", default: 0
t.datetime "selfie_view_at"
t.integer "selfie_view_count", default: 0
t.integer "selfie_submit_count", default: 0
Expand Down Expand Up @@ -244,7 +244,7 @@
t.index ["user_id", "last_sign_in_at"], name: "index_email_addresses_on_user_id_and_last_sign_in_at", order: { last_sign_in_at: :desc }
end

create_table "events", force: :cascade do |t|
create_table "events", id: :serial, force: :cascade do |t|
t.integer "user_id", null: false
t.integer "event_type", null: false
t.datetime "created_at", null: false
Expand Down Expand Up @@ -305,7 +305,7 @@
t.check_constraint "start_date IS NOT NULL", name: "iaa_orders_start_date_null"
end

create_table "identities", force: :cascade do |t|
create_table "identities", id: :serial, force: :cascade do |t|
t.string "service_provider", limit: 255
t.datetime "last_authenticated_at"
t.integer "user_id"
Expand Down Expand Up @@ -399,7 +399,7 @@
t.index ["issuer", "ial", "year_month", "user_id"], name: "index_monthly_sp_auth_counts_on_issuer_ial_month_user_id", unique: true
end

create_table "otp_requests_trackers", force: :cascade do |t|
create_table "otp_requests_trackers", id: :serial, force: :cascade do |t|
t.datetime "otp_last_sent_at"
t.integer "otp_send_count", default: 0
t.string "attribute_cost"
Expand Down Expand Up @@ -458,7 +458,7 @@
t.index ["x509_dn_uuid"], name: "index_piv_cac_configurations_on_x509_dn_uuid", unique: true
end

create_table "profiles", force: :cascade do |t|
create_table "profiles", id: :serial, force: :cascade do |t|
t.integer "user_id", null: false
t.boolean "active", default: false, null: false
t.datetime "verified_at"
Expand Down Expand Up @@ -544,7 +544,7 @@
t.index ["issuer", "ial"], name: "index_service_provider_quota_limits_on_issuer_and_ial", unique: true
end

create_table "service_providers", force: :cascade do |t|
create_table "service_providers", id: :serial, force: :cascade do |t|
t.string "issuer", null: false
t.string "friendly_name"
t.text "description"
Expand Down Expand Up @@ -572,8 +572,8 @@
t.string "push_notification_url"
t.jsonb "help_text", default: {"sign_in"=>{}, "sign_up"=>{}, "forgot_password"=>{}}
t.boolean "allow_prompt_login", default: false
t.boolean "signed_response_message_requested", default: false
t.integer "ial2_quota"
t.boolean "signed_response_message_requested", default: false
t.boolean "liveness_checking_required"
t.string "remote_logo_key"
t.date "launch_date"
Expand Down Expand Up @@ -621,7 +621,7 @@
t.index ["user_id", "throttle_type"], name: "index_throttles_on_user_id_and_throttle_type"
end

create_table "users", force: :cascade do |t|
create_table "users", id: :serial, force: :cascade do |t|
t.string "reset_password_token", limit: 255
t.datetime "reset_password_sent_at"
t.datetime "remember_created_at"
Expand Down Expand Up @@ -665,15 +665,15 @@
create_table "usps_confirmation_codes", force: :cascade do |t|
t.integer "profile_id", null: false
t.string "otp_fingerprint", null: false
t.datetime "code_sent_at", default: -> { "now()" }, null: false
t.datetime "code_sent_at", default: -> { "CURRENT_TIMESTAMP" }, null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "bounced_at"
t.index ["otp_fingerprint"], name: "index_usps_confirmation_codes_on_otp_fingerprint"
t.index ["profile_id"], name: "index_usps_confirmation_codes_on_profile_id"
end

create_table "usps_confirmations", force: :cascade do |t|
create_table "usps_confirmations", id: :serial, force: :cascade do |t|
t.text "entry", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# same as 20210804180216_create_good_jobs.rb but runs in the worker database
class CreateGoodJobsSeparateDatabase < ActiveRecord::Migration[5.2]
Comment on lines +1 to +2
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note in this one it's in db/worker_jobs_migrate instead of db/migrate or db/primary_migrate

Comment thread
zachmargolis marked this conversation as resolved.
def change
enable_extension 'pgcrypto'

create_table :good_jobs, id: :uuid do |t|
t.text :queue_name
t.integer :priority
t.jsonb :serialized_params
t.timestamp :scheduled_at
t.timestamp :performed_at
t.timestamp :finished_at
t.text :error

t.timestamps

t.uuid :active_job_id
t.text :concurrency_key
t.text :cron_key
end

add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at"
add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at
add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished
add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at
end
end
39 changes: 39 additions & 0 deletions db/worker_jobs_schema.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# This file is auto-generated from the current state of the database. Instead
# of editing this file, please use the migrations feature of Active Record to
# incrementally modify your database, and then regenerate this schema definition.
#
# This file is the source Rails uses to define your schema when running `bin/rails
# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
# be faster and is potentially less error prone than running all of your
# migrations from scratch. Old migrations may fail to apply correctly if those
# migrations use external dependencies or application code.
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema.define(version: 2021_08_09_201322) do

# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"

create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
t.text "queue_name"
t.integer "priority"
t.jsonb "serialized_params"
t.datetime "scheduled_at"
t.datetime "performed_at"
t.datetime "finished_at"
t.text "error"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.uuid "active_job_id"
t.text "concurrency_key"
t.text "cron_key"
t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at"
t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)"
t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at"
t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)"
t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)"
end

end
4 changes: 4 additions & 0 deletions lib/identity_config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ def self.build_store(config_map)
config.add(:database_statement_timeout, type: :integer)
config.add(:database_timeout, type: :integer)
config.add(:database_username, type: :string)
config.add(:database_worker_jobs_name, type: :string)
config.add(:database_worker_jobs_username, type: :string)
config.add(:database_worker_jobs_host, type: :string)
config.add(:database_worker_jobs_password, type: :string)
config.add(:deleted_user_accounts_report_configs, type: :json)
config.add(:disable_email_sending, type: :boolean)
config.add(:disallow_all_web_crawlers, type: :boolean)
Expand Down