-
Notifications
You must be signed in to change notification settings - Fork 166
Use a separate DB for worker jobs (LG-4905) #5275
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
6d828d2
64636d6
f806ecc
4cfdf14
ba3da14
40250fa
8546128
9860af1
35c279d
0e41920
090fd41
c5ea888
36c1d5b
fce66cb
1b8023e
1c623b5
2922938
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # rubocop:disable Rails/ApplicationRecord | ||
| class WorkerJobApplicationRecord < ActiveRecord::Base | ||
| self.abstract_class = true | ||
|
|
||
| connects_to database: { writing: :worker_jobs, reading: :worker_jobs } | ||
| end | ||
| # rubocop:enable Rails/ApplicationRecord | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,9 +24,14 @@ defaults: &defaults | |
| development: | ||
| primary: | ||
| <<: *defaults | ||
| migrations_paths: db/primary_migrate | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the default is |
||
| read_replica: | ||
| <<: *defaults | ||
| replica: true | ||
| worker_jobs: | ||
| <<: *defaults | ||
| database: upaya_worker_jobs_<%= Rails.env %> | ||
| migrations_paths: db/worker_jobs_migrate | ||
|
|
||
| test: | ||
| primary: &test | ||
|
|
@@ -36,9 +41,14 @@ test: | |
| database: <%= ENV['POSTGRES_DB'] || "upaya_test#{ENV['TEST_ENV_NUMBER']}" %> | ||
| user: <%= ENV['POSTGRES_USER'] %> | ||
| password: <%= ENV['POSTGRES_PASSWORD'] %> | ||
| migrations_paths: db/primary_migrate | ||
| read_replica: | ||
| <<: *test | ||
| replica: true | ||
| worker_jobs: | ||
| <<: *test | ||
| database: <%= "upaya_worker_jobs_test#{ENV['TEST_ENV_NUMBER']}" %> | ||
| migrations_paths: db/worker_jobs_migrate | ||
|
|
||
| <% | ||
| pool = if Identity::Hostdata.instance_role == 'worker' | ||
|
|
@@ -58,6 +68,7 @@ production: | |
| pool: <%= pool %> | ||
| sslmode: 'verify-full' | ||
| sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem' | ||
| migrations_paths: db/primary_migrate | ||
| read_replica: | ||
| <<: *defaults | ||
| database: <%= IdentityConfig.store.database_name %> | ||
|
|
@@ -68,3 +79,13 @@ production: | |
| sslmode: 'verify-full' | ||
| sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem' | ||
| replica: true | ||
| worker_jobs: | ||
| <<: *defaults | ||
| database: <%= IdentityConfig.store.database_worker_jobs_name %> | ||
| username: <%= IdentityConfig.store.database_worker_jobs_username %> | ||
| host: <%= IdentityConfig.store.database_worker_jobs_host %> | ||
| password: <%= IdentityConfig.store.database_worker_jobs_password %> | ||
| pool: <%= pool %> | ||
| sslmode: 'verify-full' | ||
| sslrootcert: '/usr/local/share/aws/rds-combined-ca-bundle.pem' | ||
| migrations_paths: db/worker_jobs_migrate | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,8 +71,8 @@ | |
|
|
||
| create_table "backup_code_configurations", force: :cascade do |t| | ||
| t.integer "user_id", null: false | ||
| t.string "encrypted_code", default: "", null: false | ||
| t.string "code_fingerprint", default: "", null: false | ||
| t.string "encrypted_code", default: "", null: false | ||
| t.datetime "used_at" | ||
|
Comment on lines
74
to
76
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is just from doing a full drop/recreat, basically a true-up of this file |
||
| t.datetime "created_at", null: false | ||
| t.datetime "updated_at", null: false | ||
|
|
@@ -178,12 +178,12 @@ | |
| t.integer "choose_method_view_count", default: 0 | ||
| t.datetime "present_cac_view_at" | ||
| t.integer "present_cac_view_count", default: 0 | ||
| t.integer "present_cac_submit_count", default: 0 | ||
| t.integer "present_cac_error_count", default: 0 | ||
| t.datetime "enter_info_view_at" | ||
| t.integer "enter_info_view_count", default: 0 | ||
| t.datetime "success_view_at" | ||
| t.integer "success_view_count", default: 0 | ||
| t.integer "present_cac_submit_count", default: 0 | ||
| t.integer "present_cac_error_count", default: 0 | ||
| t.datetime "selfie_view_at" | ||
| t.integer "selfie_view_count", default: 0 | ||
| t.integer "selfie_submit_count", default: 0 | ||
|
|
@@ -244,7 +244,7 @@ | |
| t.index ["user_id", "last_sign_in_at"], name: "index_email_addresses_on_user_id_and_last_sign_in_at", order: { last_sign_in_at: :desc } | ||
| end | ||
|
|
||
| create_table "events", force: :cascade do |t| | ||
| create_table "events", id: :serial, force: :cascade do |t| | ||
| t.integer "user_id", null: false | ||
| t.integer "event_type", null: false | ||
| t.datetime "created_at", null: false | ||
|
|
@@ -305,7 +305,7 @@ | |
| t.check_constraint "start_date IS NOT NULL", name: "iaa_orders_start_date_null" | ||
| end | ||
|
|
||
| create_table "identities", force: :cascade do |t| | ||
| create_table "identities", id: :serial, force: :cascade do |t| | ||
| t.string "service_provider", limit: 255 | ||
| t.datetime "last_authenticated_at" | ||
| t.integer "user_id" | ||
|
|
@@ -399,7 +399,7 @@ | |
| t.index ["issuer", "ial", "year_month", "user_id"], name: "index_monthly_sp_auth_counts_on_issuer_ial_month_user_id", unique: true | ||
| end | ||
|
|
||
| create_table "otp_requests_trackers", force: :cascade do |t| | ||
| create_table "otp_requests_trackers", id: :serial, force: :cascade do |t| | ||
| t.datetime "otp_last_sent_at" | ||
| t.integer "otp_send_count", default: 0 | ||
| t.string "attribute_cost" | ||
|
|
@@ -458,7 +458,7 @@ | |
| t.index ["x509_dn_uuid"], name: "index_piv_cac_configurations_on_x509_dn_uuid", unique: true | ||
| end | ||
|
|
||
| create_table "profiles", force: :cascade do |t| | ||
| create_table "profiles", id: :serial, force: :cascade do |t| | ||
| t.integer "user_id", null: false | ||
| t.boolean "active", default: false, null: false | ||
| t.datetime "verified_at" | ||
|
|
@@ -544,7 +544,7 @@ | |
| t.index ["issuer", "ial"], name: "index_service_provider_quota_limits_on_issuer_and_ial", unique: true | ||
| end | ||
|
|
||
| create_table "service_providers", force: :cascade do |t| | ||
| create_table "service_providers", id: :serial, force: :cascade do |t| | ||
| t.string "issuer", null: false | ||
| t.string "friendly_name" | ||
| t.text "description" | ||
|
|
@@ -572,8 +572,8 @@ | |
| t.string "push_notification_url" | ||
| t.jsonb "help_text", default: {"sign_in"=>{}, "sign_up"=>{}, "forgot_password"=>{}} | ||
| t.boolean "allow_prompt_login", default: false | ||
| t.boolean "signed_response_message_requested", default: false | ||
| t.integer "ial2_quota" | ||
| t.boolean "signed_response_message_requested", default: false | ||
| t.boolean "liveness_checking_required" | ||
| t.string "remote_logo_key" | ||
| t.date "launch_date" | ||
|
|
@@ -621,7 +621,7 @@ | |
| t.index ["user_id", "throttle_type"], name: "index_throttles_on_user_id_and_throttle_type" | ||
| end | ||
|
|
||
| create_table "users", force: :cascade do |t| | ||
| create_table "users", id: :serial, force: :cascade do |t| | ||
| t.string "reset_password_token", limit: 255 | ||
| t.datetime "reset_password_sent_at" | ||
| t.datetime "remember_created_at" | ||
|
|
@@ -665,15 +665,15 @@ | |
| create_table "usps_confirmation_codes", force: :cascade do |t| | ||
| t.integer "profile_id", null: false | ||
| t.string "otp_fingerprint", null: false | ||
| t.datetime "code_sent_at", default: -> { "now()" }, null: false | ||
| t.datetime "code_sent_at", default: -> { "CURRENT_TIMESTAMP" }, null: false | ||
| t.datetime "created_at", null: false | ||
| t.datetime "updated_at", null: false | ||
| t.datetime "bounced_at" | ||
| t.index ["otp_fingerprint"], name: "index_usps_confirmation_codes_on_otp_fingerprint" | ||
| t.index ["profile_id"], name: "index_usps_confirmation_codes_on_profile_id" | ||
| end | ||
|
|
||
| create_table "usps_confirmations", force: :cascade do |t| | ||
| create_table "usps_confirmations", id: :serial, force: :cascade do |t| | ||
| t.text "entry", null: false | ||
| t.datetime "created_at", null: false | ||
| t.datetime "updated_at", null: false | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # same as 20210804180216_create_good_jobs.rb but runs in the worker database | ||
| class CreateGoodJobsSeparateDatabase < ActiveRecord::Migration[5.2] | ||
|
Comment on lines
+1
to
+2
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. note in this one it's in
zachmargolis marked this conversation as resolved.
|
||
| def change | ||
| enable_extension 'pgcrypto' | ||
|
|
||
| create_table :good_jobs, id: :uuid do |t| | ||
| t.text :queue_name | ||
| t.integer :priority | ||
| t.jsonb :serialized_params | ||
| t.timestamp :scheduled_at | ||
| t.timestamp :performed_at | ||
| t.timestamp :finished_at | ||
| t.text :error | ||
|
|
||
| t.timestamps | ||
|
|
||
| t.uuid :active_job_id | ||
| t.text :concurrency_key | ||
| t.text :cron_key | ||
| end | ||
|
|
||
| add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at" | ||
| add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at | ||
| add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at | ||
| add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished | ||
| add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at | ||
| end | ||
| end | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # This file is auto-generated from the current state of the database. Instead | ||
| # of editing this file, please use the migrations feature of Active Record to | ||
| # incrementally modify your database, and then regenerate this schema definition. | ||
| # | ||
| # This file is the source Rails uses to define your schema when running `bin/rails | ||
| # db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to | ||
| # be faster and is potentially less error prone than running all of your | ||
| # migrations from scratch. Old migrations may fail to apply correctly if those | ||
| # migrations use external dependencies or application code. | ||
| # | ||
| # It's strongly recommended that you check this file into your version control system. | ||
|
|
||
| ActiveRecord::Schema.define(version: 2021_08_09_201322) do | ||
|
|
||
| # These are extensions that must be enabled in order to support this database | ||
| enable_extension "pgcrypto" | ||
| enable_extension "plpgsql" | ||
|
|
||
| create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| | ||
| t.text "queue_name" | ||
| t.integer "priority" | ||
| t.jsonb "serialized_params" | ||
| t.datetime "scheduled_at" | ||
| t.datetime "performed_at" | ||
| t.datetime "finished_at" | ||
| t.text "error" | ||
| t.datetime "created_at", null: false | ||
| t.datetime "updated_at", null: false | ||
| t.uuid "active_job_id" | ||
| t.text "concurrency_key" | ||
| t.text "cron_key" | ||
| t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" | ||
| t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" | ||
| t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" | ||
| t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" | ||
| t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" | ||
| end | ||
|
|
||
| end |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we need coverage here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this class is actually used, the GoodJob::Job dynamically subclasses it, however that may be set up before the code coverage is set up
If this class was not used, GoodJob would look in the wrong DB for its table