LG-228 Make migrations safer and more resilient

Gemfile

@@ -85,6 +85,7 @@ group :development, :test do
   gem 'pry-byebug'
   gem 'rspec-rails', '~> 3.5.2'
   gem 'slim_lint'
+  gem 'strong_migrations'
   gem 'thin'
 end
 

Gemfile.lock

@@ -579,6 +579,8 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     stringex (2.8.4)
+    strong_migrations (0.2.2)
+      activerecord (>= 3.2.0)
     sysexits (1.2.0)
     systemu (2.6.5)
     temple (0.8.0)
@@ -744,6 +746,7 @@ DEPENDENCIES
   slim-rails
   slim_lint
   stringex
+  strong_migrations
   thin
   timecop
   twilio-ruby

config/application.yml.example

@@ -84,6 +84,7 @@ development:
   database_pool_worker: '5'
   database_readonly_password: ''
   database_readonly_username: ''
+  database_statement_timeout: '2500'
   database_timeout: '5000'
   database_username: ''
   domain_name: 'localhost:3000'
@@ -175,6 +176,7 @@ production:
   basic_auth_password:
   disable_email_sending: 'false'
   dashboard_api_token:
+  database_statement_timeout: '2500'
   domain_name: 'login.gov'
   enable_agency_based_uuids: 'true'
   enable_identity_verification: 'false'
@@ -268,6 +270,7 @@ test:
   database_pool_worker:
   database_readonly_password: ''
   database_readonly_username: ''
+  database_statement_timeout: '2500'
   database_timeout: '5000'
   database_username: ''
   dashboard_api_token: '123ABC'

config/database.yml

@@ -21,7 +21,7 @@ defaults: &defaults
   checkout_timeout: 5
   reaping_frequency: 10
   variables:
-    statement_timeout: 2500 # ms
+    statement_timeout: <%= Figaro.env.database_statement_timeout.to_i %>
 
 development:
   <<: *defaults

config/initializers/figaro.rb

@@ -3,6 +3,7 @@
 Figaro.require_keys(
   'attribute_cost',
   'attribute_encryption_key',
+  'database_statement_timeout',
   'domain_name',
   'enable_agency_based_uuids',
   'enable_identity_verification',

config/initializers/safe_migrations.rb

@@ -0,0 +1,5 @@
+require 'deploy/migration_statement_timeout'
+
+ActiveSupport.on_load(:active_record) do
+  ActiveRecord::Migration.prepend(Deploy::MigrationStatementTimeout)
+end

deploy/migrate

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# This script is called by identity-devops cookbooks as part of the deployment
+# process. It runs any pending migrations.
+
+set -euo pipefail
+
+echo "deploy/migrate starting"
+echo "HOME: ${HOME-}"
+cd "$(dirname "$0")/.."
+
+set -x
+
+id
+which bundle
+
+export RAILS_ENV=production
+export MIGRATION_STATEMENT_TIMEOUT=60000
+
+bundle exec rake db:create db:migrate db:seed --trace
+
+set +x
+
+echo "deploy/migrate finished"

lib/deploy/migration_statement_timeout.rb

@@ -0,0 +1,13 @@
+module Deploy
+  module MigrationStatementTimeout
+    def connection
+      connection = super
+      new_statement_timeout = ENV['MIGRATION_STATEMENT_TIMEOUT']
+      if new_statement_timeout && !@migration_statement_timeout_set
+        connection.execute("SET statement_timeout = #{new_statement_timeout.to_i}")
+        @migration_statement_timeout_set = true
+      end
+      connection
+    end
+  end
+end

spec/spec_helper.rb

@@ -7,6 +7,7 @@
     add_filter '/config/'
     add_filter '/lib/rspec/formatters/user_flow_formatter.rb'
     add_filter '/lib/user_flow_exporter.rb'
+    add_filter '/lib/deploy/migration_statement_timeout.rb'
   end
 end