From 9f822dc5c693b5306ad9991c593a88b75a6a2e9d Mon Sep 17 00:00:00 2001 From: Paul Hirsch Date: Wed, 28 Dec 2022 17:43:29 -0600 Subject: [PATCH 1/3] Add RedisSessionHealthChecker changelog: Internal, Platform, Improve health checking Adds a check to attempt writing to the Redis session store then reading the key back. A default TTL of 1 second is used. The check will fail if: * Redis is not writeable * The value can not be read back from Redis * It takes more than 1 second to read back from Redis --- app/controllers/health/health_controller.rb | 1 + .../health/redis_session_controller.rb | 9 ++++++ app/services/redis_session_health_checker.rb | 30 +++++++++++++++++ .../redis_session_health_checker_spec.rb | 32 +++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 app/controllers/health/redis_session_controller.rb create mode 100644 app/services/redis_session_health_checker.rb create mode 100644 spec/services/redis_session_health_checker_spec.rb diff --git a/app/controllers/health/health_controller.rb b/app/controllers/health/health_controller.rb index 99927729aca..4990c263e00 100644 --- a/app/controllers/health/health_controller.rb +++ b/app/controllers/health/health_controller.rb @@ -5,6 +5,7 @@ class HealthController < AbstractHealthController def health_checker checkers = { database: DatabaseHealthChecker, + redis_session: RedisSessionHealthChecker, account_reset: AccountResetHealthChecker, } MultiHealthChecker.new(**checkers) diff --git a/app/controllers/health/redis_session_controller.rb b/app/controllers/health/redis_session_controller.rb new file mode 100644 index 00000000000..1cec418f771 --- /dev/null +++ b/app/controllers/health/redis_session_controller.rb @@ -0,0 +1,9 @@ +module Health + class RedisSessionController < AbstractHealthController + private + + def health_checker + RedisSessionHealthChecker + end + end +end diff --git a/app/services/redis_session_health_checker.rb b/app/services/redis_session_health_checker.rb new file mode 100644 index 00000000000..e7e23dd6f39 --- /dev/null +++ b/app/services/redis_session_health_checker.rb @@ -0,0 +1,30 @@ +module RedisSessionHealthChecker + module_function + + # @return [HealthCheckSummary] + def check + HealthCheckSummary.new(healthy: true, result: health_write_and_read) + rescue StandardError => err + NewRelic::Agent.notice_error(err) + HealthCheckSummary.new(healthy: false, result: err.message) + end + + # @api private + def health_write_and_read + REDIS_POOL.with do |client| + client.setex(health_record_key, health_record_ttl, "healthy at " + Time.now.iso8601) + client.get(health_record_key) or raise "Unable to read back #{health_record_key} from Redis" + end + end + + # @api private + def health_record_key + "healthcheck_" + Socket.gethostname + end + + # @api private + def health_record_ttl + # If we can't read back within a second that is just unacceptable + 1 + end +end diff --git a/spec/services/redis_session_health_checker_spec.rb b/spec/services/redis_session_health_checker_spec.rb new file mode 100644 index 00000000000..9a260aa09a7 --- /dev/null +++ b/spec/services/redis_session_health_checker_spec.rb @@ -0,0 +1,32 @@ +require 'rails_helper' + +RSpec.describe RedisSessionHealthChecker do + describe '.check' do + subject(:summary) { RedisSessionHealthChecker.check } + + context 'when the redis session store is healthy' do + it 'returns a healthy check' do + expect(summary.healthy).to eq(true) + expect(summary.result).to start_with("healthy at ") + end + end + + context 'when the redis session store is unhealthy' do + before do + expect(RedisSessionHealthChecker).to receive(:simple_query). + and_raise(RuntimeError.new('canceling statement due to statement timeout')) + end + + it 'returns an unhealthy check' do + expect(summary.healthy).to eq(false) + expect(summary.result).to include('canceling statement due to statement timeout') + end + + it 'notifies NewRelic' do + expect(NewRelic::Agent).to receive(:notice_error) + + summary + end + end + end +end From 9770981f5f129c41b3a5803fd61dee5e20d73010 Mon Sep 17 00:00:00 2001 From: Paul Hirsch <59626817+pauldoomgov@users.noreply.github.com> Date: Fri, 30 Dec 2022 11:47:32 -0600 Subject: [PATCH 2/3] Update app/services/redis_session_health_checker.rb Co-authored-by: Mitchell Henke --- app/services/redis_session_health_checker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/redis_session_health_checker.rb b/app/services/redis_session_health_checker.rb index e7e23dd6f39..d8d979ab6ff 100644 --- a/app/services/redis_session_health_checker.rb +++ b/app/services/redis_session_health_checker.rb @@ -13,7 +13,7 @@ def check def health_write_and_read REDIS_POOL.with do |client| client.setex(health_record_key, health_record_ttl, "healthy at " + Time.now.iso8601) - client.get(health_record_key) or raise "Unable to read back #{health_record_key} from Redis" + client.get(health_record_key) end end From 9c94c19bbef9b8409baea957236e5af50273edf1 Mon Sep 17 00:00:00 2001 From: Paul Hirsch <59626817+pauldoomgov@users.noreply.github.com> Date: Fri, 30 Dec 2022 11:47:41 -0600 Subject: [PATCH 3/3] Update app/services/redis_session_health_checker.rb Co-authored-by: Mitchell Henke --- app/services/redis_session_health_checker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/redis_session_health_checker.rb b/app/services/redis_session_health_checker.rb index d8d979ab6ff..9fef24d9609 100644 --- a/app/services/redis_session_health_checker.rb +++ b/app/services/redis_session_health_checker.rb @@ -3,7 +3,7 @@ module RedisSessionHealthChecker # @return [HealthCheckSummary] def check - HealthCheckSummary.new(healthy: true, result: health_write_and_read) + HealthCheckSummary.new(healthy: health_write_and_read.present?, result: health_write_and_read) rescue StandardError => err NewRelic::Agent.notice_error(err) HealthCheckSummary.new(healthy: false, result: err.message)